const regex = /((\\\()((?!(\\\()|(\\\))).)*(\\\)))/gm;
// Alternative syntax using RegExp constructor
// const regex = new RegExp('((\\\\\\()((?!(\\\\\\()|(\\\\\\))).)*(\\\\\\)))', 'gm')
const str = `y = x2 or \\(x = \\sqrt{y}\\) . For all \\(y \\geq 0\\), x is defined. Thus, y is set of all non-negative and real number. Hence, range is \\(y \\geq 0\\) or [0, ∞).`;
const subst = `<math>$1</math>`;
// The substituted value will be contained in the result variable
const result = str.replace(regex, subst);
console.log('Substitution result: ', result);
Please keep in mind that these code samples are automatically generated and are not guaranteed to work. If you find any syntax errors, feel free to submit a bug report. For a full regex reference for JavaScript, please visit: https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions