const regex = /({matrix}.*? \\)( .*?(?={matrix}))/gm;
// Alternative syntax using RegExp constructor
// const regex = new RegExp('({matrix}.*? \\\\)( .*?(?={matrix}))', 'gm')
const str = `matrix \\dfrac{ \\left( \\sqrt{ \\left(\\begin{matrix} 5 \\\\ \\left(\\begin{matrix} 66 \\ 11 \\end{matrix}\\right) \\ \\end{matrix}\\right) \\phantom{\\tiny{!}}} , \\sqrt[ 6 ]{ 66 \\phantom{\\tiny{!}}} \\right) }{ 6 } 5 \\dfrac{ 22 }{ 5 }`;
const subst = `$1\\$2`;
// The substituted value will be contained in the result variable
const result = str.replace(regex, subst);
console.log('Substitution result: ', result);
Please keep in mind that these code samples are automatically generated and are not guaranteed to work. If you find any syntax errors, feel free to submit a bug report. For a full regex reference for JavaScript, please visit: https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions