#include <MsgBoxConstants.au3> ; to declare the Constants of MsgBox
Local $sRegex = "(?m)({matrix}.*? \\)( .*?(?={matrix}))"
Local $sString = "matrix \dfrac{ \left( \sqrt{ \left(\begin{matrix} 5 \\ \left(\begin{matrix} 66 \ 11 \end{matrix}\right) \ \end{matrix}\right) \phantom{\tiny{!}}} , \sqrt[ 6 ]{ 66 \phantom{\tiny{!}}} \right) }{ 6 } 5 \dfrac{ 22 }{ 5 }"
Local $sSubst = "$1\\$2"
Local $sResult = StringRegExpReplace($sString, $sRegex, $sSubst)
MsgBox($MB_SYSTEMMODAL, "Result", $sResult)
Please keep in mind that these code samples are automatically generated and are not guaranteed to work. If you find any syntax errors, feel free to submit a bug report. For a full regex reference for AutoIt, please visit: https://www.autoitscript.com/autoit3/docs/functions/StringRegExp.htm