const regex = /\[((\w+)\b[^]]*)\]((?>(?!\[\2\b).|(?R))*)\[\/\2]/gs;
// Alternative syntax using RegExp constructor
// const regex = new RegExp('\\[((\\w+)\\b[^]]*)\\]((?>(?!\\[\\2\\b).|(?R))*)\\[\\\/\\2]', 'gs')
const str = `[html style="font-size: 18px;" dir="ltr"][div style="font-size: 18px;" dir="ltr"][p style="font-weight: bold;"]Hello,[/p][p]You have got a new message from [a href="https://www.example.com/"]Example.com[/a][br][br].You could check your message on [a href="https://www.example.com/en/manager/inbox.html"]Manager[/a][/p][p][img src="https://www.example.com/assets/images/logo-default-120x50.png" width="120px" height="80px"][div style="color: #D4192D; font-weight: bold;"]Example.com Team[/div][/p][/div][/html]
[div style="font-size: 18px;" dir="ltr"][p style="font-weight: bold;"]Hello,[/p][p]You have got a new message from [a href="https://www.example.com/"]Example.com[/a][br][br].You could check your message on [a href="https://www.example.com/en/manager/inbox.html"]Manager[/a][/p][p][img src="https://www.example.com/assets/images/logo-default-120x50.png" width="120px" height="80px"][div style="color: #D4192D; font-weight: bold;"]Example.com Team[/div][/p][/div]
You have got a new message from [a href="https://www.example.com/"]Example.com[/a][br][br].You could check your message on [a href="https://www.example.com/en/manager/inbox.html"]Manager[/a]`;
const subst = `<$1>$3</$2>`;
// The substituted value will be contained in the result variable
const result = str.replace(regex, subst);
console.log('Substitution result: ', result);
Please keep in mind that these code samples are automatically generated and are not guaranteed to work. If you find any syntax errors, feel free to submit a bug report. For a full regex reference for JavaScript, please visit: https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions