# coding=utf8
# the above tag defines encoding for this document and is for Python 2.x compatibility
import re
regex = r"(?:[^\p{Script=Latin}\s\w;:.,\-[\](){}'\"+\/=<>])\B|(?:(?:[\p{Script=Latin}]|[^\s\w;:.,\-[\](){}'\"+\/=<>]){2,}(?:(?:[\p{Script=Latin}]|[^\s\w+.,–:;\/\\=<>])?(?:[\p{Script=Latin}]|[^\s\w;\-:.,\[\](){}'\"+\/\\=<>]))*)|(?:(?:[0-9\p{Script=Latin}]|[^\s\w;:.,[\](){}\-'\"+\/\\=<>])(?:(?:[0-9\p{Script=Latin}]|[^\s\w+\-:;\/\\=<>])?(?:[0-9\p{Script=Latin}]|[^\s\w;:.,\-[\](){}'\"+\/\\=<>]))*)"
test_str = ("Bão Yagi (được Việt Nam định danh là bão số 3, được phía Philippines đặt tên bão Enteng - tiếng Anh: Severe Tropical Storm Enteng , nguyên văn 'Bão nhiệt đới dữ dội Enteng')\n\n"
"Some legal \"[d]ocuments\" contain corrected spel[l]ing, gram(m)ar, or simple typos; and lots of references[1]. By Extension, I included curl{e}y brackets, but not tag brack<e>ts, which are not seenin modern legal documents.\n\n"
"These.are.properly.separated.even.U.S.A., and multiple punctuations are properly ignored.\n\n"
"A. Multiple\n"
"B. Choices\n\n"
"-ABC-DEF-\n"
"-A-B-C-D-\n"
"-1-2-3-4-\n"
"-test-\n\n"
".ABC.DEF.\n"
".AB.CD.EF.D.\n"
".A.B.C.D.E.\n"
".123.456.789.\n"
".12.34.56.78.\n"
".1.2.3.4.\n\n"
"cod3 var1aBl3s\n"
"test.U.S.A.test\n\n"
"We'd want hyphenated words in cases when large words are broken for wrapping in tight column news papers/megazines, while we can still properly separate numbers such as \"30-35 pages\".\n\n"
"Non-Latin character are separated per character:\n"
"出典: フリー百科事典『ウィキペディア(Wikipedia)』\n"
"ウィキペディアには現在この名前の項目はありません。\n\n"
"Acceptable failed cases:\n\n"
"- [E]xpected: \"[\" is considered external \"wrapper/enclosure\", while the internal \"wrappers\" are included so that they can be further processed/removed in the future.\n\n"
"- test.U.S.A.test: this happens when no space trailing textContents of a block-level elemetn in HTML files.\n\n"
"- 『ウィキペディア(Wikipedia)』: mixed languages\n\n"
"- CamelCasing s not separated, but can be easily separated in \"post-processing step, even though _snake_casing_ works fine by happenstance.\n\n"
"-cod3 var1aBl3s: mixing letters and numbers. It's not intentional, but not a big deal when the side-effect is new word is always started with a number.")
matches = re.finditer(regex, test_str, re.MULTILINE | re.UNICODE | re.IGNORECASE)
for matchNum, match in enumerate(matches, start=1):
print ("Match {matchNum} was found at {start}-{end}: {match}".format(matchNum = matchNum, start = match.start(), end = match.end(), match = match.group()))
for groupNum in range(0, len(match.groups())):
groupNum = groupNum + 1
print ("Group {groupNum} found at {start}-{end}: {group}".format(groupNum = groupNum, start = match.start(groupNum), end = match.end(groupNum), group = match.group(groupNum)))
# Note: for Python 2.7 compatibility, use ur"" to prefix the regex and u"" to prefix the test string and substitution.
Please keep in mind that these code samples are automatically generated and are not guaranteed to work. If you find any syntax errors, feel free to submit a bug report. For a full regex reference for Python, please visit: https://docs.python.org/3/library/re.html