JavaScript 正则表达式匹配单词边界与变音符号

Question

I have to match in a text document, words boundaries for words having diacritics. Given a word token, my regex looks like

var wordRegex = new RegExp("\\b(" + word + ")\\b", "g");
while ((match = wordRegex.exec(text)) !== null) {
                            if (match.index > (seen.get(token) || -1)) {
                                var wordStart = match.index;
                                var wordEnd = wordStart + token.length - 1;
                                item.characterOffsetBegin = wordStart;
                                item.characterOffsetEnd = wordEnd;

                                seen.set(token, wordEnd);
                                break;
                            }
                        }

This works ok for ordinary words like ciao , casa , etc. But it will not works when I have in the text words like però , così , etc.

 const seen = new Map(); var text = "Ci son macchine nascoste e, però, nascoste male" var tokens = text.split(/[^a-zA-Z0-9àèéìíîòóùúÀÈÉÌÍÎÒÓÙÚ]+/i) tokens.forEach((token, tokenIndex) => { var item = { "index": (tokenIndex + 1), "word": token } var escaped = token.replace(/[\\-\\[\\]{}()*+?.,\\\\\\^$|#\\s]/g, "\\\\$&"); var wordRegex = new RegExp("\\\\b(" + escaped + ")\\\\b", "g"); var match = null; console.log(token, "---->", wordRegex) while ((match = wordRegex.exec(text)) !== null) { console.log("\\t---->", match.index) if (match.index > (seen.get(token) || -1)) { var wordStart = match.index; var wordEnd = wordStart + token.length - 1; item.characterOffsetBegin = wordStart; item.characterOffsetEnd = wordEnd; seen.set(token, wordEnd); break; } } })

You can see how while some words (like macchine or nascoste ) it matches, so I get the match.index , for other words (like però ) the regex does not work properly and the match variable is null :

macchine ----> /\b(macchine)\b/g
    ----> 7
nascoste ----> /\b(nascoste)\b/g
    ----> 16
e, ----> /\b(e\,)\b/g
però, ----> /\b(però\,)\b/g
nascoste ----> /\b(nascoste)\b/g
    ----> 16
    ----> 34

How to write a boundary regex that supports diacritics too then?

[UPDATE] Following the approach suggested in the comments, I have used diacritics removal for each word token before applying the Regex , and then to the whole text like:

var normalizedText = removeDiacritics(text);
// for each token...
var escaped = token.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g, "\\$&");
escaped = removeDiacritics(escaped);
var wordRegex = new RegExp("\\b(" + escaped + ")\\b", "g");
var match = null;
while ((match = wordRegex.exec( normalizedText )) !== null) 
{
                             //...

and this time I will get the words with accents captured by the \\b word boundaries. Of course this approach is not optimal, because the removeDiacritics must be applied for every token, so the best solution would be to do this once.

Answer 1

This is the solution we came up with in the comments to map words having diacritics to their index in the text:

 function removeDiacritics(text) { return _.deburr(text) } const seen = new Map(); var text = "Ci son macchine nascoste e, però, nascoste male" var tokens = text.split(/[^a-zA-Z0-9àèéìíîòóùúÀÈÉÌÍÎÒÓÙÚ]+/i) var normalizedText = removeDiacritics(text) tokens.forEach((token, tokenIndex) => { var item = { "index": (tokenIndex + 1), "word": removeDiacritics(token) } var escaped = token.replace(/[\\-\\[\\]{}()*+?.,\\\\\\^$|#\\s]/g, "\\\\$&"); escaped = removeDiacritics(escaped) var wordRegex = new RegExp("\\\\b(" + escaped + ")\\\\b", "g"); var match = null; console.log(token, "---->", wordRegex) while ((match = wordRegex.exec(normalizedText)) !== null) { console.log("\\t---->", match.index) if (match.index > (seen.get(token) || -1)) { var wordStart = match.index; var wordEnd = wordStart + token.length - 1; item.characterOffsetBegin = wordStart; item.characterOffsetEnd = wordEnd; seen.set(token, wordEnd); break; } } })

 <script src="https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.11/lodash.min.js"></script>