001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      https://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017
018package org.apache.commons.codec.language.bm;
019
020import java.util.ArrayList;
021import java.util.Arrays;
022import java.util.Collections;
023import java.util.EnumMap;
024import java.util.HashSet;
025import java.util.LinkedHashSet;
026import java.util.List;
027import java.util.Locale;
028import java.util.Map;
029import java.util.Objects;
030import java.util.Set;
031import java.util.TreeMap;
032import java.util.function.Supplier;
033import java.util.regex.Pattern;
034import java.util.stream.Collectors;
035
036import org.apache.commons.codec.language.bm.Languages.LanguageSet;
037import org.apache.commons.codec.language.bm.Rule.Phoneme;
038
039/**
040 * Converts words into potential phonetic representations.
041 * <p>
042 * This is a two-stage process. Firstly, the word is converted into a phonetic representation that takes into account the likely source language. Next, this
043 * phonetic representation is converted into a pan-European 'average' representation, allowing comparison between different versions of essentially the same
044 * word from different languages.
045 * </p>
046 * <p>
047 * This class is intentionally immutable and thread-safe. If you wish to alter the settings for a PhoneticEngine, you must make a new one with the updated
048 * settings.
049 * </p>
050 * <p>
051 * Ported from phoneticengine.php
052 * </p>
053 *
054 * @since 1.6
055 */
056public class PhoneticEngine {
057
058    /**
059     * Builder for a PhoneticEngine.
060     *
061     * @since 1.23.0
062     */
063    public static class Builder implements Supplier<PhoneticEngine> {
064
065        /** See https://en.wikipedia.org/wiki/Hubert_Blaine_Wolfeschlegelsteinhausenbergerdorff_Sr. */
066        private static final int MAX_INPUT_LENGTH = 666;
067
068        private static final int MAX_PHONEMES = 20;
069
070        private boolean concat = true;
071
072        private int maxInputLength = MAX_INPUT_LENGTH;
073
074        private int maxPhonemes = MAX_PHONEMES;
075
076        private NameType nameType = NameType.GENERIC;
077
078        private RuleType ruleType = RuleType.APPROX;
079
080        private Builder() {
081            // empty
082        }
083
084        @Override
085        public PhoneticEngine get() {
086            return new PhoneticEngine(this);
087        }
088
089        /**
090         * Sets all the properties of this builder to match those of the given engine.
091         *
092         * @param engine The engine to copy properties from.
093         * @return This builder.
094         */
095        public Builder setAll(final PhoneticEngine engine) {
096            this.nameType = engine.getNameType();
097            this.ruleType = engine.getRuleType();
098            this.concat = engine.isConcat();
099            this.maxPhonemes = engine.getMaxPhonemes();
100            return this;
101        }
102
103        /**
104         * Sets whether the engine will concatenate multiple encodings.
105         *
106         * @param concat Whether the engine will concatenate multiple encodings.
107         * @return This builder.
108         */
109        public Builder setConcat(final boolean concat) {
110            this.concat = concat;
111            return this;
112        }
113
114        /**
115         * Sets the maximum input length allowed.
116         * <p>
117         * A value less than 0 will reset the maximum input length to the default value of {@value #MAX_INPUT_LENGTH}, see
118         * <a href="https://en.wikipedia.org/wiki/Hubert_Blaine_Wolfeschlegelsteinhausenbergerdorff_Sr.">Hubert Blaine Wolfeschlegelsteinhausenbergerdorff
119         * Sr.</a>.
120         * </p>
121         *
122         * @param maxInputLength the maximum input length allowed.
123         * @return This builder.
124         */
125        public Builder setMaxInputLength(final int maxInputLength) {
126            this.maxInputLength = maxInputLength < 0 ? MAX_INPUT_LENGTH : maxInputLength;
127            return this;
128        }
129
130        /**
131         * Sets maximum number of phonemes the engine will handle.
132         * <p>
133         * A value less than 0 will reset the maximum number of phonemes to the default {@value #MAX_PHONEMES}.
134         * </p>
135         *
136         * @param maxPhonemes The maximum number of phonemes the engine will handle.
137         * @return This builder.
138         */
139        public Builder setMaxPhonemes(final int maxPhonemes) {
140            this.maxPhonemes = maxPhonemes < 0 ? MAX_PHONEMES : maxPhonemes;
141            return this;
142        }
143
144        /**
145         * Sets the name type for the engine to be built.
146         * <p>
147         * A null value will reset the name type to the default of {@link NameType#GENERIC}.
148         * </p>
149         *
150         * @param nameType The type of names the engine will use.
151         * @return This builder.
152         */
153        public Builder setNameType(final NameType nameType) {
154            this.nameType = nameType != null ? nameType : NameType.GENERIC;
155            return this;
156        }
157
158        /**
159         * Sets the rule type for the engine to be built.
160         * <p>
161         * A null value will reset the rule type to the default of {@link RuleType#APPROX}.
162         * </p>
163         *
164         * @param ruleType The type of rules the engine will use.
165         * @return This builder.
166         */
167        public Builder setRuleType(final RuleType ruleType) {
168            this.ruleType = ruleType != null ? ruleType : RuleType.APPROX;
169            return this;
170        }
171    }
172
173    /**
174     * Manipulates a set of phonemes as they are being built up. Not intended for use outside this package, and probably not outside the {@link PhoneticEngine}
175     * class.
176     *
177     * @since 1.6
178     */
179    static final class PhonemeBuilder {
180
181        /**
182         * An empty builder where all phonemes must come from some set of languages. This will contain a single phoneme of zero characters. This can then be
183         * appended to. This should be the only way to create a new phoneme from scratch.
184         *
185         * @param languages The set of languages.
186         * @return a new, empty phoneme builder.
187         */
188        public static PhonemeBuilder empty(final Languages.LanguageSet languages) {
189            return new PhonemeBuilder(new Rule.Phoneme("", languages));
190        }
191
192        private final Set<Rule.Phoneme> phonemes;
193
194        private PhonemeBuilder(final Rule.Phoneme phoneme) {
195            this.phonemes = new LinkedHashSet<>();
196            this.phonemes.add(phoneme);
197        }
198
199        private PhonemeBuilder(final Set<Rule.Phoneme> phonemes) {
200            this.phonemes = phonemes;
201        }
202
203        /**
204         * Creates a new phoneme builder containing all phonemes in this one extended by {@code str}.
205         *
206         * @param str The characters to append to the phonemes.
207         */
208        public void append(final CharSequence str) {
209            phonemes.forEach(ph -> ph.append(str));
210        }
211
212        /**
213         * Applies the given phoneme expression to all phonemes in this phoneme builder.
214         * <p>
215         * This will lengthen phonemes that have compatible language sets to the expression, and drop those that are incompatible.
216         * </p>
217         *
218         * @param phonemeExpr The expression to apply.
219         * @param maxPhonemes The maximum number of phonemes to build up.
220         */
221        public void apply(final Rule.PhonemeExpr phonemeExpr, final int maxPhonemes) {
222            final Set<Rule.Phoneme> newPhonemes = new LinkedHashSet<>(Math.min(phonemes.size() * phonemeExpr.size(), maxPhonemes));
223            EXPR: for (final Rule.Phoneme left : phonemes) {
224                for (final Rule.Phoneme right : phonemeExpr.getPhonemes()) {
225                    final LanguageSet languages = left.getLanguages().restrictTo(right.getLanguages());
226                    if (!languages.isEmpty()) {
227                        final Rule.Phoneme join = new Phoneme(left, right, languages);
228                        if (newPhonemes.size() < maxPhonemes) {
229                            newPhonemes.add(join);
230                            if (newPhonemes.size() >= maxPhonemes) {
231                                break EXPR;
232                            }
233                        }
234                    }
235                }
236            }
237            phonemes.clear();
238            phonemes.addAll(newPhonemes);
239        }
240
241        /**
242         * Gets underlying phoneme set. Please don't mutate.
243         *
244         * @return the phoneme set.
245         */
246        public Set<Rule.Phoneme> getPhonemes() {
247            return phonemes;
248        }
249
250        /**
251         * Stringifies the phoneme set. This produces a single string of the strings of each phoneme, joined with a pipe. This is explicitly provided in place
252         * of toString as it is a potentially expensive operation, which should be avoided when debugging.
253         *
254         * @return the stringified phoneme set.
255         */
256        public String makeString() {
257            return phonemes.stream().map(Rule.Phoneme::getPhonemeText).collect(Collectors.joining("|"));
258        }
259    }
260
261    /**
262     * A function closure capturing the application of a list of rules to an input sequence at a particular offset. After invocation, the values {@code i} and
263     * {@code found} are updated. {@code i} points to the index of the next char in {@code input} that must be processed next (the input up to that index having
264     * been processed already), and {@code found} indicates if a matching rule was found or not. In the case where a matching rule was found,
265     * {@code phonemeBuilder} is replaced with a new builder containing the phonemes updated by the matching rule.
266     * <p>
267     * Although this class is not thread-safe (it has mutable unprotected fields), it is not shared between threads as it is constructed as needed by the
268     * calling methods.
269     * </p>
270     */
271    private static final class RulesApplication {
272
273        private final Map<String, List<Rule>> finalRules;
274
275        private boolean found;
276
277        private int i;
278
279        private final CharSequence input;
280
281        private final int maxPhonemes;
282
283        private final PhonemeBuilder phonemeBuilder;
284
285        RulesApplication(final Map<String, List<Rule>> finalRules, final CharSequence input, final PhonemeBuilder phonemeBuilder, final int i,
286                final int maxPhonemes) {
287            this.finalRules = Objects.requireNonNull(finalRules, "finalRules");
288            this.phonemeBuilder = phonemeBuilder;
289            this.input = input;
290            this.i = i;
291            this.maxPhonemes = maxPhonemes;
292        }
293
294        public int getI() {
295            return i;
296        }
297
298        public PhonemeBuilder getPhonemeBuilder() {
299            return phonemeBuilder;
300        }
301
302        /**
303         * Invokes the rules. Loops over the rules list, stopping at the first one that has a matching context and pattern. Then applies this rule to the
304         * phoneme builder to produce updated phonemes. If there was no match, {@code i} is advanced one and the character is silently dropped from the phonetic
305         * spelling.
306         *
307         * @return {@code this}.
308         */
309        public RulesApplication invoke() {
310            found = false;
311            int patternLength = 1;
312            final List<Rule> rules = finalRules.get(input.subSequence(i, i + patternLength));
313            if (rules != null) {
314                for (final Rule rule : rules) {
315                    final String pattern = rule.getPattern();
316                    patternLength = pattern.length();
317                    if (rule.patternAndContextMatches(input, i)) {
318                        phonemeBuilder.apply(rule.getPhoneme(), maxPhonemes);
319                        found = true;
320                        break;
321                    }
322                }
323            }
324            if (!found) {
325                patternLength = 1;
326            }
327            i += patternLength;
328            return this;
329        }
330
331        public boolean isFound() {
332            return found;
333        }
334    }
335
336    private static final Map<NameType, Set<String>> NAME_PREFIXES = new EnumMap<>(NameType.class);
337
338    private static final Pattern QUOTE = Pattern.compile("'");
339    static {
340        NAME_PREFIXES.put(NameType.ASHKENAZI, Collections.unmodifiableSet(new HashSet<>(Arrays.asList("bar", "ben", "da", "de", "van", "von"))));
341        NAME_PREFIXES.put(NameType.SEPHARDIC, Collections.unmodifiableSet(
342                new HashSet<>(Arrays.asList("al", "el", "da", "dal", "de", "del", "dela", "de la", "della", "des", "di", "do", "dos", "du", "van", "von"))));
343        NAME_PREFIXES.put(NameType.GENERIC, Collections.unmodifiableSet(
344                new HashSet<>(Arrays.asList("da", "dal", "de", "del", "dela", "de la", "della", "des", "di", "do", "dos", "du", "van", "von"))));
345    }
346
347    /**
348     * Creates a new builder for a PhoneticEngine.
349     *
350     * @return a new builder for a PhoneticEngine.
351     * @since 1.23.0
352     */
353    public static Builder builder() {
354        return new Builder();
355    }
356
357    /**
358     * Joins some strings with an internal separator.
359     *
360     * @param strings Strings to join.
361     * @param sep     String to separate them with.
362     * @return A single String consisting of each element of {@code strings} interleaved by {@code sep}.
363     */
364    private static String join(final List<String> strings, final String sep) {
365        return strings.stream().collect(Collectors.joining(sep));
366    }
367
368    private final boolean concat;
369
370    private final Lang lang;
371
372    private final int maxInputLength;
373
374    private final int maxPhonemes;
375
376    private final NameType nameType;
377
378    private final RuleType ruleType;
379
380    /**
381     * Creates a new, fully-configured phonetic engine.
382     *
383     * @param builder The builder to use for configuration.
384     * @throws IllegalArgumentException Thrown if ruleType is RULES.
385     */
386    private PhoneticEngine(final Builder builder) {
387        if (builder.ruleType == RuleType.RULES) {
388            throw new IllegalArgumentException("ruleType must not be " + RuleType.RULES);
389        }
390        this.nameType = builder.nameType;
391        this.ruleType = builder.ruleType;
392        this.concat = builder.concat;
393        this.lang = Lang.instance(builder.nameType);
394        this.maxPhonemes = builder.maxPhonemes;
395        this.maxInputLength = builder.maxInputLength;
396    }
397
398    /**
399     * Generates a new, fully-configured phonetic engine.
400     *
401     * @param nameType    the type of names it will use, null is treated as {@link NameType#GENERIC}.
402     * @param ruleType    the type of rules it will apply, null is treated as {@link RuleType#APPROX}.
403     * @param concatenate if it will concatenate multiple encodings.
404     * @deprecated Use {@link #builder()} instead.
405     */
406    @Deprecated
407    public PhoneticEngine(final NameType nameType, final RuleType ruleType, final boolean concatenate) {
408        this(nameType, ruleType, concatenate, Builder.MAX_PHONEMES);
409    }
410
411    /**
412     * Generates a new, fully-configured phonetic engine.
413     *
414     * @param nameType    the type of names it will use, null is treated as {@link NameType#GENERIC}.
415     * @param ruleType    the type of rules it will apply, null is treated as {@link RuleType#APPROX}.
416     * @param concatenate if it will concatenate multiple encodings.
417     * @param maxPhonemes the maximum number of phonemes that will be handled, less than 0 will reset to the default of {@value Builder#MAX_PHONEMES}.
418     * @throws IllegalArgumentException Thrown if ruleType is RULES.
419     * @since 1.7
420     * @deprecated Use {@link #builder()} instead.
421     */
422    @Deprecated
423    public PhoneticEngine(final NameType nameType, final RuleType ruleType, final boolean concatenate, final int maxPhonemes) {
424        this(builder().setNameType(nameType).setRuleType(ruleType).setConcat(concatenate).setMaxPhonemes(maxPhonemes)
425                .setMaxInputLength(Builder.MAX_INPUT_LENGTH));
426    }
427
428    /**
429     * Applies the final rules to convert from a language-specific phonetic representation to a language-independent representation.
430     *
431     * @param phonemeBuilder The current phonemes.
432     * @param finalRules     The final rules to apply.
433     * @return The resulting phonemes.
434     */
435    private PhonemeBuilder applyFinalRules(final PhonemeBuilder phonemeBuilder, final Map<String, List<Rule>> finalRules) {
436        Objects.requireNonNull(finalRules, "finalRules");
437        if (finalRules.isEmpty()) {
438            return phonemeBuilder;
439        }
440        final Map<Rule.Phoneme, Rule.Phoneme> phonemes = new TreeMap<>(Rule.Phoneme.COMPARATOR);
441        phonemeBuilder.getPhonemes().forEach(phoneme -> {
442            PhonemeBuilder subBuilder = PhonemeBuilder.empty(phoneme.getLanguages());
443            final CharSequence phonemeText = phoneme.getPhonemeText();
444            final int length = phonemeText.length();
445            for (int i = 0; i < length;) {
446                final RulesApplication rulesApplication = new RulesApplication(finalRules, phonemeText, subBuilder, i, maxPhonemes).invoke();
447                final boolean found = rulesApplication.isFound();
448                subBuilder = rulesApplication.getPhonemeBuilder();
449                if (!found) {
450                    // not found, appending as-is
451                    subBuilder.append(phonemeText.subSequence(i, i + 1));
452                }
453                i = rulesApplication.getI();
454            }
455            // the phonemes map orders the phonemes only based on their text, but ignores the language set
456            // when adding new phonemes, check for equal phonemes and merge their language set, otherwise
457            // phonemes with the same text but different language set get lost
458            subBuilder.getPhonemes().forEach(newPhoneme -> {
459                if (phonemes.containsKey(newPhoneme)) {
460                    final Rule.Phoneme oldPhoneme = phonemes.remove(newPhoneme);
461                    final Rule.Phoneme mergedPhoneme = oldPhoneme.mergeWithLanguage(newPhoneme.getLanguages());
462                    phonemes.put(mergedPhoneme, mergedPhoneme);
463                } else {
464                    phonemes.put(newPhoneme, newPhoneme);
465                }
466            });
467        });
468        return new PhonemeBuilder(phonemes.keySet());
469    }
470
471    /**
472     * Encodes a string to its phonetic representation.
473     *
474     * @param input the String to encode, not null.
475     * @return The encoding of the input.
476     * @throws IllegalArgumentException Thrown if the input is longer than the maximum allowed length.
477     */
478    public String encode(final String input) {
479        // enforce the input length limit before language guessing runs over the input,
480        // so over-limit input cannot buy a full multi-pass scan before the guard fires
481        if (input.length() > maxInputLength) {
482            throw new IllegalArgumentException("Input is greater than maxInputLength (" + maxInputLength + ").");
483        }
484        return encode(input, lang.guessLanguages(input));
485    }
486
487    /**
488     * Encodes an input string into an output phonetic representation, given a set of possible origin languages.
489     *
490     * @param input       String to phoneticise; a String with dashes or spaces separating each word, not null.
491     * @param languageSet set of possible origin languages.
492     * @return A phonetic representation of the input; a String containing '-'-separated phonetic representations of the input.
493     * @throws IllegalArgumentException Thrown if the input is longer than the maximum allowed length.
494     */
495    public String encode(String input, final Languages.LanguageSet languageSet) {
496        if (input.length() > maxInputLength) {
497            throw new IllegalArgumentException("Input is greater than maxInputLength (" + maxInputLength + ").");
498        }
499        final Map<String, List<Rule>> rules = Rule.getInstanceMap(this.nameType, RuleType.RULES, languageSet);
500        // rules common across many (all) languages
501        final Map<String, List<Rule>> finalRules1 = Rule.getInstanceMap(this.nameType, this.ruleType, "common");
502        // rules that apply to a specific language that may be ambiguous or wrong if applied to other languages
503        final Map<String, List<Rule>> finalRules2 = Rule.getInstanceMap(this.nameType, this.ruleType, languageSet);
504        // tidy the input
505        // lower case is a locale-dependent operation
506        input = input.toLowerCase(Locale.ENGLISH).replace('-', ' ').trim();
507        if (this.nameType == NameType.GENERIC) {
508            final String dQuotePrefix = "d'";
509            final int dqpLen = dQuotePrefix.length();
510            if (input.startsWith(dQuotePrefix)) { // check for d'
511                String remainder = input.substring(dqpLen);
512                // Find remainder without allocating new string.
513                final int start = lastRepeat(remainder, dQuotePrefix, dqpLen);
514                remainder = remainder.substring(start);
515                final String combined = "d" + remainder;
516                return "(" + encode(remainder) + ")-(" + encode(combined) + ")";
517            }
518            for (final String l : NAME_PREFIXES.get(this.nameType)) {
519                // handle generic prefixes
520                if (input.startsWith(l + " ")) {
521                    // check for any prefix in the words list
522                    final String remainder = input.substring(l.length() + 1); // input without the prefix
523                    final String combined = l + remainder; // input with prefix without space
524                    return "(" + encode(remainder) + ")-(" + encode(combined) + ")";
525                }
526            }
527        }
528        final List<String> words = Arrays.asList(ResourceConstants.SPACES.split(input));
529        final List<String> words2 = new ArrayList<>();
530        // special-case handling of word prefixes based upon the name type
531        switch (this.nameType) {
532        case SEPHARDIC:
533            words.forEach(aWord -> {
534                final String[] parts = QUOTE.split(aWord, -1);
535                words2.add(parts[parts.length - 1]);
536            });
537            words2.removeAll(NAME_PREFIXES.get(this.nameType));
538            break;
539        case ASHKENAZI:
540            words2.addAll(words);
541            words2.removeAll(NAME_PREFIXES.get(this.nameType));
542            break;
543        case GENERIC:
544            words2.addAll(words);
545            break;
546        default:
547            throw new IllegalStateException("Unreachable case: " + this.nameType);
548        }
549        if (this.concat) {
550            // concat mode enabled
551            input = join(words2, " ");
552        } else if (words2.size() == 1) {
553            // not a multi-word name
554            input = words.iterator().next();
555        } else if (!words2.isEmpty()) {
556            // encode each word in a multi-word name separately (normally used for approx matches)
557            final StringBuilder result = new StringBuilder();
558            words2.forEach(word -> result.append("-").append(encode(word)));
559            // return the result without the leading "-"
560            return result.substring(1);
561        }
562        PhonemeBuilder phonemeBuilder = PhonemeBuilder.empty(languageSet);
563        // loop over each char in the input - we will handle the increment manually
564        for (int i = 0; i < input.length();) {
565            final RulesApplication rulesApplication = new RulesApplication(rules, input, phonemeBuilder, i, maxPhonemes).invoke();
566            i = rulesApplication.getI();
567            phonemeBuilder = rulesApplication.getPhonemeBuilder();
568        }
569        // Apply the general rules
570        phonemeBuilder = applyFinalRules(phonemeBuilder, finalRules1);
571        // Apply the language-specific rules
572        phonemeBuilder = applyFinalRules(phonemeBuilder, finalRules2);
573        return phonemeBuilder.makeString();
574    }
575
576    /**
577     * Gets the Lang language guessing rules being used.
578     *
579     * @return The Lang in use.
580     */
581    public Lang getLang() {
582        return this.lang;
583    }
584
585    /**
586     * Gets the maximum number of phonemes the engine will calculate for a given input.
587     *
588     * @return The maximum number of phonemes.
589     * @since 1.7
590     */
591    public int getMaxPhonemes() {
592        return this.maxPhonemes;
593    }
594
595    /**
596     * Gets the NameType being used.
597     *
598     * @return The NameType in use.
599     */
600    public NameType getNameType() {
601        return this.nameType;
602    }
603
604    /**
605     * Gets the RuleType being used.
606     *
607     * @return The RuleType in use.
608     */
609    public RuleType getRuleType() {
610        return this.ruleType;
611    }
612
613    /**
614     * Tests whether multiple phonetic encodings are concatenated or just the first one is kept.
615     *
616     * @return true if multiple phonetic encodings are returned, false if just the first is.
617     */
618    public boolean isConcat() {
619        return this.concat;
620    }
621
622    /**
623     * Finds the index past the last occurrence of a repeating prefix in a string, starting from the beginning of the string and moving forward.
624     *
625     * @param source    The source string to search within.
626     * @param prefix    The prefix to look for.
627     * @param prefixLen The length of the prefix.
628     * @return The index in the source string where the last occurrence of the repeating prefix ends.
629     */
630    private int lastRepeat(final String source, final String prefix, final int prefixLen) {
631        // Find without allocating new string.
632        int start = 0;
633        while (start + prefixLen <= source.length()) {
634            int i = 0;
635            while (i < prefixLen && source.charAt(start + i) == prefix.charAt(i)) {
636                i++;
637            }
638            if (i != prefixLen) {
639                break;
640            }
641            start += prefixLen;
642        }
643        return start;
644    }
645}