001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * https://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 018package org.apache.commons.codec.language.bm; 019 020import java.util.ArrayList; 021import java.util.Arrays; 022import java.util.Collections; 023import java.util.EnumMap; 024import java.util.HashSet; 025import java.util.LinkedHashSet; 026import java.util.List; 027import java.util.Locale; 028import java.util.Map; 029import java.util.Objects; 030import java.util.Set; 031import java.util.TreeMap; 032import java.util.function.Supplier; 033import java.util.regex.Pattern; 034import java.util.stream.Collectors; 035 036import org.apache.commons.codec.language.bm.Languages.LanguageSet; 037import org.apache.commons.codec.language.bm.Rule.Phoneme; 038 039/** 040 * Converts words into potential phonetic representations. 041 * <p> 042 * This is a two-stage process. Firstly, the word is converted into a phonetic representation that takes into account the likely source language. Next, this 043 * phonetic representation is converted into a pan-European 'average' representation, allowing comparison between different versions of essentially the same 044 * word from different languages. 045 * </p> 046 * <p> 047 * This class is intentionally immutable and thread-safe. If you wish to alter the settings for a PhoneticEngine, you must make a new one with the updated 048 * settings. 049 * </p> 050 * <p> 051 * Ported from phoneticengine.php 052 * </p> 053 * 054 * @since 1.6 055 */ 056public class PhoneticEngine { 057 058 /** 059 * Builder for a PhoneticEngine. 060 * 061 * @since 1.23.0 062 */ 063 public static class Builder implements Supplier<PhoneticEngine> { 064 065 /** See https://en.wikipedia.org/wiki/Hubert_Blaine_Wolfeschlegelsteinhausenbergerdorff_Sr. */ 066 private static final int MAX_INPUT_LENGTH = 666; 067 068 private static final int MAX_PHONEMES = 20; 069 070 private boolean concat = true; 071 072 private int maxInputLength = MAX_INPUT_LENGTH; 073 074 private int maxPhonemes = MAX_PHONEMES; 075 076 private NameType nameType = NameType.GENERIC; 077 078 private RuleType ruleType = RuleType.APPROX; 079 080 private Builder() { 081 // empty 082 } 083 084 @Override 085 public PhoneticEngine get() { 086 return new PhoneticEngine(this); 087 } 088 089 /** 090 * Sets all the properties of this builder to match those of the given engine. 091 * 092 * @param engine The engine to copy properties from. 093 * @return This builder. 094 */ 095 public Builder setAll(final PhoneticEngine engine) { 096 this.nameType = engine.getNameType(); 097 this.ruleType = engine.getRuleType(); 098 this.concat = engine.isConcat(); 099 this.maxPhonemes = engine.getMaxPhonemes(); 100 return this; 101 } 102 103 /** 104 * Sets whether the engine will concatenate multiple encodings. 105 * 106 * @param concat Whether the engine will concatenate multiple encodings. 107 * @return This builder. 108 */ 109 public Builder setConcat(final boolean concat) { 110 this.concat = concat; 111 return this; 112 } 113 114 /** 115 * Sets the maximum input length allowed. 116 * <p> 117 * A value less than 0 will reset the maximum input length to the default value of {@value #MAX_INPUT_LENGTH}, see 118 * <a href="https://en.wikipedia.org/wiki/Hubert_Blaine_Wolfeschlegelsteinhausenbergerdorff_Sr.">Hubert Blaine Wolfeschlegelsteinhausenbergerdorff 119 * Sr.</a>. 120 * </p> 121 * 122 * @param maxInputLength the maximum input length allowed. 123 * @return This builder. 124 */ 125 public Builder setMaxInputLength(final int maxInputLength) { 126 this.maxInputLength = maxInputLength < 0 ? MAX_INPUT_LENGTH : maxInputLength; 127 return this; 128 } 129 130 /** 131 * Sets maximum number of phonemes the engine will handle. 132 * <p> 133 * A value less than 0 will reset the maximum number of phonemes to the default {@value #MAX_PHONEMES}. 134 * </p> 135 * 136 * @param maxPhonemes The maximum number of phonemes the engine will handle. 137 * @return This builder. 138 */ 139 public Builder setMaxPhonemes(final int maxPhonemes) { 140 this.maxPhonemes = maxPhonemes < 0 ? MAX_PHONEMES : maxPhonemes; 141 return this; 142 } 143 144 /** 145 * Sets the name type for the engine to be built. 146 * <p> 147 * A null value will reset the name type to the default of {@link NameType#GENERIC}. 148 * </p> 149 * 150 * @param nameType The type of names the engine will use. 151 * @return This builder. 152 */ 153 public Builder setNameType(final NameType nameType) { 154 this.nameType = nameType != null ? nameType : NameType.GENERIC; 155 return this; 156 } 157 158 /** 159 * Sets the rule type for the engine to be built. 160 * <p> 161 * A null value will reset the rule type to the default of {@link RuleType#APPROX}. 162 * </p> 163 * 164 * @param ruleType The type of rules the engine will use. 165 * @return This builder. 166 */ 167 public Builder setRuleType(final RuleType ruleType) { 168 this.ruleType = ruleType != null ? ruleType : RuleType.APPROX; 169 return this; 170 } 171 } 172 173 /** 174 * Manipulates a set of phonemes as they are being built up. Not intended for use outside this package, and probably not outside the {@link PhoneticEngine} 175 * class. 176 * 177 * @since 1.6 178 */ 179 static final class PhonemeBuilder { 180 181 /** 182 * An empty builder where all phonemes must come from some set of languages. This will contain a single phoneme of zero characters. This can then be 183 * appended to. This should be the only way to create a new phoneme from scratch. 184 * 185 * @param languages The set of languages. 186 * @return a new, empty phoneme builder. 187 */ 188 public static PhonemeBuilder empty(final Languages.LanguageSet languages) { 189 return new PhonemeBuilder(new Rule.Phoneme("", languages)); 190 } 191 192 private final Set<Rule.Phoneme> phonemes; 193 194 private PhonemeBuilder(final Rule.Phoneme phoneme) { 195 this.phonemes = new LinkedHashSet<>(); 196 this.phonemes.add(phoneme); 197 } 198 199 private PhonemeBuilder(final Set<Rule.Phoneme> phonemes) { 200 this.phonemes = phonemes; 201 } 202 203 /** 204 * Creates a new phoneme builder containing all phonemes in this one extended by {@code str}. 205 * 206 * @param str The characters to append to the phonemes. 207 */ 208 public void append(final CharSequence str) { 209 phonemes.forEach(ph -> ph.append(str)); 210 } 211 212 /** 213 * Applies the given phoneme expression to all phonemes in this phoneme builder. 214 * <p> 215 * This will lengthen phonemes that have compatible language sets to the expression, and drop those that are incompatible. 216 * </p> 217 * 218 * @param phonemeExpr The expression to apply. 219 * @param maxPhonemes The maximum number of phonemes to build up. 220 */ 221 public void apply(final Rule.PhonemeExpr phonemeExpr, final int maxPhonemes) { 222 final Set<Rule.Phoneme> newPhonemes = new LinkedHashSet<>(Math.min(phonemes.size() * phonemeExpr.size(), maxPhonemes)); 223 EXPR: for (final Rule.Phoneme left : phonemes) { 224 for (final Rule.Phoneme right : phonemeExpr.getPhonemes()) { 225 final LanguageSet languages = left.getLanguages().restrictTo(right.getLanguages()); 226 if (!languages.isEmpty()) { 227 final Rule.Phoneme join = new Phoneme(left, right, languages); 228 if (newPhonemes.size() < maxPhonemes) { 229 newPhonemes.add(join); 230 if (newPhonemes.size() >= maxPhonemes) { 231 break EXPR; 232 } 233 } 234 } 235 } 236 } 237 phonemes.clear(); 238 phonemes.addAll(newPhonemes); 239 } 240 241 /** 242 * Gets underlying phoneme set. Please don't mutate. 243 * 244 * @return the phoneme set. 245 */ 246 public Set<Rule.Phoneme> getPhonemes() { 247 return phonemes; 248 } 249 250 /** 251 * Stringifies the phoneme set. This produces a single string of the strings of each phoneme, joined with a pipe. This is explicitly provided in place 252 * of toString as it is a potentially expensive operation, which should be avoided when debugging. 253 * 254 * @return the stringified phoneme set. 255 */ 256 public String makeString() { 257 return phonemes.stream().map(Rule.Phoneme::getPhonemeText).collect(Collectors.joining("|")); 258 } 259 } 260 261 /** 262 * A function closure capturing the application of a list of rules to an input sequence at a particular offset. After invocation, the values {@code i} and 263 * {@code found} are updated. {@code i} points to the index of the next char in {@code input} that must be processed next (the input up to that index having 264 * been processed already), and {@code found} indicates if a matching rule was found or not. In the case where a matching rule was found, 265 * {@code phonemeBuilder} is replaced with a new builder containing the phonemes updated by the matching rule. 266 * <p> 267 * Although this class is not thread-safe (it has mutable unprotected fields), it is not shared between threads as it is constructed as needed by the 268 * calling methods. 269 * </p> 270 */ 271 private static final class RulesApplication { 272 273 private final Map<String, List<Rule>> finalRules; 274 275 private boolean found; 276 277 private int i; 278 279 private final CharSequence input; 280 281 private final int maxPhonemes; 282 283 private final PhonemeBuilder phonemeBuilder; 284 285 RulesApplication(final Map<String, List<Rule>> finalRules, final CharSequence input, final PhonemeBuilder phonemeBuilder, final int i, 286 final int maxPhonemes) { 287 this.finalRules = Objects.requireNonNull(finalRules, "finalRules"); 288 this.phonemeBuilder = phonemeBuilder; 289 this.input = input; 290 this.i = i; 291 this.maxPhonemes = maxPhonemes; 292 } 293 294 public int getI() { 295 return i; 296 } 297 298 public PhonemeBuilder getPhonemeBuilder() { 299 return phonemeBuilder; 300 } 301 302 /** 303 * Invokes the rules. Loops over the rules list, stopping at the first one that has a matching context and pattern. Then applies this rule to the 304 * phoneme builder to produce updated phonemes. If there was no match, {@code i} is advanced one and the character is silently dropped from the phonetic 305 * spelling. 306 * 307 * @return {@code this}. 308 */ 309 public RulesApplication invoke() { 310 found = false; 311 int patternLength = 1; 312 final List<Rule> rules = finalRules.get(input.subSequence(i, i + patternLength)); 313 if (rules != null) { 314 for (final Rule rule : rules) { 315 final String pattern = rule.getPattern(); 316 patternLength = pattern.length(); 317 if (rule.patternAndContextMatches(input, i)) { 318 phonemeBuilder.apply(rule.getPhoneme(), maxPhonemes); 319 found = true; 320 break; 321 } 322 } 323 } 324 if (!found) { 325 patternLength = 1; 326 } 327 i += patternLength; 328 return this; 329 } 330 331 public boolean isFound() { 332 return found; 333 } 334 } 335 336 private static final Map<NameType, Set<String>> NAME_PREFIXES = new EnumMap<>(NameType.class); 337 338 private static final Pattern QUOTE = Pattern.compile("'"); 339 static { 340 NAME_PREFIXES.put(NameType.ASHKENAZI, Collections.unmodifiableSet(new HashSet<>(Arrays.asList("bar", "ben", "da", "de", "van", "von")))); 341 NAME_PREFIXES.put(NameType.SEPHARDIC, Collections.unmodifiableSet( 342 new HashSet<>(Arrays.asList("al", "el", "da", "dal", "de", "del", "dela", "de la", "della", "des", "di", "do", "dos", "du", "van", "von")))); 343 NAME_PREFIXES.put(NameType.GENERIC, Collections.unmodifiableSet( 344 new HashSet<>(Arrays.asList("da", "dal", "de", "del", "dela", "de la", "della", "des", "di", "do", "dos", "du", "van", "von")))); 345 } 346 347 /** 348 * Creates a new builder for a PhoneticEngine. 349 * 350 * @return a new builder for a PhoneticEngine. 351 * @since 1.23.0 352 */ 353 public static Builder builder() { 354 return new Builder(); 355 } 356 357 /** 358 * Joins some strings with an internal separator. 359 * 360 * @param strings Strings to join. 361 * @param sep String to separate them with. 362 * @return A single String consisting of each element of {@code strings} interleaved by {@code sep}. 363 */ 364 private static String join(final List<String> strings, final String sep) { 365 return strings.stream().collect(Collectors.joining(sep)); 366 } 367 368 private final boolean concat; 369 370 private final Lang lang; 371 372 private final int maxInputLength; 373 374 private final int maxPhonemes; 375 376 private final NameType nameType; 377 378 private final RuleType ruleType; 379 380 /** 381 * Creates a new, fully-configured phonetic engine. 382 * 383 * @param builder The builder to use for configuration. 384 * @throws IllegalArgumentException Thrown if ruleType is RULES. 385 */ 386 private PhoneticEngine(final Builder builder) { 387 if (builder.ruleType == RuleType.RULES) { 388 throw new IllegalArgumentException("ruleType must not be " + RuleType.RULES); 389 } 390 this.nameType = builder.nameType; 391 this.ruleType = builder.ruleType; 392 this.concat = builder.concat; 393 this.lang = Lang.instance(builder.nameType); 394 this.maxPhonemes = builder.maxPhonemes; 395 this.maxInputLength = builder.maxInputLength; 396 } 397 398 /** 399 * Generates a new, fully-configured phonetic engine. 400 * 401 * @param nameType the type of names it will use, null is treated as {@link NameType#GENERIC}. 402 * @param ruleType the type of rules it will apply, null is treated as {@link RuleType#APPROX}. 403 * @param concatenate if it will concatenate multiple encodings. 404 * @deprecated Use {@link #builder()} instead. 405 */ 406 @Deprecated 407 public PhoneticEngine(final NameType nameType, final RuleType ruleType, final boolean concatenate) { 408 this(nameType, ruleType, concatenate, Builder.MAX_PHONEMES); 409 } 410 411 /** 412 * Generates a new, fully-configured phonetic engine. 413 * 414 * @param nameType the type of names it will use, null is treated as {@link NameType#GENERIC}. 415 * @param ruleType the type of rules it will apply, null is treated as {@link RuleType#APPROX}. 416 * @param concatenate if it will concatenate multiple encodings. 417 * @param maxPhonemes the maximum number of phonemes that will be handled, less than 0 will reset to the default of {@value Builder#MAX_PHONEMES}. 418 * @throws IllegalArgumentException Thrown if ruleType is RULES. 419 * @since 1.7 420 * @deprecated Use {@link #builder()} instead. 421 */ 422 @Deprecated 423 public PhoneticEngine(final NameType nameType, final RuleType ruleType, final boolean concatenate, final int maxPhonemes) { 424 this(builder().setNameType(nameType).setRuleType(ruleType).setConcat(concatenate).setMaxPhonemes(maxPhonemes) 425 .setMaxInputLength(Builder.MAX_INPUT_LENGTH)); 426 } 427 428 /** 429 * Applies the final rules to convert from a language-specific phonetic representation to a language-independent representation. 430 * 431 * @param phonemeBuilder The current phonemes. 432 * @param finalRules The final rules to apply. 433 * @return The resulting phonemes. 434 */ 435 private PhonemeBuilder applyFinalRules(final PhonemeBuilder phonemeBuilder, final Map<String, List<Rule>> finalRules) { 436 Objects.requireNonNull(finalRules, "finalRules"); 437 if (finalRules.isEmpty()) { 438 return phonemeBuilder; 439 } 440 final Map<Rule.Phoneme, Rule.Phoneme> phonemes = new TreeMap<>(Rule.Phoneme.COMPARATOR); 441 phonemeBuilder.getPhonemes().forEach(phoneme -> { 442 PhonemeBuilder subBuilder = PhonemeBuilder.empty(phoneme.getLanguages()); 443 final CharSequence phonemeText = phoneme.getPhonemeText(); 444 final int length = phonemeText.length(); 445 for (int i = 0; i < length;) { 446 final RulesApplication rulesApplication = new RulesApplication(finalRules, phonemeText, subBuilder, i, maxPhonemes).invoke(); 447 final boolean found = rulesApplication.isFound(); 448 subBuilder = rulesApplication.getPhonemeBuilder(); 449 if (!found) { 450 // not found, appending as-is 451 subBuilder.append(phonemeText.subSequence(i, i + 1)); 452 } 453 i = rulesApplication.getI(); 454 } 455 // the phonemes map orders the phonemes only based on their text, but ignores the language set 456 // when adding new phonemes, check for equal phonemes and merge their language set, otherwise 457 // phonemes with the same text but different language set get lost 458 subBuilder.getPhonemes().forEach(newPhoneme -> { 459 if (phonemes.containsKey(newPhoneme)) { 460 final Rule.Phoneme oldPhoneme = phonemes.remove(newPhoneme); 461 final Rule.Phoneme mergedPhoneme = oldPhoneme.mergeWithLanguage(newPhoneme.getLanguages()); 462 phonemes.put(mergedPhoneme, mergedPhoneme); 463 } else { 464 phonemes.put(newPhoneme, newPhoneme); 465 } 466 }); 467 }); 468 return new PhonemeBuilder(phonemes.keySet()); 469 } 470 471 /** 472 * Encodes a string to its phonetic representation. 473 * 474 * @param input the String to encode, not null. 475 * @return The encoding of the input. 476 * @throws IllegalArgumentException Thrown if the input is longer than the maximum allowed length. 477 */ 478 public String encode(final String input) { 479 // enforce the input length limit before language guessing runs over the input, 480 // so over-limit input cannot buy a full multi-pass scan before the guard fires 481 if (input.length() > maxInputLength) { 482 throw new IllegalArgumentException("Input is greater than maxInputLength (" + maxInputLength + ")."); 483 } 484 return encode(input, lang.guessLanguages(input)); 485 } 486 487 /** 488 * Encodes an input string into an output phonetic representation, given a set of possible origin languages. 489 * 490 * @param input String to phoneticise; a String with dashes or spaces separating each word, not null. 491 * @param languageSet set of possible origin languages. 492 * @return A phonetic representation of the input; a String containing '-'-separated phonetic representations of the input. 493 * @throws IllegalArgumentException Thrown if the input is longer than the maximum allowed length. 494 */ 495 public String encode(String input, final Languages.LanguageSet languageSet) { 496 if (input.length() > maxInputLength) { 497 throw new IllegalArgumentException("Input is greater than maxInputLength (" + maxInputLength + ")."); 498 } 499 final Map<String, List<Rule>> rules = Rule.getInstanceMap(this.nameType, RuleType.RULES, languageSet); 500 // rules common across many (all) languages 501 final Map<String, List<Rule>> finalRules1 = Rule.getInstanceMap(this.nameType, this.ruleType, "common"); 502 // rules that apply to a specific language that may be ambiguous or wrong if applied to other languages 503 final Map<String, List<Rule>> finalRules2 = Rule.getInstanceMap(this.nameType, this.ruleType, languageSet); 504 // tidy the input 505 // lower case is a locale-dependent operation 506 input = input.toLowerCase(Locale.ENGLISH).replace('-', ' ').trim(); 507 if (this.nameType == NameType.GENERIC) { 508 final String dQuotePrefix = "d'"; 509 final int dqpLen = dQuotePrefix.length(); 510 if (input.startsWith(dQuotePrefix)) { // check for d' 511 String remainder = input.substring(dqpLen); 512 // Find remainder without allocating new string. 513 final int start = lastRepeat(remainder, dQuotePrefix, dqpLen); 514 remainder = remainder.substring(start); 515 final String combined = "d" + remainder; 516 return "(" + encode(remainder) + ")-(" + encode(combined) + ")"; 517 } 518 for (final String l : NAME_PREFIXES.get(this.nameType)) { 519 // handle generic prefixes 520 if (input.startsWith(l + " ")) { 521 // check for any prefix in the words list 522 final String remainder = input.substring(l.length() + 1); // input without the prefix 523 final String combined = l + remainder; // input with prefix without space 524 return "(" + encode(remainder) + ")-(" + encode(combined) + ")"; 525 } 526 } 527 } 528 final List<String> words = Arrays.asList(ResourceConstants.SPACES.split(input)); 529 final List<String> words2 = new ArrayList<>(); 530 // special-case handling of word prefixes based upon the name type 531 switch (this.nameType) { 532 case SEPHARDIC: 533 words.forEach(aWord -> { 534 final String[] parts = QUOTE.split(aWord, -1); 535 words2.add(parts[parts.length - 1]); 536 }); 537 words2.removeAll(NAME_PREFIXES.get(this.nameType)); 538 break; 539 case ASHKENAZI: 540 words2.addAll(words); 541 words2.removeAll(NAME_PREFIXES.get(this.nameType)); 542 break; 543 case GENERIC: 544 words2.addAll(words); 545 break; 546 default: 547 throw new IllegalStateException("Unreachable case: " + this.nameType); 548 } 549 if (this.concat) { 550 // concat mode enabled 551 input = join(words2, " "); 552 } else if (words2.size() == 1) { 553 // not a multi-word name 554 input = words.iterator().next(); 555 } else if (!words2.isEmpty()) { 556 // encode each word in a multi-word name separately (normally used for approx matches) 557 final StringBuilder result = new StringBuilder(); 558 words2.forEach(word -> result.append("-").append(encode(word))); 559 // return the result without the leading "-" 560 return result.substring(1); 561 } 562 PhonemeBuilder phonemeBuilder = PhonemeBuilder.empty(languageSet); 563 // loop over each char in the input - we will handle the increment manually 564 for (int i = 0; i < input.length();) { 565 final RulesApplication rulesApplication = new RulesApplication(rules, input, phonemeBuilder, i, maxPhonemes).invoke(); 566 i = rulesApplication.getI(); 567 phonemeBuilder = rulesApplication.getPhonemeBuilder(); 568 } 569 // Apply the general rules 570 phonemeBuilder = applyFinalRules(phonemeBuilder, finalRules1); 571 // Apply the language-specific rules 572 phonemeBuilder = applyFinalRules(phonemeBuilder, finalRules2); 573 return phonemeBuilder.makeString(); 574 } 575 576 /** 577 * Gets the Lang language guessing rules being used. 578 * 579 * @return The Lang in use. 580 */ 581 public Lang getLang() { 582 return this.lang; 583 } 584 585 /** 586 * Gets the maximum number of phonemes the engine will calculate for a given input. 587 * 588 * @return The maximum number of phonemes. 589 * @since 1.7 590 */ 591 public int getMaxPhonemes() { 592 return this.maxPhonemes; 593 } 594 595 /** 596 * Gets the NameType being used. 597 * 598 * @return The NameType in use. 599 */ 600 public NameType getNameType() { 601 return this.nameType; 602 } 603 604 /** 605 * Gets the RuleType being used. 606 * 607 * @return The RuleType in use. 608 */ 609 public RuleType getRuleType() { 610 return this.ruleType; 611 } 612 613 /** 614 * Tests whether multiple phonetic encodings are concatenated or just the first one is kept. 615 * 616 * @return true if multiple phonetic encodings are returned, false if just the first is. 617 */ 618 public boolean isConcat() { 619 return this.concat; 620 } 621 622 /** 623 * Finds the index past the last occurrence of a repeating prefix in a string, starting from the beginning of the string and moving forward. 624 * 625 * @param source The source string to search within. 626 * @param prefix The prefix to look for. 627 * @param prefixLen The length of the prefix. 628 * @return The index in the source string where the last occurrence of the repeating prefix ends. 629 */ 630 private int lastRepeat(final String source, final String prefix, final int prefixLen) { 631 // Find without allocating new string. 632 int start = 0; 633 while (start + prefixLen <= source.length()) { 634 int i = 0; 635 while (i < prefixLen && source.charAt(start + i) == prefix.charAt(i)) { 636 i++; 637 } 638 if (i != prefixLen) { 639 break; 640 } 641 start += prefixLen; 642 } 643 return start; 644 } 645}