001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.awt.GridBagConstraints; 008import java.awt.event.ActionEvent; 009import java.awt.event.ActionListener; 010import java.io.BufferedReader; 011import java.io.IOException; 012import java.io.InputStream; 013import java.text.MessageFormat; 014import java.util.ArrayList; 015import java.util.Arrays; 016import java.util.Collection; 017import java.util.HashMap; 018import java.util.HashSet; 019import java.util.List; 020import java.util.Locale; 021import java.util.Map; 022import java.util.Map.Entry; 023import java.util.Set; 024import java.util.regex.Matcher; 025import java.util.regex.Pattern; 026import java.util.regex.PatternSyntaxException; 027 028import javax.swing.JCheckBox; 029import javax.swing.JLabel; 030import javax.swing.JPanel; 031 032import org.openstreetmap.josm.Main; 033import org.openstreetmap.josm.command.ChangePropertyCommand; 034import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 035import org.openstreetmap.josm.command.Command; 036import org.openstreetmap.josm.command.SequenceCommand; 037import org.openstreetmap.josm.data.osm.OsmPrimitive; 038import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 039import org.openstreetmap.josm.data.osm.OsmUtils; 040import org.openstreetmap.josm.data.osm.Tag; 041import org.openstreetmap.josm.data.validation.FixableTestError; 042import org.openstreetmap.josm.data.validation.Severity; 043import org.openstreetmap.josm.data.validation.Test.TagTest; 044import org.openstreetmap.josm.data.validation.TestError; 045import org.openstreetmap.josm.data.validation.util.Entities; 046import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference; 047import org.openstreetmap.josm.gui.progress.ProgressMonitor; 048import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset; 049import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem; 050import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets; 051import org.openstreetmap.josm.gui.tagging.presets.items.Check; 052import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup; 053import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem; 054import org.openstreetmap.josm.gui.widgets.EditableList; 055import org.openstreetmap.josm.io.CachedFile; 056import org.openstreetmap.josm.io.UTFInputStreamReader; 057import org.openstreetmap.josm.tools.GBC; 058import org.openstreetmap.josm.tools.MultiMap; 059import org.openstreetmap.josm.tools.Utils; 060 061/** 062 * Check for misspelled or wrong tags 063 * 064 * @author frsantos 065 * @since 3669 066 */ 067public class TagChecker extends TagTest { 068 069 /** The config file of ignored tags */ 070 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg"; 071 /** The config file of dictionary words */ 072 public static final String SPELL_FILE = "resource://data/validator/words.cfg"; 073 074 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */ 075 private static final Map<String, String> harmonizedKeys = new HashMap<>(); 076 /** The spell check preset values */ 077 private static volatile MultiMap<String, String> presetsValueData; 078 /** The TagChecker data */ 079 private static final List<CheckerData> checkerData = new ArrayList<>(); 080 private static final List<String> ignoreDataStartsWith = new ArrayList<>(); 081 private static final List<String> ignoreDataEquals = new ArrayList<>(); 082 private static final List<String> ignoreDataEndsWith = new ArrayList<>(); 083 private static final List<Tag> ignoreDataTag = new ArrayList<>(); 084 085 /** The preferences prefix */ 086 protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName(); 087 088 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues"; 089 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys"; 090 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex"; 091 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes"; 092 093 public static final String PREF_SOURCES = PREFIX + ".source"; 094 095 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload"; 096 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload"; 097 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload"; 098 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload"; 099 100 protected boolean checkKeys; 101 protected boolean checkValues; 102 protected boolean checkComplex; 103 protected boolean checkFixmes; 104 105 protected JCheckBox prefCheckKeys; 106 protected JCheckBox prefCheckValues; 107 protected JCheckBox prefCheckComplex; 108 protected JCheckBox prefCheckFixmes; 109 protected JCheckBox prefCheckPaint; 110 111 protected JCheckBox prefCheckKeysBeforeUpload; 112 protected JCheckBox prefCheckValuesBeforeUpload; 113 protected JCheckBox prefCheckComplexBeforeUpload; 114 protected JCheckBox prefCheckFixmesBeforeUpload; 115 protected JCheckBox prefCheckPaintBeforeUpload; 116 117 // CHECKSTYLE.OFF: SingleSpaceSeparator 118 protected static final int EMPTY_VALUES = 1200; 119 protected static final int INVALID_KEY = 1201; 120 protected static final int INVALID_VALUE = 1202; 121 protected static final int FIXME = 1203; 122 protected static final int INVALID_SPACE = 1204; 123 protected static final int INVALID_KEY_SPACE = 1205; 124 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */ 125 protected static final int LONG_VALUE = 1208; 126 protected static final int LONG_KEY = 1209; 127 protected static final int LOW_CHAR_VALUE = 1210; 128 protected static final int LOW_CHAR_KEY = 1211; 129 protected static final int MISSPELLED_VALUE = 1212; 130 protected static final int MISSPELLED_KEY = 1213; 131 protected static final int MULTIPLE_SPACES = 1214; 132 // CHECKSTYLE.ON: SingleSpaceSeparator 133 // 1250 and up is used by tagcheck 134 135 protected EditableList sourcesList; 136 137 private static final Set<String> DEFAULT_SOURCES = new HashSet<>(Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE)); 138 139 /** 140 * Constructor 141 */ 142 public TagChecker() { 143 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values.")); 144 } 145 146 @Override 147 public void initialize() throws IOException { 148 initializeData(); 149 initializePresets(); 150 } 151 152 /** 153 * Reads the spellcheck file into a HashMap. 154 * The data file is a list of words, beginning with +/-. If it starts with +, 155 * the word is valid, but if it starts with -, the word should be replaced 156 * by the nearest + word before this. 157 * 158 * @throws IOException if any I/O error occurs 159 */ 160 private static void initializeData() throws IOException { 161 checkerData.clear(); 162 ignoreDataStartsWith.clear(); 163 ignoreDataEquals.clear(); 164 ignoreDataEndsWith.clear(); 165 ignoreDataTag.clear(); 166 harmonizedKeys.clear(); 167 168 StringBuilder errorSources = new StringBuilder(); 169 for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) { 170 try ( 171 CachedFile cf = new CachedFile(source); 172 InputStream s = cf.getInputStream(); 173 BufferedReader reader = new BufferedReader(UTFInputStreamReader.create(s)); 174 ) { 175 String okValue = null; 176 boolean tagcheckerfile = false; 177 boolean ignorefile = false; 178 boolean isFirstLine = true; 179 String line; 180 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) { 181 if (line.startsWith("#")) { 182 if (line.startsWith("# JOSM TagChecker")) { 183 tagcheckerfile = true; 184 if (!DEFAULT_SOURCES.contains(source)) { 185 Main.info(tr("Adding {0} to tag checker", source)); 186 } 187 } else 188 if (line.startsWith("# JOSM IgnoreTags")) { 189 ignorefile = true; 190 if (!DEFAULT_SOURCES.contains(source)) { 191 Main.info(tr("Adding {0} to ignore tags", source)); 192 } 193 } 194 } else if (ignorefile) { 195 line = line.trim(); 196 if (line.length() < 4) { 197 continue; 198 } 199 200 String key = line.substring(0, 2); 201 line = line.substring(2); 202 203 switch (key) { 204 case "S:": 205 ignoreDataStartsWith.add(line); 206 break; 207 case "E:": 208 ignoreDataEquals.add(line); 209 break; 210 case "F:": 211 ignoreDataEndsWith.add(line); 212 break; 213 case "K:": 214 ignoreDataTag.add(Tag.ofString(line)); 215 break; 216 default: 217 if (!key.startsWith(";")) { 218 Main.warn("Unsupported TagChecker key: " + key); 219 } 220 } 221 } else if (tagcheckerfile) { 222 if (!line.isEmpty()) { 223 CheckerData d = new CheckerData(); 224 String err = d.getData(line); 225 226 if (err == null) { 227 checkerData.add(d); 228 } else { 229 Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line)); 230 } 231 } 232 } else if (line.charAt(0) == '+') { 233 okValue = line.substring(1); 234 } else if (line.charAt(0) == '-' && okValue != null) { 235 harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue); 236 } else { 237 Main.error(tr("Invalid spellcheck line: {0}", line)); 238 } 239 if (isFirstLine) { 240 isFirstLine = false; 241 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) { 242 Main.info(tr("Adding {0} to spellchecker", source)); 243 } 244 } 245 } 246 } catch (IOException e) { 247 Main.error(e); 248 errorSources.append(source).append('\n'); 249 } 250 } 251 252 if (errorSources.length() > 0) 253 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources)); 254 } 255 256 /** 257 * Reads the presets data. 258 * 259 */ 260 public static void initializePresets() { 261 262 if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true)) 263 return; 264 265 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets(); 266 if (!presets.isEmpty()) { 267 presetsValueData = new MultiMap<>(); 268 for (String a : OsmPrimitive.getUninterestingKeys()) { 269 presetsValueData.putVoid(a); 270 } 271 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead) 272 for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys", 273 Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) { 274 presetsValueData.putVoid(a); 275 } 276 for (TaggingPreset p : presets) { 277 for (TaggingPresetItem i : p.data) { 278 if (i instanceof KeyedItem) { 279 addPresetValue(p, (KeyedItem) i); 280 } else if (i instanceof CheckGroup) { 281 for (Check c : ((CheckGroup) i).checks) { 282 addPresetValue(p, c); 283 } 284 } 285 } 286 } 287 } 288 } 289 290 private static void addPresetValue(TaggingPreset p, KeyedItem ky) { 291 Collection<String> values = ky.getValues(); 292 if (ky.key != null && values != null) { 293 try { 294 presetsValueData.putAll(ky.key, values); 295 harmonizedKeys.put(harmonizeKey(ky.key), ky.key); 296 } catch (NullPointerException e) { 297 Main.error(e, p+": Unable to initialize "+ky+'.'); 298 } 299 } 300 } 301 302 /** 303 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters) 304 * @param s string to check 305 * @return {@code true} if {@code s} contains characters with code below 0x20 306 */ 307 private static boolean containsLow(String s) { 308 if (s == null) 309 return false; 310 for (int i = 0; i < s.length(); i++) { 311 if (s.charAt(i) < 0x20) 312 return true; 313 } 314 return false; 315 } 316 317 /** 318 * Determines if the given key is in internal presets. 319 * @param key key 320 * @return {@code true} if the given key is in internal presets 321 * @since 9023 322 */ 323 public static boolean isKeyInPresets(String key) { 324 return presetsValueData.get(key) != null; 325 } 326 327 /** 328 * Determines if the given tag is in internal presets. 329 * @param key key 330 * @param value value 331 * @return {@code true} if the given tag is in internal presets 332 * @since 9023 333 */ 334 public static boolean isTagInPresets(String key, String value) { 335 final Set<String> values = presetsValueData.get(key); 336 return values != null && (values.isEmpty() || values.contains(value)); 337 } 338 339 /** 340 * Returns the list of ignored tags. 341 * @return the list of ignored tags 342 * @since 9023 343 */ 344 public static List<Tag> getIgnoredTags() { 345 return new ArrayList<>(ignoreDataTag); 346 } 347 348 /** 349 * Determines if the given tag is ignored for checks "key/tag not in presets". 350 * @param key key 351 * @param value value 352 * @return {@code true} if the given tag is ignored 353 * @since 9023 354 */ 355 public static boolean isTagIgnored(String key, String value) { 356 boolean tagInPresets = isTagInPresets(key, value); 357 boolean ignore = false; 358 359 for (String a : ignoreDataStartsWith) { 360 if (key.startsWith(a)) { 361 ignore = true; 362 } 363 } 364 for (String a : ignoreDataEquals) { 365 if (key.equals(a)) { 366 ignore = true; 367 } 368 } 369 for (String a : ignoreDataEndsWith) { 370 if (key.endsWith(a)) { 371 ignore = true; 372 } 373 } 374 375 if (!tagInPresets) { 376 for (Tag a : ignoreDataTag) { 377 if (key.equals(a.getKey()) && value.equals(a.getValue())) { 378 ignore = true; 379 } 380 } 381 } 382 return ignore; 383 } 384 385 /** 386 * Checks the primitive tags 387 * @param p The primitive to check 388 */ 389 @Override 390 public void check(OsmPrimitive p) { 391 // Just a collection to know if a primitive has been already marked with error 392 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>(); 393 394 if (checkComplex) { 395 Map<String, String> keys = p.getKeys(); 396 for (CheckerData d : checkerData) { 397 if (d.match(p, keys)) { 398 errors.add(new TestError(this, d.getSeverity(), tr("Suspicious tag/value combinations"), 399 d.getDescription(), d.getDescriptionOrig(), d.getCode(), p)); 400 withErrors.put(p, "TC"); 401 } 402 } 403 } 404 405 for (Entry<String, String> prop : p.getKeys().entrySet()) { 406 String s = marktr("Key ''{0}'' invalid."); 407 String key = prop.getKey(); 408 String value = prop.getValue(); 409 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) { 410 errors.add(new TestError(this, Severity.WARNING, tr("Tag value contains character with code less than 0x20"), 411 tr(s, key), MessageFormat.format(s, key), LOW_CHAR_VALUE, p)); 412 withErrors.put(p, "ICV"); 413 } 414 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) { 415 errors.add(new TestError(this, Severity.WARNING, tr("Tag key contains character with code less than 0x20"), 416 tr(s, key), MessageFormat.format(s, key), LOW_CHAR_KEY, p)); 417 withErrors.put(p, "ICK"); 418 } 419 if (checkValues && (value != null && value.length() > 255) && !withErrors.contains(p, "LV")) { 420 errors.add(new TestError(this, Severity.ERROR, tr("Tag value longer than allowed"), 421 tr(s, key), MessageFormat.format(s, key), LONG_VALUE, p)); 422 withErrors.put(p, "LV"); 423 } 424 if (checkKeys && (key != null && key.length() > 255) && !withErrors.contains(p, "LK")) { 425 errors.add(new TestError(this, Severity.ERROR, tr("Tag key longer than allowed"), 426 tr(s, key), MessageFormat.format(s, key), LONG_KEY, p)); 427 withErrors.put(p, "LK"); 428 } 429 if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) { 430 errors.add(new TestError(this, Severity.WARNING, tr("Tags with empty values"), 431 tr(s, key), MessageFormat.format(s, key), EMPTY_VALUES, p)); 432 withErrors.put(p, "EV"); 433 } 434 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) { 435 errors.add(new TestError(this, Severity.WARNING, tr("Invalid white space in property key"), 436 tr(s, key), MessageFormat.format(s, key), INVALID_KEY_SPACE, p)); 437 withErrors.put(p, "IPK"); 438 } 439 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) { 440 errors.add(new TestError(this, Severity.WARNING, tr("Property values start or end with white space"), 441 tr(s, key), MessageFormat.format(s, key), INVALID_SPACE, p)); 442 withErrors.put(p, "SPACE"); 443 } 444 if (checkValues && value != null && value.contains(" ") && !withErrors.contains(p, "SPACE")) { 445 errors.add(new TestError(this, Severity.WARNING, tr("Property values contain multiple white spaces"), 446 tr(s, key), MessageFormat.format(s, key), MULTIPLE_SPACES, p)); 447 withErrors.put(p, "SPACE"); 448 } 449 if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) { 450 errors.add(new TestError(this, Severity.OTHER, tr("Property values contain HTML entity"), 451 tr(s, key), MessageFormat.format(s, key), INVALID_HTML, p)); 452 withErrors.put(p, "HTML"); 453 } 454 if (checkValues && key != null && value != null && !value.isEmpty() && presetsValueData != null) { 455 if (!isTagIgnored(key, value)) { 456 if (!isKeyInPresets(key)) { 457 String prettifiedKey = harmonizeKey(key); 458 String fixedKey = harmonizedKeys.get(prettifiedKey); 459 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) { 460 // misspelled preset key 461 String i = marktr("Key ''{0}'' looks like ''{1}''."); 462 final TestError error; 463 if (p.hasKey(fixedKey)) { 464 error = new TestError(this, Severity.WARNING, tr("Misspelled property key"), 465 tr(i, key, fixedKey), 466 MessageFormat.format(i, key, fixedKey), MISSPELLED_KEY, p); 467 } else { 468 error = new FixableTestError(this, Severity.WARNING, tr("Misspelled property key"), 469 tr(i, key, fixedKey), 470 MessageFormat.format(i, key, fixedKey), MISSPELLED_KEY, p, 471 new ChangePropertyKeyCommand(p, key, fixedKey)); 472 } 473 errors.add(error); 474 withErrors.put(p, "WPK"); 475 } else { 476 String i = marktr("Key ''{0}'' not in presets."); 477 errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property key"), 478 tr(i, key), MessageFormat.format(i, key), INVALID_VALUE, p)); 479 withErrors.put(p, "UPK"); 480 } 481 } else if (!isTagInPresets(key, value)) { 482 // try to fix common typos and check again if value is still unknown 483 String fixedValue = harmonizeValue(prop.getValue()); 484 Map<String, String> possibleValues = getPossibleValues(presetsValueData.get(key)); 485 if (possibleValues.containsKey(fixedValue)) { 486 fixedValue = possibleValues.get(fixedValue); 487 // misspelled preset value 488 String i = marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."); 489 errors.add(new FixableTestError(this, Severity.WARNING, tr("Misspelled property value"), 490 tr(i, prop.getValue(), key, fixedValue), MessageFormat.format(i, prop.getValue(), fixedValue), 491 MISSPELLED_VALUE, p, new ChangePropertyCommand(p, key, fixedValue))); 492 withErrors.put(p, "WPV"); 493 } else { 494 // unknown preset value 495 String i = marktr("Value ''{0}'' for key ''{1}'' not in presets."); 496 errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property value"), 497 tr(i, prop.getValue(), key), MessageFormat.format(i, prop.getValue(), key), INVALID_VALUE, p)); 498 withErrors.put(p, "UPV"); 499 } 500 } 501 } 502 } 503 if (checkFixmes && key != null && value != null && !value.isEmpty()) { 504 if ((value.toLowerCase(Locale.ENGLISH).contains("fixme") 505 || value.contains("check and delete") 506 || key.contains("todo") || key.toLowerCase(Locale.ENGLISH).contains("fixme")) 507 && !withErrors.contains(p, "FIXME")) { 508 errors.add(new TestError(this, Severity.OTHER, 509 tr("FIXMES"), FIXME, p)); 510 withErrors.put(p, "FIXME"); 511 } 512 } 513 } 514 } 515 516 private static Map<String, String> getPossibleValues(Set<String> values) { 517 // generate a map with common typos 518 Map<String, String> map = new HashMap<>(); 519 if (values != null) { 520 for (String value : values) { 521 map.put(value, value); 522 if (value.contains("_")) { 523 map.put(value.replace("_", ""), value); 524 } 525 } 526 } 527 return map; 528 } 529 530 private static String harmonizeKey(String key) { 531 key = key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'); 532 return Utils.strip(key, "-_;:,"); 533 } 534 535 private static String harmonizeValue(String value) { 536 value = value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'); 537 return Utils.strip(value, "-_;:,"); 538 } 539 540 @Override 541 public void startTest(ProgressMonitor monitor) { 542 super.startTest(monitor); 543 checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true); 544 if (isBeforeUpload) { 545 checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true); 546 } 547 548 checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true); 549 if (isBeforeUpload) { 550 checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true); 551 } 552 553 checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true); 554 if (isBeforeUpload) { 555 checkComplex = checkComplex && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true); 556 } 557 558 checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true); 559 if (isBeforeUpload) { 560 checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true); 561 } 562 } 563 564 @Override 565 public void visit(Collection<OsmPrimitive> selection) { 566 if (checkKeys || checkValues || checkComplex || checkFixmes) { 567 super.visit(selection); 568 } 569 } 570 571 @Override 572 public void addGui(JPanel testPanel) { 573 GBC a = GBC.eol(); 574 a.anchor = GridBagConstraints.EAST; 575 576 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0)); 577 578 prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true)); 579 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words.")); 580 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0)); 581 582 prefCheckKeysBeforeUpload = new JCheckBox(); 583 prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true)); 584 testPanel.add(prefCheckKeysBeforeUpload, a); 585 586 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true)); 587 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules.")); 588 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0)); 589 590 prefCheckComplexBeforeUpload = new JCheckBox(); 591 prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true)); 592 testPanel.add(prefCheckComplexBeforeUpload, a); 593 594 final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES); 595 sourcesList = new EditableList(tr("TagChecker source")); 596 sourcesList.setItems(sources); 597 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0)); 598 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0)); 599 600 ActionListener disableCheckActionListener = new ActionListener() { 601 @Override 602 public void actionPerformed(ActionEvent e) { 603 handlePrefEnable(); 604 } 605 }; 606 prefCheckKeys.addActionListener(disableCheckActionListener); 607 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener); 608 prefCheckComplex.addActionListener(disableCheckActionListener); 609 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener); 610 611 handlePrefEnable(); 612 613 prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true)); 614 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets.")); 615 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0)); 616 617 prefCheckValuesBeforeUpload = new JCheckBox(); 618 prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true)); 619 testPanel.add(prefCheckValuesBeforeUpload, a); 620 621 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true)); 622 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value.")); 623 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0)); 624 625 prefCheckFixmesBeforeUpload = new JCheckBox(); 626 prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true)); 627 testPanel.add(prefCheckFixmesBeforeUpload, a); 628 } 629 630 public void handlePrefEnable() { 631 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected() 632 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 633 sourcesList.setEnabled(selected); 634 } 635 636 @Override 637 public boolean ok() { 638 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected(); 639 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected() 640 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 641 642 Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected()); 643 Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected()); 644 Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected()); 645 Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected()); 646 Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected()); 647 Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected()); 648 Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected()); 649 Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected()); 650 return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems()); 651 } 652 653 @Override 654 public Command fixError(TestError testError) { 655 List<Command> commands = new ArrayList<>(50); 656 657 if (testError instanceof FixableTestError) { 658 commands.add(testError.getFix()); 659 } else { 660 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives(); 661 for (OsmPrimitive p : primitives) { 662 Map<String, String> tags = p.getKeys(); 663 if (tags == null || tags.isEmpty()) { 664 continue; 665 } 666 667 for (Entry<String, String> prop: tags.entrySet()) { 668 String key = prop.getKey(); 669 String value = prop.getValue(); 670 if (value == null || value.trim().isEmpty()) { 671 commands.add(new ChangePropertyCommand(p, key, null)); 672 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) { 673 commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value))); 674 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) { 675 commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key))); 676 } else { 677 String evalue = Entities.unescape(value); 678 if (!evalue.equals(value)) { 679 commands.add(new ChangePropertyCommand(p, key, evalue)); 680 } 681 } 682 } 683 } 684 } 685 686 if (commands.isEmpty()) 687 return null; 688 if (commands.size() == 1) 689 return commands.get(0); 690 691 return new SequenceCommand(tr("Fix tags"), commands); 692 } 693 694 @Override 695 public boolean isFixable(TestError testError) { 696 if (testError.getTester() instanceof TagChecker) { 697 int code = testError.getCode(); 698 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE || 699 code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE || 700 code == MULTIPLE_SPACES; 701 } 702 703 return false; 704 } 705 706 protected static class CheckerData { 707 private String description; 708 protected List<CheckerElement> data = new ArrayList<>(); 709 private OsmPrimitiveType type; 710 private int code; 711 protected Severity severity; 712 // CHECKSTYLE.OFF: SingleSpaceSeparator 713 protected static final int TAG_CHECK_ERROR = 1250; 714 protected static final int TAG_CHECK_WARN = 1260; 715 protected static final int TAG_CHECK_INFO = 1270; 716 // CHECKSTYLE.ON: SingleSpaceSeparator 717 718 protected static class CheckerElement { 719 public Object tag; 720 public Object value; 721 public boolean noMatch; 722 public boolean tagAll; 723 public boolean valueAll; 724 public boolean valueBool; 725 726 private static Pattern getPattern(String str) throws PatternSyntaxException { 727 if (str.endsWith("/i")) 728 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE); 729 if (str.endsWith("/")) 730 return Pattern.compile(str.substring(1, str.length()-1)); 731 732 throw new IllegalStateException(); 733 } 734 735 public CheckerElement(String exp) throws PatternSyntaxException { 736 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp); 737 m.matches(); 738 739 String n = m.group(1).trim(); 740 741 if ("*".equals(n)) { 742 tagAll = true; 743 } else { 744 tag = n.startsWith("/") ? getPattern(n) : n; 745 noMatch = "!=".equals(m.group(2)); 746 n = m.group(3).trim(); 747 if ("*".equals(n)) { 748 valueAll = true; 749 } else if ("BOOLEAN_TRUE".equals(n)) { 750 valueBool = true; 751 value = OsmUtils.trueval; 752 } else if ("BOOLEAN_FALSE".equals(n)) { 753 valueBool = true; 754 value = OsmUtils.falseval; 755 } else { 756 value = n.startsWith("/") ? getPattern(n) : n; 757 } 758 } 759 } 760 761 public boolean match(Map<String, String> keys) { 762 for (Entry<String, String> prop: keys.entrySet()) { 763 String key = prop.getKey(); 764 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue(); 765 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag))) 766 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value)))) 767 return !noMatch; 768 } 769 return noMatch; 770 } 771 } 772 773 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$"); 774 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *"); 775 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *"); 776 777 public String getData(final String str) { 778 Matcher m = CLEAN_STR_PATTERN.matcher(str); 779 String trimmed = m.replaceFirst("").trim(); 780 try { 781 description = m.group(1); 782 if (description != null && description.isEmpty()) { 783 description = null; 784 } 785 } catch (IllegalStateException e) { 786 Main.error(e); 787 description = null; 788 } 789 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3); 790 switch (n[0]) { 791 case "way": 792 type = OsmPrimitiveType.WAY; 793 break; 794 case "node": 795 type = OsmPrimitiveType.NODE; 796 break; 797 case "relation": 798 type = OsmPrimitiveType.RELATION; 799 break; 800 case "*": 801 type = null; 802 break; 803 default: 804 return tr("Could not find element type"); 805 } 806 if (n.length != 3) 807 return tr("Incorrect number of parameters"); 808 809 switch (n[1]) { 810 case "W": 811 severity = Severity.WARNING; 812 code = TAG_CHECK_WARN; 813 break; 814 case "E": 815 severity = Severity.ERROR; 816 code = TAG_CHECK_ERROR; 817 break; 818 case "I": 819 severity = Severity.OTHER; 820 code = TAG_CHECK_INFO; 821 break; 822 default: 823 return tr("Could not find warning level"); 824 } 825 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) { 826 try { 827 data.add(new CheckerElement(exp)); 828 } catch (IllegalStateException e) { 829 Main.trace(e); 830 return tr("Illegal expression ''{0}''", exp); 831 } catch (PatternSyntaxException e) { 832 Main.trace(e); 833 return tr("Illegal regular expression ''{0}''", exp); 834 } 835 } 836 return null; 837 } 838 839 public boolean match(OsmPrimitive osm, Map<String, String> keys) { 840 if (type != null && OsmPrimitiveType.from(osm) != type) 841 return false; 842 843 for (CheckerElement ce : data) { 844 if (!ce.match(keys)) 845 return false; 846 } 847 return true; 848 } 849 850 public String getDescription() { 851 return tr(description); 852 } 853 854 public String getDescriptionOrig() { 855 return description; 856 } 857 858 public Severity getSeverity() { 859 return severity; 860 } 861 862 public int getCode() { 863 if (type == null) 864 return code; 865 866 return code + type.ordinal() + 1; 867 } 868 } 869}