001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006
007import java.awt.GridBagConstraints;
008import java.awt.event.ActionEvent;
009import java.awt.event.ActionListener;
010import java.io.BufferedReader;
011import java.io.IOException;
012import java.io.InputStream;
013import java.text.MessageFormat;
014import java.util.ArrayList;
015import java.util.Arrays;
016import java.util.Collection;
017import java.util.HashMap;
018import java.util.HashSet;
019import java.util.List;
020import java.util.Locale;
021import java.util.Map;
022import java.util.Map.Entry;
023import java.util.Set;
024import java.util.regex.Matcher;
025import java.util.regex.Pattern;
026import java.util.regex.PatternSyntaxException;
027
028import javax.swing.JCheckBox;
029import javax.swing.JLabel;
030import javax.swing.JPanel;
031
032import org.openstreetmap.josm.Main;
033import org.openstreetmap.josm.command.ChangePropertyCommand;
034import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
035import org.openstreetmap.josm.command.Command;
036import org.openstreetmap.josm.command.SequenceCommand;
037import org.openstreetmap.josm.data.osm.OsmPrimitive;
038import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
039import org.openstreetmap.josm.data.osm.OsmUtils;
040import org.openstreetmap.josm.data.osm.Tag;
041import org.openstreetmap.josm.data.validation.FixableTestError;
042import org.openstreetmap.josm.data.validation.Severity;
043import org.openstreetmap.josm.data.validation.Test.TagTest;
044import org.openstreetmap.josm.data.validation.TestError;
045import org.openstreetmap.josm.data.validation.util.Entities;
046import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference;
047import org.openstreetmap.josm.gui.progress.ProgressMonitor;
048import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
049import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
050import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
051import org.openstreetmap.josm.gui.tagging.presets.items.Check;
052import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
053import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
054import org.openstreetmap.josm.gui.widgets.EditableList;
055import org.openstreetmap.josm.io.CachedFile;
056import org.openstreetmap.josm.io.UTFInputStreamReader;
057import org.openstreetmap.josm.tools.GBC;
058import org.openstreetmap.josm.tools.MultiMap;
059import org.openstreetmap.josm.tools.Utils;
060
061/**
062 * Check for misspelled or wrong tags
063 *
064 * @author frsantos
065 * @since 3669
066 */
067public class TagChecker extends TagTest {
068
069    /** The config file of ignored tags */
070    public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
071    /** The config file of dictionary words */
072    public static final String SPELL_FILE = "resource://data/validator/words.cfg";
073
074    /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
075    private static final Map<String, String> harmonizedKeys = new HashMap<>();
076    /** The spell check preset values */
077    private static volatile MultiMap<String, String> presetsValueData;
078    /** The TagChecker data */
079    private static final List<CheckerData> checkerData = new ArrayList<>();
080    private static final List<String> ignoreDataStartsWith = new ArrayList<>();
081    private static final List<String> ignoreDataEquals = new ArrayList<>();
082    private static final List<String> ignoreDataEndsWith = new ArrayList<>();
083    private static final List<Tag> ignoreDataTag = new ArrayList<>();
084
085    /** The preferences prefix */
086    protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName();
087
088    public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
089    public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
090    public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
091    public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
092
093    public static final String PREF_SOURCES = PREFIX + ".source";
094
095    public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
096    public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
097    public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
098    public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
099
100    protected boolean checkKeys;
101    protected boolean checkValues;
102    protected boolean checkComplex;
103    protected boolean checkFixmes;
104
105    protected JCheckBox prefCheckKeys;
106    protected JCheckBox prefCheckValues;
107    protected JCheckBox prefCheckComplex;
108    protected JCheckBox prefCheckFixmes;
109    protected JCheckBox prefCheckPaint;
110
111    protected JCheckBox prefCheckKeysBeforeUpload;
112    protected JCheckBox prefCheckValuesBeforeUpload;
113    protected JCheckBox prefCheckComplexBeforeUpload;
114    protected JCheckBox prefCheckFixmesBeforeUpload;
115    protected JCheckBox prefCheckPaintBeforeUpload;
116
117    // CHECKSTYLE.OFF: SingleSpaceSeparator
118    protected static final int EMPTY_VALUES      = 1200;
119    protected static final int INVALID_KEY       = 1201;
120    protected static final int INVALID_VALUE     = 1202;
121    protected static final int FIXME             = 1203;
122    protected static final int INVALID_SPACE     = 1204;
123    protected static final int INVALID_KEY_SPACE = 1205;
124    protected static final int INVALID_HTML      = 1206; /* 1207 was PAINT */
125    protected static final int LONG_VALUE        = 1208;
126    protected static final int LONG_KEY          = 1209;
127    protected static final int LOW_CHAR_VALUE    = 1210;
128    protected static final int LOW_CHAR_KEY      = 1211;
129    protected static final int MISSPELLED_VALUE  = 1212;
130    protected static final int MISSPELLED_KEY    = 1213;
131    protected static final int MULTIPLE_SPACES   = 1214;
132    // CHECKSTYLE.ON: SingleSpaceSeparator
133    // 1250 and up is used by tagcheck
134
135    protected EditableList sourcesList;
136
137    private static final Set<String> DEFAULT_SOURCES = new HashSet<>(Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE));
138
139    /**
140     * Constructor
141     */
142    public TagChecker() {
143        super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
144    }
145
146    @Override
147    public void initialize() throws IOException {
148        initializeData();
149        initializePresets();
150    }
151
152    /**
153     * Reads the spellcheck file into a HashMap.
154     * The data file is a list of words, beginning with +/-. If it starts with +,
155     * the word is valid, but if it starts with -, the word should be replaced
156     * by the nearest + word before this.
157     *
158     * @throws IOException if any I/O error occurs
159     */
160    private static void initializeData() throws IOException {
161        checkerData.clear();
162        ignoreDataStartsWith.clear();
163        ignoreDataEquals.clear();
164        ignoreDataEndsWith.clear();
165        ignoreDataTag.clear();
166        harmonizedKeys.clear();
167
168        StringBuilder errorSources = new StringBuilder();
169        for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) {
170            try (
171                CachedFile cf = new CachedFile(source);
172                InputStream s = cf.getInputStream();
173                BufferedReader reader = new BufferedReader(UTFInputStreamReader.create(s));
174            ) {
175                String okValue = null;
176                boolean tagcheckerfile = false;
177                boolean ignorefile = false;
178                boolean isFirstLine = true;
179                String line;
180                while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) {
181                    if (line.startsWith("#")) {
182                        if (line.startsWith("# JOSM TagChecker")) {
183                            tagcheckerfile = true;
184                            if (!DEFAULT_SOURCES.contains(source)) {
185                                Main.info(tr("Adding {0} to tag checker", source));
186                            }
187                        } else
188                        if (line.startsWith("# JOSM IgnoreTags")) {
189                            ignorefile = true;
190                            if (!DEFAULT_SOURCES.contains(source)) {
191                                Main.info(tr("Adding {0} to ignore tags", source));
192                            }
193                        }
194                    } else if (ignorefile) {
195                        line = line.trim();
196                        if (line.length() < 4) {
197                            continue;
198                        }
199
200                        String key = line.substring(0, 2);
201                        line = line.substring(2);
202
203                        switch (key) {
204                        case "S:":
205                            ignoreDataStartsWith.add(line);
206                            break;
207                        case "E:":
208                            ignoreDataEquals.add(line);
209                            break;
210                        case "F:":
211                            ignoreDataEndsWith.add(line);
212                            break;
213                        case "K:":
214                            ignoreDataTag.add(Tag.ofString(line));
215                            break;
216                        default:
217                            if (!key.startsWith(";")) {
218                                Main.warn("Unsupported TagChecker key: " + key);
219                            }
220                        }
221                    } else if (tagcheckerfile) {
222                        if (!line.isEmpty()) {
223                            CheckerData d = new CheckerData();
224                            String err = d.getData(line);
225
226                            if (err == null) {
227                                checkerData.add(d);
228                            } else {
229                                Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
230                            }
231                        }
232                    } else if (line.charAt(0) == '+') {
233                        okValue = line.substring(1);
234                    } else if (line.charAt(0) == '-' && okValue != null) {
235                        harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue);
236                    } else {
237                        Main.error(tr("Invalid spellcheck line: {0}", line));
238                    }
239                    if (isFirstLine) {
240                        isFirstLine = false;
241                        if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
242                            Main.info(tr("Adding {0} to spellchecker", source));
243                        }
244                    }
245                }
246            } catch (IOException e) {
247                Main.error(e);
248                errorSources.append(source).append('\n');
249            }
250        }
251
252        if (errorSources.length() > 0)
253            throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
254    }
255
256    /**
257     * Reads the presets data.
258     *
259     */
260    public static void initializePresets() {
261
262        if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true))
263            return;
264
265        Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
266        if (!presets.isEmpty()) {
267            presetsValueData = new MultiMap<>();
268            for (String a : OsmPrimitive.getUninterestingKeys()) {
269                presetsValueData.putVoid(a);
270            }
271            // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
272            for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys",
273                    Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) {
274                presetsValueData.putVoid(a);
275            }
276            for (TaggingPreset p : presets) {
277                for (TaggingPresetItem i : p.data) {
278                    if (i instanceof KeyedItem) {
279                        addPresetValue(p, (KeyedItem) i);
280                    } else if (i instanceof CheckGroup) {
281                        for (Check c : ((CheckGroup) i).checks) {
282                            addPresetValue(p, c);
283                        }
284                    }
285                }
286            }
287        }
288    }
289
290    private static void addPresetValue(TaggingPreset p, KeyedItem ky) {
291        Collection<String> values = ky.getValues();
292        if (ky.key != null && values != null) {
293            try {
294                presetsValueData.putAll(ky.key, values);
295                harmonizedKeys.put(harmonizeKey(ky.key), ky.key);
296            } catch (NullPointerException e) {
297                Main.error(e, p+": Unable to initialize "+ky+'.');
298            }
299        }
300    }
301
302    /**
303     * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
304     * @param s string to check
305     * @return {@code true} if {@code s} contains characters with code below 0x20
306     */
307    private static boolean containsLow(String s) {
308        if (s == null)
309            return false;
310        for (int i = 0; i < s.length(); i++) {
311            if (s.charAt(i) < 0x20)
312                return true;
313        }
314        return false;
315    }
316
317    /**
318     * Determines if the given key is in internal presets.
319     * @param key key
320     * @return {@code true} if the given key is in internal presets
321     * @since 9023
322     */
323    public static boolean isKeyInPresets(String key) {
324        return presetsValueData.get(key) != null;
325    }
326
327    /**
328     * Determines if the given tag is in internal presets.
329     * @param key key
330     * @param value value
331     * @return {@code true} if the given tag is in internal presets
332     * @since 9023
333     */
334    public static boolean isTagInPresets(String key, String value) {
335        final Set<String> values = presetsValueData.get(key);
336        return values != null && (values.isEmpty() || values.contains(value));
337    }
338
339    /**
340     * Returns the list of ignored tags.
341     * @return the list of ignored tags
342     * @since 9023
343     */
344    public static List<Tag> getIgnoredTags() {
345        return new ArrayList<>(ignoreDataTag);
346    }
347
348    /**
349     * Determines if the given tag is ignored for checks "key/tag not in presets".
350     * @param key key
351     * @param value value
352     * @return {@code true} if the given tag is ignored
353     * @since 9023
354     */
355    public static boolean isTagIgnored(String key, String value) {
356        boolean tagInPresets = isTagInPresets(key, value);
357        boolean ignore = false;
358
359        for (String a : ignoreDataStartsWith) {
360            if (key.startsWith(a)) {
361                ignore = true;
362            }
363        }
364        for (String a : ignoreDataEquals) {
365            if (key.equals(a)) {
366                ignore = true;
367            }
368        }
369        for (String a : ignoreDataEndsWith) {
370            if (key.endsWith(a)) {
371                ignore = true;
372            }
373        }
374
375        if (!tagInPresets) {
376            for (Tag a : ignoreDataTag) {
377                if (key.equals(a.getKey()) && value.equals(a.getValue())) {
378                    ignore = true;
379                }
380            }
381        }
382        return ignore;
383    }
384
385    /**
386     * Checks the primitive tags
387     * @param p The primitive to check
388     */
389    @Override
390    public void check(OsmPrimitive p) {
391        // Just a collection to know if a primitive has been already marked with error
392        MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
393
394        if (checkComplex) {
395            Map<String, String> keys = p.getKeys();
396            for (CheckerData d : checkerData) {
397                if (d.match(p, keys)) {
398                    errors.add(new TestError(this, d.getSeverity(), tr("Suspicious tag/value combinations"),
399                            d.getDescription(), d.getDescriptionOrig(), d.getCode(), p));
400                    withErrors.put(p, "TC");
401                }
402            }
403        }
404
405        for (Entry<String, String> prop : p.getKeys().entrySet()) {
406            String s = marktr("Key ''{0}'' invalid.");
407            String key = prop.getKey();
408            String value = prop.getValue();
409            if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
410                errors.add(new TestError(this, Severity.WARNING, tr("Tag value contains character with code less than 0x20"),
411                        tr(s, key), MessageFormat.format(s, key), LOW_CHAR_VALUE, p));
412                withErrors.put(p, "ICV");
413            }
414            if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
415                errors.add(new TestError(this, Severity.WARNING, tr("Tag key contains character with code less than 0x20"),
416                        tr(s, key), MessageFormat.format(s, key), LOW_CHAR_KEY, p));
417                withErrors.put(p, "ICK");
418            }
419            if (checkValues && (value != null && value.length() > 255) && !withErrors.contains(p, "LV")) {
420                errors.add(new TestError(this, Severity.ERROR, tr("Tag value longer than allowed"),
421                        tr(s, key), MessageFormat.format(s, key), LONG_VALUE, p));
422                withErrors.put(p, "LV");
423            }
424            if (checkKeys && (key != null && key.length() > 255) && !withErrors.contains(p, "LK")) {
425                errors.add(new TestError(this, Severity.ERROR, tr("Tag key longer than allowed"),
426                        tr(s, key), MessageFormat.format(s, key), LONG_KEY, p));
427                withErrors.put(p, "LK");
428            }
429            if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
430                errors.add(new TestError(this, Severity.WARNING, tr("Tags with empty values"),
431                        tr(s, key), MessageFormat.format(s, key), EMPTY_VALUES, p));
432                withErrors.put(p, "EV");
433            }
434            if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
435                errors.add(new TestError(this, Severity.WARNING, tr("Invalid white space in property key"),
436                        tr(s, key), MessageFormat.format(s, key), INVALID_KEY_SPACE, p));
437                withErrors.put(p, "IPK");
438            }
439            if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
440                errors.add(new TestError(this, Severity.WARNING, tr("Property values start or end with white space"),
441                        tr(s, key), MessageFormat.format(s, key), INVALID_SPACE, p));
442                withErrors.put(p, "SPACE");
443            }
444            if (checkValues && value != null && value.contains("  ") && !withErrors.contains(p, "SPACE")) {
445                errors.add(new TestError(this, Severity.WARNING, tr("Property values contain multiple white spaces"),
446                        tr(s, key), MessageFormat.format(s, key), MULTIPLE_SPACES, p));
447                withErrors.put(p, "SPACE");
448            }
449            if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
450                errors.add(new TestError(this, Severity.OTHER, tr("Property values contain HTML entity"),
451                        tr(s, key), MessageFormat.format(s, key), INVALID_HTML, p));
452                withErrors.put(p, "HTML");
453            }
454            if (checkValues && key != null && value != null && !value.isEmpty() && presetsValueData != null) {
455                if (!isTagIgnored(key, value)) {
456                    if (!isKeyInPresets(key)) {
457                        String prettifiedKey = harmonizeKey(key);
458                        String fixedKey = harmonizedKeys.get(prettifiedKey);
459                        if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
460                            // misspelled preset key
461                            String i = marktr("Key ''{0}'' looks like ''{1}''.");
462                            final TestError error;
463                            if (p.hasKey(fixedKey)) {
464                                error = new TestError(this, Severity.WARNING, tr("Misspelled property key"),
465                                        tr(i, key, fixedKey),
466                                        MessageFormat.format(i, key, fixedKey), MISSPELLED_KEY, p);
467                            } else {
468                                error = new FixableTestError(this, Severity.WARNING, tr("Misspelled property key"),
469                                        tr(i, key, fixedKey),
470                                        MessageFormat.format(i, key, fixedKey), MISSPELLED_KEY, p,
471                                        new ChangePropertyKeyCommand(p, key, fixedKey));
472                            }
473                            errors.add(error);
474                            withErrors.put(p, "WPK");
475                        } else {
476                            String i = marktr("Key ''{0}'' not in presets.");
477                            errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property key"),
478                                    tr(i, key), MessageFormat.format(i, key), INVALID_VALUE, p));
479                            withErrors.put(p, "UPK");
480                        }
481                    } else if (!isTagInPresets(key, value)) {
482                        // try to fix common typos and check again if value is still unknown
483                        String fixedValue = harmonizeValue(prop.getValue());
484                        Map<String, String> possibleValues = getPossibleValues(presetsValueData.get(key));
485                        if (possibleValues.containsKey(fixedValue)) {
486                            fixedValue = possibleValues.get(fixedValue);
487                            // misspelled preset value
488                            String i = marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''.");
489                            errors.add(new FixableTestError(this, Severity.WARNING, tr("Misspelled property value"),
490                                    tr(i, prop.getValue(), key, fixedValue), MessageFormat.format(i, prop.getValue(), fixedValue),
491                                    MISSPELLED_VALUE, p, new ChangePropertyCommand(p, key, fixedValue)));
492                            withErrors.put(p, "WPV");
493                        } else {
494                            // unknown preset value
495                            String i = marktr("Value ''{0}'' for key ''{1}'' not in presets.");
496                            errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property value"),
497                                    tr(i, prop.getValue(), key), MessageFormat.format(i, prop.getValue(), key), INVALID_VALUE, p));
498                            withErrors.put(p, "UPV");
499                        }
500                    }
501                }
502            }
503            if (checkFixmes && key != null && value != null && !value.isEmpty()) {
504                if ((value.toLowerCase(Locale.ENGLISH).contains("fixme")
505                        || value.contains("check and delete")
506                        || key.contains("todo") || key.toLowerCase(Locale.ENGLISH).contains("fixme"))
507                        && !withErrors.contains(p, "FIXME")) {
508                    errors.add(new TestError(this, Severity.OTHER,
509                            tr("FIXMES"), FIXME, p));
510                    withErrors.put(p, "FIXME");
511                }
512            }
513        }
514    }
515
516    private static Map<String, String> getPossibleValues(Set<String> values) {
517        // generate a map with common typos
518        Map<String, String> map = new HashMap<>();
519        if (values != null) {
520            for (String value : values) {
521                map.put(value, value);
522                if (value.contains("_")) {
523                    map.put(value.replace("_", ""), value);
524                }
525            }
526        }
527        return map;
528    }
529
530    private static String harmonizeKey(String key) {
531        key = key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_');
532        return Utils.strip(key, "-_;:,");
533    }
534
535    private static String harmonizeValue(String value) {
536        value = value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_');
537        return Utils.strip(value, "-_;:,");
538    }
539
540    @Override
541    public void startTest(ProgressMonitor monitor) {
542        super.startTest(monitor);
543        checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true);
544        if (isBeforeUpload) {
545            checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
546        }
547
548        checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true);
549        if (isBeforeUpload) {
550            checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
551        }
552
553        checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true);
554        if (isBeforeUpload) {
555            checkComplex = checkComplex && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
556        }
557
558        checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true);
559        if (isBeforeUpload) {
560            checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
561        }
562    }
563
564    @Override
565    public void visit(Collection<OsmPrimitive> selection) {
566        if (checkKeys || checkValues || checkComplex || checkFixmes) {
567            super.visit(selection);
568        }
569    }
570
571    @Override
572    public void addGui(JPanel testPanel) {
573        GBC a = GBC.eol();
574        a.anchor = GridBagConstraints.EAST;
575
576        testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
577
578        prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true));
579        prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
580        testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
581
582        prefCheckKeysBeforeUpload = new JCheckBox();
583        prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
584        testPanel.add(prefCheckKeysBeforeUpload, a);
585
586        prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true));
587        prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
588        testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
589
590        prefCheckComplexBeforeUpload = new JCheckBox();
591        prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
592        testPanel.add(prefCheckComplexBeforeUpload, a);
593
594        final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES);
595        sourcesList = new EditableList(tr("TagChecker source"));
596        sourcesList.setItems(sources);
597        testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
598        testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
599
600        ActionListener disableCheckActionListener = new ActionListener() {
601            @Override
602            public void actionPerformed(ActionEvent e) {
603                handlePrefEnable();
604            }
605        };
606        prefCheckKeys.addActionListener(disableCheckActionListener);
607        prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
608        prefCheckComplex.addActionListener(disableCheckActionListener);
609        prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
610
611        handlePrefEnable();
612
613        prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true));
614        prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
615        testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
616
617        prefCheckValuesBeforeUpload = new JCheckBox();
618        prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
619        testPanel.add(prefCheckValuesBeforeUpload, a);
620
621        prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true));
622        prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
623        testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
624
625        prefCheckFixmesBeforeUpload = new JCheckBox();
626        prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
627        testPanel.add(prefCheckFixmesBeforeUpload, a);
628    }
629
630    public void handlePrefEnable() {
631        boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
632                || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
633        sourcesList.setEnabled(selected);
634    }
635
636    @Override
637    public boolean ok() {
638        enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
639        testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
640                || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
641
642        Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected());
643        Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
644        Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
645        Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
646        Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
647        Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
648        Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
649        Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
650        return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems());
651    }
652
653    @Override
654    public Command fixError(TestError testError) {
655        List<Command> commands = new ArrayList<>(50);
656
657        if (testError instanceof FixableTestError) {
658            commands.add(testError.getFix());
659        } else {
660            Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
661            for (OsmPrimitive p : primitives) {
662                Map<String, String> tags = p.getKeys();
663                if (tags == null || tags.isEmpty()) {
664                    continue;
665                }
666
667                for (Entry<String, String> prop: tags.entrySet()) {
668                    String key = prop.getKey();
669                    String value = prop.getValue();
670                    if (value == null || value.trim().isEmpty()) {
671                        commands.add(new ChangePropertyCommand(p, key, null));
672                    } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains("  ")) {
673                        commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value)));
674                    } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains("  ")) {
675                        commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key)));
676                    } else {
677                        String evalue = Entities.unescape(value);
678                        if (!evalue.equals(value)) {
679                            commands.add(new ChangePropertyCommand(p, key, evalue));
680                        }
681                    }
682                }
683            }
684        }
685
686        if (commands.isEmpty())
687            return null;
688        if (commands.size() == 1)
689            return commands.get(0);
690
691        return new SequenceCommand(tr("Fix tags"), commands);
692    }
693
694    @Override
695    public boolean isFixable(TestError testError) {
696        if (testError.getTester() instanceof TagChecker) {
697            int code = testError.getCode();
698            return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
699                   code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE ||
700                   code == MULTIPLE_SPACES;
701        }
702
703        return false;
704    }
705
706    protected static class CheckerData {
707        private String description;
708        protected List<CheckerElement> data = new ArrayList<>();
709        private OsmPrimitiveType type;
710        private int code;
711        protected Severity severity;
712        // CHECKSTYLE.OFF: SingleSpaceSeparator
713        protected static final int TAG_CHECK_ERROR = 1250;
714        protected static final int TAG_CHECK_WARN  = 1260;
715        protected static final int TAG_CHECK_INFO  = 1270;
716        // CHECKSTYLE.ON: SingleSpaceSeparator
717
718        protected static class CheckerElement {
719            public Object tag;
720            public Object value;
721            public boolean noMatch;
722            public boolean tagAll;
723            public boolean valueAll;
724            public boolean valueBool;
725
726            private static Pattern getPattern(String str) throws PatternSyntaxException {
727                if (str.endsWith("/i"))
728                    return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE);
729                if (str.endsWith("/"))
730                    return Pattern.compile(str.substring(1, str.length()-1));
731
732                throw new IllegalStateException();
733            }
734
735            public CheckerElement(String exp) throws PatternSyntaxException {
736                Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
737                m.matches();
738
739                String n = m.group(1).trim();
740
741                if ("*".equals(n)) {
742                    tagAll = true;
743                } else {
744                    tag = n.startsWith("/") ? getPattern(n) : n;
745                    noMatch = "!=".equals(m.group(2));
746                    n = m.group(3).trim();
747                    if ("*".equals(n)) {
748                        valueAll = true;
749                    } else if ("BOOLEAN_TRUE".equals(n)) {
750                        valueBool = true;
751                        value = OsmUtils.trueval;
752                    } else if ("BOOLEAN_FALSE".equals(n)) {
753                        valueBool = true;
754                        value = OsmUtils.falseval;
755                    } else {
756                        value = n.startsWith("/") ? getPattern(n) : n;
757                    }
758                }
759            }
760
761            public boolean match(Map<String, String> keys) {
762                for (Entry<String, String> prop: keys.entrySet()) {
763                    String key = prop.getKey();
764                    String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
765                    if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
766                            && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
767                        return !noMatch;
768                }
769                return noMatch;
770            }
771        }
772
773        private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
774        private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
775        private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
776
777        public String getData(final String str) {
778            Matcher m = CLEAN_STR_PATTERN.matcher(str);
779            String trimmed = m.replaceFirst("").trim();
780            try {
781                description = m.group(1);
782                if (description != null && description.isEmpty()) {
783                    description = null;
784                }
785            } catch (IllegalStateException e) {
786                Main.error(e);
787                description = null;
788            }
789            String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
790            switch (n[0]) {
791            case "way":
792                type = OsmPrimitiveType.WAY;
793                break;
794            case "node":
795                type = OsmPrimitiveType.NODE;
796                break;
797            case "relation":
798                type = OsmPrimitiveType.RELATION;
799                break;
800            case "*":
801                type = null;
802                break;
803            default:
804                return tr("Could not find element type");
805            }
806            if (n.length != 3)
807                return tr("Incorrect number of parameters");
808
809            switch (n[1]) {
810            case "W":
811                severity = Severity.WARNING;
812                code = TAG_CHECK_WARN;
813                break;
814            case "E":
815                severity = Severity.ERROR;
816                code = TAG_CHECK_ERROR;
817                break;
818            case "I":
819                severity = Severity.OTHER;
820                code = TAG_CHECK_INFO;
821                break;
822            default:
823                return tr("Could not find warning level");
824            }
825            for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
826                try {
827                    data.add(new CheckerElement(exp));
828                } catch (IllegalStateException e) {
829                    Main.trace(e);
830                    return tr("Illegal expression ''{0}''", exp);
831                } catch (PatternSyntaxException e) {
832                    Main.trace(e);
833                    return tr("Illegal regular expression ''{0}''", exp);
834                }
835            }
836            return null;
837        }
838
839        public boolean match(OsmPrimitive osm, Map<String, String> keys) {
840            if (type != null && OsmPrimitiveType.from(osm) != type)
841                return false;
842
843            for (CheckerElement ce : data) {
844                if (!ce.match(keys))
845                    return false;
846            }
847            return true;
848        }
849
850        public String getDescription() {
851            return tr(description);
852        }
853
854        public String getDescriptionOrig() {
855            return description;
856        }
857
858        public Severity getSeverity() {
859            return severity;
860        }
861
862        public int getCode() {
863            if (type == null)
864                return code;
865
866            return code + type.ordinal() + 1;
867        }
868    }
869}