001// License: GPL. See LICENSE file for details.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006
007import java.awt.GridBagConstraints;
008import java.awt.event.ActionEvent;
009import java.awt.event.ActionListener;
010import java.io.BufferedReader;
011import java.io.FileNotFoundException;
012import java.io.IOException;
013import java.io.InputStream;
014import java.text.MessageFormat;
015import java.util.ArrayList;
016import java.util.Arrays;
017import java.util.Collection;
018import java.util.HashMap;
019import java.util.List;
020import java.util.Map;
021import java.util.Map.Entry;
022import java.util.Set;
023import java.util.regex.Matcher;
024import java.util.regex.Pattern;
025import java.util.regex.PatternSyntaxException;
026
027import javax.swing.JCheckBox;
028import javax.swing.JLabel;
029import javax.swing.JPanel;
030
031import org.openstreetmap.josm.Main;
032import org.openstreetmap.josm.command.ChangePropertyCommand;
033import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
034import org.openstreetmap.josm.command.Command;
035import org.openstreetmap.josm.command.SequenceCommand;
036import org.openstreetmap.josm.data.osm.OsmPrimitive;
037import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
038import org.openstreetmap.josm.data.osm.OsmUtils;
039import org.openstreetmap.josm.data.osm.Tag;
040import org.openstreetmap.josm.data.validation.Severity;
041import org.openstreetmap.josm.data.validation.Test;
042import org.openstreetmap.josm.data.validation.TestError;
043import org.openstreetmap.josm.data.validation.util.Entities;
044import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference;
045import org.openstreetmap.josm.gui.progress.ProgressMonitor;
046import org.openstreetmap.josm.gui.tagging.TaggingPreset;
047import org.openstreetmap.josm.gui.tagging.TaggingPresetItem;
048import org.openstreetmap.josm.gui.tagging.TaggingPresetItems.Check;
049import org.openstreetmap.josm.gui.tagging.TaggingPresetItems.CheckGroup;
050import org.openstreetmap.josm.gui.tagging.TaggingPresetItems.KeyedItem;
051import org.openstreetmap.josm.gui.tagging.TaggingPresets;
052import org.openstreetmap.josm.gui.widgets.EditableList;
053import org.openstreetmap.josm.io.CachedFile;
054import org.openstreetmap.josm.io.UTFInputStreamReader;
055import org.openstreetmap.josm.tools.GBC;
056import org.openstreetmap.josm.tools.MultiMap;
057
058/**
059 * Check for misspelled or wrong tags
060 *
061 * @author frsantos
062 */
063public class TagChecker extends Test.TagTest {
064
065    /** The default data file of tagchecker rules */
066    public static final String DATA_FILE = "resource://data/validator/tagchecker.cfg";
067    /** The config file of ignored tags */
068    public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
069    /** The config file of dictionary words */
070    public static final String SPELL_FILE = "resource://data/validator/words.cfg";
071
072    /** The spell check key substitutions: the key should be substituted by the value */
073    private static Map<String, String> spellCheckKeyData;
074    /** The spell check preset values */
075    private static MultiMap<String, String> presetsValueData;
076    /** The TagChecker data */
077    private static final List<CheckerData> checkerData = new ArrayList<>();
078    private static final List<String> ignoreDataStartsWith = new ArrayList<>();
079    private static final List<String> ignoreDataEquals = new ArrayList<>();
080    private static final List<String> ignoreDataEndsWith = new ArrayList<>();
081    private static final List<IgnoreKeyPair> ignoreDataKeyPair = new ArrayList<>();
082
083    /** The preferences prefix */
084    protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName();
085
086    public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
087    public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
088    public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
089    public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
090
091    public static final String PREF_SOURCES = PREFIX + ".source";
092
093    public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
094    public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
095    public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
096    public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
097
098    protected boolean checkKeys = false;
099    protected boolean checkValues = false;
100    protected boolean checkComplex = false;
101    protected boolean checkFixmes = false;
102
103    protected JCheckBox prefCheckKeys;
104    protected JCheckBox prefCheckValues;
105    protected JCheckBox prefCheckComplex;
106    protected JCheckBox prefCheckFixmes;
107    protected JCheckBox prefCheckPaint;
108
109    protected JCheckBox prefCheckKeysBeforeUpload;
110    protected JCheckBox prefCheckValuesBeforeUpload;
111    protected JCheckBox prefCheckComplexBeforeUpload;
112    protected JCheckBox prefCheckFixmesBeforeUpload;
113    protected JCheckBox prefCheckPaintBeforeUpload;
114
115    protected static final int EMPTY_VALUES      = 1200;
116    protected static final int INVALID_KEY       = 1201;
117    protected static final int INVALID_VALUE     = 1202;
118    protected static final int FIXME             = 1203;
119    protected static final int INVALID_SPACE     = 1204;
120    protected static final int INVALID_KEY_SPACE = 1205;
121    protected static final int INVALID_HTML      = 1206; /* 1207 was PAINT */
122    protected static final int LONG_VALUE        = 1208;
123    protected static final int LONG_KEY          = 1209;
124    protected static final int LOW_CHAR_VALUE    = 1210;
125    protected static final int LOW_CHAR_KEY      = 1211;
126    /** 1250 and up is used by tagcheck */
127
128    protected EditableList sourcesList;
129
130    protected static final Entities entities = new Entities();
131
132    static final List<String> DEFAULT_SOURCES = Arrays.asList(DATA_FILE, IGNORE_FILE, SPELL_FILE);
133
134    /**
135     * Constructor
136     */
137    public TagChecker() {
138        super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
139    }
140
141    @Override
142    public void initialize() throws IOException {
143        initializeData();
144        initializePresets();
145    }
146
147    /**
148     * Reads the spellcheck file into a HashMap.
149     * The data file is a list of words, beginning with +/-. If it starts with +,
150     * the word is valid, but if it starts with -, the word should be replaced
151     * by the nearest + word before this.
152     *
153     * @throws FileNotFoundException
154     * @throws IOException
155     */
156    private static void initializeData() throws IOException {
157        checkerData.clear();
158        ignoreDataStartsWith.clear();
159        ignoreDataEquals.clear();
160        ignoreDataEndsWith.clear();
161        ignoreDataKeyPair.clear();
162
163        spellCheckKeyData = new HashMap<>();
164
165        String errorSources = "";
166        for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) {
167            try (
168                InputStream s = new CachedFile(source).getInputStream();
169                BufferedReader reader = new BufferedReader(UTFInputStreamReader.create(s));
170            ) {
171                String okValue = null;
172                boolean tagcheckerfile = false;
173                boolean ignorefile = false;
174                boolean isFirstLine = true;
175                String line;
176                while ((line = reader.readLine()) != null && (tagcheckerfile || line.length() != 0)) {
177                    if (line.startsWith("#")) {
178                        if (line.startsWith("# JOSM TagChecker")) {
179                            tagcheckerfile = true;
180                            if (!DEFAULT_SOURCES.contains(source)) {
181                                Main.info(tr("Adding {0} to tag checker", source));
182                            }
183                        } else
184                        if (line.startsWith("# JOSM IgnoreTags")) {
185                            ignorefile = true;
186                            if (!DEFAULT_SOURCES.contains(source)) {
187                                Main.info(tr("Adding {0} to ignore tags", source));
188                            }
189                        }
190                    } else if (ignorefile) {
191                        line = line.trim();
192                        if (line.length() < 4) {
193                            continue;
194                        }
195
196                        String key = line.substring(0, 2);
197                        line = line.substring(2);
198
199                        switch (key) {
200                        case "S:":
201                            ignoreDataStartsWith.add(line);
202                            break;
203                        case "E:":
204                            ignoreDataEquals.add(line);
205                            break;
206                        case "F:":
207                            ignoreDataEndsWith.add(line);
208                            break;
209                        case "K:":
210                            IgnoreKeyPair tmp = new IgnoreKeyPair();
211                            int mid = line.indexOf('=');
212                            tmp.key = line.substring(0, mid);
213                            tmp.value = line.substring(mid+1);
214                            ignoreDataKeyPair.add(tmp);
215                        }
216                    } else if (tagcheckerfile) {
217                        if (line.length() > 0) {
218                            CheckerData d = new CheckerData();
219                            String err = d.getData(line);
220
221                            if (err == null) {
222                                checkerData.add(d);
223                            } else {
224                                Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
225                            }
226                        }
227                    } else if (line.charAt(0) == '+') {
228                        okValue = line.substring(1);
229                    } else if (line.charAt(0) == '-' && okValue != null) {
230                        spellCheckKeyData.put(line.substring(1), okValue);
231                    } else {
232                        Main.error(tr("Invalid spellcheck line: {0}", line));
233                    }
234                    if (isFirstLine) {
235                        isFirstLine = false;
236                        if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
237                            Main.info(tr("Adding {0} to spellchecker", source));
238                        }
239                    }
240                }
241            } catch (IOException e) {
242                errorSources += source + "\n";
243            }
244        }
245
246        if (errorSources.length() > 0)
247            throw new IOException( tr("Could not access data file(s):\n{0}", errorSources) );
248    }
249
250    /**
251     * Reads the presets data.
252     *
253     */
254    public static void initializePresets() {
255
256        if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true))
257            return;
258
259        Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
260        if (!presets.isEmpty()) {
261            presetsValueData = new MultiMap<>();
262            for (String a : OsmPrimitive.getUninterestingKeys()) {
263                presetsValueData.putVoid(a);
264            }
265            // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
266            /*  for(String a : OsmPrimitive.getDirectionKeys())
267                presetsValueData.add(a);
268             */
269            for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys",
270                    Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) {
271                presetsValueData.putVoid(a);
272            }
273            for (TaggingPreset p : presets) {
274                for (TaggingPresetItem i : p.data) {
275                    if (i instanceof KeyedItem) {
276                        addPresetValue(p, (KeyedItem) i);
277                    } else if (i instanceof CheckGroup) {
278                        for (Check c : ((CheckGroup) i).checks) {
279                            addPresetValue(p, c);
280                        }
281                    }
282                }
283            }
284        }
285    }
286
287    private static void addPresetValue(TaggingPreset p, KeyedItem ky) {
288        Collection<String> values = ky.getValues();
289        if (ky.key != null && values != null) {
290            try {
291                presetsValueData.putAll(ky.key, values);
292            } catch (NullPointerException e) {
293                Main.error(p+": Unable to initialize "+ky);
294            }
295        }
296    }
297
298    /**
299     * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
300     * @param s string to check
301     */
302    private boolean containsLow(String s) {
303        if (s == null)
304            return false;
305        for (int i = 0; i < s.length(); i++) {
306            if (s.charAt(i) < 0x20)
307                return true;
308        }
309        return false;
310    }
311
312    /**
313     * Checks the primitive tags
314     * @param p The primitive to check
315     */
316    @Override
317    public void check(OsmPrimitive p) {
318        // Just a collection to know if a primitive has been already marked with error
319        MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
320
321        if (checkComplex) {
322            Map<String, String> keys = p.getKeys();
323            for (CheckerData d : checkerData) {
324                if (d.match(p, keys)) {
325                    errors.add( new TestError(this, d.getSeverity(), tr("Suspicious tag/value combinations"),
326                            d.getDescription(), d.getDescriptionOrig(), d.getCode(), p) );
327                    withErrors.put(p, "TC");
328                }
329            }
330        }
331
332        for (Entry<String, String> prop : p.getKeys().entrySet()) {
333            String s = marktr("Key ''{0}'' invalid.");
334            String key = prop.getKey();
335            String value = prop.getValue();
336            if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
337                errors.add( new TestError(this, Severity.WARNING, tr("Tag value contains character with code less than 0x20"),
338                        tr(s, key), MessageFormat.format(s, key), LOW_CHAR_VALUE, p) );
339                withErrors.put(p, "ICV");
340            }
341            if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
342                errors.add( new TestError(this, Severity.WARNING, tr("Tag key contains character with code less than 0x20"),
343                        tr(s, key), MessageFormat.format(s, key), LOW_CHAR_KEY, p) );
344                withErrors.put(p, "ICK");
345            }
346            if (checkValues && (value!=null && value.length() > 255) && !withErrors.contains(p, "LV")) {
347                errors.add( new TestError(this, Severity.ERROR, tr("Tag value longer than allowed"),
348                        tr(s, key), MessageFormat.format(s, key), LONG_VALUE, p) );
349                withErrors.put(p, "LV");
350            }
351            if (checkKeys && (key!=null && key.length() > 255) && !withErrors.contains(p, "LK")) {
352                errors.add( new TestError(this, Severity.ERROR, tr("Tag key longer than allowed"),
353                        tr(s, key), MessageFormat.format(s, key), LONG_KEY, p) );
354                withErrors.put(p, "LK");
355            }
356            if (checkValues && (value==null || value.trim().length() == 0) && !withErrors.contains(p, "EV")) {
357                errors.add( new TestError(this, Severity.WARNING, tr("Tags with empty values"),
358                        tr(s, key), MessageFormat.format(s, key), EMPTY_VALUES, p) );
359                withErrors.put(p, "EV");
360            }
361            if (checkKeys && spellCheckKeyData.containsKey(key) && !withErrors.contains(p, "IPK")) {
362                errors.add( new TestError(this, Severity.WARNING, tr("Invalid property key"),
363                        tr(s, key), MessageFormat.format(s, key), INVALID_KEY, p) );
364                withErrors.put(p, "IPK");
365            }
366            if (checkKeys && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
367                errors.add( new TestError(this, Severity.WARNING, tr("Invalid white space in property key"),
368                        tr(s, key), MessageFormat.format(s, key), INVALID_KEY_SPACE, p) );
369                withErrors.put(p, "IPK");
370            }
371            if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
372                errors.add( new TestError(this, Severity.WARNING, tr("Property values start or end with white space"),
373                        tr(s, key), MessageFormat.format(s, key), INVALID_SPACE, p) );
374                withErrors.put(p, "SPACE");
375            }
376            if (checkValues && value != null && !value.equals(entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
377                errors.add( new TestError(this, Severity.OTHER, tr("Property values contain HTML entity"),
378                        tr(s, key), MessageFormat.format(s, key), INVALID_HTML, p) );
379                withErrors.put(p, "HTML");
380            }
381            if (checkValues && value != null && value.length() > 0 && presetsValueData != null) {
382                final Set<String> values = presetsValueData.get(key);
383                final boolean keyInPresets = values != null;
384                final boolean tagInPresets = values != null && (values.isEmpty() || values.contains(prop.getValue()));
385
386                boolean ignore = false;
387                for (String a : ignoreDataStartsWith) {
388                    if (key.startsWith(a)) {
389                        ignore = true;
390                    }
391                }
392                for (String a : ignoreDataEquals) {
393                    if(key.equals(a)) {
394                        ignore = true;
395                    }
396                }
397                for (String a : ignoreDataEndsWith) {
398                    if(key.endsWith(a)) {
399                        ignore = true;
400                    }
401                }
402
403                if (!tagInPresets) {
404                    for (IgnoreKeyPair a : ignoreDataKeyPair) {
405                        if (key.equals(a.key) && value.equals(a.value)) {
406                            ignore = true;
407                        }
408                    }
409                }
410
411                if (!ignore) {
412                    if (!keyInPresets) {
413                        String i = marktr("Key ''{0}'' not in presets.");
414                        errors.add( new TestError(this, Severity.OTHER, tr("Presets do not contain property key"),
415                                tr(i, key), MessageFormat.format(i, key), INVALID_VALUE, p) );
416                        withErrors.put(p, "UPK");
417                    } else if (!tagInPresets) {
418                        String i = marktr("Value ''{0}'' for key ''{1}'' not in presets.");
419                        errors.add( new TestError(this, Severity.OTHER, tr("Presets do not contain property value"),
420                                tr(i, prop.getValue(), key), MessageFormat.format(i, prop.getValue(), key), INVALID_VALUE, p) );
421                        withErrors.put(p, "UPV");
422                    }
423                }
424            }
425            if (checkFixmes && value != null && value.length() > 0) {
426                if ((value.toLowerCase().contains("fixme")
427                        || value.contains("check and delete")
428                        || key.contains("todo") || key.toLowerCase().contains("fixme"))
429                        && !withErrors.contains(p, "FIXME")) {
430                    errors.add(new TestError(this, Severity.OTHER,
431                            tr("FIXMES"), FIXME, p));
432                    withErrors.put(p, "FIXME");
433                }
434            }
435        }
436    }
437
438    @Override
439    public void startTest(ProgressMonitor monitor) {
440        super.startTest(monitor);
441        checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true);
442        if (isBeforeUpload) {
443            checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
444        }
445
446        checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true);
447        if (isBeforeUpload) {
448            checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
449        }
450
451        checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true);
452        if (isBeforeUpload) {
453            checkComplex = checkValues && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
454        }
455
456        checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true);
457        if (isBeforeUpload) {
458            checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
459        }
460    }
461
462    @Override
463    public void visit(Collection<OsmPrimitive> selection) {
464        if (checkKeys || checkValues || checkComplex || checkFixmes) {
465            super.visit(selection);
466        }
467    }
468
469    @Override
470    public void addGui(JPanel testPanel) {
471        GBC a = GBC.eol();
472        a.anchor = GridBagConstraints.EAST;
473
474        testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3,0,0,0));
475
476        prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true));
477        prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
478        testPanel.add(prefCheckKeys, GBC.std().insets(20,0,0,0));
479
480        prefCheckKeysBeforeUpload = new JCheckBox();
481        prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
482        testPanel.add(prefCheckKeysBeforeUpload, a);
483
484        prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true));
485        prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
486        testPanel.add(prefCheckComplex, GBC.std().insets(20,0,0,0));
487
488        prefCheckComplexBeforeUpload = new JCheckBox();
489        prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
490        testPanel.add(prefCheckComplexBeforeUpload, a);
491
492        final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, Arrays.asList(DATA_FILE, IGNORE_FILE, SPELL_FILE));
493        sourcesList = new EditableList(tr("TagChecker source"));
494        sourcesList.setItems(sources);
495        testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
496        testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
497
498        ActionListener disableCheckActionListener = new ActionListener() {
499            @Override
500            public void actionPerformed(ActionEvent e) {
501                handlePrefEnable();
502            }
503        };
504        prefCheckKeys.addActionListener(disableCheckActionListener);
505        prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
506        prefCheckComplex.addActionListener(disableCheckActionListener);
507        prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
508
509        handlePrefEnable();
510
511        prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true));
512        prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
513        testPanel.add(prefCheckValues, GBC.std().insets(20,0,0,0));
514
515        prefCheckValuesBeforeUpload = new JCheckBox();
516        prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
517        testPanel.add(prefCheckValuesBeforeUpload, a);
518
519        prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true));
520        prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
521        testPanel.add(prefCheckFixmes, GBC.std().insets(20,0,0,0));
522
523        prefCheckFixmesBeforeUpload = new JCheckBox();
524        prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
525        testPanel.add(prefCheckFixmesBeforeUpload, a);
526    }
527
528    public void handlePrefEnable() {
529        boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
530                || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
531        sourcesList.setEnabled(selected);
532    }
533
534    @Override
535    public boolean ok() {
536        enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
537        testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
538                || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
539
540        Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected());
541        Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
542        Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
543        Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
544        Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
545        Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
546        Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
547        Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
548        return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems());
549    }
550
551    @Override
552    public Command fixError(TestError testError) {
553        List<Command> commands = new ArrayList<>(50);
554
555        Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
556        for (OsmPrimitive p : primitives) {
557            Map<String, String> tags = p.getKeys();
558            if (tags == null || tags.isEmpty()) {
559                continue;
560            }
561
562            for (Entry<String, String> prop: tags.entrySet()) {
563                String key = prop.getKey();
564                String value = prop.getValue();
565                if (value == null || value.trim().length() == 0) {
566                    commands.add(new ChangePropertyCommand(p, key, null));
567                } else if (value.startsWith(" ") || value.endsWith(" ")) {
568                    commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value)));
569                } else if (key.startsWith(" ") || key.endsWith(" ")) {
570                    commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key)));
571                } else {
572                    String evalue = entities.unescape(value);
573                    if (!evalue.equals(value)) {
574                        commands.add(new ChangePropertyCommand(p, key, evalue));
575                    } else {
576                        String replacementKey = spellCheckKeyData.get(key);
577                        if (replacementKey != null) {
578                            commands.add(new ChangePropertyKeyCommand(p, key, replacementKey));
579                        }
580                    }
581                }
582            }
583        }
584
585        if (commands.isEmpty())
586            return null;
587        if (commands.size() == 1)
588            return commands.get(0);
589
590        return new SequenceCommand(tr("Fix tags"), commands);
591    }
592
593    @Override
594    public boolean isFixable(TestError testError) {
595        if (testError.getTester() instanceof TagChecker) {
596            int code = testError.getCode();
597            return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE || code == INVALID_KEY_SPACE || code == INVALID_HTML;
598        }
599
600        return false;
601    }
602
603    protected static class IgnoreKeyPair {
604        public String key;
605        public String value;
606    }
607
608    protected static class CheckerData {
609        private String description;
610        protected List<CheckerElement> data = new ArrayList<>();
611        private OsmPrimitiveType type;
612        private int code;
613        protected Severity severity;
614        protected static final int TAG_CHECK_ERROR  = 1250;
615        protected static final int TAG_CHECK_WARN   = 1260;
616        protected static final int TAG_CHECK_INFO   = 1270;
617
618        protected static class CheckerElement {
619            public Object tag;
620            public Object value;
621            public boolean noMatch;
622            public boolean tagAll = false;
623            public boolean valueAll = false;
624            public boolean valueBool = false;
625
626            private Pattern getPattern(String str) throws IllegalStateException, PatternSyntaxException {
627                if (str.endsWith("/i"))
628                    return Pattern.compile(str.substring(1,str.length()-2), Pattern.CASE_INSENSITIVE);
629                if (str.endsWith("/"))
630                    return Pattern.compile(str.substring(1,str.length()-1));
631
632                throw new IllegalStateException();
633            }
634            public CheckerElement(String exp) throws IllegalStateException, PatternSyntaxException {
635                Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
636                m.matches();
637
638                String n = m.group(1).trim();
639
640                if ("*".equals(n)) {
641                    tagAll = true;
642                } else {
643                    tag = n.startsWith("/") ? getPattern(n) : n;
644                    noMatch = "!=".equals(m.group(2));
645                    n = m.group(3).trim();
646                    if ("*".equals(n)) {
647                        valueAll = true;
648                    } else if ("BOOLEAN_TRUE".equals(n)) {
649                        valueBool = true;
650                        value = OsmUtils.trueval;
651                    } else if ("BOOLEAN_FALSE".equals(n)) {
652                        valueBool = true;
653                        value = OsmUtils.falseval;
654                    } else {
655                        value = n.startsWith("/") ? getPattern(n) : n;
656                    }
657                }
658            }
659
660            public boolean match(OsmPrimitive osm, Map<String, String> keys) {
661                for (Entry<String, String> prop: keys.entrySet()) {
662                    String key = prop.getKey();
663                    String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
664                    if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
665                            && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
666                        return !noMatch;
667                }
668                return noMatch;
669            }
670        }
671
672        private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
673        private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
674        private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
675
676        public String getData(final String str) {
677            Matcher m = CLEAN_STR_PATTERN.matcher(str);
678            String trimmed = m.replaceFirst("").trim();
679            try {
680                description = m.group(1);
681                if (description != null && description.length() == 0) {
682                    description = null;
683                }
684            } catch (IllegalStateException e) {
685                description = null;
686            }
687            String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
688            switch (n[0]) {
689            case "way":
690                type = OsmPrimitiveType.WAY;
691                break;
692            case "node":
693                type = OsmPrimitiveType.NODE;
694                break;
695            case "relation":
696                type = OsmPrimitiveType.RELATION;
697                break;
698            case "*":
699                type = null;
700                break;
701            default:
702                return tr("Could not find element type");
703            }
704            if (n.length != 3)
705                return tr("Incorrect number of parameters");
706
707            switch (n[1]) {
708            case "W":
709                severity = Severity.WARNING;
710                code = TAG_CHECK_WARN;
711                break;
712            case "E":
713                severity = Severity.ERROR;
714                code = TAG_CHECK_ERROR;
715                break;
716            case "I":
717                severity = Severity.OTHER;
718                code = TAG_CHECK_INFO;
719                break;
720            default:
721                return tr("Could not find warning level");
722            }
723            for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
724                try {
725                    data.add(new CheckerElement(exp));
726                } catch (IllegalStateException e) {
727                    return tr("Illegal expression ''{0}''", exp);
728                } catch (PatternSyntaxException e) {
729                    return tr("Illegal regular expression ''{0}''", exp);
730                }
731            }
732            return null;
733        }
734
735        public boolean match(OsmPrimitive osm, Map<String, String> keys) {
736            if (type != null && OsmPrimitiveType.from(osm) != type)
737                return false;
738
739            for (CheckerElement ce : data) {
740                if (!ce.match(osm, keys))
741                    return false;
742            }
743            return true;
744        }
745
746        public String getDescription() {
747            return tr(description);
748        }
749
750        public String getDescriptionOrig() {
751            return description;
752        }
753
754        public Severity getSeverity() {
755            return severity;
756        }
757
758        public int getCode() {
759            if (type == null)
760                return code;
761
762            return code + type.ordinal() + 1;
763        }
764    }
765}