001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.BufferedReader;
007import java.io.IOException;
008import java.io.InputStream;
009import java.io.Reader;
010import java.io.StringReader;
011import java.lang.reflect.Method;
012import java.text.MessageFormat;
013import java.util.ArrayList;
014import java.util.Collection;
015import java.util.Collections;
016import java.util.HashMap;
017import java.util.HashSet;
018import java.util.Iterator;
019import java.util.LinkedHashMap;
020import java.util.LinkedHashSet;
021import java.util.LinkedList;
022import java.util.List;
023import java.util.Locale;
024import java.util.Map;
025import java.util.Objects;
026import java.util.Optional;
027import java.util.Set;
028import java.util.function.Predicate;
029import java.util.regex.Matcher;
030import java.util.regex.Pattern;
031
032import org.openstreetmap.josm.command.ChangePropertyCommand;
033import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
034import org.openstreetmap.josm.command.Command;
035import org.openstreetmap.josm.command.DeleteCommand;
036import org.openstreetmap.josm.command.SequenceCommand;
037import org.openstreetmap.josm.data.coor.LatLon;
038import org.openstreetmap.josm.data.osm.DataSet;
039import org.openstreetmap.josm.data.osm.IPrimitive;
040import org.openstreetmap.josm.data.osm.OsmPrimitive;
041import org.openstreetmap.josm.data.osm.OsmUtils;
042import org.openstreetmap.josm.data.osm.Relation;
043import org.openstreetmap.josm.data.osm.Tag;
044import org.openstreetmap.josm.data.osm.Way;
045import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
046import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
047import org.openstreetmap.josm.data.validation.OsmValidator;
048import org.openstreetmap.josm.data.validation.Severity;
049import org.openstreetmap.josm.data.validation.Test;
050import org.openstreetmap.josm.data.validation.TestError;
051import org.openstreetmap.josm.gui.mappaint.Environment;
052import org.openstreetmap.josm.gui.mappaint.Keyword;
053import org.openstreetmap.josm.gui.mappaint.MultiCascade;
054import org.openstreetmap.josm.gui.mappaint.mapcss.Condition;
055import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition;
056import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ExpressionCondition;
057import org.openstreetmap.josm.gui.mappaint.mapcss.Expression;
058import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.ParameterFunction;
059import org.openstreetmap.josm.gui.mappaint.mapcss.Functions;
060import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction;
061import org.openstreetmap.josm.gui.mappaint.mapcss.LiteralExpression;
062import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
063import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration;
064import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
065import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex;
066import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
067import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector;
068import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector;
069import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector;
070import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
071import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
072import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError;
073import org.openstreetmap.josm.gui.progress.ProgressMonitor;
074import org.openstreetmap.josm.io.CachedFile;
075import org.openstreetmap.josm.io.FileWatcher;
076import org.openstreetmap.josm.io.IllegalDataException;
077import org.openstreetmap.josm.io.UTFInputStreamReader;
078import org.openstreetmap.josm.spi.preferences.Config;
079import org.openstreetmap.josm.tools.CheckParameterUtil;
080import org.openstreetmap.josm.tools.DefaultGeoProperty;
081import org.openstreetmap.josm.tools.GeoProperty;
082import org.openstreetmap.josm.tools.GeoPropertyIndex;
083import org.openstreetmap.josm.tools.I18n;
084import org.openstreetmap.josm.tools.Logging;
085import org.openstreetmap.josm.tools.MultiMap;
086import org.openstreetmap.josm.tools.Territories;
087import org.openstreetmap.josm.tools.Utils;
088
089/**
090 * MapCSS-based tag checker/fixer.
091 * @since 6506
092 */
093public class MapCSSTagChecker extends Test.TagTest {
094    MapCSSTagCheckerIndex indexData;
095    final Set<OsmPrimitive> tested = new HashSet<>();
096
097
098    /**
099    * A grouped MapCSSRule with multiple selectors for a single declaration.
100    * @see MapCSSRule
101    */
102    public static class GroupedMapCSSRule {
103        /** MapCSS selectors **/
104        public final List<Selector> selectors;
105        /** MapCSS declaration **/
106        public final Declaration declaration;
107
108        /**
109         * Constructs a new {@code GroupedMapCSSRule}.
110         * @param selectors MapCSS selectors
111         * @param declaration MapCSS declaration
112         */
113        public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) {
114            this.selectors = selectors;
115            this.declaration = declaration;
116        }
117
118        @Override
119        public int hashCode() {
120            return Objects.hash(selectors, declaration);
121        }
122
123        @Override
124        public boolean equals(Object obj) {
125            if (this == obj) return true;
126            if (obj == null || getClass() != obj.getClass()) return false;
127            GroupedMapCSSRule that = (GroupedMapCSSRule) obj;
128            return Objects.equals(selectors, that.selectors) &&
129                    Objects.equals(declaration, that.declaration);
130        }
131
132        @Override
133        public String toString() {
134            return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']';
135        }
136    }
137
138    /**
139     * The preference key for tag checker source entries.
140     * @since 6670
141     */
142    public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
143
144    /**
145     * Constructs a new {@code MapCSSTagChecker}.
146     */
147    public MapCSSTagChecker() {
148        super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
149    }
150
151    /**
152     * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}.
153     */
154    @FunctionalInterface
155    interface FixCommand {
156        /**
157         * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders
158         * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}).
159         * @param p OSM primitive
160         * @param matchingSelector  matching selector
161         * @return fix command
162         */
163        Command createCommand(OsmPrimitive p, Selector matchingSelector);
164
165        /**
166         * Checks that object is either an {@link Expression} or a {@link String}.
167         * @param obj object to check
168         * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String}
169         */
170        static void checkObject(final Object obj) {
171            CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String,
172                    () -> "instance of Exception or String expected, but got " + obj);
173        }
174
175        /**
176         * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}.
177         * @param obj object to evaluate ({@link Expression} or {@link String})
178         * @param p OSM primitive
179         * @param matchingSelector matching selector
180         * @return result string
181         */
182        static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) {
183            final String s;
184            if (obj instanceof Expression) {
185                s = (String) ((Expression) obj).evaluate(new Environment(p));
186            } else if (obj instanceof String) {
187                s = (String) obj;
188            } else {
189                return null;
190            }
191            return TagCheck.insertArguments(matchingSelector, s, p);
192        }
193
194        /**
195         * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag.
196         * @param obj object to evaluate ({@link Expression} or {@link String})
197         * @return created fix command
198         */
199        static FixCommand fixAdd(final Object obj) {
200            checkObject(obj);
201            return new FixCommand() {
202                @Override
203                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
204                    final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector));
205                    return new ChangePropertyCommand(p, tag.getKey(), tag.getValue());
206                }
207
208                @Override
209                public String toString() {
210                    return "fixAdd: " + obj;
211                }
212            };
213        }
214
215        /**
216         * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key.
217         * @param obj object to evaluate ({@link Expression} or {@link String})
218         * @return created fix command
219         */
220        static FixCommand fixRemove(final Object obj) {
221            checkObject(obj);
222            return new FixCommand() {
223                @Override
224                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
225                    final String key = evaluateObject(obj, p, matchingSelector);
226                    return new ChangePropertyCommand(p, key, "");
227                }
228
229                @Override
230                public String toString() {
231                    return "fixRemove: " + obj;
232                }
233            };
234        }
235
236        /**
237         * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys.
238         * @param oldKey old key
239         * @param newKey new key
240         * @return created fix command
241         */
242        static FixCommand fixChangeKey(final String oldKey, final String newKey) {
243            return new FixCommand() {
244                @Override
245                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
246                    return new ChangePropertyKeyCommand(p,
247                            TagCheck.insertArguments(matchingSelector, oldKey, p),
248                            TagCheck.insertArguments(matchingSelector, newKey, p));
249                }
250
251                @Override
252                public String toString() {
253                    return "fixChangeKey: " + oldKey + " => " + newKey;
254                }
255            };
256        }
257    }
258
259    final MultiMap<String, TagCheck> checks = new MultiMap<>();
260
261    /**
262     * Result of {@link TagCheck#readMapCSS}
263     * @since 8936
264     */
265    public static class ParseResult {
266        /** Checks successfully parsed */
267        public final List<TagCheck> parseChecks;
268        /** Errors that occurred during parsing */
269        public final Collection<Throwable> parseErrors;
270
271        /**
272         * Constructs a new {@code ParseResult}.
273         * @param parseChecks Checks successfully parsed
274         * @param parseErrors Errors that occurred during parsing
275         */
276        public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) {
277            this.parseChecks = parseChecks;
278            this.parseErrors = parseErrors;
279        }
280    }
281
282    /**
283     * Tag check.
284     */
285    public static class TagCheck implements Predicate<OsmPrimitive> {
286        /** The selector of this {@code TagCheck} */
287        protected final GroupedMapCSSRule rule;
288        /** Commands to apply in order to fix a matching primitive */
289        protected final List<FixCommand> fixCommands = new ArrayList<>();
290        /** Tags (or arbitraty strings) of alternatives to be presented to the user */
291        protected final List<String> alternatives = new ArrayList<>();
292        /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair.
293         * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */
294        protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>();
295        /** Unit tests */
296        protected final Map<String, Boolean> assertions = new HashMap<>();
297        /** MapCSS Classes to set on matching primitives */
298        protected final Set<String> setClassExpressions = new HashSet<>();
299        /** Denotes whether the object should be deleted for fixing it */
300        protected boolean deletion;
301        /** A string used to group similar tests */
302        protected String group;
303
304        TagCheck(GroupedMapCSSRule rule) {
305            this.rule = rule;
306        }
307
308        private static final String POSSIBLE_THROWS = possibleThrows();
309
310        static final String possibleThrows() {
311            StringBuilder sb = new StringBuilder();
312            for (Severity s : Severity.values()) {
313                if (sb.length() > 0) {
314                    sb.append('/');
315                }
316                sb.append("throw")
317                .append(s.name().charAt(0))
318                .append(s.name().substring(1).toLowerCase(Locale.ENGLISH));
319            }
320            return sb.toString();
321        }
322
323        static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException {
324            final TagCheck check = new TagCheck(rule);
325            for (Instruction i : rule.declaration.instructions) {
326                if (i instanceof Instruction.AssignmentInstruction) {
327                    final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i;
328                    if (ai.isSetInstruction) {
329                        check.setClassExpressions.add(ai.key);
330                        continue;
331                    }
332                    try {
333                        final String val = ai.val instanceof Expression
334                                ? Optional.ofNullable(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null)
335                                : ai.val instanceof String
336                                ? (String) ai.val
337                                : ai.val instanceof Keyword
338                                ? ((Keyword) ai.val).val
339                                : null;
340                        if (ai.key.startsWith("throw")) {
341                            try {
342                                check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH)));
343                            } catch (IllegalArgumentException e) {
344                                Logging.log(Logging.LEVEL_WARN,
345                                        "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e);
346                            }
347                        } else if ("fixAdd".equals(ai.key)) {
348                            check.fixCommands.add(FixCommand.fixAdd(ai.val));
349                        } else if ("fixRemove".equals(ai.key)) {
350                            CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")),
351                                    "Unexpected '='. Please only specify the key to remove in: " + ai);
352                            check.fixCommands.add(FixCommand.fixRemove(ai.val));
353                        } else if (val != null && "fixChangeKey".equals(ai.key)) {
354                            CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!");
355                            final String[] x = val.split("=>", 2);
356                            check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1])));
357                        } else if (val != null && "fixDeleteObject".equals(ai.key)) {
358                            CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'");
359                            check.deletion = true;
360                        } else if (val != null && "suggestAlternative".equals(ai.key)) {
361                            check.alternatives.add(val);
362                        } else if (val != null && "assertMatch".equals(ai.key)) {
363                            check.assertions.put(val, Boolean.TRUE);
364                        } else if (val != null && "assertNoMatch".equals(ai.key)) {
365                            check.assertions.put(val, Boolean.FALSE);
366                        } else if (val != null && "group".equals(ai.key)) {
367                            check.group = val;
368                        } else if (ai.key.startsWith("-")) {
369                            Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val);
370                        } else {
371                            throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!');
372                        }
373                    } catch (IllegalArgumentException e) {
374                        throw new IllegalDataException(e);
375                    }
376                }
377            }
378            if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) {
379                throw new IllegalDataException(
380                        "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors);
381            } else if (check.errors.size() > 1) {
382                throw new IllegalDataException(
383                        "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for "
384                                + rule.selectors);
385            }
386            return check;
387        }
388
389        static ParseResult readMapCSS(Reader css) throws ParseException {
390            CheckParameterUtil.ensureParameterNotNull(css, "css");
391
392            final MapCSSStyleSource source = new MapCSSStyleSource("");
393            final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR);
394            final StringReader mapcss = new StringReader(preprocessor.pp_root(source));
395            final MapCSSParser parser = new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT);
396            parser.sheet(source);
397            // Ignore "meta" rule(s) from external rules of JOSM wiki
398            source.removeMetaRules();
399            // group rules with common declaration block
400            Map<Declaration, List<Selector>> g = new LinkedHashMap<>();
401            for (MapCSSRule rule : source.rules) {
402                if (!g.containsKey(rule.declaration)) {
403                    List<Selector> sels = new ArrayList<>();
404                    sels.add(rule.selector);
405                    g.put(rule.declaration, sels);
406                } else {
407                    g.get(rule.declaration).add(rule.selector);
408                }
409            }
410            List<TagCheck> parseChecks = new ArrayList<>();
411            for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) {
412                try {
413                    parseChecks.add(TagCheck.ofMapCSSRule(
414                            new GroupedMapCSSRule(map.getValue(), map.getKey())));
415                } catch (IllegalDataException e) {
416                    Logging.error("Cannot add MapCss rule: "+e.getMessage());
417                    source.logError(e);
418                }
419            }
420            return new ParseResult(parseChecks, source.getErrors());
421        }
422
423        @Override
424        public boolean test(OsmPrimitive primitive) {
425            // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker.
426            return whichSelectorMatchesPrimitive(primitive) != null;
427        }
428
429        Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) {
430            return whichSelectorMatchesEnvironment(new Environment(primitive));
431        }
432
433        Selector whichSelectorMatchesEnvironment(Environment env) {
434            for (Selector i : rule.selectors) {
435                env.clearSelectorMatchingInformation();
436                if (i.matches(env)) {
437                    return i;
438                }
439            }
440            return null;
441        }
442
443        /**
444         * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the
445         * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}.
446         * @param matchingSelector matching selector
447         * @param index index
448         * @param type selector type ("key", "value" or "tag")
449         * @param p OSM primitive
450         * @return argument value, can be {@code null}
451         */
452        static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) {
453            try {
454                final Condition c = matchingSelector.getConditions().get(index);
455                final Tag tag = c instanceof Condition.ToTagConvertable
456                        ? ((Condition.ToTagConvertable) c).asTag(p)
457                        : null;
458                if (tag == null) {
459                    return null;
460                } else if ("key".equals(type)) {
461                    return tag.getKey();
462                } else if ("value".equals(type)) {
463                    return tag.getValue();
464                } else if ("tag".equals(type)) {
465                    return tag.toString();
466                }
467            } catch (IndexOutOfBoundsException ignore) {
468                Logging.debug(ignore);
469            }
470            return null;
471        }
472
473        /**
474         * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding
475         * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}.
476         * @param matchingSelector matching selector
477         * @param s any string
478         * @param p OSM primitive
479         * @return string with arguments inserted
480         */
481        static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) {
482            if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) {
483                return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p);
484            } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) {
485                return s;
486            }
487            final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s);
488            final StringBuffer sb = new StringBuffer();
489            while (m.find()) {
490                final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector,
491                        Integer.parseInt(m.group(1)), m.group(2), p);
492                try {
493                    // Perform replacement with null-safe + regex-safe handling
494                    m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", ""));
495                } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
496                    Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e);
497                }
498            }
499            m.appendTail(sb);
500            return sb.toString();
501        }
502
503        /**
504         * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive}
505         * if the error is fixable, or {@code null} otherwise.
506         *
507         * @param p the primitive to construct the fix for
508         * @return the fix or {@code null}
509         */
510        Command fixPrimitive(OsmPrimitive p) {
511            if (fixCommands.isEmpty() && !deletion) {
512                return null;
513            }
514            try {
515                final Selector matchingSelector = whichSelectorMatchesPrimitive(p);
516                Collection<Command> cmds = new LinkedList<>();
517                for (FixCommand fixCommand : fixCommands) {
518                    cmds.add(fixCommand.createCommand(p, matchingSelector));
519                }
520                if (deletion && !p.isDeleted()) {
521                    cmds.add(new DeleteCommand(p));
522                }
523                return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds);
524            } catch (IllegalArgumentException e) {
525                Logging.error(e);
526                return null;
527            }
528        }
529
530        /**
531         * Constructs a (localized) message for this deprecation check.
532         * @param p OSM primitive
533         *
534         * @return a message
535         */
536        String getMessage(OsmPrimitive p) {
537            if (errors.isEmpty()) {
538                // Return something to avoid NPEs
539                return rule.declaration.toString();
540            } else {
541                final Object val = errors.keySet().iterator().next().val;
542                return String.valueOf(
543                        val instanceof Expression
544                                ? ((Expression) val).evaluate(new Environment(p))
545                                : val
546                );
547            }
548        }
549
550        /**
551         * Constructs a (localized) description for this deprecation check.
552         * @param p OSM primitive
553         *
554         * @return a description (possibly with alternative suggestions)
555         * @see #getDescriptionForMatchingSelector
556         */
557        String getDescription(OsmPrimitive p) {
558            if (alternatives.isEmpty()) {
559                return getMessage(p);
560            } else {
561                /* I18N: {0} is the test error message and {1} is an alternative */
562                return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives));
563            }
564        }
565
566        /**
567         * Constructs a (localized) description for this deprecation check
568         * where any placeholders are replaced by values of the matched selector.
569         *
570         * @param matchingSelector matching selector
571         * @param p OSM primitive
572         * @return a description (possibly with alternative suggestions)
573         */
574        String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) {
575            return insertArguments(matchingSelector, getDescription(p), p);
576        }
577
578        Severity getSeverity() {
579            return errors.isEmpty() ? null : errors.values().iterator().next();
580        }
581
582        @Override
583        public String toString() {
584            return getDescription(null);
585        }
586
587        /**
588         * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error.
589         *
590         * @param p the primitive to construct the error for
591         * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error.
592         */
593        List<TestError> getErrorsForPrimitive(OsmPrimitive p) {
594            final Environment env = new Environment(p);
595            return getErrorsForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null);
596        }
597
598        private List<TestError> getErrorsForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) {
599            List<TestError> res = new ArrayList<>();
600            if (matchingSelector != null && !errors.isEmpty()) {
601                final Command fix = fixPrimitive(p);
602                final String description = getDescriptionForMatchingSelector(p, matchingSelector);
603                final String description1 = group == null ? description : group;
604                final String description2 = group == null ? null : description;
605                TestError.Builder errorBuilder = TestError.builder(tester, getSeverity(), 3000)
606                        .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString());
607                if (fix != null) {
608                    errorBuilder = errorBuilder.fix(() -> fix);
609                }
610                if (env.child instanceof OsmPrimitive) {
611                    res.add(errorBuilder.primitives(p, (OsmPrimitive) env.child).build());
612                } else if (env.children != null) {
613                    for (IPrimitive c : env.children) {
614                        if (c instanceof OsmPrimitive) {
615                            errorBuilder = TestError.builder(tester, getSeverity(), 3000)
616                                    .messageWithManuallyTranslatedDescription(description1, description2,
617                                            matchingSelector.toString());
618                            if (fix != null) {
619                                errorBuilder = errorBuilder.fix(() -> fix);
620                            }
621                            res.add(errorBuilder.primitives(p, (OsmPrimitive) c).build());
622                        }
623                    }
624                } else {
625                    res.add(errorBuilder.primitives(p).build());
626                }
627            }
628            return res;
629        }
630
631        /**
632         * Returns the set of tagchecks on which this check depends on.
633         * @param schecks the collection of tagcheks to search in
634         * @return the set of tagchecks on which this check depends on
635         * @since 7881
636         */
637        public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) {
638            Set<TagCheck> result = new HashSet<>();
639            Set<String> classes = getClassesIds();
640            if (schecks != null && !classes.isEmpty()) {
641                for (TagCheck tc : schecks) {
642                    if (this.equals(tc)) {
643                        continue;
644                    }
645                    for (String id : tc.setClassExpressions) {
646                        if (classes.contains(id)) {
647                            result.add(tc);
648                            break;
649                        }
650                    }
651                }
652            }
653            return result;
654        }
655
656        /**
657         * Returns the list of ids of all MapCSS classes referenced in the rule selectors.
658         * @return the list of ids of all MapCSS classes referenced in the rule selectors
659         * @since 7881
660         */
661        public Set<String> getClassesIds() {
662            Set<String> result = new HashSet<>();
663            for (Selector s : rule.selectors) {
664                if (s instanceof AbstractSelector) {
665                    for (Condition c : ((AbstractSelector) s).getConditions()) {
666                        if (c instanceof ClassCondition) {
667                            result.add(((ClassCondition) c).id);
668                        }
669                    }
670                }
671            }
672            return result;
673        }
674    }
675
676    static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
677        public final GroupedMapCSSRule rule;
678
679        MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) {
680            this.rule = rule;
681        }
682
683        @Override
684        public synchronized boolean equals(Object obj) {
685            return super.equals(obj)
686                    || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule))
687                    || (obj instanceof GroupedMapCSSRule && rule.equals(obj));
688        }
689
690        @Override
691        public synchronized int hashCode() {
692            return Objects.hash(super.hashCode(), rule);
693        }
694
695        @Override
696        public String toString() {
697            return "MapCSSTagCheckerAndRule [rule=" + rule + ']';
698        }
699    }
700
701    /**
702     * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
703     * @param p The OSM primitive
704     * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
705     * @return all errors for the given primitive, with or without those of "info" severity
706     */
707    public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
708        final List<TestError> res = new ArrayList<>();
709        if (indexData == null) {
710            indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverity, MapCSSTagCheckerIndex.ALL_TESTS);
711        }
712
713        MapCSSRuleIndex matchingRuleIndex = indexData.get(p);
714
715        Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
716        // the declaration indices are sorted, so it suffices to save the last used index
717        Declaration lastDeclUsed = null;
718
719        Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p);
720        while (candidates.hasNext()) {
721            MapCSSRule r = candidates.next();
722            env.clearSelectorMatchingInformation();
723            if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector)
724                TagCheck check = indexData.getCheck(r);
725                if (check != null) {
726                    if (r.declaration == lastDeclUsed)
727                        continue; // don't apply one declaration more than once
728                    lastDeclUsed = r.declaration;
729
730                    r.declaration.execute(env);
731                    if (!check.errors.isEmpty()) {
732                        for (TestError e: check.getErrorsForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule))) {
733                            addIfNotSimilar(e, res);
734                        }
735                    }
736                }
737            }
738        }
739        return res;
740    }
741
742    /**
743     * See #12627
744     * Add error to given list if list doesn't already contain a similar error.
745     * Similar means same code and description and same combination of primitives and same combination of highlighted objects,
746     * but maybe with different orders.
747     * @param toAdd the error to add
748     * @param errors the list of errors
749     */
750    private static void addIfNotSimilar(TestError toAdd, List<TestError> errors) {
751        boolean isDup = false;
752        if (toAdd.getPrimitives().size() >= 2) {
753            for (TestError e : errors) {
754                if (e.getCode() == toAdd.getCode() && e.getMessage().equals(toAdd.getMessage())
755                        && e.getPrimitives().size() == toAdd.getPrimitives().size()
756                        && e.getPrimitives().containsAll(toAdd.getPrimitives())
757                        && e.getHighlighted().size() == toAdd.getHighlighted().size()
758                        && e.getHighlighted().containsAll(toAdd.getHighlighted())) {
759                    isDup = true;
760                    break;
761                }
762            }
763        }
764        if (!isDup)
765            errors.add(toAdd);
766    }
767
768    private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity,
769            Collection<Set<TagCheck>> checksCol) {
770        final List<TestError> r = new ArrayList<>();
771        final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
772        for (Set<TagCheck> schecks : checksCol) {
773            for (TagCheck check : schecks) {
774                boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
775                // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
776                if (ignoreError && check.setClassExpressions.isEmpty()) {
777                    continue;
778                }
779                final Selector selector = check.whichSelectorMatchesEnvironment(env);
780                if (selector != null) {
781                    check.rule.declaration.execute(env);
782                    if (!ignoreError && !check.errors.isEmpty()) {
783                        r.addAll(check.getErrorsForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule)));
784                    }
785                }
786            }
787        }
788        return r;
789    }
790
791    /**
792     * Visiting call for primitives.
793     *
794     * @param p The primitive to inspect.
795     */
796    @Override
797    public void check(OsmPrimitive p) {
798        for (TestError e : getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get())) {
799            addIfNotSimilar(e, errors);
800        }
801        if (partialSelection && p.isTagged()) {
802            tested.add(p);
803        }
804    }
805
806    /**
807     * Adds a new MapCSS config file from the given URL.
808     * @param url The unique URL of the MapCSS config file
809     * @return List of tag checks and parsing errors, or null
810     * @throws ParseException if the config file does not match MapCSS syntax
811     * @throws IOException if any I/O error occurs
812     * @since 7275
813     */
814    public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException {
815        CheckParameterUtil.ensureParameterNotNull(url, "url");
816        ParseResult result;
817        try (CachedFile cache = new CachedFile(url);
818             InputStream zip = cache.findZipEntryInputStream("validator.mapcss", "");
819             InputStream s = zip != null ? zip : cache.getInputStream();
820             Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) {
821            if (zip != null)
822                I18n.addTexts(cache.getFile());
823            result = TagCheck.readMapCSS(reader);
824            checks.remove(url);
825            checks.putAll(url, result.parseChecks);
826            indexData = null;
827            // Check assertions, useful for development of local files
828            if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) {
829                for (String msg : checkAsserts(result.parseChecks)) {
830                    Logging.warn(msg);
831                }
832            }
833        }
834        return result;
835    }
836
837    @Override
838    public synchronized void initialize() throws Exception {
839        checks.clear();
840        indexData = null;
841        for (SourceEntry source : new ValidatorPrefHelper().get()) {
842            if (!source.active) {
843                continue;
844            }
845            String i = source.url;
846            try {
847                if (!i.startsWith("resource:")) {
848                    Logging.info(tr("Adding {0} to tag checker", i));
849                } else if (Logging.isDebugEnabled()) {
850                    Logging.debug(tr("Adding {0} to tag checker", i));
851                }
852                addMapCSS(i);
853                if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
854                    FileWatcher.getDefaultInstance().registerSource(source);
855                }
856            } catch (IOException | IllegalStateException | IllegalArgumentException ex) {
857                Logging.warn(tr("Failed to add {0} to tag checker", i));
858                Logging.log(Logging.LEVEL_WARN, ex);
859            } catch (ParseException | TokenMgrError ex) {
860                Logging.warn(tr("Failed to add {0} to tag checker", i));
861                Logging.warn(ex);
862            }
863        }
864    }
865
866    private static Method getFunctionMethod(String method) {
867        try {
868            return Functions.class.getDeclaredMethod(method, Environment.class, String.class);
869        } catch (NoSuchMethodException | SecurityException e) {
870            Logging.error(e);
871            return null;
872        }
873    }
874
875    private static Optional<String> getFirstInsideCountry(TagCheck check, Method insideMethod) {
876        return check.rule.selectors.stream()
877                .filter(s -> s instanceof GeneralSelector)
878                .flatMap(s -> ((GeneralSelector) s).getConditions().stream())
879                .filter(c -> c instanceof ExpressionCondition)
880                .map(c -> ((ExpressionCondition) c).getExpression())
881                .filter(c -> c instanceof ParameterFunction)
882                .map(c -> (ParameterFunction) c)
883                .filter(c -> c.getMethod().equals(insideMethod))
884                .flatMap(c -> c.getArgs().stream())
885                .filter(e -> e instanceof LiteralExpression)
886                .map(e -> ((LiteralExpression) e).getLiteral())
887                .filter(l -> l instanceof String)
888                .map(l -> ((String) l).split(",")[0])
889                .findFirst();
890    }
891
892    private static LatLon getLocation(TagCheck check, Method insideMethod) {
893        Optional<String> inside = getFirstInsideCountry(check, insideMethod);
894        if (inside.isPresent()) {
895            GeoPropertyIndex<Boolean> index = Territories.getGeoPropertyIndex(inside.get());
896            if (index != null) {
897                GeoProperty<Boolean> prop = index.getGeoProperty();
898                if (prop instanceof DefaultGeoProperty) {
899                    return ((DefaultGeoProperty) prop).getRandomLatLon();
900                }
901            }
902        }
903        return LatLon.ZERO;
904    }
905
906    /**
907     * Checks that rule assertions are met for the given set of TagChecks.
908     * @param schecks The TagChecks for which assertions have to be checked
909     * @return A set of error messages, empty if all assertions are met
910     * @since 7356
911     */
912    public Set<String> checkAsserts(final Collection<TagCheck> schecks) {
913        Set<String> assertionErrors = new LinkedHashSet<>();
914        final Method insideMethod = getFunctionMethod("inside");
915        final DataSet ds = new DataSet();
916        for (final TagCheck check : schecks) {
917            Logging.debug("Check: {0}", check);
918            for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) {
919                Logging.debug("- Assertion: {0}", i);
920                final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey(), getLocation(check, insideMethod), true);
921                // Build minimal ordered list of checks to run to test the assertion
922                List<Set<TagCheck>> checksToRun = new ArrayList<>();
923                Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks);
924                if (!checkDependencies.isEmpty()) {
925                    checksToRun.add(checkDependencies);
926                }
927                checksToRun.add(Collections.singleton(check));
928                // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors
929                addPrimitive(ds, p);
930                final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun);
931                Logging.debug("- Errors: {0}", pErrors);
932                @SuppressWarnings({"EqualsBetweenInconvertibleTypes", "EqualsIncompatibleType"})
933                final boolean isError = pErrors.stream().anyMatch(e -> e.getTester().equals(check.rule));
934                if (isError != i.getValue()) {
935                    final String error = MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})",
936                            check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys());
937                    assertionErrors.add(error);
938                }
939                ds.removePrimitive(p);
940            }
941        }
942        return assertionErrors;
943    }
944
945    private static void addPrimitive(DataSet ds, OsmPrimitive p) {
946        if (p instanceof Way) {
947            ((Way) p).getNodes().forEach(n -> addPrimitive(ds, n));
948        } else if (p instanceof Relation) {
949            ((Relation) p).getMembers().forEach(m -> addPrimitive(ds, m.getMember()));
950        }
951        ds.addPrimitive(p);
952    }
953
954    @Override
955    public synchronized int hashCode() {
956        return Objects.hash(super.hashCode(), checks);
957    }
958
959    @Override
960    public synchronized boolean equals(Object obj) {
961        if (this == obj) return true;
962        if (obj == null || getClass() != obj.getClass()) return false;
963        if (!super.equals(obj)) return false;
964        MapCSSTagChecker that = (MapCSSTagChecker) obj;
965        return Objects.equals(checks, that.checks);
966    }
967
968    /**
969     * Reload tagchecker rule.
970     * @param rule tagchecker rule to reload
971     * @since 12825
972     */
973    public static void reloadRule(SourceEntry rule) {
974        MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class);
975        if (tagChecker != null) {
976            try {
977                tagChecker.addMapCSS(rule.url);
978            } catch (IOException | ParseException | TokenMgrError e) {
979                Logging.warn(e);
980            }
981        }
982    }
983
984    @Override
985    public synchronized void startTest(ProgressMonitor progressMonitor) {
986        super.startTest(progressMonitor);
987        super.setShowElements(true);
988        if (indexData == null) {
989            indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ALL_TESTS);
990        }
991        tested.clear();
992    }
993
994    @Override
995    public synchronized void endTest() {
996        if (partialSelection && !tested.isEmpty()) {
997            // #14287: see https://josm.openstreetmap.de/ticket/14287#comment:15
998            // execute tests for objects which might contain or cross previously tested elements
999
1000            // rebuild index with a reduced set of rules (those that use ChildOrParentSelector) and thus may have left selectors
1001            // matching the previously tested elements
1002            indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ONLY_SELECTED_TESTS);
1003
1004            Set<OsmPrimitive> surrounding = new HashSet<>();
1005            for (OsmPrimitive p : tested) {
1006                if (p.getDataSet() != null) {
1007                    surrounding.addAll(p.getDataSet().searchWays(p.getBBox()));
1008                    surrounding.addAll(p.getDataSet().searchRelations(p.getBBox()));
1009                }
1010            }
1011            final boolean includeOtherSeverity = includeOtherSeverityChecks();
1012            for (OsmPrimitive p : surrounding) {
1013                if (tested.contains(p))
1014                    continue;
1015                Collection<TestError> additionalErrors = getErrorsForPrimitive(p, includeOtherSeverity);
1016                for (TestError e : additionalErrors) {
1017                    if (e.getPrimitives().stream().anyMatch(tested::contains))
1018                        addIfNotSimilar(e, errors);
1019                }
1020            }
1021            tested.clear();
1022        }
1023        super.endTest();
1024        // no need to keep the index, it is quickly build and doubles the memory needs
1025        indexData = null;
1026    }
1027
1028    private boolean includeOtherSeverityChecks() {
1029        return isBeforeUpload ? ValidatorPrefHelper.PREF_OTHER_UPLOAD.get() : ValidatorPrefHelper.PREF_OTHER.get();
1030    }
1031
1032}