001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedReader; 007import java.io.IOException; 008import java.io.InputStream; 009import java.io.Reader; 010import java.io.StringReader; 011import java.lang.reflect.Method; 012import java.text.MessageFormat; 013import java.util.ArrayList; 014import java.util.Collection; 015import java.util.Collections; 016import java.util.HashMap; 017import java.util.HashSet; 018import java.util.Iterator; 019import java.util.LinkedHashMap; 020import java.util.LinkedHashSet; 021import java.util.LinkedList; 022import java.util.List; 023import java.util.Locale; 024import java.util.Map; 025import java.util.Objects; 026import java.util.Optional; 027import java.util.Set; 028import java.util.function.Predicate; 029import java.util.regex.Matcher; 030import java.util.regex.Pattern; 031 032import org.openstreetmap.josm.command.ChangePropertyCommand; 033import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 034import org.openstreetmap.josm.command.Command; 035import org.openstreetmap.josm.command.DeleteCommand; 036import org.openstreetmap.josm.command.SequenceCommand; 037import org.openstreetmap.josm.data.coor.LatLon; 038import org.openstreetmap.josm.data.osm.DataSet; 039import org.openstreetmap.josm.data.osm.IPrimitive; 040import org.openstreetmap.josm.data.osm.OsmPrimitive; 041import org.openstreetmap.josm.data.osm.OsmUtils; 042import org.openstreetmap.josm.data.osm.Relation; 043import org.openstreetmap.josm.data.osm.Tag; 044import org.openstreetmap.josm.data.osm.Way; 045import org.openstreetmap.josm.data.preferences.sources.SourceEntry; 046import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 047import org.openstreetmap.josm.data.validation.OsmValidator; 048import org.openstreetmap.josm.data.validation.Severity; 049import org.openstreetmap.josm.data.validation.Test; 050import org.openstreetmap.josm.data.validation.TestError; 051import org.openstreetmap.josm.gui.mappaint.Environment; 052import org.openstreetmap.josm.gui.mappaint.Keyword; 053import org.openstreetmap.josm.gui.mappaint.MultiCascade; 054import org.openstreetmap.josm.gui.mappaint.mapcss.Condition; 055import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition; 056import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ExpressionCondition; 057import org.openstreetmap.josm.gui.mappaint.mapcss.Expression; 058import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.ParameterFunction; 059import org.openstreetmap.josm.gui.mappaint.mapcss.Functions; 060import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction; 061import org.openstreetmap.josm.gui.mappaint.mapcss.LiteralExpression; 062import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule; 063import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration; 064import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource; 065import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex; 066import org.openstreetmap.josm.gui.mappaint.mapcss.Selector; 067import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector; 068import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector; 069import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector; 070import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser; 071import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException; 072import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError; 073import org.openstreetmap.josm.gui.progress.ProgressMonitor; 074import org.openstreetmap.josm.io.CachedFile; 075import org.openstreetmap.josm.io.FileWatcher; 076import org.openstreetmap.josm.io.IllegalDataException; 077import org.openstreetmap.josm.io.UTFInputStreamReader; 078import org.openstreetmap.josm.spi.preferences.Config; 079import org.openstreetmap.josm.tools.CheckParameterUtil; 080import org.openstreetmap.josm.tools.DefaultGeoProperty; 081import org.openstreetmap.josm.tools.GeoProperty; 082import org.openstreetmap.josm.tools.GeoPropertyIndex; 083import org.openstreetmap.josm.tools.I18n; 084import org.openstreetmap.josm.tools.Logging; 085import org.openstreetmap.josm.tools.MultiMap; 086import org.openstreetmap.josm.tools.Territories; 087import org.openstreetmap.josm.tools.Utils; 088 089/** 090 * MapCSS-based tag checker/fixer. 091 * @since 6506 092 */ 093public class MapCSSTagChecker extends Test.TagTest { 094 MapCSSTagCheckerIndex indexData; 095 final Set<OsmPrimitive> tested = new HashSet<>(); 096 097 /** 098 * A grouped MapCSSRule with multiple selectors for a single declaration. 099 * @see MapCSSRule 100 */ 101 public static class GroupedMapCSSRule { 102 /** MapCSS selectors **/ 103 public final List<Selector> selectors; 104 /** MapCSS declaration **/ 105 public final Declaration declaration; 106 107 /** 108 * Constructs a new {@code GroupedMapCSSRule}. 109 * @param selectors MapCSS selectors 110 * @param declaration MapCSS declaration 111 */ 112 public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) { 113 this.selectors = selectors; 114 this.declaration = declaration; 115 } 116 117 @Override 118 public int hashCode() { 119 return Objects.hash(selectors, declaration); 120 } 121 122 @Override 123 public boolean equals(Object obj) { 124 if (this == obj) return true; 125 if (obj == null || getClass() != obj.getClass()) return false; 126 GroupedMapCSSRule that = (GroupedMapCSSRule) obj; 127 return Objects.equals(selectors, that.selectors) && 128 Objects.equals(declaration, that.declaration); 129 } 130 131 @Override 132 public String toString() { 133 return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']'; 134 } 135 } 136 137 /** 138 * The preference key for tag checker source entries. 139 * @since 6670 140 */ 141 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries"; 142 143 /** 144 * Constructs a new {@code MapCSSTagChecker}. 145 */ 146 public MapCSSTagChecker() { 147 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values.")); 148 } 149 150 /** 151 * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}. 152 */ 153 @FunctionalInterface 154 interface FixCommand { 155 /** 156 * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders 157 * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}). 158 * @param p OSM primitive 159 * @param matchingSelector matching selector 160 * @return fix command 161 */ 162 Command createCommand(OsmPrimitive p, Selector matchingSelector); 163 164 /** 165 * Checks that object is either an {@link Expression} or a {@link String}. 166 * @param obj object to check 167 * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String} 168 */ 169 static void checkObject(final Object obj) { 170 CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String, 171 () -> "instance of Exception or String expected, but got " + obj); 172 } 173 174 /** 175 * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}. 176 * @param obj object to evaluate ({@link Expression} or {@link String}) 177 * @param p OSM primitive 178 * @param matchingSelector matching selector 179 * @return result string 180 */ 181 static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) { 182 final String s; 183 if (obj instanceof Expression) { 184 s = (String) ((Expression) obj).evaluate(new Environment(p)); 185 } else if (obj instanceof String) { 186 s = (String) obj; 187 } else { 188 return null; 189 } 190 return TagCheck.insertArguments(matchingSelector, s, p); 191 } 192 193 /** 194 * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag. 195 * @param obj object to evaluate ({@link Expression} or {@link String}) 196 * @return created fix command 197 */ 198 static FixCommand fixAdd(final Object obj) { 199 checkObject(obj); 200 return new FixCommand() { 201 @Override 202 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 203 final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector)); 204 return new ChangePropertyCommand(p, tag.getKey(), tag.getValue()); 205 } 206 207 @Override 208 public String toString() { 209 return "fixAdd: " + obj; 210 } 211 }; 212 } 213 214 /** 215 * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key. 216 * @param obj object to evaluate ({@link Expression} or {@link String}) 217 * @return created fix command 218 */ 219 static FixCommand fixRemove(final Object obj) { 220 checkObject(obj); 221 return new FixCommand() { 222 @Override 223 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 224 final String key = evaluateObject(obj, p, matchingSelector); 225 return new ChangePropertyCommand(p, key, ""); 226 } 227 228 @Override 229 public String toString() { 230 return "fixRemove: " + obj; 231 } 232 }; 233 } 234 235 /** 236 * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys. 237 * @param oldKey old key 238 * @param newKey new key 239 * @return created fix command 240 */ 241 static FixCommand fixChangeKey(final String oldKey, final String newKey) { 242 return new FixCommand() { 243 @Override 244 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 245 return new ChangePropertyKeyCommand(p, 246 TagCheck.insertArguments(matchingSelector, oldKey, p), 247 TagCheck.insertArguments(matchingSelector, newKey, p)); 248 } 249 250 @Override 251 public String toString() { 252 return "fixChangeKey: " + oldKey + " => " + newKey; 253 } 254 }; 255 } 256 } 257 258 final MultiMap<String, TagCheck> checks = new MultiMap<>(); 259 260 /** 261 * Result of {@link TagCheck#readMapCSS} 262 * @since 8936 263 */ 264 public static class ParseResult { 265 /** Checks successfully parsed */ 266 public final List<TagCheck> parseChecks; 267 /** Errors that occurred during parsing */ 268 public final Collection<Throwable> parseErrors; 269 270 /** 271 * Constructs a new {@code ParseResult}. 272 * @param parseChecks Checks successfully parsed 273 * @param parseErrors Errors that occurred during parsing 274 */ 275 public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) { 276 this.parseChecks = parseChecks; 277 this.parseErrors = parseErrors; 278 } 279 } 280 281 /** 282 * Tag check. 283 */ 284 public static class TagCheck implements Predicate<OsmPrimitive> { 285 /** The selector of this {@code TagCheck} */ 286 protected final GroupedMapCSSRule rule; 287 /** Commands to apply in order to fix a matching primitive */ 288 protected final List<FixCommand> fixCommands = new ArrayList<>(); 289 /** Tags (or arbitraty strings) of alternatives to be presented to the user */ 290 protected final List<String> alternatives = new ArrayList<>(); 291 /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair. 292 * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */ 293 protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>(); 294 /** Unit tests */ 295 protected final Map<String, Boolean> assertions = new HashMap<>(); 296 /** MapCSS Classes to set on matching primitives */ 297 protected final Set<String> setClassExpressions = new HashSet<>(); 298 /** Denotes whether the object should be deleted for fixing it */ 299 protected boolean deletion; 300 /** A string used to group similar tests */ 301 protected String group; 302 303 TagCheck(GroupedMapCSSRule rule) { 304 this.rule = rule; 305 } 306 307 private static final String POSSIBLE_THROWS = possibleThrows(); 308 309 static final String possibleThrows() { 310 StringBuilder sb = new StringBuilder(); 311 for (Severity s : Severity.values()) { 312 if (sb.length() > 0) { 313 sb.append('/'); 314 } 315 sb.append("throw") 316 .append(s.name().charAt(0)) 317 .append(s.name().substring(1).toLowerCase(Locale.ENGLISH)); 318 } 319 return sb.toString(); 320 } 321 322 static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException { 323 final TagCheck check = new TagCheck(rule); 324 for (Instruction i : rule.declaration.instructions) { 325 if (i instanceof Instruction.AssignmentInstruction) { 326 final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i; 327 if (ai.isSetInstruction) { 328 check.setClassExpressions.add(ai.key); 329 continue; 330 } 331 try { 332 final String val = ai.val instanceof Expression 333 ? Optional.ofNullable(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null) 334 : ai.val instanceof String 335 ? (String) ai.val 336 : ai.val instanceof Keyword 337 ? ((Keyword) ai.val).val 338 : null; 339 if (ai.key.startsWith("throw")) { 340 try { 341 check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH))); 342 } catch (IllegalArgumentException e) { 343 Logging.log(Logging.LEVEL_WARN, 344 "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e); 345 } 346 } else if ("fixAdd".equals(ai.key)) { 347 check.fixCommands.add(FixCommand.fixAdd(ai.val)); 348 } else if ("fixRemove".equals(ai.key)) { 349 CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")), 350 "Unexpected '='. Please only specify the key to remove in: " + ai); 351 check.fixCommands.add(FixCommand.fixRemove(ai.val)); 352 } else if (val != null && "fixChangeKey".equals(ai.key)) { 353 CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!"); 354 final String[] x = val.split("=>", 2); 355 check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1]))); 356 } else if (val != null && "fixDeleteObject".equals(ai.key)) { 357 CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'"); 358 check.deletion = true; 359 } else if (val != null && "suggestAlternative".equals(ai.key)) { 360 check.alternatives.add(val); 361 } else if (val != null && "assertMatch".equals(ai.key)) { 362 check.assertions.put(val, Boolean.TRUE); 363 } else if (val != null && "assertNoMatch".equals(ai.key)) { 364 check.assertions.put(val, Boolean.FALSE); 365 } else if (val != null && "group".equals(ai.key)) { 366 check.group = val; 367 } else if (ai.key.startsWith("-")) { 368 Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val); 369 } else { 370 throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!'); 371 } 372 } catch (IllegalArgumentException e) { 373 throw new IllegalDataException(e); 374 } 375 } 376 } 377 if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) { 378 throw new IllegalDataException( 379 "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors); 380 } else if (check.errors.size() > 1) { 381 throw new IllegalDataException( 382 "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for " 383 + rule.selectors); 384 } 385 return check; 386 } 387 388 static ParseResult readMapCSS(Reader css) throws ParseException { 389 CheckParameterUtil.ensureParameterNotNull(css, "css"); 390 391 final MapCSSStyleSource source = new MapCSSStyleSource(""); 392 final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR); 393 try (StringReader mapcss = new StringReader(preprocessor.pp_root(source))) { 394 new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT).sheet(source); 395 } 396 // Ignore "meta" rule(s) from external rules of JOSM wiki 397 source.removeMetaRules(); 398 // group rules with common declaration block 399 Map<Declaration, List<Selector>> g = new LinkedHashMap<>(); 400 for (MapCSSRule rule : source.rules) { 401 if (!g.containsKey(rule.declaration)) { 402 List<Selector> sels = new ArrayList<>(); 403 sels.add(rule.selector); 404 g.put(rule.declaration, sels); 405 } else { 406 g.get(rule.declaration).add(rule.selector); 407 } 408 } 409 List<TagCheck> parseChecks = new ArrayList<>(); 410 for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) { 411 try { 412 parseChecks.add(TagCheck.ofMapCSSRule( 413 new GroupedMapCSSRule(map.getValue(), map.getKey()))); 414 } catch (IllegalDataException e) { 415 Logging.error("Cannot add MapCss rule: "+e.getMessage()); 416 source.logError(e); 417 } 418 } 419 return new ParseResult(parseChecks, source.getErrors()); 420 } 421 422 @Override 423 public boolean test(OsmPrimitive primitive) { 424 // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker. 425 return whichSelectorMatchesPrimitive(primitive) != null; 426 } 427 428 Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) { 429 return whichSelectorMatchesEnvironment(new Environment(primitive)); 430 } 431 432 Selector whichSelectorMatchesEnvironment(Environment env) { 433 for (Selector i : rule.selectors) { 434 env.clearSelectorMatchingInformation(); 435 if (i.matches(env)) { 436 return i; 437 } 438 } 439 return null; 440 } 441 442 /** 443 * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the 444 * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}. 445 * @param matchingSelector matching selector 446 * @param index index 447 * @param type selector type ("key", "value" or "tag") 448 * @param p OSM primitive 449 * @return argument value, can be {@code null} 450 */ 451 static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) { 452 try { 453 final Condition c = matchingSelector.getConditions().get(index); 454 final Tag tag = c instanceof Condition.ToTagConvertable 455 ? ((Condition.ToTagConvertable) c).asTag(p) 456 : null; 457 if (tag == null) { 458 return null; 459 } else if ("key".equals(type)) { 460 return tag.getKey(); 461 } else if ("value".equals(type)) { 462 return tag.getValue(); 463 } else if ("tag".equals(type)) { 464 return tag.toString(); 465 } 466 } catch (IndexOutOfBoundsException ignore) { 467 Logging.debug(ignore); 468 } 469 return null; 470 } 471 472 /** 473 * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding 474 * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}. 475 * @param matchingSelector matching selector 476 * @param s any string 477 * @param p OSM primitive 478 * @return string with arguments inserted 479 */ 480 static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) { 481 if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) { 482 return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p); 483 } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) { 484 return s; 485 } 486 final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s); 487 final StringBuffer sb = new StringBuffer(); 488 while (m.find()) { 489 final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector, 490 Integer.parseInt(m.group(1)), m.group(2), p); 491 try { 492 // Perform replacement with null-safe + regex-safe handling 493 m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", "")); 494 } catch (IndexOutOfBoundsException | IllegalArgumentException e) { 495 Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e); 496 } 497 } 498 m.appendTail(sb); 499 return sb.toString(); 500 } 501 502 /** 503 * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive} 504 * if the error is fixable, or {@code null} otherwise. 505 * 506 * @param p the primitive to construct the fix for 507 * @return the fix or {@code null} 508 */ 509 Command fixPrimitive(OsmPrimitive p) { 510 if (fixCommands.isEmpty() && !deletion) { 511 return null; 512 } 513 try { 514 final Selector matchingSelector = whichSelectorMatchesPrimitive(p); 515 Collection<Command> cmds = new LinkedList<>(); 516 for (FixCommand fixCommand : fixCommands) { 517 cmds.add(fixCommand.createCommand(p, matchingSelector)); 518 } 519 if (deletion && !p.isDeleted()) { 520 cmds.add(new DeleteCommand(p)); 521 } 522 return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds); 523 } catch (IllegalArgumentException e) { 524 Logging.error(e); 525 return null; 526 } 527 } 528 529 /** 530 * Constructs a (localized) message for this deprecation check. 531 * @param p OSM primitive 532 * 533 * @return a message 534 */ 535 String getMessage(OsmPrimitive p) { 536 if (errors.isEmpty()) { 537 // Return something to avoid NPEs 538 return rule.declaration.toString(); 539 } else { 540 final Object val = errors.keySet().iterator().next().val; 541 return String.valueOf( 542 val instanceof Expression 543 ? ((Expression) val).evaluate(new Environment(p)) 544 : val 545 ); 546 } 547 } 548 549 /** 550 * Constructs a (localized) description for this deprecation check. 551 * @param p OSM primitive 552 * 553 * @return a description (possibly with alternative suggestions) 554 * @see #getDescriptionForMatchingSelector 555 */ 556 String getDescription(OsmPrimitive p) { 557 if (alternatives.isEmpty()) { 558 return getMessage(p); 559 } else { 560 /* I18N: {0} is the test error message and {1} is an alternative */ 561 return tr("{0}, use {1} instead", getMessage(p), String.join(tr(" or "), alternatives)); 562 } 563 } 564 565 /** 566 * Constructs a (localized) description for this deprecation check 567 * where any placeholders are replaced by values of the matched selector. 568 * 569 * @param matchingSelector matching selector 570 * @param p OSM primitive 571 * @return a description (possibly with alternative suggestions) 572 */ 573 String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) { 574 return insertArguments(matchingSelector, getDescription(p), p); 575 } 576 577 Severity getSeverity() { 578 return errors.isEmpty() ? null : errors.values().iterator().next(); 579 } 580 581 @Override 582 public String toString() { 583 return getDescription(null); 584 } 585 586 /** 587 * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error. 588 * 589 * @param p the primitive to construct the error for 590 * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error. 591 */ 592 List<TestError> getErrorsForPrimitive(OsmPrimitive p) { 593 final Environment env = new Environment(p); 594 return getErrorsForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null); 595 } 596 597 private List<TestError> getErrorsForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) { 598 List<TestError> res = new ArrayList<>(); 599 if (matchingSelector != null && !errors.isEmpty()) { 600 final Command fix = fixPrimitive(p); 601 final String description = getDescriptionForMatchingSelector(p, matchingSelector); 602 final String description1 = group == null ? description : group; 603 final String description2 = group == null ? null : description; 604 TestError.Builder errorBuilder = TestError.builder(tester, getSeverity(), 3000) 605 .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString()); 606 if (fix != null) { 607 errorBuilder = errorBuilder.fix(() -> fix); 608 } 609 if (env.child instanceof OsmPrimitive) { 610 res.add(errorBuilder.primitives(p, (OsmPrimitive) env.child).build()); 611 } else if (env.children != null) { 612 for (IPrimitive c : env.children) { 613 if (c instanceof OsmPrimitive) { 614 errorBuilder = TestError.builder(tester, getSeverity(), 3000) 615 .messageWithManuallyTranslatedDescription(description1, description2, 616 matchingSelector.toString()); 617 if (fix != null) { 618 errorBuilder = errorBuilder.fix(() -> fix); 619 } 620 res.add(errorBuilder.primitives(p, (OsmPrimitive) c).build()); 621 } 622 } 623 } else { 624 res.add(errorBuilder.primitives(p).build()); 625 } 626 } 627 return res; 628 } 629 630 /** 631 * Returns the set of tagchecks on which this check depends on. 632 * @param schecks the collection of tagcheks to search in 633 * @return the set of tagchecks on which this check depends on 634 * @since 7881 635 */ 636 public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) { 637 Set<TagCheck> result = new HashSet<>(); 638 Set<String> classes = getClassesIds(); 639 if (schecks != null && !classes.isEmpty()) { 640 for (TagCheck tc : schecks) { 641 if (this.equals(tc)) { 642 continue; 643 } 644 for (String id : tc.setClassExpressions) { 645 if (classes.contains(id)) { 646 result.add(tc); 647 break; 648 } 649 } 650 } 651 } 652 return result; 653 } 654 655 /** 656 * Returns the list of ids of all MapCSS classes referenced in the rule selectors. 657 * @return the list of ids of all MapCSS classes referenced in the rule selectors 658 * @since 7881 659 */ 660 public Set<String> getClassesIds() { 661 Set<String> result = new HashSet<>(); 662 for (Selector s : rule.selectors) { 663 if (s instanceof AbstractSelector) { 664 for (Condition c : ((AbstractSelector) s).getConditions()) { 665 if (c instanceof ClassCondition) { 666 result.add(((ClassCondition) c).id); 667 } 668 } 669 } 670 } 671 return result; 672 } 673 } 674 675 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker { 676 public final GroupedMapCSSRule rule; 677 678 MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) { 679 this.rule = rule; 680 } 681 682 @Override 683 public String toString() { 684 return "MapCSSTagCheckerAndRule [rule=" + rule + ']'; 685 } 686 } 687 688 /** 689 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}. 690 * @param p The OSM primitive 691 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned 692 * @return all errors for the given primitive, with or without those of "info" severity 693 */ 694 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) { 695 final List<TestError> res = new ArrayList<>(); 696 if (indexData == null) { 697 indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverity, MapCSSTagCheckerIndex.ALL_TESTS); 698 } 699 700 MapCSSRuleIndex matchingRuleIndex = indexData.get(p); 701 702 Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 703 // the declaration indices are sorted, so it suffices to save the last used index 704 Declaration lastDeclUsed = null; 705 706 Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p); 707 while (candidates.hasNext()) { 708 MapCSSRule r = candidates.next(); 709 env.clearSelectorMatchingInformation(); 710 if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector) 711 TagCheck check = indexData.getCheck(r); 712 if (check != null) { 713 if (r.declaration == lastDeclUsed) 714 continue; // don't apply one declaration more than once 715 lastDeclUsed = r.declaration; 716 717 r.declaration.execute(env); 718 if (!check.errors.isEmpty()) { 719 for (TestError e: check.getErrorsForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule))) { 720 addIfNotSimilar(e, res); 721 } 722 } 723 } 724 } 725 } 726 return res; 727 } 728 729 /** 730 * See #12627 731 * Add error to given list if list doesn't already contain a similar error. 732 * Similar means same code and description and same combination of primitives and same combination of highlighted objects, 733 * but maybe with different orders. 734 * @param toAdd the error to add 735 * @param errors the list of errors 736 */ 737 private static void addIfNotSimilar(TestError toAdd, List<TestError> errors) { 738 boolean isDup = false; 739 if (toAdd.getPrimitives().size() >= 2) { 740 for (TestError e : errors) { 741 if (e.getCode() == toAdd.getCode() && e.getMessage().equals(toAdd.getMessage()) 742 && e.getPrimitives().size() == toAdd.getPrimitives().size() 743 && e.getPrimitives().containsAll(toAdd.getPrimitives()) 744 && e.getHighlighted().size() == toAdd.getHighlighted().size() 745 && e.getHighlighted().containsAll(toAdd.getHighlighted())) { 746 isDup = true; 747 break; 748 } 749 } 750 } 751 if (!isDup) 752 errors.add(toAdd); 753 } 754 755 private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity, 756 Collection<Set<TagCheck>> checksCol) { 757 final List<TestError> r = new ArrayList<>(); 758 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 759 for (Set<TagCheck> schecks : checksCol) { 760 for (TagCheck check : schecks) { 761 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity; 762 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class 763 if (ignoreError && check.setClassExpressions.isEmpty()) { 764 continue; 765 } 766 final Selector selector = check.whichSelectorMatchesEnvironment(env); 767 if (selector != null) { 768 check.rule.declaration.execute(env); 769 if (!ignoreError && !check.errors.isEmpty()) { 770 r.addAll(check.getErrorsForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule))); 771 } 772 } 773 } 774 } 775 return r; 776 } 777 778 /** 779 * Visiting call for primitives. 780 * 781 * @param p The primitive to inspect. 782 */ 783 @Override 784 public void check(OsmPrimitive p) { 785 for (TestError e : getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get())) { 786 addIfNotSimilar(e, errors); 787 } 788 if (partialSelection) { 789 tested.add(p); 790 } 791 } 792 793 /** 794 * Adds a new MapCSS config file from the given URL. 795 * @param url The unique URL of the MapCSS config file 796 * @return List of tag checks and parsing errors, or null 797 * @throws ParseException if the config file does not match MapCSS syntax 798 * @throws IOException if any I/O error occurs 799 * @since 7275 800 */ 801 public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException { 802 CheckParameterUtil.ensureParameterNotNull(url, "url"); 803 ParseResult result; 804 try (CachedFile cache = new CachedFile(url); 805 InputStream zip = cache.findZipEntryInputStream("validator.mapcss", ""); 806 InputStream s = zip != null ? zip : cache.getInputStream(); 807 Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) { 808 if (zip != null) 809 I18n.addTexts(cache.getFile()); 810 result = TagCheck.readMapCSS(reader); 811 checks.remove(url); 812 checks.putAll(url, result.parseChecks); 813 indexData = null; 814 // Check assertions, useful for development of local files 815 if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) { 816 for (String msg : checkAsserts(result.parseChecks)) { 817 Logging.warn(msg); 818 } 819 } 820 } 821 return result; 822 } 823 824 @Override 825 public synchronized void initialize() throws Exception { 826 checks.clear(); 827 indexData = null; 828 for (SourceEntry source : new ValidatorPrefHelper().get()) { 829 if (!source.active) { 830 continue; 831 } 832 String i = source.url; 833 try { 834 if (!i.startsWith("resource:")) { 835 Logging.info(tr("Adding {0} to tag checker", i)); 836 } else if (Logging.isDebugEnabled()) { 837 Logging.debug(tr("Adding {0} to tag checker", i)); 838 } 839 addMapCSS(i); 840 if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) { 841 FileWatcher.getDefaultInstance().registerSource(source); 842 } 843 } catch (IOException | IllegalStateException | IllegalArgumentException ex) { 844 Logging.warn(tr("Failed to add {0} to tag checker", i)); 845 Logging.log(Logging.LEVEL_WARN, ex); 846 } catch (ParseException | TokenMgrError ex) { 847 Logging.warn(tr("Failed to add {0} to tag checker", i)); 848 Logging.warn(ex); 849 } 850 } 851 } 852 853 private static Method getFunctionMethod(String method) { 854 try { 855 return Functions.class.getDeclaredMethod(method, Environment.class, String.class); 856 } catch (NoSuchMethodException | SecurityException e) { 857 Logging.error(e); 858 return null; 859 } 860 } 861 862 private static Optional<String> getFirstInsideCountry(TagCheck check, Method insideMethod) { 863 return check.rule.selectors.stream() 864 .filter(s -> s instanceof GeneralSelector) 865 .flatMap(s -> ((GeneralSelector) s).getConditions().stream()) 866 .filter(c -> c instanceof ExpressionCondition) 867 .map(c -> ((ExpressionCondition) c).getExpression()) 868 .filter(c -> c instanceof ParameterFunction) 869 .map(c -> (ParameterFunction) c) 870 .filter(c -> c.getMethod().equals(insideMethod)) 871 .flatMap(c -> c.getArgs().stream()) 872 .filter(e -> e instanceof LiteralExpression) 873 .map(e -> ((LiteralExpression) e).getLiteral()) 874 .filter(l -> l instanceof String) 875 .map(l -> ((String) l).split(",")[0]) 876 .findFirst(); 877 } 878 879 private static LatLon getLocation(TagCheck check, Method insideMethod) { 880 Optional<String> inside = getFirstInsideCountry(check, insideMethod); 881 if (inside.isPresent()) { 882 GeoPropertyIndex<Boolean> index = Territories.getGeoPropertyIndex(inside.get()); 883 if (index != null) { 884 GeoProperty<Boolean> prop = index.getGeoProperty(); 885 if (prop instanceof DefaultGeoProperty) { 886 return ((DefaultGeoProperty) prop).getRandomLatLon(); 887 } 888 } 889 } 890 return LatLon.ZERO; 891 } 892 893 /** 894 * Checks that rule assertions are met for the given set of TagChecks. 895 * @param schecks The TagChecks for which assertions have to be checked 896 * @return A set of error messages, empty if all assertions are met 897 * @since 7356 898 */ 899 public Set<String> checkAsserts(final Collection<TagCheck> schecks) { 900 Set<String> assertionErrors = new LinkedHashSet<>(); 901 final Method insideMethod = getFunctionMethod("inside"); 902 final DataSet ds = new DataSet(); 903 for (final TagCheck check : schecks) { 904 Logging.debug("Check: {0}", check); 905 for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) { 906 Logging.debug("- Assertion: {0}", i); 907 final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey(), getLocation(check, insideMethod), true); 908 // Build minimal ordered list of checks to run to test the assertion 909 List<Set<TagCheck>> checksToRun = new ArrayList<>(); 910 Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks); 911 if (!checkDependencies.isEmpty()) { 912 checksToRun.add(checkDependencies); 913 } 914 checksToRun.add(Collections.singleton(check)); 915 // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors 916 addPrimitive(ds, p); 917 final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun); 918 Logging.debug("- Errors: {0}", pErrors); 919 final boolean isError = pErrors.stream().anyMatch(e -> e.getTester() instanceof MapCSSTagCheckerAndRule 920 && ((MapCSSTagCheckerAndRule) e.getTester()).rule.equals(check.rule)); 921 if (isError != i.getValue()) { 922 assertionErrors.add(MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})", 923 check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys())); 924 } 925 if (isError) { 926 // Check that autofix works as expected 927 Command fix = check.fixPrimitive(p); 928 if (fix != null && fix.executeCommand() && !getErrorsForPrimitive(p, true, checksToRun).isEmpty()) { 929 assertionErrors.add(MessageFormat.format("Autofix does not work for test ''{0}'' (i.e., {1})", 930 check.getMessage(p), check.rule.selectors)); 931 } 932 } 933 ds.removePrimitive(p); 934 } 935 } 936 return assertionErrors; 937 } 938 939 private static void addPrimitive(DataSet ds, OsmPrimitive p) { 940 if (p instanceof Way) { 941 ((Way) p).getNodes().forEach(n -> addPrimitive(ds, n)); 942 } else if (p instanceof Relation) { 943 ((Relation) p).getMembers().forEach(m -> addPrimitive(ds, m.getMember())); 944 } 945 ds.addPrimitive(p); 946 } 947 948 /** 949 * Reload tagchecker rule. 950 * @param rule tagchecker rule to reload 951 * @since 12825 952 */ 953 public static void reloadRule(SourceEntry rule) { 954 MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class); 955 if (tagChecker != null) { 956 try { 957 tagChecker.addMapCSS(rule.url); 958 } catch (IOException | ParseException | TokenMgrError e) { 959 Logging.warn(e); 960 } 961 } 962 } 963 964 @Override 965 public synchronized void startTest(ProgressMonitor progressMonitor) { 966 super.startTest(progressMonitor); 967 super.setShowElements(true); 968 if (indexData == null) { 969 indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ALL_TESTS); 970 } 971 tested.clear(); 972 } 973 974 @Override 975 public synchronized void endTest() { 976 if (partialSelection && !tested.isEmpty()) { 977 // #14287: see https://josm.openstreetmap.de/ticket/14287#comment:15 978 // execute tests for objects which might contain or cross previously tested elements 979 980 // rebuild index with a reduced set of rules (those that use ChildOrParentSelector) and thus may have left selectors 981 // matching the previously tested elements 982 indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ONLY_SELECTED_TESTS); 983 984 Set<OsmPrimitive> surrounding = new HashSet<>(); 985 for (OsmPrimitive p : tested) { 986 if (p.getDataSet() != null) { 987 surrounding.addAll(p.getDataSet().searchWays(p.getBBox())); 988 surrounding.addAll(p.getDataSet().searchRelations(p.getBBox())); 989 } 990 } 991 final boolean includeOtherSeverity = includeOtherSeverityChecks(); 992 for (OsmPrimitive p : surrounding) { 993 if (tested.contains(p)) 994 continue; 995 Collection<TestError> additionalErrors = getErrorsForPrimitive(p, includeOtherSeverity); 996 for (TestError e : additionalErrors) { 997 if (e.getPrimitives().stream().anyMatch(tested::contains)) 998 addIfNotSimilar(e, errors); 999 } 1000 } 1001 tested.clear(); 1002 } 1003 super.endTest(); 1004 // no need to keep the index, it is quickly build and doubles the memory needs 1005 indexData = null; 1006 } 1007}