001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006
007import java.util.ArrayList;
008import java.util.Arrays;
009import java.util.Collection;
010import java.util.Collections;
011import java.util.HashMap;
012import java.util.HashSet;
013import java.util.List;
014import java.util.Locale;
015import java.util.Map;
016import java.util.Map.Entry;
017import java.util.Objects;
018import java.util.Set;
019import java.util.stream.Collectors;
020import java.util.stream.Stream;
021
022import org.openstreetmap.josm.command.Command;
023import org.openstreetmap.josm.command.DeleteCommand;
024import org.openstreetmap.josm.data.coor.EastNorth;
025import org.openstreetmap.josm.data.coor.LatLon;
026import org.openstreetmap.josm.data.osm.Node;
027import org.openstreetmap.josm.data.osm.OsmPrimitive;
028import org.openstreetmap.josm.data.osm.Relation;
029import org.openstreetmap.josm.data.osm.RelationMember;
030import org.openstreetmap.josm.data.osm.TagMap;
031import org.openstreetmap.josm.data.osm.Way;
032import org.openstreetmap.josm.data.preferences.DoubleProperty;
033import org.openstreetmap.josm.data.validation.Severity;
034import org.openstreetmap.josm.data.validation.Test;
035import org.openstreetmap.josm.data.validation.TestError;
036import org.openstreetmap.josm.tools.Geometry;
037import org.openstreetmap.josm.tools.Logging;
038import org.openstreetmap.josm.tools.Pair;
039import org.openstreetmap.josm.tools.SubclassFilteredCollection;
040import org.openstreetmap.josm.tools.Territories;
041import org.openstreetmap.josm.tools.Utils;
042
043/**
044 * Performs validation tests on addresses (addr:housenumber) and associatedStreet relations.
045 * @since 5644
046 */
047public class Addresses extends Test {
048
049    protected static final int HOUSE_NUMBER_WITHOUT_STREET = 2601;
050    protected static final int DUPLICATE_HOUSE_NUMBER = 2602;
051    protected static final int MULTIPLE_STREET_NAMES = 2603;
052    protected static final int MULTIPLE_STREET_RELATIONS = 2604;
053    protected static final int HOUSE_NUMBER_TOO_FAR = 2605;
054    protected static final int OBSOLETE_RELATION = 2606;
055
056    protected static final DoubleProperty MAX_DUPLICATE_DISTANCE = new DoubleProperty("validator.addresses.max_duplicate_distance", 200.0);
057    protected static final DoubleProperty MAX_STREET_DISTANCE = new DoubleProperty("validator.addresses.max_street_distance", 200.0);
058
059    // CHECKSTYLE.OFF: SingleSpaceSeparator
060    protected static final String ADDR_HOUSE_NUMBER  = "addr:housenumber";
061    protected static final String ADDR_INTERPOLATION = "addr:interpolation";
062    protected static final String ADDR_NEIGHBOURHOOD = "addr:neighbourhood";
063    protected static final String ADDR_PLACE         = "addr:place";
064    protected static final String ADDR_STREET        = "addr:street";
065    protected static final String ADDR_SUBURB        = "addr:suburb";
066    protected static final String ADDR_CITY          = "addr:city";
067    protected static final String ADDR_UNIT          = "addr:unit";
068    protected static final String ADDR_FLATS         = "addr:flats";
069    protected static final String ADDR_HOUSE_NAME    = "addr:housename";
070    protected static final String ADDR_POSTCODE      = "addr:postcode";
071    protected static final String ASSOCIATED_STREET  = "associatedStreet";
072    // CHECKSTYLE.ON: SingleSpaceSeparator
073
074    private Map<String, Collection<OsmPrimitive>> knownAddresses;
075    private Set<String> ignoredAddresses;
076
077    /**
078     * Constructor
079     */
080    public Addresses() {
081        super(tr("Addresses"), tr("Checks for errors in addresses and associatedStreet relations."));
082    }
083
084    protected List<Relation> getAndCheckAssociatedStreets(OsmPrimitive p) {
085        final List<Relation> list = p.referrers(Relation.class)
086                .filter(r -> r.hasTag("type", ASSOCIATED_STREET))
087                .collect(Collectors.toList());
088        if (list.size() > 1) {
089            Severity level;
090            // warning level only if several relations have different names, see #10945
091            final String name = list.get(0).get("name");
092            if (name == null || SubclassFilteredCollection.filter(list, r -> r.hasTag("name", name)).size() < list.size()) {
093                level = Severity.WARNING;
094            } else {
095                level = Severity.OTHER;
096            }
097            List<OsmPrimitive> errorList = new ArrayList<>(list);
098            errorList.add(0, p);
099            errors.add(TestError.builder(this, level, MULTIPLE_STREET_RELATIONS)
100                    .message(tr("Multiple associatedStreet relations"))
101                    .primitives(errorList)
102                    .build());
103        }
104        return list;
105    }
106
107    /**
108     * Checks for house numbers for which the street is unknown.
109     * @param p primitive to test
110     * @return error found, or null
111     */
112    protected TestError checkHouseNumbersWithoutStreet(OsmPrimitive p) {
113        // Find house number without proper location
114        // (neither addr:street, associatedStreet, addr:place, addr:neighbourhood or addr:interpolation)
115        if (p.hasKey(ADDR_HOUSE_NUMBER) && !p.hasKey(ADDR_STREET, ADDR_PLACE, ADDR_NEIGHBOURHOOD)
116            && getAndCheckAssociatedStreets(p).isEmpty()
117            && p.referrers(Way.class).noneMatch(w -> w.hasKey(ADDR_INTERPOLATION) && w.hasKey(ADDR_STREET))) {
118            // no street found
119            TestError e = TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_WITHOUT_STREET)
120                .message(tr("House number without street"))
121                .primitives(p)
122                .build();
123            errors.add(e);
124            return e;
125        }
126        return null;
127    }
128
129    static boolean isPOI(OsmPrimitive p) {
130        return p.hasKey("shop", "amenity", "tourism", "leisure", "emergency", "craft", "office", "name");
131    }
132
133    static boolean hasAddress(OsmPrimitive p) {
134        return p.hasKey(ADDR_HOUSE_NUMBER) && p.hasKey(ADDR_STREET, ADDR_PLACE);
135    }
136
137    /**
138     * adds the OsmPrimitive to the address map if it complies to the restrictions
139     * @param p OsmPrimitive that has an address
140     */
141    private void collectAddress(OsmPrimitive p) {
142        if (!isPOI(p)) {
143            for (String simplifiedAddress : getSimplifiedAddresses(p)) {
144                if (!ignoredAddresses.contains(simplifiedAddress)) {
145                    knownAddresses.computeIfAbsent(simplifiedAddress, x -> new ArrayList<>()).add(p);
146                }
147            }
148        }
149    }
150
151    protected void initAddressMap(OsmPrimitive primitive) {
152        knownAddresses = new HashMap<>();
153        ignoredAddresses = new HashSet<>();
154        for (OsmPrimitive p : primitive.getDataSet().allNonDeletedPrimitives()) {
155            if (p instanceof Node && p.hasKey(ADDR_UNIT, ADDR_FLATS)) {
156                for (OsmPrimitive r : p.getReferrers()) {
157                    if (hasAddress(r)) {
158                        // ignore addresses of buildings that are connected to addr:unit nodes
159                        // it's quite reasonable that there are more buildings with this address
160                        for (String simplifiedAddress : getSimplifiedAddresses(r)) {
161                            if (!ignoredAddresses.contains(simplifiedAddress)) {
162                                ignoredAddresses.add(simplifiedAddress);
163                            } else if (knownAddresses.containsKey(simplifiedAddress)) {
164                                knownAddresses.remove(simplifiedAddress);
165                            }
166                        }
167                    }
168                }
169            }
170            if (hasAddress(p)) {
171                collectAddress(p);
172            }
173        }
174    }
175
176    @Override
177    public void endTest() {
178        knownAddresses = null;
179        ignoredAddresses = null;
180        super.endTest();
181    }
182
183    protected List<TestError> checkForDuplicate(OsmPrimitive p) {
184        if (knownAddresses == null) {
185            initAddressMap(p);
186        }
187        if (!isPOI(p) && hasAddress(p)) {
188            List<TestError> result = new ArrayList<>();
189            for (String simplifiedAddress : getSimplifiedAddresses(p)) {
190                if (!ignoredAddresses.contains(simplifiedAddress) && knownAddresses.containsKey(simplifiedAddress)) {
191                    double maxDistance = MAX_DUPLICATE_DISTANCE.get();
192                    for (OsmPrimitive p2 : knownAddresses.get(simplifiedAddress)) {
193                        if (p == p2) {
194                            continue;
195                        }
196                        Severity severityLevel;
197                        String city1 = p.get(ADDR_CITY);
198                        String city2 = p2.get(ADDR_CITY);
199                        double distance = getDistance(p, p2);
200                        if (city1 != null && city2 != null) {
201                            if (city1.equals(city2)) {
202                                if ((!p.hasKey(ADDR_POSTCODE) || !p2.hasKey(ADDR_POSTCODE)
203                                        || p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE)))
204                                        && (!p.hasKey(ADDR_SUBURB) || !p2.hasKey(ADDR_SUBURB)
205                                                || p.get(ADDR_SUBURB).equals(p2.get(ADDR_SUBURB)))) {
206                                    severityLevel = Severity.WARNING;
207                                } else {
208                                    // address including city identical but postcode or suburb differs
209                                    // most likely perfectly fine
210                                    severityLevel = Severity.OTHER;
211                                }
212                            } else {
213                                // address differs only by city - notify if very close, otherwise ignore
214                                if (distance < maxDistance) {
215                                    severityLevel = Severity.OTHER;
216                                } else {
217                                    continue;
218                                }
219                            }
220                        } else {
221                            // at least one address has no city specified
222                            if (p.hasKey(ADDR_POSTCODE) && p2.hasKey(ADDR_POSTCODE)
223                                    && p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) {
224                                // address including postcode identical
225                                severityLevel = Severity.WARNING;
226                            } else {
227                                // city/postcode unclear - warn if very close, otherwise only notify
228                                // TODO: get city from surrounding boundaries?
229                                if (distance < maxDistance) {
230                                    severityLevel = Severity.WARNING;
231                                } else {
232                                    severityLevel = Severity.OTHER;
233                                }
234                            }
235                        }
236                        result.add(TestError.builder(this, severityLevel, DUPLICATE_HOUSE_NUMBER)
237                                .message(tr("Duplicate house numbers"), marktr("''{0}'' ({1}m)"), simplifiedAddress, (int) distance)
238                                .primitives(Arrays.asList(p, p2)).build());
239                    }
240                    knownAddresses.get(simplifiedAddress).remove(p); // otherwise we would get every warning two times
241                }
242            }
243            errors.addAll(result);
244            return result;
245        }
246        return Collections.emptyList();
247    }
248
249    static List<String> getSimplifiedAddresses(OsmPrimitive p) {
250        String simplifiedStreetName = p.hasKey(ADDR_STREET) ? p.get(ADDR_STREET) : p.get(ADDR_PLACE);
251        // ignore whitespaces and dashes in street name, so that "Mozart-Gasse", "Mozart Gasse" and "Mozartgasse" are all seen as equal
252        return expandHouseNumber(p.get(ADDR_HOUSE_NUMBER)).stream().map(addrHouseNumber -> Utils.strip(Stream.of(
253                simplifiedStreetName.replaceAll("[ -]", ""),
254                addrHouseNumber,
255                p.get(ADDR_HOUSE_NAME),
256                p.get(ADDR_UNIT),
257                p.get(ADDR_FLATS))
258            .filter(Objects::nonNull)
259            .collect(Collectors.joining(" ")))
260                .toUpperCase(Locale.ENGLISH)).collect(Collectors.toList());
261    }
262
263    /**
264     * Split addr:housenumber on , and ; (common separators)
265     *
266     * @param houseNumber The housenumber to be split
267     * @return A list of addr:housenumber equivalents
268     */
269    static List<String> expandHouseNumber(String houseNumber) {
270        return Arrays.asList(houseNumber.split(",|;"));
271    }
272
273    @Override
274    public void visit(Node n) {
275        checkHouseNumbersWithoutStreet(n);
276        checkForDuplicate(n);
277    }
278
279    @Override
280    public void visit(Way w) {
281        checkHouseNumbersWithoutStreet(w);
282        checkForDuplicate(w);
283    }
284
285    @Override
286    public void visit(Relation r) {
287        checkHouseNumbersWithoutStreet(r);
288        checkForDuplicate(r);
289        if (r.hasTag("type", ASSOCIATED_STREET)) {
290            checkIfObsolete(r);
291            // Used to count occurrences of each house number in order to find duplicates
292            Map<String, List<OsmPrimitive>> map = new HashMap<>();
293            // Used to detect different street names
294            String relationName = r.get("name");
295            Set<OsmPrimitive> wrongStreetNames = new HashSet<>();
296            // Used to check distance
297            Set<OsmPrimitive> houses = new HashSet<>();
298            Set<Way> street = new HashSet<>();
299            for (RelationMember m : r.getMembers()) {
300                String role = m.getRole();
301                OsmPrimitive p = m.getMember();
302                if ("house".equals(role)) {
303                    houses.add(p);
304                    String number = p.get(ADDR_HOUSE_NUMBER);
305                    if (number != null) {
306                        number = number.trim().toUpperCase(Locale.ENGLISH);
307                        List<OsmPrimitive> list = map.get(number);
308                        if (list == null) {
309                            list = new ArrayList<>();
310                            map.put(number, list);
311                        }
312                        list.add(p);
313                    }
314                    if (relationName != null && p.hasKey(ADDR_STREET) && !relationName.equals(p.get(ADDR_STREET))) {
315                        if (wrongStreetNames.isEmpty()) {
316                            wrongStreetNames.add(r);
317                        }
318                        wrongStreetNames.add(p);
319                    }
320                } else if ("street".equals(role)) {
321                    if (p instanceof Way) {
322                        street.add((Way) p);
323                    }
324                    if (relationName != null && p.hasTagDifferent("name", relationName)) {
325                        if (wrongStreetNames.isEmpty()) {
326                            wrongStreetNames.add(r);
327                        }
328                        wrongStreetNames.add(p);
329                    }
330                }
331            }
332            // Report duplicate house numbers
333            for (Entry<String, List<OsmPrimitive>> entry : map.entrySet()) {
334                List<OsmPrimitive> list = entry.getValue();
335                if (list.size() > 1) {
336                    errors.add(TestError.builder(this, Severity.WARNING, DUPLICATE_HOUSE_NUMBER)
337                            .message(tr("Duplicate house numbers"), marktr("House number ''{0}'' duplicated"), entry.getKey())
338                            .primitives(list)
339                            .build());
340                }
341            }
342            // Report wrong street names
343            if (!wrongStreetNames.isEmpty()) {
344                errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_STREET_NAMES)
345                        .message(tr("Multiple street names in relation"))
346                        .primitives(wrongStreetNames)
347                        .build());
348            }
349            // Report addresses too far away
350            if (!street.isEmpty()) {
351                for (OsmPrimitive house : houses) {
352                    if (house.isUsable()) {
353                        checkDistance(house, street);
354                    }
355                }
356            }
357        }
358    }
359
360    /**
361     * returns rough distance between two OsmPrimitives
362     * @param a primitive a
363     * @param b primitive b
364     * @return distance of center of bounding boxes in meters
365     */
366    static double getDistance(OsmPrimitive a, OsmPrimitive b) {
367        LatLon centerA = a.getBBox().getCenter();
368        LatLon centerB = b.getBBox().getCenter();
369        return (centerA.greatCircleDistance(centerB));
370    }
371
372    protected void checkDistance(OsmPrimitive house, Collection<Way> street) {
373        EastNorth centroid;
374        if (house instanceof Node) {
375            centroid = ((Node) house).getEastNorth();
376        } else if (house instanceof Way) {
377            List<Node> nodes = ((Way) house).getNodes();
378            if (house.hasKey(ADDR_INTERPOLATION)) {
379                for (Node n : nodes) {
380                    if (n.hasKey(ADDR_HOUSE_NUMBER)) {
381                        checkDistance(n, street);
382                    }
383                }
384                return;
385            }
386            centroid = Geometry.getCentroid(nodes);
387        } else {
388            return; // TODO handle multipolygon houses ?
389        }
390        if (centroid == null) return; // fix #8305
391        double maxDistance = MAX_STREET_DISTANCE.get();
392        boolean hasIncompleteWays = false;
393        for (Way streetPart : street) {
394            for (Pair<Node, Node> chunk : streetPart.getNodePairs(false)) {
395                EastNorth p1 = chunk.a.getEastNorth();
396                EastNorth p2 = chunk.b.getEastNorth();
397                if (p1 != null && p2 != null) {
398                    EastNorth closest = Geometry.closestPointToSegment(p1, p2, centroid);
399                    if (closest.distance(centroid) <= maxDistance) {
400                        return;
401                    }
402                } else {
403                    Logging.warn("Addresses test skipped chunck "+chunk+" for street part "+streetPart+" because p1 or p2 is null");
404                }
405            }
406            if (!hasIncompleteWays && streetPart.isIncomplete()) {
407                hasIncompleteWays = true;
408            }
409        }
410        // No street segment found near this house, report error on if the relation does not contain incomplete street ways (fix #8314)
411        if (hasIncompleteWays) return;
412        List<OsmPrimitive> errorList = new ArrayList<>(street);
413        errorList.add(0, house);
414        errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_TOO_FAR)
415                .message(tr("House number too far from street"))
416                .primitives(errorList)
417                .build());
418    }
419
420    /**
421     * Check if an associatedStreet Relation is obsolete. This test marks only those relations which
422     * are complete and don't contain any information which isn't also tagged on the members.
423     * The strategy is to avoid any false positive.
424     * @param r the relation
425     */
426    private void checkIfObsolete(Relation r) {
427        if (r.isIncomplete())
428            return;
429        /** array of country codes for which the test should be performed. For now, only Germany */
430        String[] countryCodes = {"DE"};
431        TagMap neededtagsForHouse = new TagMap();
432        for (Entry<String, String> tag : r.getKeys().entrySet()) {
433            String key = tag.getKey();
434            if (key.startsWith("name:")) {
435                return; // maybe check if all members have corresponding tags?
436            } else if (key.startsWith("addr:")) {
437                neededtagsForHouse.put(key, tag.getValue());
438            } else {
439                switch (key) {
440                case "name":
441                case "type":
442                case "source":
443                    break;
444                default:
445                    // unexpected tag in relation
446                    return;
447                }
448            }
449        }
450
451        for (RelationMember m : r.getMembers()) {
452            if (m.getMember().isIncomplete() || !isInWarnCountry(m, countryCodes))
453                return;
454
455            String role = m.getRole();
456            if ("".equals(role)) {
457                if (m.isWay() && m.getMember().hasKey("highway")) {
458                    role = "street";
459                } else if (m.getMember().hasTag("building"))
460                    role = "house";
461            }
462            switch (role) {
463            case "house":
464            case "addr:houselink":
465            case "address":
466                if (!m.getMember().hasTag(ADDR_STREET) || !m.getMember().hasTag(ADDR_HOUSE_NUMBER))
467                    return;
468                for (Entry<String, String> tag : neededtagsForHouse.entrySet()) {
469                    if (!m.getMember().hasTag(tag.getKey(), tag.getValue()))
470                        return;
471                }
472                break;
473            case "street":
474                if (!m.getMember().hasTag("name") && r.hasTag("name"))
475                    return;
476                break;
477            default:
478                // unknown role: don't create auto-fix
479                return;
480            }
481        }
482        errors.add(TestError.builder(this, Severity.WARNING, OBSOLETE_RELATION)
483                .message(tr("Relation is obsolete"))
484                .primitives(r)
485                .build());
486    }
487
488    private static boolean isInWarnCountry(RelationMember m, String[] countryCodes) {
489        if (countryCodes.length == 0)
490            return true;
491        LatLon center = null;
492
493        if (m.isNode()) {
494            center = m.getNode().getCoor();
495        } else if (m.isWay()) {
496            center = m.getWay().getBBox().getCenter();
497        } else if (m.isRelation() && m.getRelation().isMultipolygon()) {
498            center = m.getRelation().getBBox().getCenter();
499        }
500        if (center == null)
501            return false;
502        for (String country : countryCodes) {
503            if (Territories.isIso3166Code(country, center))
504                return true;
505        }
506        return false;
507    }
508
509    /**
510     * remove obsolete relation.
511     */
512    @Override
513    public Command fixError(TestError testError) {
514        return new DeleteCommand(testError.getPrimitives());
515    }
516
517    @Override
518    public boolean isFixable(TestError testError) {
519        if (!(testError.getTester() instanceof Addresses))
520            return false;
521        return testError.getCode() == OBSOLETE_RELATION;
522    }
523
524}