001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005import static org.openstreetmap.josm.tools.I18n.trn; 006 007import java.io.IOException; 008import java.io.InputStream; 009import java.net.HttpURLConnection; 010import java.util.ArrayList; 011import java.util.Collection; 012import java.util.Collections; 013import java.util.HashSet; 014import java.util.Iterator; 015import java.util.LinkedHashSet; 016import java.util.List; 017import java.util.Set; 018import java.util.concurrent.Callable; 019import java.util.concurrent.CompletionService; 020import java.util.concurrent.ExecutionException; 021import java.util.concurrent.ExecutorCompletionService; 022import java.util.concurrent.ExecutorService; 023import java.util.concurrent.Executors; 024import java.util.concurrent.Future; 025 026import org.openstreetmap.josm.Main; 027import org.openstreetmap.josm.data.osm.DataSet; 028import org.openstreetmap.josm.data.osm.DataSetMerger; 029import org.openstreetmap.josm.data.osm.Node; 030import org.openstreetmap.josm.data.osm.OsmPrimitive; 031import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 032import org.openstreetmap.josm.data.osm.PrimitiveId; 033import org.openstreetmap.josm.data.osm.Relation; 034import org.openstreetmap.josm.data.osm.RelationMember; 035import org.openstreetmap.josm.data.osm.SimplePrimitiveId; 036import org.openstreetmap.josm.data.osm.Way; 037import org.openstreetmap.josm.gui.preferences.server.OverpassServerPreference; 038import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 039import org.openstreetmap.josm.gui.progress.ProgressMonitor; 040import org.openstreetmap.josm.tools.Utils; 041 042/** 043 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called 044 * Multi Fetch API. 045 * 046 * Usage: 047 * <pre> 048 * MultiFetchServerObjectReader reader = MultiFetchServerObjectReader() 049 * .append(2345,2334,4444) 050 * .append(new Node(72343)); 051 * reader.parseOsm(); 052 * if (!reader.getMissingPrimitives().isEmpty()) { 053 * Main.info("There are missing primitives: " + reader.getMissingPrimitives()); 054 * } 055 * if (!reader.getSkippedWays().isEmpty()) { 056 * Main.info("There are skipped ways: " + reader.getMissingPrimitives()); 057 * } 058 * </pre> 059 */ 060public class MultiFetchServerObjectReader extends OsmServerReader { 061 /** 062 * the max. number of primitives retrieved in one step. Assuming IDs with 7 digits, 063 * this leads to a max. request URL of ~ 1600 Bytes ((7 digits + 1 Separator) * 200), 064 * which should be safe according to the 065 * <a href="http://www.boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>. 066 */ 067 private static final int MAX_IDS_PER_REQUEST = 200; 068 069 private final Set<Long> nodes; 070 private final Set<Long> ways; 071 private final Set<Long> relations; 072 private Set<PrimitiveId> missingPrimitives; 073 private final DataSet outputDataSet; 074 075 /** 076 * Constructs a {@code MultiFetchServerObjectReader}. 077 */ 078 protected MultiFetchServerObjectReader() { 079 nodes = new LinkedHashSet<>(); 080 ways = new LinkedHashSet<>(); 081 relations = new LinkedHashSet<>(); 082 this.outputDataSet = new DataSet(); 083 this.missingPrimitives = new LinkedHashSet<>(); 084 } 085 086 /** 087 * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader} 088 * depending on the {@link OverpassServerPreference#useForMultiFetch preference}. 089 * 090 * @return a new instance 091 * @since 9241 092 */ 093 public static MultiFetchServerObjectReader create() { 094 return create(OverpassServerPreference.useForMultiFetch()); 095 } 096 097 /** 098 * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader} 099 * depending on the {@code fromMirror} parameter. 100 * 101 * @param fromMirror {@code false} for {@link MultiFetchServerObjectReader}, {@code true} for {@link MultiFetchOverpassObjectReader} 102 * @return a new instance 103 * @since 9241 104 */ 105 static MultiFetchServerObjectReader create(final boolean fromMirror) { 106 if (fromMirror) { 107 return new MultiFetchOverpassObjectReader(); 108 } else { 109 return new MultiFetchServerObjectReader(); 110 } 111 } 112 113 /** 114 * Remembers an {@link OsmPrimitive}'s id. The id will 115 * later be fetched as part of a Multi Get request. 116 * 117 * Ignore the id if it represents a new primitives. 118 * 119 * @param id the id 120 */ 121 protected void remember(PrimitiveId id) { 122 if (id.isNew()) return; 123 switch(id.getType()) { 124 case NODE: nodes.add(id.getUniqueId()); break; 125 case WAY: ways.add(id.getUniqueId()); break; 126 case RELATION: relations.add(id.getUniqueId()); break; 127 default: throw new AssertionError(); 128 } 129 } 130 131 /** 132 * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server. 133 * 134 * @param ds the {@link DataSet} to which the primitive belongs 135 * @param id the primitive id 136 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 137 * {@link OsmPrimitiveType#RELATION RELATION} 138 * @return this 139 */ 140 public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) { 141 OsmPrimitive p = ds.getPrimitiveById(id, type); 142 switch(type) { 143 case NODE: 144 return appendNode((Node) p); 145 case WAY: 146 return appendWay((Way) p); 147 case RELATION: 148 return appendRelation((Relation) p); 149 } 150 return this; 151 } 152 153 /** 154 * appends a {@link Node} id to the list of ids which will be fetched from the server. 155 * 156 * @param node the node (ignored, if null) 157 * @return this 158 */ 159 public MultiFetchServerObjectReader appendNode(Node node) { 160 if (node == null) return this; 161 remember(node.getPrimitiveId()); 162 return this; 163 } 164 165 /** 166 * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server. 167 * 168 * @param way the way (ignored, if null) 169 * @return this 170 */ 171 public MultiFetchServerObjectReader appendWay(Way way) { 172 if (way == null) return this; 173 if (way.isNew()) return this; 174 for (Node node: !recursesDown() ? way.getNodes() : Collections.<Node>emptyList()) { 175 if (!node.isNew()) { 176 remember(node.getPrimitiveId()); 177 } 178 } 179 remember(way.getPrimitiveId()); 180 return this; 181 } 182 183 /** 184 * appends a {@link Relation} id to the list of ids which will be fetched from the server. 185 * 186 * @param relation the relation (ignored, if null) 187 * @return this 188 */ 189 protected MultiFetchServerObjectReader appendRelation(Relation relation) { 190 if (relation == null) return this; 191 if (relation.isNew()) return this; 192 remember(relation.getPrimitiveId()); 193 for (RelationMember member : !recursesDown() ? relation.getMembers() : Collections.<RelationMember>emptyList()) { 194 if (OsmPrimitiveType.from(member.getMember()).equals(OsmPrimitiveType.RELATION)) { 195 // avoid infinite recursion in case of cyclic dependencies in relations 196 // 197 if (relations.contains(member.getMember().getId())) { 198 continue; 199 } 200 } 201 if (!member.getMember().isIncomplete()) { 202 append(member.getMember()); 203 } 204 } 205 return this; 206 } 207 208 /** 209 * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server. 210 * @param primitive the primitive 211 * @return this 212 */ 213 public MultiFetchServerObjectReader append(OsmPrimitive primitive) { 214 if (primitive != null) { 215 switch (OsmPrimitiveType.from(primitive)) { 216 case NODE: return appendNode((Node) primitive); 217 case WAY: return appendWay((Way) primitive); 218 case RELATION: return appendRelation((Relation) primitive); 219 } 220 } 221 return this; 222 } 223 224 /** 225 * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server. 226 * 227 * @param primitives the list of primitives (ignored, if null) 228 * @return this 229 * 230 * @see #append(OsmPrimitive) 231 */ 232 public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) { 233 if (primitives == null) return this; 234 for (OsmPrimitive primitive : primitives) { 235 append(primitive); 236 } 237 return this; 238 } 239 240 /** 241 * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and 242 * replies the subset. The extracted subset is removed from <code>ids</code>. 243 * 244 * @param ids a set of ids 245 * @return the subset of ids 246 */ 247 protected Set<Long> extractIdPackage(Set<Long> ids) { 248 Set<Long> pkg = new HashSet<>(); 249 if (ids.isEmpty()) 250 return pkg; 251 if (ids.size() > MAX_IDS_PER_REQUEST) { 252 Iterator<Long> it = ids.iterator(); 253 for (int i = 0; i < MAX_IDS_PER_REQUEST; i++) { 254 pkg.add(it.next()); 255 } 256 ids.removeAll(pkg); 257 } else { 258 pkg.addAll(ids); 259 ids.clear(); 260 } 261 return pkg; 262 } 263 264 /** 265 * builds the Multi Get request string for a set of ids and a given {@link OsmPrimitiveType}. 266 * 267 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 268 * {@link OsmPrimitiveType#RELATION RELATION} 269 * @param idPackage the package of ids 270 * @return the request string 271 */ 272 protected String buildRequestString(final OsmPrimitiveType type, Set<Long> idPackage) { 273 return type.getAPIName() + "s?" + type.getAPIName() + "s=" + Utils.join(",", idPackage); 274 } 275 276 @Override 277 protected String getBaseUrl() { 278 return super.getBaseUrl(); 279 } 280 281 protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) { 282 for (Way w: from.getWays()) { 283 if (w.hasIncompleteNodes()) { 284 for (Node n: w.getNodes()) { 285 if (n.isIncomplete()) { 286 nodes.add(n.getId()); 287 } 288 } 289 } 290 } 291 } 292 293 /** 294 * merges the dataset <code>from</code> to {@link #outputDataSet}. 295 * 296 * @param from the other dataset 297 */ 298 protected void merge(DataSet from) { 299 final DataSetMerger visitor = new DataSetMerger(outputDataSet, from); 300 visitor.merge(); 301 } 302 303 /** 304 * fetches a set of ids of a given {@link OsmPrimitiveType} from the server 305 * 306 * @param ids the set of ids 307 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 308 * {@link OsmPrimitiveType#RELATION RELATION} 309 * @param progressMonitor progress monitor 310 * @throws OsmTransferException if an error occurs while communicating with the API server 311 */ 312 protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException { 313 String msg; 314 final String baseUrl = getBaseUrl(); 315 switch (type) { 316 case NODE: msg = tr("Fetching a package of nodes from ''{0}''", baseUrl); break; 317 case WAY: msg = tr("Fetching a package of ways from ''{0}''", baseUrl); break; 318 case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break; 319 default: throw new AssertionError(); 320 } 321 progressMonitor.setTicksCount(ids.size()); 322 progressMonitor.setTicks(0); 323 // The complete set containing all primitives to fetch 324 Set<Long> toFetch = new HashSet<>(ids); 325 // Build a list of fetchers that will download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each. 326 // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers. 327 int threadsNumber = Main.pref.getInteger("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS); 328 threadsNumber = Math.min(Math.max(threadsNumber, 1), OsmApi.MAX_DOWNLOAD_THREADS); 329 final ExecutorService exec = Executors.newFixedThreadPool( 330 threadsNumber, Utils.newThreadFactory(getClass() + "-%d", Thread.NORM_PRIORITY)); 331 CompletionService<FetchResult> ecs = new ExecutorCompletionService<>(exec); 332 List<Future<FetchResult>> jobs = new ArrayList<>(); 333 while (!toFetch.isEmpty()) { 334 jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor))); 335 } 336 // Run the fetchers 337 for (int i = 0; i < jobs.size() && !isCanceled(); i++) { 338 progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + '/' + progressMonitor.getTicksCount()); 339 try { 340 FetchResult result = ecs.take().get(); 341 if (result.missingPrimitives != null) { 342 missingPrimitives.addAll(result.missingPrimitives); 343 } 344 if (result.dataSet != null && !isCanceled()) { 345 rememberNodesOfIncompleteWaysToLoad(result.dataSet); 346 merge(result.dataSet); 347 } 348 } catch (InterruptedException | ExecutionException e) { 349 Main.error(e); 350 } 351 } 352 exec.shutdown(); 353 // Cancel requests if the user chose to 354 if (isCanceled()) { 355 for (Future<FetchResult> job : jobs) { 356 job.cancel(true); 357 } 358 } 359 } 360 361 /** 362 * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies 363 * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too! 364 * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies 365 * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if 366 * visible==false). 367 * 368 * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been 369 * found on the server (the server response code was 404) 370 * 371 * @return the parsed data 372 * @throws OsmTransferException if an error occurs while communicating with the API server 373 * @see #getMissingPrimitives() 374 * 375 */ 376 @Override 377 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 378 int n = nodes.size() + ways.size() + relations.size(); 379 progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''", 380 "Downloading {0} objects from ''{1}''", n, n, OsmApi.getOsmApi().getBaseUrl())); 381 try { 382 missingPrimitives = new HashSet<>(); 383 if (isCanceled()) return null; 384 fetchPrimitives(ways, OsmPrimitiveType.WAY, progressMonitor); 385 if (isCanceled()) return null; 386 fetchPrimitives(nodes, OsmPrimitiveType.NODE, progressMonitor); 387 if (isCanceled()) return null; 388 fetchPrimitives(relations, OsmPrimitiveType.RELATION, progressMonitor); 389 if (outputDataSet != null) { 390 outputDataSet.deleteInvisible(); 391 } 392 return outputDataSet; 393 } finally { 394 progressMonitor.finishTask(); 395 } 396 } 397 398 /** 399 * replies the set of ids of all primitives for which a fetch request to the 400 * server was submitted but which are not available from the server (the server 401 * replied a return code of 404) 402 * 403 * @return the set of ids of missing primitives 404 */ 405 public Set<PrimitiveId> getMissingPrimitives() { 406 return missingPrimitives; 407 } 408 409 /** 410 * Whether this reader fetches nodes when loading ways, or members when loading relations. 411 * 412 * @return {@code true} if the reader recurses down 413 */ 414 protected boolean recursesDown() { 415 return false; 416 } 417 418 /** 419 * The class holding the results given by {@link Fetcher}. 420 * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded. 421 */ 422 protected static class FetchResult { 423 424 /** 425 * The resulting data set 426 */ 427 public final DataSet dataSet; 428 429 /** 430 * The collection of primitive ids that could not have been loaded 431 */ 432 public final Set<PrimitiveId> missingPrimitives; 433 434 /** 435 * Constructs a {@code FetchResult} 436 * @param dataSet The resulting data set 437 * @param missingPrimitives The collection of primitive ids that could not have been loaded 438 */ 439 public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) { 440 this.dataSet = dataSet; 441 this.missingPrimitives = missingPrimitives; 442 } 443 } 444 445 /** 446 * The class that actually download data from OSM API. 447 * Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch). 448 * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance. 449 * @see FetchResult 450 */ 451 protected class Fetcher extends OsmServerReader implements Callable<FetchResult> { 452 453 private final Set<Long> pkg; 454 private final OsmPrimitiveType type; 455 private final ProgressMonitor progressMonitor; 456 457 /** 458 * Constructs a {@code Fetcher} 459 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 460 * {@link OsmPrimitiveType#RELATION RELATION} 461 * @param idsPackage The set of primitives ids to fetch 462 * @param progressMonitor The progress monitor 463 */ 464 public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) { 465 this.pkg = idsPackage; 466 this.type = type; 467 this.progressMonitor = progressMonitor; 468 } 469 470 @Override 471 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 472 // This method is implemented because of the OsmServerReader inheritance, but not used, 473 // as the main target of this class is the call() method. 474 return fetch(progressMonitor).dataSet; 475 } 476 477 @Override 478 public FetchResult call() throws Exception { 479 return fetch(progressMonitor); 480 } 481 482 /** 483 * fetches the requested primitives and updates the specified progress monitor. 484 * @param progressMonitor the progress monitor 485 * @return the {@link FetchResult} of this operation 486 * @throws OsmTransferException if an error occurs while communicating with the API server 487 */ 488 protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException { 489 try { 490 return multiGetIdPackage(type, pkg, progressMonitor); 491 } catch (OsmApiException e) { 492 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 493 Main.info(tr("Server replied with response code 404, retrying with an individual request for each object.")); 494 return singleGetIdPackage(type, pkg, progressMonitor); 495 } else { 496 throw e; 497 } 498 } 499 } 500 501 @Override 502 protected String getBaseUrl() { 503 return MultiFetchServerObjectReader.this.getBaseUrl(); 504 } 505 506 /** 507 * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}. 508 * The retrieved primitives are merged to {@link #outputDataSet}. 509 * 510 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 511 * {@link OsmPrimitiveType#RELATION RELATION} 512 * @param pkg the package of ids 513 * @param progressMonitor progress monitor 514 * @return the {@link FetchResult} of this operation 515 * @throws OsmTransferException if an error occurs while communicating with the API server 516 */ 517 protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) 518 throws OsmTransferException { 519 String request = buildRequestString(type, pkg); 520 FetchResult result = null; 521 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 522 if (in == null) return null; 523 progressMonitor.subTask(tr("Downloading OSM data...")); 524 try { 525 result = new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null); 526 } catch (IllegalDataException e) { 527 throw new OsmTransferException(e); 528 } 529 } catch (IOException ex) { 530 Main.warn(ex); 531 } 532 return result; 533 } 534 535 /** 536 * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}. 537 * The retrieved primitive is merged to {@link #outputDataSet}. 538 * 539 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 540 * {@link OsmPrimitiveType#RELATION RELATION} 541 * @param id the id 542 * @param progressMonitor progress monitor 543 * @return the {@link DataSet} resulting of this operation 544 * @throws OsmTransferException if an error occurs while communicating with the API server 545 */ 546 protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException { 547 String request = buildRequestString(type, Collections.singleton(id)); 548 DataSet result = null; 549 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 550 if (in == null) return null; 551 progressMonitor.subTask(tr("Downloading OSM data...")); 552 try { 553 result = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false)); 554 } catch (IllegalDataException e) { 555 throw new OsmTransferException(e); 556 } 557 } catch (IOException ex) { 558 Main.warn(ex); 559 } 560 return result; 561 } 562 563 /** 564 * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}. 565 * The retrieved primitives are merged to {@link #outputDataSet}. 566 * 567 * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404). 568 * If the set is fetched with this method it is possible to find out which of the ids doesn't exist. 569 * Unfortunately, the server does not provide an error header or an error body for a 404 reply. 570 * 571 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 572 * {@link OsmPrimitiveType#RELATION RELATION} 573 * @param pkg the set of ids 574 * @param progressMonitor progress monitor 575 * @return the {@link FetchResult} of this operation 576 * @throws OsmTransferException if an error occurs while communicating with the API server 577 */ 578 protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) 579 throws OsmTransferException { 580 FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>()); 581 String baseUrl = OsmApi.getOsmApi().getBaseUrl(); 582 for (long id : pkg) { 583 try { 584 String msg; 585 switch (type) { 586 case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, baseUrl); break; 587 case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, baseUrl); break; 588 case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break; 589 default: throw new AssertionError(); 590 } 591 progressMonitor.setCustomText(msg); 592 result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor)); 593 } catch (OsmApiException e) { 594 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 595 Main.info(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id))); 596 result.missingPrimitives.add(new SimplePrimitiveId(id, type)); 597 } else { 598 throw e; 599 } 600 } 601 } 602 return result; 603 } 604 } 605}