001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.actions.downloadtasks; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.net.MalformedURLException; 008import java.net.URL; 009import java.util.ArrayList; 010import java.util.Arrays; 011import java.util.Collection; 012import java.util.Collections; 013import java.util.HashSet; 014import java.util.Objects; 015import java.util.Optional; 016import java.util.Set; 017import java.util.concurrent.Future; 018import java.util.regex.Matcher; 019import java.util.regex.Pattern; 020import java.util.stream.Stream; 021 022import org.openstreetmap.josm.data.Bounds; 023import org.openstreetmap.josm.data.DataSource; 024import org.openstreetmap.josm.data.ProjectionBounds; 025import org.openstreetmap.josm.data.ViewportData; 026import org.openstreetmap.josm.data.coor.LatLon; 027import org.openstreetmap.josm.data.osm.DataSet; 028import org.openstreetmap.josm.data.osm.OsmPrimitive; 029import org.openstreetmap.josm.data.osm.Relation; 030import org.openstreetmap.josm.data.osm.Way; 031import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor; 032import org.openstreetmap.josm.gui.MainApplication; 033import org.openstreetmap.josm.gui.MapFrame; 034import org.openstreetmap.josm.gui.PleaseWaitRunnable; 035import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask; 036import org.openstreetmap.josm.gui.layer.OsmDataLayer; 037import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 038import org.openstreetmap.josm.gui.progress.ProgressMonitor; 039import org.openstreetmap.josm.io.BoundingBoxDownloader; 040import org.openstreetmap.josm.io.Compression; 041import org.openstreetmap.josm.io.OsmServerLocationReader; 042import org.openstreetmap.josm.io.OsmServerReader; 043import org.openstreetmap.josm.io.OsmTransferCanceledException; 044import org.openstreetmap.josm.io.OsmTransferException; 045import org.openstreetmap.josm.io.OverpassDownloadReader; 046import org.openstreetmap.josm.io.UrlPatterns.OsmUrlPattern; 047import org.openstreetmap.josm.tools.Logging; 048import org.openstreetmap.josm.tools.Utils; 049import org.xml.sax.SAXException; 050 051/** 052 * Open the download dialog and download the data. 053 * Run in the worker thread. 054 */ 055public class DownloadOsmTask extends AbstractDownloadTask<DataSet> { 056 057 protected Bounds currentBounds; 058 protected DownloadTask downloadTask; 059 060 protected String newLayerName; 061 062 /** This allows subclasses to ignore this warning */ 063 protected boolean warnAboutEmptyArea = true; 064 065 protected static final String OVERPASS_INTERPRETER_DATA = "interpreter?data="; 066 067 private static final String NO_DATA_FOUND = tr("No data found in this area."); 068 static { 069 PostDownloadHandler.addNoDataErrorMessage(NO_DATA_FOUND); 070 } 071 072 @Override 073 public String[] getPatterns() { 074 if (this.getClass() == DownloadOsmTask.class) { 075 return patterns(OsmUrlPattern.class); 076 } else { 077 return super.getPatterns(); 078 } 079 } 080 081 @Override 082 public String getTitle() { 083 if (this.getClass() == DownloadOsmTask.class) { 084 return tr("Download OSM"); 085 } else { 086 return super.getTitle(); 087 } 088 } 089 090 @Override 091 public Future<?> download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { 092 return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor); 093 } 094 095 /** 096 * Asynchronously launches the download task for a given bounding box. 097 * 098 * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor. 099 * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to 100 * be discarded. 101 * 102 * You can wait for the asynchronous download task to finish by synchronizing on the returned 103 * {@link Future}, but make sure not to freeze up JOSM. Example: 104 * <pre> 105 * Future<?> future = task.download(...); 106 * // DON'T run this on the Swing EDT or JOSM will freeze 107 * future.get(); // waits for the dowload task to complete 108 * </pre> 109 * 110 * The following example uses a pattern which is better suited if a task is launched from 111 * the Swing EDT: 112 * <pre> 113 * final Future<?> future = task.download(...); 114 * Runnable runAfterTask = new Runnable() { 115 * public void run() { 116 * // this is not strictly necessary because of the type of executor service 117 * // Main.worker is initialized with, but it doesn't harm either 118 * // 119 * future.get(); // wait for the download task to complete 120 * doSomethingAfterTheTaskCompleted(); 121 * } 122 * } 123 * MainApplication.worker.submit(runAfterTask); 124 * </pre> 125 * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm}) 126 * @param settings download settings 127 * @param downloadArea the area to download 128 * @param progressMonitor the progressMonitor 129 * @return the future representing the asynchronous task 130 * @since 13927 131 */ 132 public Future<?> download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { 133 return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea); 134 } 135 136 protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) { 137 this.downloadTask = downloadTask; 138 this.currentBounds = new Bounds(downloadArea); 139 // We need submit instead of execute so we can wait for it to finish and get the error 140 // message if necessary. If no one calls getErrorMessage() it just behaves like execute. 141 return MainApplication.worker.submit(downloadTask); 142 } 143 144 /** 145 * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed. 146 * @param url the original URL 147 * @return the modified URL 148 */ 149 protected String modifyUrlBeforeLoad(String url) { 150 return url; 151 } 152 153 /** 154 * Loads a given URL from the OSM Server 155 * @param settings download settings 156 * @param url The URL as String 157 */ 158 @Override 159 public Future<?> loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) { 160 String newUrl = modifyUrlBeforeLoad(url); 161 Optional<OsmUrlPattern> urlPattern = Arrays.stream(OsmUrlPattern.values()).filter(p -> p.matches(newUrl)).findFirst(); 162 downloadTask = new DownloadTask(settings, getOsmServerReader(newUrl), progressMonitor, true, Compression.byExtension(newUrl)); 163 currentBounds = null; 164 // Extract .osm filename from URL to set the new layer name 165 extractOsmFilename(settings, urlPattern.orElse(OsmUrlPattern.EXTERNAL_OSM_FILE).pattern(), newUrl); 166 return MainApplication.worker.submit(downloadTask); 167 } 168 169 protected OsmServerReader getOsmServerReader(String url) { 170 try { 171 String host = new URL(url).getHost(); 172 for (String knownOverpassServer : OverpassDownloadReader.OVERPASS_SERVER_HISTORY.get()) { 173 if (host.equals(new URL(knownOverpassServer).getHost())) { 174 int index = url.indexOf(OVERPASS_INTERPRETER_DATA); 175 if (index > 0) { 176 return new OverpassDownloadReader(new Bounds(LatLon.ZERO), knownOverpassServer, 177 Utils.decodeUrl(url.substring(index + OVERPASS_INTERPRETER_DATA.length()))); 178 } 179 } 180 } 181 } catch (MalformedURLException e) { 182 Logging.error(e); 183 } 184 return new OsmServerLocationReader(url); 185 } 186 187 protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) { 188 newLayerName = settings.getLayerName(); 189 if (newLayerName == null || newLayerName.isEmpty()) { 190 Matcher matcher = Pattern.compile(pattern).matcher(url); 191 newLayerName = matcher.matches() && matcher.groupCount() > 0 ? matcher.group(1) : null; 192 } 193 } 194 195 @Override 196 public void cancel() { 197 if (downloadTask != null) { 198 downloadTask.cancel(); 199 } 200 } 201 202 @Override 203 public boolean isSafeForRemotecontrolRequests() { 204 return true; 205 } 206 207 @Override 208 public ProjectionBounds getDownloadProjectionBounds() { 209 return downloadTask != null ? downloadTask.computeBbox(currentBounds).orElse(null) : null; 210 } 211 212 protected Collection<OsmPrimitive> searchPotentiallyDeletedPrimitives(DataSet ds) { 213 return downloadTask.searchPrimitivesToUpdate(currentBounds, ds); 214 } 215 216 /** 217 * Superclass of internal download task. 218 * @since 7636 219 */ 220 public abstract static class AbstractInternalTask extends PleaseWaitRunnable { 221 222 protected final DownloadParams settings; 223 protected final boolean zoomAfterDownload; 224 protected DataSet dataSet; 225 226 /** 227 * Constructs a new {@code AbstractInternalTask}. 228 * @param settings download settings 229 * @param title message for the user 230 * @param ignoreException If true, exception will be propagated to calling code. If false then 231 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 232 * then use false unless you read result of task (because exception will get lost if you don't) 233 * @param zoomAfterDownload If true, the map view will zoom to download area after download 234 */ 235 public AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) { 236 super(title, ignoreException); 237 this.settings = Objects.requireNonNull(settings); 238 this.zoomAfterDownload = zoomAfterDownload; 239 } 240 241 /** 242 * Constructs a new {@code AbstractInternalTask}. 243 * @param settings download settings 244 * @param title message for the user 245 * @param progressMonitor progress monitor 246 * @param ignoreException If true, exception will be propagated to calling code. If false then 247 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 248 * then use false unless you read result of task (because exception will get lost if you don't) 249 * @param zoomAfterDownload If true, the map view will zoom to download area after download 250 */ 251 public AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException, 252 boolean zoomAfterDownload) { 253 super(title, progressMonitor, ignoreException); 254 this.settings = Objects.requireNonNull(settings); 255 this.zoomAfterDownload = zoomAfterDownload; 256 } 257 258 protected OsmDataLayer getEditLayer() { 259 return MainApplication.getLayerManager().getEditLayer(); 260 } 261 262 private static Stream<OsmDataLayer> getModifiableDataLayers() { 263 return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class) 264 .stream().filter(OsmDataLayer::isDownloadable); 265 } 266 267 /** 268 * Returns the number of modifiable data layers 269 * @return number of modifiable data layers 270 * @since 13434 271 */ 272 protected long getNumModifiableDataLayers() { 273 return getModifiableDataLayers().count(); 274 } 275 276 /** 277 * Returns the first modifiable data layer 278 * @return the first modifiable data layer 279 * @since 13434 280 */ 281 protected OsmDataLayer getFirstModifiableDataLayer() { 282 return getModifiableDataLayers().findFirst().orElse(null); 283 } 284 285 /** 286 * Creates a name for a new layer by utilizing the settings ({@link DownloadParams#getLayerName()}) or 287 * {@link OsmDataLayer#createNewName()} if the former option is {@code null}. 288 * 289 * @return a name for a new layer 290 * @since 14347 291 */ 292 protected String generateLayerName() { 293 return Optional.ofNullable(settings.getLayerName()) 294 .filter(layerName -> !Utils.isStripEmpty(layerName)) 295 .orElse(OsmDataLayer.createNewName()); 296 } 297 298 /** 299 * Can be overridden (e.g. by plugins) if a subclass of {@link OsmDataLayer} is needed. 300 * If you want to change how the name is determined, consider overriding 301 * {@link #generateLayerName()} instead. 302 * 303 * @param ds the dataset on which the layer is based, must be non-null 304 * @param layerName the name of the new layer, must be either non-blank or non-present 305 * @return a new instance of {@link OsmDataLayer} constructed with the given arguments 306 * @since 14347 307 */ 308 protected OsmDataLayer createNewLayer(final DataSet ds, final Optional<String> layerName) { 309 if (layerName.filter(Utils::isStripEmpty).isPresent()) { 310 throw new IllegalArgumentException("Blank layer name!"); 311 } 312 return new OsmDataLayer( 313 Objects.requireNonNull(ds, "dataset parameter"), 314 layerName.orElseGet(this::generateLayerName), 315 null 316 ); 317 } 318 319 /** 320 * Convenience method for {@link #createNewLayer(DataSet, Optional)}, uses the dataset 321 * from field {@link #dataSet} and applies the settings from field {@link #settings}. 322 * 323 * @param layerName an optional layer name, must be non-blank if the [Optional] is present 324 * @return a newly constructed layer 325 * @since 14347 326 */ 327 protected final OsmDataLayer createNewLayer(final Optional<String> layerName) { 328 Optional.ofNullable(settings.getDownloadPolicy()) 329 .ifPresent(dataSet::setDownloadPolicy); 330 Optional.ofNullable(settings.getUploadPolicy()) 331 .ifPresent(dataSet::setUploadPolicy); 332 if (dataSet.isLocked() && !settings.isLocked()) { 333 dataSet.unlock(); 334 } else if (!dataSet.isLocked() && settings.isLocked()) { 335 dataSet.lock(); 336 } 337 return createNewLayer(dataSet, layerName); 338 } 339 340 protected Optional<ProjectionBounds> computeBbox(Bounds bounds) { 341 BoundingXYVisitor v = new BoundingXYVisitor(); 342 if (bounds != null) { 343 v.visit(bounds); 344 } else { 345 v.computeBoundingBox(dataSet.getNodes()); 346 } 347 return Optional.ofNullable(v.getBounds()); 348 } 349 350 protected OsmDataLayer addNewLayerIfRequired(String newLayerName) { 351 long numDataLayers = getNumModifiableDataLayers(); 352 if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) { 353 // the user explicitly wants a new layer, we don't have any layer at all 354 // or it is not clear which layer to merge to 355 final OsmDataLayer layer = createNewLayer(Optional.ofNullable(newLayerName).filter(it -> !Utils.isStripEmpty(it))); 356 MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload); 357 return layer; 358 } 359 return null; 360 } 361 362 protected void loadData(String newLayerName, Bounds bounds) { 363 OsmDataLayer layer = addNewLayerIfRequired(newLayerName); 364 if (layer == null) { 365 layer = getEditLayer(); 366 if (layer == null || !layer.isDownloadable()) { 367 layer = getFirstModifiableDataLayer(); 368 } 369 Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet()); 370 layer.mergeFrom(dataSet); 371 MapFrame map = MainApplication.getMap(); 372 if (map != null && zoomAfterDownload) { 373 computeBbox(bounds).map(ViewportData::new).ifPresent(map.mapView::zoomTo); 374 } 375 if (!primitivesToUpdate.isEmpty()) { 376 MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate)); 377 } 378 layer.onPostDownloadFromServer(); 379 } 380 } 381 382 /** 383 * Look for primitives deleted on server (thus absent from downloaded data) 384 * but still present in existing data layer 385 * @param bounds download bounds 386 * @param ds existing data set 387 * @return the primitives to update 388 */ 389 protected Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) { 390 if (bounds == null) 391 return Collections.emptySet(); 392 Collection<OsmPrimitive> col = new ArrayList<>(); 393 ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add); 394 if (!col.isEmpty()) { 395 Set<Way> ways = new HashSet<>(); 396 Set<Relation> rels = new HashSet<>(); 397 for (OsmPrimitive n : col) { 398 for (OsmPrimitive ref : n.getReferrers()) { 399 if (ref.isNew()) { 400 continue; 401 } else if (ref instanceof Way) { 402 ways.add((Way) ref); 403 } else if (ref instanceof Relation) { 404 rels.add((Relation) ref); 405 } 406 } 407 } 408 ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add); 409 rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add); 410 } 411 return col; 412 } 413 } 414 415 protected class DownloadTask extends AbstractInternalTask { 416 protected final OsmServerReader reader; 417 protected final Compression compression; 418 419 /** 420 * Constructs a new {@code DownloadTask}. 421 * @param settings download settings 422 * @param reader OSM data reader 423 * @param progressMonitor progress monitor 424 * @since 13927 425 */ 426 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) { 427 this(settings, reader, progressMonitor, true); 428 } 429 430 /** 431 * Constructs a new {@code DownloadTask}. 432 * @param settings download settings 433 * @param reader OSM data reader 434 * @param progressMonitor progress monitor 435 * @param zoomAfterDownload If true, the map view will zoom to download area after download 436 * @since 13927 437 */ 438 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) { 439 this(settings, reader, progressMonitor, zoomAfterDownload, Compression.NONE); 440 } 441 442 /** 443 * Constructs a new {@code DownloadTask}. 444 * @param settings download settings 445 * @param reader OSM data reader 446 * @param progressMonitor progress monitor 447 * @param zoomAfterDownload If true, the map view will zoom to download area after download 448 * @param compression compression to use 449 * @since 15784 450 */ 451 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload, 452 Compression compression) { 453 super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload); 454 this.reader = Objects.requireNonNull(reader); 455 this.compression = compression; 456 } 457 458 protected DataSet parseDataSet() throws OsmTransferException { 459 ProgressMonitor subTaskMonitor = progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false); 460 // Don't call parseOsm signature with compression if not needed, too many implementations to update before to avoid side effects 461 return compression != null && compression != Compression.NONE ? 462 reader.parseOsm(subTaskMonitor, compression) : reader.parseOsm(subTaskMonitor); 463 } 464 465 @Override 466 public void realRun() throws IOException, SAXException, OsmTransferException { 467 try { 468 if (isCanceled()) 469 return; 470 dataSet = parseDataSet(); 471 } catch (OsmTransferException e) { 472 if (isCanceled()) { 473 Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString())); 474 return; 475 } 476 if (e instanceof OsmTransferCanceledException) { 477 setCanceled(true); 478 return; 479 } else { 480 rememberException(e); 481 } 482 DownloadOsmTask.this.setFailed(true); 483 } 484 } 485 486 @Override 487 protected void finish() { 488 if (isFailed() || isCanceled()) 489 return; 490 if (dataSet == null) 491 return; // user canceled download or error occurred 492 if (dataSet.allPrimitives().isEmpty()) { 493 if (warnAboutEmptyArea) { 494 rememberErrorMessage(NO_DATA_FOUND); 495 } 496 String remark = dataSet.getRemark(); 497 if (remark != null && !remark.isEmpty()) { 498 rememberErrorMessage(remark); 499 } 500 if (!(reader instanceof BoundingBoxDownloader) 501 || ((BoundingBoxDownloader) reader).considerAsFullDownload()) { 502 // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work 503 dataSet.addDataSource(new DataSource( 504 currentBounds != null ? currentBounds : new Bounds(LatLon.ZERO), "OpenStreetMap server")); 505 } 506 } 507 508 rememberDownloadedData(dataSet); 509 loadData(newLayerName, currentBounds); 510 } 511 512 @Override 513 protected void cancel() { 514 setCanceled(true); 515 if (reader != null) { 516 reader.cancel(); 517 } 518 } 519 } 520 521 @Override 522 public String getConfirmationMessage(URL url) { 523 if (OsmUrlPattern.OSM_API_URL.matches(url)) { 524 Collection<String> items = new ArrayList<>(); 525 items.add(tr("OSM Server URL:") + ' ' + url.getHost()); 526 items.add(tr("Command")+": "+url.getPath()); 527 if (url.getQuery() != null) { 528 items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", "))); 529 } 530 return Utils.joinAsHtmlUnorderedList(items); 531 } 532 return null; 533 } 534}