001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.actions.downloadtasks; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.net.MalformedURLException; 008import java.net.URL; 009import java.util.ArrayList; 010import java.util.Arrays; 011import java.util.Collection; 012import java.util.Collections; 013import java.util.HashSet; 014import java.util.Objects; 015import java.util.Optional; 016import java.util.Set; 017import java.util.concurrent.Future; 018import java.util.regex.Matcher; 019import java.util.regex.Pattern; 020import java.util.stream.Stream; 021 022import org.openstreetmap.josm.data.Bounds; 023import org.openstreetmap.josm.data.DataSource; 024import org.openstreetmap.josm.data.ProjectionBounds; 025import org.openstreetmap.josm.data.ViewportData; 026import org.openstreetmap.josm.data.coor.LatLon; 027import org.openstreetmap.josm.data.osm.DataSet; 028import org.openstreetmap.josm.data.osm.OsmPrimitive; 029import org.openstreetmap.josm.data.osm.Relation; 030import org.openstreetmap.josm.data.osm.Way; 031import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor; 032import org.openstreetmap.josm.gui.MainApplication; 033import org.openstreetmap.josm.gui.MapFrame; 034import org.openstreetmap.josm.gui.PleaseWaitRunnable; 035import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask; 036import org.openstreetmap.josm.gui.layer.OsmDataLayer; 037import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 038import org.openstreetmap.josm.gui.progress.ProgressMonitor; 039import org.openstreetmap.josm.io.BoundingBoxDownloader; 040import org.openstreetmap.josm.io.OsmServerLocationReader; 041import org.openstreetmap.josm.io.OsmServerLocationReader.OsmUrlPattern; 042import org.openstreetmap.josm.io.OsmServerReader; 043import org.openstreetmap.josm.io.OsmTransferCanceledException; 044import org.openstreetmap.josm.io.OsmTransferException; 045import org.openstreetmap.josm.io.OverpassDownloadReader; 046import org.openstreetmap.josm.tools.Logging; 047import org.openstreetmap.josm.tools.Utils; 048import org.xml.sax.SAXException; 049 050/** 051 * Open the download dialog and download the data. 052 * Run in the worker thread. 053 */ 054public class DownloadOsmTask extends AbstractDownloadTask<DataSet> { 055 056 protected Bounds currentBounds; 057 protected DownloadTask downloadTask; 058 059 protected String newLayerName; 060 061 /** This allows subclasses to ignore this warning */ 062 protected boolean warnAboutEmptyArea = true; 063 064 protected static final String OVERPASS_INTERPRETER_DATA = "interpreter?data="; 065 066 private static final String NO_DATA_FOUND = tr("No data found in this area."); 067 static { 068 PostDownloadHandler.addNoDataErrorMessage(NO_DATA_FOUND); 069 } 070 071 @Override 072 public String[] getPatterns() { 073 if (this.getClass() == DownloadOsmTask.class) { 074 return Arrays.stream(OsmUrlPattern.values()).map(OsmUrlPattern::pattern).toArray(String[]::new); 075 } else { 076 return super.getPatterns(); 077 } 078 } 079 080 @Override 081 public String getTitle() { 082 if (this.getClass() == DownloadOsmTask.class) { 083 return tr("Download OSM"); 084 } else { 085 return super.getTitle(); 086 } 087 } 088 089 @Override 090 public Future<?> download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { 091 return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor); 092 } 093 094 /** 095 * Asynchronously launches the download task for a given bounding box. 096 * 097 * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor. 098 * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to 099 * be discarded. 100 * 101 * You can wait for the asynchronous download task to finish by synchronizing on the returned 102 * {@link Future}, but make sure not to freeze up JOSM. Example: 103 * <pre> 104 * Future<?> future = task.download(...); 105 * // DON'T run this on the Swing EDT or JOSM will freeze 106 * future.get(); // waits for the dowload task to complete 107 * </pre> 108 * 109 * The following example uses a pattern which is better suited if a task is launched from 110 * the Swing EDT: 111 * <pre> 112 * final Future<?> future = task.download(...); 113 * Runnable runAfterTask = new Runnable() { 114 * public void run() { 115 * // this is not strictly necessary because of the type of executor service 116 * // Main.worker is initialized with, but it doesn't harm either 117 * // 118 * future.get(); // wait for the download task to complete 119 * doSomethingAfterTheTaskCompleted(); 120 * } 121 * } 122 * MainApplication.worker.submit(runAfterTask); 123 * </pre> 124 * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm}) 125 * @param settings download settings 126 * @param downloadArea the area to download 127 * @param progressMonitor the progressMonitor 128 * @return the future representing the asynchronous task 129 * @since 13927 130 */ 131 public Future<?> download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { 132 return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea); 133 } 134 135 protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) { 136 this.downloadTask = downloadTask; 137 this.currentBounds = new Bounds(downloadArea); 138 // We need submit instead of execute so we can wait for it to finish and get the error 139 // message if necessary. If no one calls getErrorMessage() it just behaves like execute. 140 return MainApplication.worker.submit(downloadTask); 141 } 142 143 /** 144 * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed. 145 * @param url the original URL 146 * @return the modified URL 147 */ 148 protected String modifyUrlBeforeLoad(String url) { 149 return url; 150 } 151 152 /** 153 * Loads a given URL from the OSM Server 154 * @param settings download settings 155 * @param url The URL as String 156 */ 157 @Override 158 public Future<?> loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) { 159 String newUrl = modifyUrlBeforeLoad(url); 160 downloadTask = new DownloadTask(settings, getOsmServerReader(newUrl), progressMonitor); 161 currentBounds = null; 162 // Extract .osm filename from URL to set the new layer name 163 extractOsmFilename(settings, "https?://.*/(.*\\.osm)", newUrl); 164 return MainApplication.worker.submit(downloadTask); 165 } 166 167 protected OsmServerReader getOsmServerReader(String url) { 168 try { 169 String host = new URL(url).getHost(); 170 for (String knownOverpassServer : OverpassDownloadReader.OVERPASS_SERVER_HISTORY.get()) { 171 if (host.equals(new URL(knownOverpassServer).getHost())) { 172 int index = url.indexOf(OVERPASS_INTERPRETER_DATA); 173 if (index > 0) { 174 return new OverpassDownloadReader(new Bounds(LatLon.ZERO), knownOverpassServer, 175 Utils.decodeUrl(url.substring(index + OVERPASS_INTERPRETER_DATA.length()))); 176 } 177 } 178 } 179 } catch (MalformedURLException e) { 180 Logging.error(e); 181 } 182 return new OsmServerLocationReader(url); 183 } 184 185 protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) { 186 newLayerName = settings.getLayerName(); 187 if (newLayerName == null || newLayerName.isEmpty()) { 188 Matcher matcher = Pattern.compile(pattern).matcher(url); 189 newLayerName = matcher.matches() ? matcher.group(1) : null; 190 } 191 } 192 193 @Override 194 public void cancel() { 195 if (downloadTask != null) { 196 downloadTask.cancel(); 197 } 198 } 199 200 @Override 201 public boolean isSafeForRemotecontrolRequests() { 202 return true; 203 } 204 205 @Override 206 public ProjectionBounds getDownloadProjectionBounds() { 207 return downloadTask != null ? downloadTask.computeBbox(currentBounds).orElse(null) : null; 208 } 209 210 protected Collection<OsmPrimitive> searchPotentiallyDeletedPrimitives(DataSet ds) { 211 return downloadTask.searchPrimitivesToUpdate(currentBounds, ds); 212 } 213 214 /** 215 * Superclass of internal download task. 216 * @since 7636 217 */ 218 public abstract static class AbstractInternalTask extends PleaseWaitRunnable { 219 220 protected final DownloadParams settings; 221 protected final boolean zoomAfterDownload; 222 protected DataSet dataSet; 223 224 /** 225 * Constructs a new {@code AbstractInternalTask}. 226 * @param settings download settings 227 * @param title message for the user 228 * @param ignoreException If true, exception will be propagated to calling code. If false then 229 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 230 * then use false unless you read result of task (because exception will get lost if you don't) 231 * @param zoomAfterDownload If true, the map view will zoom to download area after download 232 */ 233 public AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) { 234 super(title, ignoreException); 235 this.settings = Objects.requireNonNull(settings); 236 this.zoomAfterDownload = zoomAfterDownload; 237 } 238 239 /** 240 * Constructs a new {@code AbstractInternalTask}. 241 * @param settings download settings 242 * @param title message for the user 243 * @param progressMonitor progress monitor 244 * @param ignoreException If true, exception will be propagated to calling code. If false then 245 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 246 * then use false unless you read result of task (because exception will get lost if you don't) 247 * @param zoomAfterDownload If true, the map view will zoom to download area after download 248 */ 249 public AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException, 250 boolean zoomAfterDownload) { 251 super(title, progressMonitor, ignoreException); 252 this.settings = Objects.requireNonNull(settings); 253 this.zoomAfterDownload = zoomAfterDownload; 254 } 255 256 protected OsmDataLayer getEditLayer() { 257 return MainApplication.getLayerManager().getEditLayer(); 258 } 259 260 private static Stream<OsmDataLayer> getModifiableDataLayers() { 261 return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class) 262 .stream().filter(OsmDataLayer::isDownloadable); 263 } 264 265 /** 266 * Returns the number of modifiable data layers 267 * @return number of modifiable data layers 268 * @since 13434 269 */ 270 protected long getNumModifiableDataLayers() { 271 return getModifiableDataLayers().count(); 272 } 273 274 /** 275 * Returns the first modifiable data layer 276 * @return the first modifiable data layer 277 * @since 13434 278 */ 279 protected OsmDataLayer getFirstModifiableDataLayer() { 280 return getModifiableDataLayers().findFirst().orElse(null); 281 } 282 283 /** 284 * Creates a name for a new layer by utilizing the settings ({@link DownloadParams#getLayerName()}) or 285 * {@link OsmDataLayer#createNewName()} if the former option is {@code null}. 286 * 287 * @return a name for a new layer 288 * @since 14347 289 */ 290 protected String generateLayerName() { 291 return Optional.ofNullable(settings.getLayerName()) 292 .filter(layerName -> !Utils.isStripEmpty(layerName)) 293 .orElse(OsmDataLayer.createNewName()); 294 } 295 296 /** 297 * Can be overridden (e.g. by plugins) if a subclass of {@link OsmDataLayer} is needed. 298 * If you want to change how the name is determined, consider overriding 299 * {@link #generateLayerName()} instead. 300 * 301 * @param ds the dataset on which the layer is based, must be non-null 302 * @param layerName the name of the new layer, must be either non-blank or non-present 303 * @return a new instance of {@link OsmDataLayer} constructed with the given arguments 304 * @since 14347 305 */ 306 protected OsmDataLayer createNewLayer(final DataSet ds, final Optional<String> layerName) { 307 if (layerName.filter(Utils::isStripEmpty).isPresent()) { 308 throw new IllegalArgumentException("Blank layer name!"); 309 } 310 return new OsmDataLayer( 311 Objects.requireNonNull(ds, "dataset parameter"), 312 layerName.orElseGet(this::generateLayerName), 313 null 314 ); 315 } 316 317 /** 318 * Convenience method for {@link #createNewLayer(DataSet, Optional)}, uses the dataset 319 * from field {@link #dataSet} and applies the settings from field {@link #settings}. 320 * 321 * @param layerName an optional layer name, must be non-blank if the [Optional] is present 322 * @return a newly constructed layer 323 * @since 14347 324 */ 325 protected final OsmDataLayer createNewLayer(final Optional<String> layerName) { 326 Optional.ofNullable(settings.getDownloadPolicy()) 327 .ifPresent(dataSet::setDownloadPolicy); 328 Optional.ofNullable(settings.getUploadPolicy()) 329 .ifPresent(dataSet::setUploadPolicy); 330 if (dataSet.isLocked() && !settings.isLocked()) { 331 dataSet.unlock(); 332 } else if (!dataSet.isLocked() && settings.isLocked()) { 333 dataSet.lock(); 334 } 335 return createNewLayer(dataSet, layerName); 336 } 337 338 protected Optional<ProjectionBounds> computeBbox(Bounds bounds) { 339 BoundingXYVisitor v = new BoundingXYVisitor(); 340 if (bounds != null) { 341 v.visit(bounds); 342 } else { 343 v.computeBoundingBox(dataSet.getNodes()); 344 } 345 return Optional.ofNullable(v.getBounds()); 346 } 347 348 protected OsmDataLayer addNewLayerIfRequired(String newLayerName) { 349 long numDataLayers = getNumModifiableDataLayers(); 350 if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) { 351 // the user explicitly wants a new layer, we don't have any layer at all 352 // or it is not clear which layer to merge to 353 final OsmDataLayer layer = createNewLayer(Optional.ofNullable(newLayerName).filter(it -> !Utils.isStripEmpty(it))); 354 MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload); 355 return layer; 356 } 357 return null; 358 } 359 360 protected void loadData(String newLayerName, Bounds bounds) { 361 OsmDataLayer layer = addNewLayerIfRequired(newLayerName); 362 if (layer == null) { 363 layer = getEditLayer(); 364 if (layer == null || !layer.isDownloadable()) { 365 layer = getFirstModifiableDataLayer(); 366 } 367 Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet()); 368 layer.mergeFrom(dataSet); 369 MapFrame map = MainApplication.getMap(); 370 if (map != null && zoomAfterDownload) { 371 computeBbox(bounds).map(ViewportData::new).ifPresent(map.mapView::zoomTo); 372 } 373 if (!primitivesToUpdate.isEmpty()) { 374 MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate)); 375 } 376 layer.onPostDownloadFromServer(); 377 } 378 } 379 380 /** 381 * Look for primitives deleted on server (thus absent from downloaded data) 382 * but still present in existing data layer 383 * @param bounds download bounds 384 * @param ds existing data set 385 * @return the primitives to update 386 */ 387 protected Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) { 388 if (bounds == null) 389 return Collections.emptySet(); 390 Collection<OsmPrimitive> col = new ArrayList<>(); 391 ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add); 392 if (!col.isEmpty()) { 393 Set<Way> ways = new HashSet<>(); 394 Set<Relation> rels = new HashSet<>(); 395 for (OsmPrimitive n : col) { 396 for (OsmPrimitive ref : n.getReferrers()) { 397 if (ref.isNew()) { 398 continue; 399 } else if (ref instanceof Way) { 400 ways.add((Way) ref); 401 } else if (ref instanceof Relation) { 402 rels.add((Relation) ref); 403 } 404 } 405 } 406 ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add); 407 rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add); 408 } 409 return col; 410 } 411 } 412 413 protected class DownloadTask extends AbstractInternalTask { 414 protected final OsmServerReader reader; 415 416 /** 417 * Constructs a new {@code DownloadTask}. 418 * @param settings download settings 419 * @param reader OSM data reader 420 * @param progressMonitor progress monitor 421 * @since 13927 422 */ 423 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) { 424 this(settings, reader, progressMonitor, true); 425 } 426 427 /** 428 * Constructs a new {@code DownloadTask}. 429 * @param settings download settings 430 * @param reader OSM data reader 431 * @param progressMonitor progress monitor 432 * @param zoomAfterDownload If true, the map view will zoom to download area after download 433 * @since 13927 434 */ 435 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) { 436 super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload); 437 this.reader = reader; 438 } 439 440 protected DataSet parseDataSet() throws OsmTransferException { 441 return reader.parseOsm(progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false)); 442 } 443 444 @Override 445 public void realRun() throws IOException, SAXException, OsmTransferException { 446 try { 447 if (isCanceled()) 448 return; 449 dataSet = parseDataSet(); 450 } catch (OsmTransferException e) { 451 if (isCanceled()) { 452 Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString())); 453 return; 454 } 455 if (e instanceof OsmTransferCanceledException) { 456 setCanceled(true); 457 return; 458 } else { 459 rememberException(e); 460 } 461 DownloadOsmTask.this.setFailed(true); 462 } 463 } 464 465 @Override 466 protected void finish() { 467 if (isFailed() || isCanceled()) 468 return; 469 if (dataSet == null) 470 return; // user canceled download or error occurred 471 if (dataSet.allPrimitives().isEmpty()) { 472 if (warnAboutEmptyArea) { 473 rememberErrorMessage(NO_DATA_FOUND); 474 } 475 String remark = dataSet.getRemark(); 476 if (remark != null && !remark.isEmpty()) { 477 rememberErrorMessage(remark); 478 } 479 // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work 480 dataSet.addDataSource(new DataSource(currentBounds != null ? currentBounds : 481 new Bounds(LatLon.ZERO), "OpenStreetMap server")); 482 } 483 484 rememberDownloadedData(dataSet); 485 loadData(newLayerName, currentBounds); 486 } 487 488 @Override 489 protected void cancel() { 490 setCanceled(true); 491 if (reader != null) { 492 reader.cancel(); 493 } 494 } 495 } 496 497 @Override 498 public String getConfirmationMessage(URL url) { 499 if (url != null) { 500 String urlString = url.toExternalForm(); 501 if (urlString.matches(OsmUrlPattern.OSM_API_URL.pattern())) { 502 // TODO: proper i18n after stabilization 503 Collection<String> items = new ArrayList<>(); 504 items.add(tr("OSM Server URL:") + ' ' + url.getHost()); 505 items.add(tr("Command")+": "+url.getPath()); 506 if (url.getQuery() != null) { 507 items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", "))); 508 } 509 return Utils.joinAsHtmlUnorderedList(items); 510 } 511 // TODO: other APIs 512 } 513 return null; 514 } 515}