001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedReader; 007import java.io.Closeable; 008import java.io.File; 009import java.io.IOException; 010import java.io.InputStream; 011import java.math.BigInteger; 012import java.net.HttpURLConnection; 013import java.net.MalformedURLException; 014import java.net.URL; 015import java.nio.charset.StandardCharsets; 016import java.nio.file.Files; 017import java.nio.file.InvalidPathException; 018import java.nio.file.StandardCopyOption; 019import java.security.MessageDigest; 020import java.security.NoSuchAlgorithmException; 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.Enumeration; 024import java.util.List; 025import java.util.Map; 026import java.util.concurrent.ConcurrentHashMap; 027import java.util.concurrent.TimeUnit; 028import java.util.zip.ZipEntry; 029import java.util.zip.ZipFile; 030 031import org.openstreetmap.josm.data.Preferences; 032import org.openstreetmap.josm.spi.preferences.Config; 033import org.openstreetmap.josm.tools.HttpClient; 034import org.openstreetmap.josm.tools.Logging; 035import org.openstreetmap.josm.tools.Pair; 036import org.openstreetmap.josm.tools.PlatformManager; 037import org.openstreetmap.josm.tools.Utils; 038 039/** 040 * Downloads a file and caches it on disk in order to reduce network load. 041 * 042 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get 043 * resources from the current *.jar file. (Local caching is only done for URLs.) 044 * <p> 045 * The mirrored file is only downloaded if it has been more than 7 days since 046 * last download. (Time can be configured.) 047 * <p> 048 * The file content is normally accessed with {@link #getInputStream()}, but 049 * you can also get the mirrored copy with {@link #getFile()}. 050 */ 051public class CachedFile implements Closeable { 052 053 /** 054 * Caching strategy. 055 */ 056 public enum CachingStrategy { 057 /** 058 * If cached file on disk is older than a certain time (7 days by default), 059 * consider the cache stale and try to download the file again. 060 */ 061 MaxAge, 062 /** 063 * Similar to MaxAge, considers the cache stale when a certain age is 064 * exceeded. In addition, a If-Modified-Since HTTP header is added. 065 * When the server replies "304 Not Modified", this is considered the same 066 * as a full download. 067 */ 068 IfModifiedSince 069 } 070 071 protected String name; 072 protected long maxAge; 073 protected String destDir; 074 protected String httpAccept; 075 protected CachingStrategy cachingStrategy; 076 077 private boolean fastFail; 078 private HttpClient activeConnection; 079 protected File cacheFile; 080 protected boolean initialized; 081 protected String parameter; 082 083 public static final long DEFAULT_MAXTIME = -1L; 084 public static final long DAYS = TimeUnit.DAYS.toSeconds(1); // factor to get caching time in days 085 086 private final Map<String, String> httpHeaders = new ConcurrentHashMap<>(); 087 088 /** 089 * Constructs a CachedFile object from a given filename, URL or internal resource. 090 * 091 * @param name can be:<ul> 092 * <li>relative or absolute file name</li> 093 * <li>{@code file:///SOME/FILE} the same as above</li> 094 * <li>{@code http://...} a URL. It will be cached on disk.</li> 095 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 096 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 097 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 098 */ 099 public CachedFile(String name) { 100 this.name = name; 101 } 102 103 /** 104 * Set the name of the resource. 105 * @param name can be:<ul> 106 * <li>relative or absolute file name</li> 107 * <li>{@code file:///SOME/FILE} the same as above</li> 108 * <li>{@code http://...} a URL. It will be cached on disk.</li> 109 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 110 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 111 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 112 * @return this object 113 */ 114 public CachedFile setName(String name) { 115 this.name = name; 116 return this; 117 } 118 119 /** 120 * Set maximum age of cache file. Only applies to URLs. 121 * When this time has passed after the last download of the file, the 122 * cache is considered stale and a new download will be attempted. 123 * @param maxAge the maximum cache age in seconds 124 * @return this object 125 */ 126 public CachedFile setMaxAge(long maxAge) { 127 this.maxAge = maxAge; 128 return this; 129 } 130 131 /** 132 * Set the destination directory for the cache file. Only applies to URLs. 133 * @param destDir the destination directory 134 * @return this object 135 */ 136 public CachedFile setDestDir(String destDir) { 137 this.destDir = destDir; 138 return this; 139 } 140 141 /** 142 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs. 143 * @param httpAccept the accepted MIME types 144 * @return this object 145 */ 146 public CachedFile setHttpAccept(String httpAccept) { 147 this.httpAccept = httpAccept; 148 return this; 149 } 150 151 /** 152 * Set the caching strategy. Only applies to URLs. 153 * @param cachingStrategy caching strategy 154 * @return this object 155 */ 156 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) { 157 this.cachingStrategy = cachingStrategy; 158 return this; 159 } 160 161 /** 162 * Sets the http headers. Only applies to URL pointing to http or https resources 163 * @param headers that should be sent together with request 164 * @return this object 165 */ 166 public CachedFile setHttpHeaders(Map<String, String> headers) { 167 this.httpHeaders.putAll(headers); 168 return this; 169 } 170 171 /** 172 * Sets whether opening HTTP connections should fail fast, i.e., whether a 173 * {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used. 174 * @param fastFail whether opening HTTP connections should fail fast 175 */ 176 public void setFastFail(boolean fastFail) { 177 this.fastFail = fastFail; 178 } 179 180 /** 181 * Sets additional URL parameter (used e.g. for maps) 182 * @param parameter the URL parameter 183 * @since 13536 184 */ 185 public void setParam(String parameter) { 186 this.parameter = parameter; 187 } 188 189 public String getName() { 190 if (parameter != null) 191 return name.replaceAll("%<(.*)>", ""); 192 return name; 193 } 194 195 /** 196 * Returns maximum age of cache file. Only applies to URLs. 197 * When this time has passed after the last download of the file, the 198 * cache is considered stale and a new download will be attempted. 199 * @return the maximum cache age in seconds 200 */ 201 public long getMaxAge() { 202 return maxAge; 203 } 204 205 public String getDestDir() { 206 return destDir; 207 } 208 209 public String getHttpAccept() { 210 return httpAccept; 211 } 212 213 public CachingStrategy getCachingStrategy() { 214 return cachingStrategy; 215 } 216 217 /** 218 * Get InputStream to the requested resource. 219 * @return the InputStream 220 * @throws IOException when the resource with the given name could not be retrieved 221 * @throws InvalidPathException if a Path object cannot be constructed from the inner file path 222 */ 223 public InputStream getInputStream() throws IOException { 224 File file = getFile(); 225 if (file == null) { 226 if (name != null && name.startsWith("resource://")) { 227 String resourceName = name.substring("resource:/".length()); 228 InputStream is = Utils.getResourceAsStream(getClass(), resourceName); 229 if (is == null) { 230 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name)); 231 } 232 return is; 233 } else { 234 throw new IOException("No file found for: "+name); 235 } 236 } 237 return Files.newInputStream(file.toPath()); 238 } 239 240 /** 241 * Get the full content of the requested resource as a byte array. 242 * @return the full content of the requested resource as byte array 243 * @throws IOException in case of an I/O error 244 */ 245 public byte[] getByteContent() throws IOException { 246 return Utils.readBytesFromStream(getInputStream()); 247 } 248 249 /** 250 * Returns {@link #getInputStream()} wrapped in a buffered reader. 251 * <p> 252 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}. 253 * 254 * @return buffered reader 255 * @throws IOException if any I/O error occurs 256 * @since 9411 257 */ 258 public BufferedReader getContentReader() throws IOException { 259 return new BufferedReader(UTFInputStreamReader.create(getInputStream())); 260 } 261 262 /** 263 * Get local file for the requested resource. 264 * @return The local cache file for URLs. If the resource is a local file, 265 * returns just that file. 266 * @throws IOException when the resource with the given name could not be retrieved 267 */ 268 public synchronized File getFile() throws IOException { 269 if (initialized) 270 return cacheFile; 271 initialized = true; 272 if (name == null || name.startsWith("resource://")) { 273 return null; 274 } 275 URL url; 276 try { 277 url = new URL(name); 278 if ("file".equals(url.getProtocol())) { 279 cacheFile = new File(name.substring("file:/".length() - 1)); 280 if (!cacheFile.exists()) { 281 cacheFile = new File(name.substring("file://".length() - 1)); 282 } 283 } else { 284 try { 285 cacheFile = checkLocal(url); 286 } catch (SecurityException e) { 287 throw new IOException(e); 288 } 289 } 290 } catch (MalformedURLException e) { 291 if (name.startsWith("josmdir://")) { 292 cacheFile = new File(Config.getDirs().getUserDataDirectory(false), name.substring("josmdir://".length())); 293 } else if (name.startsWith("josmplugindir://")) { 294 cacheFile = new File(Preferences.main().getPluginsDirectory(), name.substring("josmplugindir://".length())); 295 } else { 296 cacheFile = new File(name); 297 } 298 } 299 if (cacheFile == null) 300 throw new IOException("Unable to get cache file for "+getName()); 301 return cacheFile; 302 } 303 304 /** 305 * Looks for a certain entry inside a zip file and returns the entry path. 306 * 307 * Replies a file in the top level directory of the ZIP file which has an 308 * extension <code>extension</code>. If more than one files have this 309 * extension, the last file whose name includes <code>namepart</code> 310 * is opened. 311 * 312 * @param extension the extension of the file we're looking for 313 * @param namepart the name part 314 * @return The zip entry path of the matching file. <code>null</code> if this cached file 315 * doesn't represent a zip file or if there was no matching 316 * file in the ZIP file. 317 */ 318 public String findZipEntryPath(String extension, String namepart) { 319 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 320 if (ze == null) return null; 321 return ze.a; 322 } 323 324 /** 325 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream. 326 * @param extension the extension of the file we're looking for 327 * @param namepart the name part 328 * @return InputStream to the matching file. <code>null</code> if this cached file 329 * doesn't represent a zip file or if there was no matching 330 * file in the ZIP file. 331 * @since 6148 332 */ 333 public InputStream findZipEntryInputStream(String extension, String namepart) { 334 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 335 if (ze == null) return null; 336 return ze.b; 337 } 338 339 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) { 340 File file = null; 341 try { 342 file = getFile(); 343 } catch (IOException ex) { 344 Logging.log(Logging.LEVEL_WARN, ex); 345 } 346 if (file == null) 347 return null; 348 Pair<String, InputStream> res = null; 349 try { 350 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8); 351 ZipEntry resentry = null; 352 Enumeration<? extends ZipEntry> entries = zipFile.entries(); 353 while (entries.hasMoreElements()) { 354 ZipEntry entry = entries.nextElement(); 355 // choose any file with correct extension. When more than one file, prefer the one which matches namepart 356 if (entry.getName().endsWith('.' + extension) && (resentry == null || entry.getName().indexOf(namepart) >= 0)) { 357 resentry = entry; 358 } 359 } 360 if (resentry != null) { 361 InputStream is = zipFile.getInputStream(resentry); 362 res = Pair.create(resentry.getName(), is); 363 } else { 364 Utils.close(zipFile); 365 } 366 } catch (IOException e) { 367 if (file.getName().endsWith(".zip")) { 368 Logging.log(Logging.LEVEL_WARN, 369 tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}", 370 file.getName(), e.toString(), extension, namepart), e); 371 } 372 } 373 return res; 374 } 375 376 /** 377 * Clear the cache for the given resource. 378 * This forces a fresh download. 379 * @param name the URL 380 */ 381 public static void cleanup(String name) { 382 cleanup(name, null); 383 } 384 385 /** 386 * Clear the cache for the given resource. 387 * This forces a fresh download. 388 * @param name the URL 389 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)}) 390 */ 391 public static void cleanup(String name, String destDir) { 392 URL url; 393 try { 394 url = new URL(name); 395 if (!"file".equals(url.getProtocol())) { 396 String prefKey = getPrefKey(url, destDir); 397 List<String> localPath = new ArrayList<>(Config.getPref().getList(prefKey)); 398 if (localPath.size() == 2) { 399 File lfile = new File(localPath.get(1)); 400 if (lfile.exists()) { 401 Utils.deleteFile(lfile); 402 } 403 } 404 Config.getPref().putList(prefKey, null); 405 } 406 } catch (MalformedURLException e) { 407 Logging.warn(e); 408 } 409 } 410 411 /** 412 * Get preference key to store the location and age of the cached file. 413 * 2 resources that point to the same url, but that are to be stored in different 414 * directories will not share a cache file. 415 * @param url URL 416 * @param destDir destination directory 417 * @return Preference key 418 */ 419 private static String getPrefKey(URL url, String destDir) { 420 StringBuilder prefKey = new StringBuilder("mirror."); 421 if (destDir != null) { 422 prefKey.append(destDir).append('.'); 423 } 424 prefKey.append(url.toString().replaceAll("%<(.*)>", "")); 425 return prefKey.toString().replace("=", "_"); 426 } 427 428 private File checkLocal(URL url) throws IOException { 429 String prefKey = getPrefKey(url, destDir); 430 String urlStr = url.toExternalForm(); 431 if (parameter != null) 432 urlStr = urlStr.replaceAll("%<(.*)>", ""); 433 long age = 0L; 434 long maxAgeMillis = TimeUnit.SECONDS.toMillis(maxAge); 435 Long ifModifiedSince = null; 436 File localFile = null; 437 List<String> localPathEntry = new ArrayList<>(Config.getPref().getList(prefKey)); 438 boolean offline = false; 439 try { 440 checkOfflineAccess(urlStr); 441 } catch (OfflineAccessException e) { 442 Logging.trace(e); 443 offline = true; 444 } 445 if (localPathEntry.size() == 2) { 446 localFile = new File(localPathEntry.get(1)); 447 if (!localFile.exists()) { 448 localFile = null; 449 } else { 450 if (maxAge == DEFAULT_MAXTIME 451 || maxAge <= 0 // arbitrary value <= 0 is deprecated 452 ) { 453 maxAgeMillis = TimeUnit.SECONDS.toMillis(Config.getPref().getLong("mirror.maxtime", TimeUnit.DAYS.toSeconds(7))); 454 } 455 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0)); 456 if (offline || age < maxAgeMillis) { 457 return localFile; 458 } 459 if (cachingStrategy == CachingStrategy.IfModifiedSince) { 460 ifModifiedSince = Long.valueOf(localPathEntry.get(0)); 461 } 462 } 463 } 464 if (destDir == null) { 465 destDir = Config.getDirs().getCacheDirectory(true).getPath(); 466 } 467 468 File destDirFile = new File(destDir); 469 if (!destDirFile.exists()) { 470 Utils.mkDirs(destDirFile); 471 } 472 473 // No local file + offline => nothing to do 474 if (offline) { 475 return null; 476 } 477 478 if (parameter != null) { 479 String u = url.toExternalForm(); 480 String uc; 481 if (parameter.isEmpty()) { 482 uc = u.replaceAll("%<(.*)>", ""); 483 } else { 484 uc = u.replaceAll("%<(.*)>", "$1" + Utils.encodeUrl(parameter)); 485 } 486 if (!uc.equals(u)) 487 url = new URL(uc); 488 } 489 490 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_"); 491 String localPath = "mirror_" + a; 492 localPath = truncatePath(destDir, localPath); 493 destDirFile = new File(destDir, localPath + ".tmp"); 494 try { 495 activeConnection = HttpClient.create(url) 496 .setAccept(httpAccept) 497 .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince) 498 .setHeaders(httpHeaders); 499 if (fastFail) { 500 activeConnection.setReadTimeout(1000); 501 } 502 final HttpClient.Response con = activeConnection.connect(); 503 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { 504 Logging.debug("304 Not Modified ({0})", urlStr); 505 if (localFile == null) 506 throw new AssertionError(); 507 Config.getPref().putList(prefKey, 508 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1))); 509 return localFile; 510 } else if (con.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 511 throw new IOException(tr("The requested URL {0} was not found", urlStr)); 512 } 513 try (InputStream is = con.getContent()) { 514 Files.copy(is, destDirFile.toPath(), StandardCopyOption.REPLACE_EXISTING); 515 } 516 activeConnection = null; 517 localFile = new File(destDir, localPath); 518 if (PlatformManager.getPlatform().rename(destDirFile, localFile)) { 519 Config.getPref().putList(prefKey, 520 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString())); 521 } else { 522 Logging.warn(tr("Failed to rename file {0} to {1}.", 523 destDirFile.getPath(), localFile.getPath())); 524 } 525 } catch (IOException e) { 526 if (age >= maxAgeMillis && age < maxAgeMillis*2) { 527 Logging.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e)); 528 return localFile; 529 } else { 530 throw e; 531 } 532 } 533 534 return localFile; 535 } 536 537 private static void checkOfflineAccess(String urlString) { 538 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Config.getUrls().getJOSMWebsite()); 539 OnlineResource.OSM_API.checkOfflineAccess(urlString, OsmApi.getOsmApi().getServerUrl()); 540 } 541 542 private static String truncatePath(String directory, String fileName) { 543 if (directory.length() + fileName.length() > 255) { 544 // Windows doesn't support paths longer than 260, leave 5 chars as safe buffer, 4 will be used by ".tmp" 545 // TODO: what about filename size on other systems? 255? 546 if (directory.length() > 191 && PlatformManager.isPlatformWindows()) { 547 // digest length + name prefix == 64 548 // 255 - 64 = 191 549 // TODO: use this check only on Windows? 550 throw new IllegalArgumentException("Path " + directory + " too long to cached files"); 551 } 552 553 MessageDigest md; 554 try { 555 md = MessageDigest.getInstance("SHA-256"); 556 md.update(fileName.getBytes(StandardCharsets.UTF_8)); 557 String digest = String.format("%064x", new BigInteger(1, md.digest())); 558 return fileName.substring(0, Math.min(fileName.length(), 32)) + digest.substring(0, 32); 559 } catch (NoSuchAlgorithmException e) { 560 Logging.error(e); 561 // TODO: what better can we do here? 562 throw new IllegalArgumentException("Missing digest algorithm SHA-256", e); 563 } 564 } 565 return fileName; 566 } 567 568 /** 569 * Attempts to disconnect an URL connection. 570 * @see HttpClient#disconnect() 571 * @since 9411 572 */ 573 @Override 574 public void close() { 575 if (activeConnection != null) { 576 activeConnection.disconnect(); 577 } 578 } 579 580 /** 581 * Clears the cached file 582 * @throws IOException if any I/O error occurs 583 * @since 10993 584 */ 585 public void clear() throws IOException { 586 URL url; 587 try { 588 url = new URL(name); 589 if ("file".equals(url.getProtocol())) { 590 return; // this is local file - do not delete it 591 } 592 } catch (MalformedURLException e) { 593 return; // if it's not a URL, then it still might be a local file - better not to delete 594 } 595 File f = getFile(); 596 if (f != null && f.exists()) { 597 Utils.deleteFile(f); 598 } 599 } 600}