001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.tools; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedOutputStream; 007import java.io.BufferedReader; 008import java.io.IOException; 009import java.io.InputStream; 010import java.io.OutputStream; 011import java.net.CookieHandler; 012import java.net.CookieManager; 013import java.net.HttpURLConnection; 014import java.net.URL; 015import java.util.Collections; 016import java.util.List; 017import java.util.Locale; 018import java.util.Map; 019import java.util.Map.Entry; 020import java.util.Scanner; 021import java.util.TreeMap; 022import java.util.regex.Matcher; 023import java.util.regex.Pattern; 024import java.util.zip.GZIPInputStream; 025 026import org.openstreetmap.josm.Main; 027import org.openstreetmap.josm.data.Version; 028import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 029import org.openstreetmap.josm.gui.progress.ProgressMonitor; 030import org.openstreetmap.josm.io.Compression; 031import org.openstreetmap.josm.io.ProgressInputStream; 032import org.openstreetmap.josm.io.ProgressOutputStream; 033import org.openstreetmap.josm.io.UTFInputStreamReader; 034import org.openstreetmap.josm.io.auth.DefaultAuthenticator; 035 036/** 037 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}. 038 * @since 9168 039 */ 040public final class HttpClient { 041 042 private URL url; 043 private final String requestMethod; 044 private int connectTimeout = Main.pref.getInteger("socket.timeout.connect", 15) * 1000; 045 private int readTimeout = Main.pref.getInteger("socket.timeout.read", 30) * 1000; 046 private byte[] requestBody; 047 private long ifModifiedSince; 048 private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); 049 private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5); 050 private boolean useCache; 051 private String reasonForRequest; 052 private HttpURLConnection connection; // to allow disconnecting before `response` is set 053 private Response response; 054 private boolean finishOnCloseOutput = true; 055 056 static { 057 CookieHandler.setDefault(new CookieManager()); 058 } 059 060 private HttpClient(URL url, String requestMethod) { 061 this.url = url; 062 this.requestMethod = requestMethod; 063 this.headers.put("Accept-Encoding", "gzip"); 064 } 065 066 /** 067 * Opens the HTTP connection. 068 * @return HTTP response 069 * @throws IOException if any I/O error occurs 070 */ 071 public Response connect() throws IOException { 072 return connect(null); 073 } 074 075 /** 076 * Opens the HTTP connection. 077 * @param progressMonitor progress monitor 078 * @return HTTP response 079 * @throws IOException if any I/O error occurs 080 * @since 9179 081 */ 082 public Response connect(ProgressMonitor progressMonitor) throws IOException { 083 if (progressMonitor == null) { 084 progressMonitor = NullProgressMonitor.INSTANCE; 085 } 086 final HttpURLConnection connection = (HttpURLConnection) url.openConnection(); 087 this.connection = connection; 088 connection.setRequestMethod(requestMethod); 089 connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString()); 090 connection.setConnectTimeout(connectTimeout); 091 connection.setReadTimeout(readTimeout); 092 connection.setInstanceFollowRedirects(false); // we do that ourselves 093 if (ifModifiedSince > 0) { 094 connection.setIfModifiedSince(ifModifiedSince); 095 } 096 connection.setUseCaches(useCache); 097 if (!useCache) { 098 connection.setRequestProperty("Cache-Control", "no-cache"); 099 } 100 for (Map.Entry<String, String> header : headers.entrySet()) { 101 if (header.getValue() != null) { 102 connection.setRequestProperty(header.getKey(), header.getValue()); 103 } 104 } 105 106 progressMonitor.beginTask(tr("Contacting Server..."), 1); 107 progressMonitor.indeterminateSubTask(null); 108 109 if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) { 110 Main.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault())); 111 connection.setFixedLengthStreamingMode(requestBody.length); 112 connection.setDoOutput(true); 113 try (OutputStream out = new BufferedOutputStream( 114 new ProgressOutputStream(connection.getOutputStream(), requestBody.length, progressMonitor, finishOnCloseOutput))) { 115 out.write(requestBody); 116 } 117 } 118 119 boolean successfulConnection = false; 120 try { 121 try { 122 connection.connect(); 123 final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty(); 124 Main.info("{0} {1}{2} -> {3}{4}", 125 requestMethod, url, hasReason ? (" (" + reasonForRequest + ')') : "", 126 connection.getResponseCode(), 127 connection.getContentLengthLong() > 0 128 ? (" (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ')') 129 : "" 130 ); 131 if (Main.isDebugEnabled()) { 132 Main.debug("RESPONSE: " + connection.getHeaderFields()); 133 } 134 if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) { 135 DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost()); 136 } 137 } catch (IOException e) { 138 Main.info("{0} {1} -> !!!", requestMethod, url); 139 Main.warn(e); 140 //noinspection ThrowableResultOfMethodCallIgnored 141 Main.addNetworkError(url, Utils.getRootCause(e)); 142 throw e; 143 } 144 if (isRedirect(connection.getResponseCode())) { 145 final String redirectLocation = connection.getHeaderField("Location"); 146 if (redirectLocation == null) { 147 /* I18n: argument is HTTP response code */ 148 String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." + 149 " Can''t redirect. Aborting.", connection.getResponseCode()); 150 throw new IOException(msg); 151 } else if (maxRedirects > 0) { 152 url = new URL(url, redirectLocation); 153 maxRedirects--; 154 Main.info(tr("Download redirected to ''{0}''", redirectLocation)); 155 return connect(); 156 } else if (maxRedirects == 0) { 157 String msg = tr("Too many redirects to the download URL detected. Aborting."); 158 throw new IOException(msg); 159 } 160 } 161 response = new Response(connection, progressMonitor); 162 successfulConnection = true; 163 return response; 164 } finally { 165 if (!successfulConnection) { 166 connection.disconnect(); 167 } 168 } 169 } 170 171 /** 172 * Returns the HTTP response which is set only after calling {@link #connect()}. 173 * Calling this method again, returns the identical object (unless another {@link #connect()} is performed). 174 * 175 * @return the HTTP response 176 * @since 9309 177 */ 178 public Response getResponse() { 179 return response; 180 } 181 182 /** 183 * A wrapper for the HTTP response. 184 */ 185 public static final class Response { 186 private final HttpURLConnection connection; 187 private final ProgressMonitor monitor; 188 private final int responseCode; 189 private final String responseMessage; 190 private boolean uncompress; 191 private boolean uncompressAccordingToContentDisposition; 192 193 private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException { 194 CheckParameterUtil.ensureParameterNotNull(connection, "connection"); 195 CheckParameterUtil.ensureParameterNotNull(monitor, "monitor"); 196 this.connection = connection; 197 this.monitor = monitor; 198 this.responseCode = connection.getResponseCode(); 199 this.responseMessage = connection.getResponseMessage(); 200 } 201 202 /** 203 * Sets whether {@link #getContent()} should uncompress the input stream if necessary. 204 * 205 * @param uncompress whether the input stream should be uncompressed if necessary 206 * @return {@code this} 207 */ 208 public Response uncompress(boolean uncompress) { 209 this.uncompress = uncompress; 210 return this; 211 } 212 213 /** 214 * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition} 215 * HTTP header. 216 * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to 217 * {@code Content-Disposition} 218 * @return {@code this} 219 * @since 9172 220 */ 221 public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) { 222 this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition; 223 return this; 224 } 225 226 /** 227 * Returns the URL. 228 * @return the URL 229 * @see HttpURLConnection#getURL() 230 * @since 9172 231 */ 232 public URL getURL() { 233 return connection.getURL(); 234 } 235 236 /** 237 * Returns the request method. 238 * @return the HTTP request method 239 * @see HttpURLConnection#getRequestMethod() 240 * @since 9172 241 */ 242 public String getRequestMethod() { 243 return connection.getRequestMethod(); 244 } 245 246 /** 247 * Returns an input stream that reads from this HTTP connection, or, 248 * error stream if the connection failed but the server sent useful data. 249 * <p> 250 * Note: the return value can be null, if both the input and the error stream are null. 251 * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887 252 * @return input or error stream 253 * @throws IOException if any I/O error occurs 254 * 255 * @see HttpURLConnection#getInputStream() 256 * @see HttpURLConnection#getErrorStream() 257 */ 258 @SuppressWarnings("resource") 259 public InputStream getContent() throws IOException { 260 InputStream in; 261 try { 262 in = connection.getInputStream(); 263 } catch (IOException ioe) { 264 in = connection.getErrorStream(); 265 } 266 if (in != null) { 267 in = new ProgressInputStream(in, getContentLength(), monitor); 268 in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in; 269 Compression compression = Compression.NONE; 270 if (uncompress) { 271 final String contentType = getContentType(); 272 Main.debug("Uncompressing input stream according to Content-Type header: {0}", contentType); 273 compression = Compression.forContentType(contentType); 274 } 275 if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) { 276 final String contentDisposition = getHeaderField("Content-Disposition"); 277 final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher( 278 contentDisposition != null ? contentDisposition : ""); 279 if (matcher.find()) { 280 Main.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition); 281 compression = Compression.byExtension(matcher.group(1)); 282 } 283 } 284 in = compression.getUncompressedInputStream(in); 285 } 286 return in; 287 } 288 289 /** 290 * Returns {@link #getContent()} wrapped in a buffered reader. 291 * 292 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}. 293 * @return buffered reader 294 * @throws IOException if any I/O error occurs 295 */ 296 public BufferedReader getContentReader() throws IOException { 297 return new BufferedReader( 298 UTFInputStreamReader.create(getContent()) 299 ); 300 } 301 302 /** 303 * Fetches the HTTP response as String. 304 * @return the response 305 * @throws IOException if any I/O error occurs 306 */ 307 @SuppressWarnings("resource") 308 public String fetchContent() throws IOException { 309 try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) { 310 return scanner.hasNext() ? scanner.next() : ""; 311 } 312 } 313 314 /** 315 * Gets the response code from this HTTP connection. 316 * @return HTTP response code 317 * 318 * @see HttpURLConnection#getResponseCode() 319 */ 320 public int getResponseCode() { 321 return responseCode; 322 } 323 324 /** 325 * Gets the response message from this HTTP connection. 326 * @return HTTP response message 327 * 328 * @see HttpURLConnection#getResponseMessage() 329 * @since 9172 330 */ 331 public String getResponseMessage() { 332 return responseMessage; 333 } 334 335 /** 336 * Returns the {@code Content-Encoding} header. 337 * @return {@code Content-Encoding} HTTP header 338 * @see HttpURLConnection#getContentEncoding() 339 */ 340 public String getContentEncoding() { 341 return connection.getContentEncoding(); 342 } 343 344 /** 345 * Returns the {@code Content-Type} header. 346 * @return {@code Content-Type} HTTP header 347 */ 348 public String getContentType() { 349 return connection.getHeaderField("Content-Type"); 350 } 351 352 /** 353 * Returns the {@code Expire} header. 354 * @return {@code Expire} HTTP header 355 * @see HttpURLConnection#getExpiration() 356 * @since 9232 357 */ 358 public long getExpiration() { 359 return connection.getExpiration(); 360 } 361 362 /** 363 * Returns the {@code Last-Modified} header. 364 * @return {@code Last-Modified} HTTP header 365 * @see HttpURLConnection#getLastModified() 366 * @since 9232 367 */ 368 public long getLastModified() { 369 return connection.getLastModified(); 370 } 371 372 /** 373 * Returns the {@code Content-Length} header. 374 * @return {@code Content-Length} HTTP header 375 * @see HttpURLConnection#getContentLengthLong() 376 */ 377 public long getContentLength() { 378 return connection.getContentLengthLong(); 379 } 380 381 /** 382 * Returns the value of the named header field. 383 * @param name the name of a header field 384 * @return the value of the named header field, or {@code null} if there is no such field in the header 385 * @see HttpURLConnection#getHeaderField(String) 386 * @since 9172 387 */ 388 public String getHeaderField(String name) { 389 return connection.getHeaderField(name); 390 } 391 392 /** 393 * Returns an unmodifiable Map mapping header keys to a List of header values. 394 * As per RFC 2616, section 4.2 header names are case insensitive, so returned map is also case insensitive 395 * @return unmodifiable Map mapping header keys to a List of header values 396 * @see HttpURLConnection#getHeaderFields() 397 * @since 9232 398 */ 399 public Map<String, List<String>> getHeaderFields() { 400 // returned map from HttpUrlConnection is case sensitive, use case insensitive TreeMap to conform to RFC 2616 401 Map<String, List<String>> ret = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); 402 for (Entry<String, List<String>> e: connection.getHeaderFields().entrySet()) { 403 if (e.getKey() != null) { 404 ret.put(e.getKey(), e.getValue()); 405 } 406 } 407 return Collections.unmodifiableMap(ret); 408 } 409 410 /** 411 * @see HttpURLConnection#disconnect() 412 */ 413 public void disconnect() { 414 HttpClient.disconnect(connection); 415 } 416 } 417 418 /** 419 * Creates a new instance for the given URL and a {@code GET} request 420 * 421 * @param url the URL 422 * @return a new instance 423 */ 424 public static HttpClient create(URL url) { 425 return create(url, "GET"); 426 } 427 428 /** 429 * Creates a new instance for the given URL and a {@code GET} request 430 * 431 * @param url the URL 432 * @param requestMethod the HTTP request method to perform when calling 433 * @return a new instance 434 */ 435 public static HttpClient create(URL url, String requestMethod) { 436 return new HttpClient(url, requestMethod); 437 } 438 439 /** 440 * Returns the URL set for this connection. 441 * @return the URL 442 * @see #create(URL) 443 * @see #create(URL, String) 444 * @since 9172 445 */ 446 public URL getURL() { 447 return url; 448 } 449 450 /** 451 * Returns the request method set for this connection. 452 * @return the HTTP request method 453 * @see #create(URL, String) 454 * @since 9172 455 */ 456 public String getRequestMethod() { 457 return requestMethod; 458 } 459 460 /** 461 * Returns the set value for the given {@code header}. 462 * @param header HTTP header name 463 * @return HTTP header value 464 * @since 9172 465 */ 466 public String getRequestHeader(String header) { 467 return headers.get(header); 468 } 469 470 /** 471 * Sets whether not to set header {@code Cache-Control=no-cache} 472 * 473 * @param useCache whether not to set header {@code Cache-Control=no-cache} 474 * @return {@code this} 475 * @see HttpURLConnection#setUseCaches(boolean) 476 */ 477 public HttpClient useCache(boolean useCache) { 478 this.useCache = useCache; 479 return this; 480 } 481 482 /** 483 * Sets whether not to set header {@code Connection=close} 484 * <p> 485 * This might fix #7640, see 486 * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>. 487 * 488 * @param keepAlive whether not to set header {@code Connection=close} 489 * @return {@code this} 490 */ 491 public HttpClient keepAlive(boolean keepAlive) { 492 return setHeader("Connection", keepAlive ? null : "close"); 493 } 494 495 /** 496 * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced 497 * by this URLConnection. If the timeout expires before the connection can be established, a 498 * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout. 499 * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds 500 * @return {@code this} 501 * @see HttpURLConnection#setConnectTimeout(int) 502 */ 503 public HttpClient setConnectTimeout(int connectTimeout) { 504 this.connectTimeout = connectTimeout; 505 return this; 506 } 507 508 /** 509 * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from 510 * input stream when a connection is established to a resource. If the timeout expires before there is data available for 511 * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout. 512 * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds 513 * @return {@code this} 514 * @see HttpURLConnection#setReadTimeout(int) 515 */ 516 public HttpClient setReadTimeout(int readTimeout) { 517 this.readTimeout = readTimeout; 518 return this; 519 } 520 521 /** 522 * Sets the {@code Accept} header. 523 * @param accept header value 524 * 525 * @return {@code this} 526 */ 527 public HttpClient setAccept(String accept) { 528 return setHeader("Accept", accept); 529 } 530 531 /** 532 * Sets the request body for {@code PUT}/{@code POST} requests. 533 * @param requestBody request body 534 * 535 * @return {@code this} 536 */ 537 public HttpClient setRequestBody(byte[] requestBody) { 538 this.requestBody = requestBody; 539 return this; 540 } 541 542 /** 543 * Sets the {@code If-Modified-Since} header. 544 * @param ifModifiedSince header value 545 * 546 * @return {@code this} 547 */ 548 public HttpClient setIfModifiedSince(long ifModifiedSince) { 549 this.ifModifiedSince = ifModifiedSince; 550 return this; 551 } 552 553 /** 554 * Sets the maximum number of redirections to follow. 555 * 556 * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e., 557 * to not throw an {@link IOException} in {@link #connect()}. 558 * @param maxRedirects header value 559 * 560 * @return {@code this} 561 */ 562 public HttpClient setMaxRedirects(int maxRedirects) { 563 this.maxRedirects = maxRedirects; 564 return this; 565 } 566 567 /** 568 * Sets an arbitrary HTTP header. 569 * @param key header name 570 * @param value header value 571 * 572 * @return {@code this} 573 */ 574 public HttpClient setHeader(String key, String value) { 575 this.headers.put(key, value); 576 return this; 577 } 578 579 /** 580 * Sets arbitrary HTTP headers. 581 * @param headers HTTP headers 582 * 583 * @return {@code this} 584 */ 585 public HttpClient setHeaders(Map<String, String> headers) { 586 this.headers.putAll(headers); 587 return this; 588 } 589 590 /** 591 * Sets a reason to show on console. Can be {@code null} if no reason is given. 592 * @param reasonForRequest Reason to show 593 * @return {@code this} 594 * @since 9172 595 */ 596 public HttpClient setReasonForRequest(String reasonForRequest) { 597 this.reasonForRequest = reasonForRequest; 598 return this; 599 } 600 601 /** 602 * Sets whether the progress monitor task will be finished when the output stream is closed. This is {@code true} by default. 603 * @param finishOnCloseOutput whether the progress monitor task will be finished when the output stream is closed 604 * @return {@code this} 605 * @since 10302 606 */ 607 public HttpClient setFinishOnCloseOutput(boolean finishOnCloseOutput) { 608 this.finishOnCloseOutput = finishOnCloseOutput; 609 return this; 610 } 611 612 private static boolean isRedirect(final int statusCode) { 613 switch (statusCode) { 614 case HttpURLConnection.HTTP_MOVED_PERM: // 301 615 case HttpURLConnection.HTTP_MOVED_TEMP: // 302 616 case HttpURLConnection.HTTP_SEE_OTHER: // 303 617 case 307: // TEMPORARY_REDIRECT: 618 case 308: // PERMANENT_REDIRECT: 619 return true; 620 default: 621 return false; 622 } 623 } 624 625 /** 626 * @see HttpURLConnection#disconnect() 627 * @since 9309 628 */ 629 public void disconnect() { 630 HttpClient.disconnect(connection); 631 } 632 633 private static void disconnect(final HttpURLConnection connection) { 634 // Fix upload aborts - see #263 635 connection.setConnectTimeout(100); 636 connection.setReadTimeout(100); 637 try { 638 Thread.sleep(100); 639 } catch (InterruptedException ex) { 640 Main.warn("InterruptedException in " + HttpClient.class + " during cancel"); 641 } 642 connection.disconnect(); 643 } 644}