001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.tools; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedOutputStream; 007import java.io.BufferedReader; 008import java.io.IOException; 009import java.io.InputStream; 010import java.io.OutputStream; 011import java.net.CookieHandler; 012import java.net.CookieManager; 013import java.net.HttpURLConnection; 014import java.net.URL; 015import java.util.Collections; 016import java.util.List; 017import java.util.Locale; 018import java.util.Map; 019import java.util.Map.Entry; 020import java.util.Scanner; 021import java.util.TreeMap; 022import java.util.regex.Matcher; 023import java.util.regex.Pattern; 024import java.util.zip.GZIPInputStream; 025 026import org.openstreetmap.josm.Main; 027import org.openstreetmap.josm.data.Version; 028import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 029import org.openstreetmap.josm.gui.progress.ProgressMonitor; 030import org.openstreetmap.josm.io.Compression; 031import org.openstreetmap.josm.io.ProgressInputStream; 032import org.openstreetmap.josm.io.ProgressOutputStream; 033import org.openstreetmap.josm.io.UTFInputStreamReader; 034import org.openstreetmap.josm.io.auth.DefaultAuthenticator; 035 036/** 037 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}. 038 * @since 9168 039 */ 040public final class HttpClient { 041 042 private URL url; 043 private final String requestMethod; 044 private int connectTimeout = Main.pref.getInteger("socket.timeout.connect", 15) * 1000; 045 private int readTimeout = Main.pref.getInteger("socket.timeout.read", 30) * 1000; 046 private byte[] requestBody; 047 private long ifModifiedSince; 048 private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); 049 private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5); 050 private boolean useCache; 051 private String reasonForRequest; 052 private HttpURLConnection connection; // to allow disconnecting before `response` is set 053 private Response response; 054 055 static { 056 CookieHandler.setDefault(new CookieManager()); 057 } 058 059 private HttpClient(URL url, String requestMethod) { 060 this.url = url; 061 this.requestMethod = requestMethod; 062 this.headers.put("Accept-Encoding", "gzip"); 063 } 064 065 /** 066 * Opens the HTTP connection. 067 * @return HTTP response 068 * @throws IOException if any I/O error occurs 069 */ 070 public Response connect() throws IOException { 071 return connect(null); 072 } 073 074 /** 075 * Opens the HTTP connection. 076 * @param progressMonitor progress monitor 077 * @return HTTP response 078 * @throws IOException if any I/O error occurs 079 * @since 9179 080 */ 081 public Response connect(ProgressMonitor progressMonitor) throws IOException { 082 if (progressMonitor == null) { 083 progressMonitor = NullProgressMonitor.INSTANCE; 084 } 085 final HttpURLConnection connection = (HttpURLConnection) url.openConnection(); 086 this.connection = connection; 087 connection.setRequestMethod(requestMethod); 088 connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString()); 089 connection.setConnectTimeout(connectTimeout); 090 connection.setReadTimeout(readTimeout); 091 connection.setInstanceFollowRedirects(false); // we do that ourselves 092 if (ifModifiedSince > 0) { 093 connection.setIfModifiedSince(ifModifiedSince); 094 } 095 connection.setUseCaches(useCache); 096 if (!useCache) { 097 connection.setRequestProperty("Cache-Control", "no-cache"); 098 } 099 for (Map.Entry<String, String> header : headers.entrySet()) { 100 if (header.getValue() != null) { 101 connection.setRequestProperty(header.getKey(), header.getValue()); 102 } 103 } 104 105 progressMonitor.beginTask(tr("Contacting Server..."), 1); 106 progressMonitor.indeterminateSubTask(null); 107 108 if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) { 109 Main.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault())); 110 connection.setFixedLengthStreamingMode(requestBody.length); 111 connection.setDoOutput(true); 112 final ProgressMonitor subTaskMonitor = progressMonitor.createSubTaskMonitor(1, false); 113 try (OutputStream out = new BufferedOutputStream( 114 new ProgressOutputStream(connection.getOutputStream(), requestBody.length, subTaskMonitor))) { 115 out.write(requestBody); 116 } 117 } 118 119 boolean successfulConnection = false; 120 try { 121 try { 122 connection.connect(); 123 final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty(); 124 Main.info("{0} {1}{2} -> {3}{4}", 125 requestMethod, url, hasReason ? " (" + reasonForRequest + ")" : "", 126 connection.getResponseCode(), 127 connection.getContentLengthLong() > 0 128 ? " (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ")" 129 : "" 130 ); 131 if (Main.isDebugEnabled()) { 132 Main.debug("RESPONSE: " + connection.getHeaderFields()); 133 } 134 if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) { 135 DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost()); 136 } 137 } catch (IOException e) { 138 Main.info("{0} {1} -> !!!", requestMethod, url); 139 Main.warn(e); 140 //noinspection ThrowableResultOfMethodCallIgnored 141 Main.addNetworkError(url, Utils.getRootCause(e)); 142 throw e; 143 } 144 if (isRedirect(connection.getResponseCode())) { 145 final String redirectLocation = connection.getHeaderField("Location"); 146 if (redirectLocation == null) { 147 /* I18n: argument is HTTP response code */ 148 String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." + 149 " Can''t redirect. Aborting.", connection.getResponseCode()); 150 throw new IOException(msg); 151 } else if (maxRedirects > 0) { 152 url = new URL(url, redirectLocation); 153 maxRedirects--; 154 Main.info(tr("Download redirected to ''{0}''", redirectLocation)); 155 return connect(); 156 } else if (maxRedirects == 0) { 157 String msg = tr("Too many redirects to the download URL detected. Aborting."); 158 throw new IOException(msg); 159 } 160 } 161 response = new Response(connection, progressMonitor); 162 successfulConnection = true; 163 return response; 164 } finally { 165 if (!successfulConnection) { 166 connection.disconnect(); 167 } 168 } 169 } 170 171 /** 172 * Returns the HTTP response which is set only after calling {@link #connect()}. 173 * Calling this method again, returns the identical object (unless another {@link #connect()} is performed). 174 * 175 * @return the HTTP response 176 * @since 9309 177 */ 178 public Response getResponse() { 179 return response; 180 } 181 182 /** 183 * A wrapper for the HTTP response. 184 */ 185 public static final class Response { 186 private final HttpURLConnection connection; 187 private final ProgressMonitor monitor; 188 private final int responseCode; 189 private final String responseMessage; 190 private boolean uncompress; 191 private boolean uncompressAccordingToContentDisposition; 192 193 private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException { 194 CheckParameterUtil.ensureParameterNotNull(connection, "connection"); 195 CheckParameterUtil.ensureParameterNotNull(monitor, "monitor"); 196 this.connection = connection; 197 this.monitor = monitor; 198 this.responseCode = connection.getResponseCode(); 199 this.responseMessage = connection.getResponseMessage(); 200 } 201 202 /** 203 * Sets whether {@link #getContent()} should uncompress the input stream if necessary. 204 * 205 * @param uncompress whether the input stream should be uncompressed if necessary 206 * @return {@code this} 207 */ 208 public Response uncompress(boolean uncompress) { 209 this.uncompress = uncompress; 210 return this; 211 } 212 213 /** 214 * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition} 215 * HTTP header. 216 * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to 217 * {@code Content-Disposition} 218 * @return {@code this} 219 * @since 9172 220 */ 221 public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) { 222 this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition; 223 return this; 224 } 225 226 /** 227 * Returns the URL. 228 * @return the URL 229 * @see HttpURLConnection#getURL() 230 * @since 9172 231 */ 232 public URL getURL() { 233 return connection.getURL(); 234 } 235 236 /** 237 * Returns the request method. 238 * @return the HTTP request method 239 * @see HttpURLConnection#getRequestMethod() 240 * @since 9172 241 */ 242 public String getRequestMethod() { 243 return connection.getRequestMethod(); 244 } 245 246 /** 247 * Returns an input stream that reads from this HTTP connection, or, 248 * error stream if the connection failed but the server sent useful data. 249 * <p> 250 * Note: the return value can be null, if both the input and the error stream are null. 251 * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887 252 * @return input or error stream 253 * @throws IOException if any I/O error occurs 254 * 255 * @see HttpURLConnection#getInputStream() 256 * @see HttpURLConnection#getErrorStream() 257 */ 258 @SuppressWarnings("resource") 259 public InputStream getContent() throws IOException { 260 InputStream in; 261 try { 262 in = connection.getInputStream(); 263 } catch (IOException ioe) { 264 in = connection.getErrorStream(); 265 } 266 monitor.subTask(tr("Fetching content...")); 267 if (in != null) { 268 in = new ProgressInputStream(in, getContentLength(), monitor.createSubTaskMonitor(1, false)); 269 in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in; 270 Compression compression = Compression.NONE; 271 if (uncompress) { 272 final String contentType = getContentType(); 273 Main.debug("Uncompressing input stream according to Content-Type header: {0}", contentType); 274 compression = Compression.forContentType(contentType); 275 } 276 if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) { 277 final String contentDisposition = getHeaderField("Content-Disposition"); 278 final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher( 279 contentDisposition != null ? contentDisposition : ""); 280 if (matcher.find()) { 281 Main.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition); 282 compression = Compression.byExtension(matcher.group(1)); 283 } 284 } 285 in = compression.getUncompressedInputStream(in); 286 } 287 return in; 288 } 289 290 /** 291 * Returns {@link #getContent()} wrapped in a buffered reader. 292 * 293 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}. 294 * @return buffered reader 295 * @throws IOException if any I/O error occurs 296 */ 297 public BufferedReader getContentReader() throws IOException { 298 return new BufferedReader( 299 UTFInputStreamReader.create(getContent()) 300 ); 301 } 302 303 /** 304 * Fetches the HTTP response as String. 305 * @return the response 306 * @throws IOException if any I/O error occurs 307 */ 308 @SuppressWarnings("resource") 309 public String fetchContent() throws IOException { 310 try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) { 311 return scanner.hasNext() ? scanner.next() : ""; 312 } 313 } 314 315 /** 316 * Gets the response code from this HTTP connection. 317 * @return HTTP response code 318 * 319 * @see HttpURLConnection#getResponseCode() 320 */ 321 public int getResponseCode() { 322 return responseCode; 323 } 324 325 /** 326 * Gets the response message from this HTTP connection. 327 * @return HTTP response message 328 * 329 * @see HttpURLConnection#getResponseMessage() 330 * @since 9172 331 */ 332 public String getResponseMessage() { 333 return responseMessage; 334 } 335 336 /** 337 * Returns the {@code Content-Encoding} header. 338 * @return {@code Content-Encoding} HTTP header 339 * @see HttpURLConnection#getContentEncoding() 340 */ 341 public String getContentEncoding() { 342 return connection.getContentEncoding(); 343 } 344 345 /** 346 * Returns the {@code Content-Type} header. 347 * @return {@code Content-Type} HTTP header 348 */ 349 public String getContentType() { 350 return connection.getHeaderField("Content-Type"); 351 } 352 353 /** 354 * Returns the {@code Expire} header. 355 * @return {@code Expire} HTTP header 356 * @see HttpURLConnection#getExpiration() 357 * @since 9232 358 */ 359 public long getExpiration() { 360 return connection.getExpiration(); 361 } 362 363 /** 364 * Returns the {@code Last-Modified} header. 365 * @return {@code Last-Modified} HTTP header 366 * @see HttpURLConnection#getLastModified() 367 * @since 9232 368 */ 369 public long getLastModified() { 370 return connection.getLastModified(); 371 } 372 373 /** 374 * Returns the {@code Content-Length} header. 375 * @return {@code Content-Length} HTTP header 376 * @see HttpURLConnection#getContentLengthLong() 377 */ 378 public long getContentLength() { 379 return connection.getContentLengthLong(); 380 } 381 382 /** 383 * Returns the value of the named header field. 384 * @param name the name of a header field 385 * @return the value of the named header field, or {@code null} if there is no such field in the header 386 * @see HttpURLConnection#getHeaderField(String) 387 * @since 9172 388 */ 389 public String getHeaderField(String name) { 390 return connection.getHeaderField(name); 391 } 392 393 /** 394 * Returns an unmodifiable Map mapping header keys to a List of header values. 395 * As per RFC 2616, section 4.2 header names are case insensitive, so returned map is also case insensitive 396 * @return unmodifiable Map mapping header keys to a List of header values 397 * @see HttpURLConnection#getHeaderFields() 398 * @since 9232 399 */ 400 public Map<String, List<String>> getHeaderFields() { 401 // returned map from HttpUrlConnection is case sensitive, use case insensitive TreeMap to conform to RFC 2616 402 Map<String, List<String>> ret = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); 403 for (Entry<String, List<String>> e: connection.getHeaderFields().entrySet()) { 404 if (e.getKey() != null) { 405 ret.put(e.getKey(), e.getValue()); 406 } 407 } 408 return Collections.unmodifiableMap(ret); 409 } 410 411 /** 412 * @see HttpURLConnection#disconnect() 413 */ 414 public void disconnect() { 415 HttpClient.disconnect(connection); 416 } 417 } 418 419 /** 420 * Creates a new instance for the given URL and a {@code GET} request 421 * 422 * @param url the URL 423 * @return a new instance 424 */ 425 public static HttpClient create(URL url) { 426 return create(url, "GET"); 427 } 428 429 /** 430 * Creates a new instance for the given URL and a {@code GET} request 431 * 432 * @param url the URL 433 * @param requestMethod the HTTP request method to perform when calling 434 * @return a new instance 435 */ 436 public static HttpClient create(URL url, String requestMethod) { 437 return new HttpClient(url, requestMethod); 438 } 439 440 /** 441 * Returns the URL set for this connection. 442 * @return the URL 443 * @see #create(URL) 444 * @see #create(URL, String) 445 * @since 9172 446 */ 447 public URL getURL() { 448 return url; 449 } 450 451 /** 452 * Returns the request method set for this connection. 453 * @return the HTTP request method 454 * @see #create(URL, String) 455 * @since 9172 456 */ 457 public String getRequestMethod() { 458 return requestMethod; 459 } 460 461 /** 462 * Returns the set value for the given {@code header}. 463 * @param header HTTP header name 464 * @return HTTP header value 465 * @since 9172 466 */ 467 public String getRequestHeader(String header) { 468 return headers.get(header); 469 } 470 471 /** 472 * Sets whether not to set header {@code Cache-Control=no-cache} 473 * 474 * @param useCache whether not to set header {@code Cache-Control=no-cache} 475 * @return {@code this} 476 * @see HttpURLConnection#setUseCaches(boolean) 477 */ 478 public HttpClient useCache(boolean useCache) { 479 this.useCache = useCache; 480 return this; 481 } 482 483 /** 484 * Sets whether not to set header {@code Connection=close} 485 * <p> 486 * This might fix #7640, see 487 * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>. 488 * 489 * @param keepAlive whether not to set header {@code Connection=close} 490 * @return {@code this} 491 */ 492 public HttpClient keepAlive(boolean keepAlive) { 493 return setHeader("Connection", keepAlive ? null : "close"); 494 } 495 496 /** 497 * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced 498 * by this URLConnection. If the timeout expires before the connection can be established, a 499 * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout. 500 * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds 501 * @return {@code this} 502 * @see HttpURLConnection#setConnectTimeout(int) 503 */ 504 public HttpClient setConnectTimeout(int connectTimeout) { 505 this.connectTimeout = connectTimeout; 506 return this; 507 } 508 509 /** 510 * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from 511 * input stream when a connection is established to a resource. If the timeout expires before there is data available for 512 * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout. 513 * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds 514 * @return {@code this} 515 * @see HttpURLConnection#setReadTimeout(int) 516 */ 517 public HttpClient setReadTimeout(int readTimeout) { 518 this.readTimeout = readTimeout; 519 return this; 520 } 521 522 /** 523 * Sets the {@code Accept} header. 524 * @param accept header value 525 * 526 * @return {@code this} 527 */ 528 public HttpClient setAccept(String accept) { 529 return setHeader("Accept", accept); 530 } 531 532 /** 533 * Sets the request body for {@code PUT}/{@code POST} requests. 534 * @param requestBody request body 535 * 536 * @return {@code this} 537 */ 538 public HttpClient setRequestBody(byte[] requestBody) { 539 this.requestBody = requestBody; 540 return this; 541 } 542 543 /** 544 * Sets the {@code If-Modified-Since} header. 545 * @param ifModifiedSince header value 546 * 547 * @return {@code this} 548 */ 549 public HttpClient setIfModifiedSince(long ifModifiedSince) { 550 this.ifModifiedSince = ifModifiedSince; 551 return this; 552 } 553 554 /** 555 * Sets the maximum number of redirections to follow. 556 * 557 * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e., 558 * to not throw an {@link IOException} in {@link #connect()}. 559 * @param maxRedirects header value 560 * 561 * @return {@code this} 562 */ 563 public HttpClient setMaxRedirects(int maxRedirects) { 564 this.maxRedirects = maxRedirects; 565 return this; 566 } 567 568 /** 569 * Sets an arbitrary HTTP header. 570 * @param key header name 571 * @param value header value 572 * 573 * @return {@code this} 574 */ 575 public HttpClient setHeader(String key, String value) { 576 this.headers.put(key, value); 577 return this; 578 } 579 580 /** 581 * Sets arbitrary HTTP headers. 582 * @param headers HTTP headers 583 * 584 * @return {@code this} 585 */ 586 public HttpClient setHeaders(Map<String, String> headers) { 587 this.headers.putAll(headers); 588 return this; 589 } 590 591 /** 592 * Sets a reason to show on console. Can be {@code null} if no reason is given. 593 * @param reasonForRequest Reason to show 594 * @return {@code this} 595 * @since 9172 596 */ 597 public HttpClient setReasonForRequest(String reasonForRequest) { 598 this.reasonForRequest = reasonForRequest; 599 return this; 600 } 601 602 private static boolean isRedirect(final int statusCode) { 603 switch (statusCode) { 604 case HttpURLConnection.HTTP_MOVED_PERM: // 301 605 case HttpURLConnection.HTTP_MOVED_TEMP: // 302 606 case HttpURLConnection.HTTP_SEE_OTHER: // 303 607 case 307: // TEMPORARY_REDIRECT: 608 case 308: // PERMANENT_REDIRECT: 609 return true; 610 default: 611 return false; 612 } 613 } 614 615 /** 616 * @see HttpURLConnection#disconnect() 617 * @since 9309 618 */ 619 public void disconnect() { 620 HttpClient.disconnect(connection); 621 } 622 623 private static void disconnect(final HttpURLConnection connection) { 624 // Fix upload aborts - see #263 625 connection.setConnectTimeout(100); 626 connection.setReadTimeout(100); 627 try { 628 Thread.sleep(100); 629 } catch (InterruptedException ex) { 630 Main.warn("InterruptedException in " + HttpClient.class + " during cancel"); 631 } 632 connection.disconnect(); 633 } 634}