001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.io.InputStream;
008import java.net.Authenticator.RequestorType;
009import java.net.HttpURLConnection;
010import java.net.MalformedURLException;
011import java.net.URL;
012import java.util.List;
013
014import javax.xml.parsers.ParserConfigurationException;
015
016import org.openstreetmap.josm.data.gpx.GpxData;
017import org.openstreetmap.josm.data.notes.Note;
018import org.openstreetmap.josm.data.osm.DataSet;
019import org.openstreetmap.josm.gui.progress.ProgressMonitor;
020import org.openstreetmap.josm.io.auth.CredentialsAgentException;
021import org.openstreetmap.josm.io.auth.CredentialsManager;
022import org.openstreetmap.josm.spi.preferences.Config;
023import org.openstreetmap.josm.tools.HttpClient;
024import org.openstreetmap.josm.tools.Logging;
025import org.openstreetmap.josm.tools.Utils;
026import org.openstreetmap.josm.tools.XmlParsingException;
027import org.openstreetmap.josm.tools.XmlUtils;
028import org.w3c.dom.Document;
029import org.w3c.dom.Node;
030import org.xml.sax.SAXException;
031
032/**
033 * This DataReader reads directly from the REST API of the osm server.
034 *
035 * It supports plain text transfer as well as gzip or deflate encoded transfers;
036 * if compressed transfers are unwanted, set property osm-server.use-compression
037 * to false.
038 *
039 * @author imi
040 */
041public abstract class OsmServerReader extends OsmConnection {
042    private final OsmApi api = OsmApi.getOsmApi();
043    private boolean doAuthenticate;
044    protected boolean gpxParsedProperly;
045    protected String contentType;
046
047    /**
048     * Constructs a new {@code OsmServerReader}.
049     */
050    public OsmServerReader() {
051        try {
052            doAuthenticate = OsmApi.isUsingOAuth() && CredentialsManager.getInstance().lookupOAuthAccessToken() != null;
053        } catch (CredentialsAgentException e) {
054            Logging.warn(e);
055        }
056    }
057
058    /**
059     * Open a connection to the given url and return a reader on the input stream
060     * from that connection. In case of user cancel, return <code>null</code>.
061     * Relative URL's are directed to API base URL.
062     * @param urlStr The url to connect to.
063     * @param progressMonitor progress monitoring and abort handler
064     * @return A reader reading the input stream (servers answer) or <code>null</code>.
065     * @throws OsmTransferException if data transfer errors occur
066     */
067    protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
068        return getInputStream(urlStr, progressMonitor, null);
069    }
070
071    /**
072     * Open a connection to the given url and return a reader on the input stream
073     * from that connection. In case of user cancel, return <code>null</code>.
074     * Relative URL's are directed to API base URL.
075     * @param urlStr The url to connect to.
076     * @param progressMonitor progress monitoring and abort handler
077     * @param reason The reason to show on console. Can be {@code null} if no reason is given
078     * @return A reader reading the input stream (servers answer) or <code>null</code>.
079     * @throws OsmTransferException if data transfer errors occur
080     */
081    protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
082        try {
083            api.initialize(progressMonitor);
084            String url = urlStr.startsWith("http") ? urlStr : (getBaseUrl() + urlStr);
085            return getInputStreamRaw(url, progressMonitor, reason);
086        } finally {
087            progressMonitor.invalidate();
088        }
089    }
090
091    /**
092     * Return the base URL for relative URL requests
093     * @return base url of API
094     */
095    protected String getBaseUrl() {
096        return api.getBaseUrl();
097    }
098
099    /**
100     * Open a connection to the given url and return a reader on the input stream
101     * from that connection. In case of user cancel, return <code>null</code>.
102     * @param urlStr The exact url to connect to.
103     * @param progressMonitor progress monitoring and abort handler
104     * @return An reader reading the input stream (servers answer) or <code>null</code>.
105     * @throws OsmTransferException if data transfer errors occur
106     */
107    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
108        return getInputStreamRaw(urlStr, progressMonitor, null);
109    }
110
111    /**
112     * Open a connection to the given url and return a reader on the input stream
113     * from that connection. In case of user cancel, return <code>null</code>.
114     * @param urlStr The exact url to connect to.
115     * @param progressMonitor progress monitoring and abort handler
116     * @param reason The reason to show on console. Can be {@code null} if no reason is given
117     * @return An reader reading the input stream (servers answer) or <code>null</code>.
118     * @throws OsmTransferException if data transfer errors occur
119     */
120    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
121        return getInputStreamRaw(urlStr, progressMonitor, reason, false);
122    }
123
124    /**
125     * Open a connection to the given url (if HTTP, trough a GET request) and return a reader on the input stream
126     * from that connection. In case of user cancel, return <code>null</code>.
127     * @param urlStr The exact url to connect to.
128     * @param progressMonitor progress monitoring and abort handler
129     * @param reason The reason to show on console. Can be {@code null} if no reason is given
130     * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
131     *                                                for {@code filename} and uncompress a gzip/bzip2/xz/zip stream.
132     * @return An reader reading the input stream (servers answer) or <code>null</code>.
133     * @throws OsmTransferException if data transfer errors occur
134     */
135    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
136            boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
137        return getInputStreamRaw(urlStr, progressMonitor, reason, uncompressAccordingToContentDisposition, "GET", null);
138    }
139
140    /**
141     * Open a connection to the given url (if HTTP, with the specified method) and return a reader on the input stream
142     * from that connection. In case of user cancel, return <code>null</code>.
143     * @param urlStr The exact url to connect to.
144     * @param progressMonitor progress monitoring and abort handler
145     * @param reason The reason to show on console. Can be {@code null} if no reason is given
146     * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
147     *                                                for {@code filename} and uncompress a gzip/bzip2/xz/zip stream.
148     * @param httpMethod HTTP method ("GET", "POST" or "PUT")
149     * @param requestBody HTTP request body (for "POST" and "PUT" methods only). Must be null for "GET" method.
150     * @return An reader reading the input stream (servers answer) or <code>null</code>.
151     * @throws OsmTransferException if data transfer errors occur
152     * @since 12596
153     */
154    @SuppressWarnings("resource")
155    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
156            boolean uncompressAccordingToContentDisposition, String httpMethod, byte[] requestBody) throws OsmTransferException {
157        try {
158            OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlStr, Config.getUrls().getJOSMWebsite());
159            OnlineResource.OSM_API.checkOfflineAccess(urlStr, OsmApi.getOsmApi().getServerUrl());
160
161            URL url = null;
162            try {
163                url = new URL(urlStr.replace(" ", "%20"));
164            } catch (MalformedURLException e) {
165                throw new OsmTransferException(e);
166            }
167
168            String protocol = url.getProtocol();
169            if ("file".equals(protocol) || "jar".equals(protocol)) {
170                try {
171                    return Utils.openStream(url);
172                } catch (IOException e) {
173                    throw new OsmTransferException(e);
174                }
175            }
176
177            final HttpClient client = HttpClient.create(url, httpMethod)
178                    .setFinishOnCloseOutput(false)
179                    .setReasonForRequest(reason)
180                    .setOutputMessage(tr("Downloading data..."))
181                    .setRequestBody(requestBody);
182            activeConnection = client;
183            adaptRequest(client);
184            if (doAuthenticate) {
185                addAuth(client);
186            }
187            if (cancel)
188                throw new OsmTransferCanceledException("Operation canceled");
189
190            final HttpClient.Response response;
191            try {
192                response = client.connect(progressMonitor);
193                contentType = response.getContentType();
194            } catch (IOException e) {
195                Logging.error(e);
196                OsmTransferException ote = new OsmTransferException(
197                        tr("Could not connect to the OSM server. Please check your internet connection."), e);
198                ote.setUrl(url.toString());
199                throw ote;
200            }
201            try {
202                if (response.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
203                    CredentialsManager.getInstance().purgeCredentialsCache(RequestorType.SERVER);
204                    throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED, null, null);
205                }
206
207                if (response.getResponseCode() == HttpURLConnection.HTTP_PROXY_AUTH)
208                    throw new OsmTransferCanceledException("Proxy Authentication Required");
209
210                if (response.getResponseCode() != HttpURLConnection.HTTP_OK) {
211                    String errorHeader = response.getHeaderField("Error");
212                    String errorBody = fetchResponseText(response);
213                    throw new OsmApiException(response.getResponseCode(), errorHeader, errorBody, url.toString(), null,
214                            contentType);
215                }
216
217                response.uncompressAccordingToContentDisposition(uncompressAccordingToContentDisposition);
218                return response.getContent();
219            } catch (OsmTransferException e) {
220                throw e;
221            } catch (IOException e) {
222                throw new OsmTransferException(e);
223            }
224        } finally {
225            progressMonitor.invalidate();
226        }
227    }
228
229    private static String fetchResponseText(final HttpClient.Response response) {
230        try {
231            return response.fetchContent();
232        } catch (IOException e) {
233            Logging.error(e);
234            return tr("Reading error text failed.");
235        }
236    }
237
238    /**
239     * Allows subclasses to modify the request.
240     * @param request the prepared request
241     * @since 9308
242     */
243    protected void adaptRequest(HttpClient request) {
244    }
245
246    /**
247     * Download OSM files from somewhere
248     * @param progressMonitor The progress monitor
249     * @return The corresponding dataset
250     * @throws OsmTransferException if any error occurs
251     */
252    public abstract DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException;
253
254    /**
255     * Download compressed OSM files from somewhere
256     * @param progressMonitor The progress monitor
257     * @param compression compression to use
258     * @return The corresponding dataset
259     * @throws OsmTransferException if any error occurs
260     * @since 13352
261     */
262    public DataSet parseOsm(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
263        throw new UnsupportedOperationException();
264    }
265
266    /**
267     * Download OSM Change uncompressed files from somewhere
268     * @param progressMonitor The progress monitor
269     * @return The corresponding dataset
270     * @throws OsmTransferException if any error occurs
271     */
272    public DataSet parseOsmChange(ProgressMonitor progressMonitor) throws OsmTransferException {
273        return null;
274    }
275
276    /**
277     * Download OSM Change compressed files from somewhere
278     * @param progressMonitor The progress monitor
279     * @param compression compression to use
280     * @return The corresponding dataset
281     * @throws OsmTransferException if any error occurs
282     * @since 13352
283     */
284    public DataSet parseOsmChange(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
285        throw new UnsupportedOperationException();
286    }
287
288    /**
289     * Retrieve raw gps waypoints from the server API.
290     * @param progressMonitor The progress monitor
291     * @return The corresponding GPX tracks
292     * @throws OsmTransferException if any error occurs
293     */
294    public GpxData parseRawGps(ProgressMonitor progressMonitor) throws OsmTransferException {
295        return null;
296    }
297
298    /**
299     * Retrieve compressed GPX files from somewhere.
300     * @param progressMonitor The progress monitor
301     * @param compression compression to use
302     * @return The corresponding GPX tracks
303     * @throws OsmTransferException if any error occurs
304     * @since 13352
305     */
306    public GpxData parseRawGps(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
307        throw new UnsupportedOperationException();
308    }
309
310    /**
311     * Returns true if this reader is adding authentication credentials to the read
312     * request sent to the server.
313     *
314     * @return true if this reader is adding authentication credentials to the read
315     * request sent to the server
316     */
317    public boolean isDoAuthenticate() {
318        return doAuthenticate;
319    }
320
321    /**
322     * Sets whether this reader adds authentication credentials to the read
323     * request sent to the server.
324     *
325     * @param doAuthenticate  true if  this reader adds authentication credentials to the read
326     * request sent to the server
327     */
328    public void setDoAuthenticate(boolean doAuthenticate) {
329        this.doAuthenticate = doAuthenticate;
330    }
331
332    /**
333     * Determines if the GPX data has been parsed properly.
334     * @return true if the GPX data has been parsed properly, false otherwise
335     * @see GpxReader#parse
336     */
337    public final boolean isGpxParsedProperly() {
338        return gpxParsedProperly;
339    }
340
341    /**
342     * Downloads notes from the API, given API limit parameters
343     *
344     * @param noteLimit How many notes to download.
345     * @param daysClosed Return notes closed this many days in the past. -1 means all notes, ever. 0 means only unresolved notes.
346     * @param progressMonitor Progress monitor for user feedback
347     * @return List of notes returned by the API
348     * @throws OsmTransferException if any errors happen
349     */
350    public List<Note> parseNotes(int noteLimit, int daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException {
351        return null;
352    }
353
354    /**
355     * Downloads notes from a given raw URL. The URL is assumed to be complete and no API limits are added
356     *
357     * @param progressMonitor progress monitor
358     * @return A list of notes parsed from the URL
359     * @throws OsmTransferException if any error occurs during dialog with OSM API
360     */
361    public List<Note> parseRawNotes(final ProgressMonitor progressMonitor) throws OsmTransferException {
362        return null;
363    }
364
365    /**
366     * Download notes from a URL that contains a compressed notes dump file
367     * @param progressMonitor progress monitor
368     * @param compression compression to use
369     * @return A list of notes parsed from the URL
370     * @throws OsmTransferException if any error occurs during dialog with OSM API
371     * @since 13352
372     */
373    public List<Note> parseRawNotes(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
374        throw new UnsupportedOperationException();
375    }
376
377    /**
378     * Returns an attribute from the given DOM node.
379     * @param node DOM node
380     * @param name attribute name
381     * @return attribute value for the given attribute
382     * @since 12510
383     */
384    protected static String getAttribute(Node node, String name) {
385        return node.getAttributes().getNamedItem(name).getNodeValue();
386    }
387
388    /**
389     * DOM document parser.
390     * @param <R> resulting type
391     * @since 12510
392     */
393    @FunctionalInterface
394    protected interface DomParser<R> {
395        /**
396         * Parses a given DOM document.
397         * @param doc DOM document
398         * @return parsed data
399         * @throws XmlParsingException if an XML parsing error occurs
400         */
401        R parse(Document doc) throws XmlParsingException;
402    }
403
404    /**
405     * Fetches generic data from the DOM document resulting an API call.
406     * @param api the OSM API call
407     * @param subtask the subtask translated message
408     * @param parser the parser converting the DOM document (OSM API result)
409     * @param <T> data type
410     * @param monitor The progress monitor
411     * @param reason The reason to show on console. Can be {@code null} if no reason is given
412     * @return The converted data
413     * @throws OsmTransferException if something goes wrong
414     * @since 12510
415     */
416    public <T> T fetchData(String api, String subtask, DomParser<T> parser, ProgressMonitor monitor, String reason)
417            throws OsmTransferException {
418        try {
419            monitor.beginTask("");
420            monitor.indeterminateSubTask(subtask);
421            try (InputStream in = getInputStream(api, monitor.createSubTaskMonitor(1, true), reason)) {
422                return parser.parse(XmlUtils.parseSafeDOM(in));
423            }
424        } catch (OsmTransferException e) {
425            throw e;
426        } catch (IOException | ParserConfigurationException | SAXException e) {
427            throw new OsmTransferException(e);
428        } finally {
429            monitor.finishTask();
430        }
431    }
432}