00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027 #include "config.h"
00028
00029 static char rcsid[] not_used =
00030 { "$Id: HTTPConnect.cc 18763 2008-05-20 19:22:36Z jimg $"
00031 };
00032
00033 #ifdef HAVE_UNISTD_H
00034 #include <unistd.h>
00035 #endif
00036
00037 #ifdef WIN32
00038 #include <io.h>
00039 #endif
00040
00041 #include <string>
00042 #include <vector>
00043 #include <functional>
00044 #include <algorithm>
00045 #include <sstream>
00046 #include <iterator>
00047 #include <cstdlib>
00048 #include <cstring>
00049
00050
00051
00052
00053 #include "debug.h"
00054 #include "GNURegex.h"
00055 #include "HTTPCache.h"
00056 #include "HTTPConnect.h"
00057 #include "RCReader.h"
00058 #include "HTTPResponse.h"
00059 #include "HTTPCacheResponse.h"
00060
00061 using namespace std;
00062
00063 namespace libdap {
00064
00065
00066
00067
00068
00069
00070
00071 int www_trace = 0;
00072
00073
00074 int dods_keep_temps = 0;
00075
00076 #define CLIENT_ERR_MIN 400
00077 #define CLIENT_ERR_MAX 417
00078 static char *http_client_errors[CLIENT_ERR_MAX - CLIENT_ERR_MIN +1] =
00079 {
00080 "Bad Request:",
00081 "Unauthorized: Contact the server administrator.",
00082 "Payment Required.",
00083 "Forbidden: Contact the server administrator.",
00084 "Not Found: The data source or server could not be found.\n\
00085 Often this means that the OPeNDAP server is missing or needs attention;\n\
00086 Please contact the server administrator.",
00087 "Method Not Allowed.",
00088 "Not Acceptable.",
00089 "Proxy Authentication Required.",
00090 "Request Time-out.",
00091 "Conflict.",
00092 "Gone:.",
00093 "Length Required.",
00094 "Precondition Failed.",
00095 "Request Entity Too Large.",
00096 "Request URI Too Large.",
00097 "Unsupported Media Type.",
00098 "Requested Range Not Satisfiable.",
00099 "Expectation Failed."
00100 };
00101
00102 #define SERVER_ERR_MIN 500
00103 #define SERVER_ERR_MAX 505
00104 static char *http_server_errors[SERVER_ERR_MAX - SERVER_ERR_MIN +1] =
00105 {
00106 "Internal Server Error.",
00107 "Not Implemented.",
00108 "Bad Gateway.",
00109 "Service Unavailable.",
00110 "Gateway Time-out.",
00111 "HTTP Version Not Supported."
00112 };
00113
00116 static string
00117 http_status_to_string(int status)
00118 {
00119 if (status >= CLIENT_ERR_MIN && status <= CLIENT_ERR_MAX)
00120 return string(http_client_errors[status - CLIENT_ERR_MIN]);
00121 else if (status >= SERVER_ERR_MIN && status <= SERVER_ERR_MAX)
00122 return string(http_server_errors[status - SERVER_ERR_MIN]);
00123 else
00124 return string("Unknown Error: This indicates a problem with libdap++.\nPlease report this to support@opendap.org.");
00125 }
00126
00131 class ParseHeader : public unary_function<const string &, void>
00132 {
00133 ObjectType type;
00134 string server;
00135 string protocol;
00136 string location;
00137
00138 public:
00139 ParseHeader() : type(unknown_type), server("dods/0.0"), protocol("2.0")
00140 { }
00141
00142 void operator()(const string &header)
00143 {
00144 std::istringstream line(header);
00145
00146 string name;
00147 line >> name;
00148 downcase(name);
00149 if (name == "content-description:") {
00150 string value;
00151 line >> value;
00152 downcase(value);
00153 DBG2(cout << name << ": " << value << endl);
00154 type = get_type(value);
00155 }
00156
00157
00158
00159 else if (name == "xdods-server:" && server == "dods/0.0") {
00160 string value;
00161 line >> value;
00162 downcase(value);
00163 DBG2(cout << name << ": " << value << endl);
00164 server = value;
00165 }
00166 else if (name == "xopendap-server:") {
00167 string value;
00168 line >> value;
00169 downcase(value);
00170 DBG2(cout << name << ": " << value << endl);
00171 server = value;
00172 }
00173 else if (name == "xdap:") {
00174 string value;
00175 line >> value;
00176 downcase(value);
00177 DBG2(cout << name << ": " << value << endl);
00178 protocol = value;
00179 }
00180 else if (server == "dods/0.0" && name == "server:") {
00181 string value;
00182 line >> value;
00183 downcase(value);
00184 DBG2(cout << name << ": " << value << endl);
00185 server = value;
00186 }
00187 else if (name == "location:") {
00188 string value;
00189 line >> value;
00190 DBG2(cout << name << ": " << value << endl);
00191 location = value;
00192 }
00193 else if (type == unknown_type && name == "content-type:"
00194 && line.str().find("text/html") != string::npos) {
00195 DBG2(cout << name << ": text/html..." << endl);
00196 type = web_error;
00197 }
00198 }
00199
00200 ObjectType get_object_type()
00201 {
00202 return type;
00203 }
00204
00205 string get_server()
00206 {
00207 return server;
00208 }
00209
00210 string get_protocol()
00211 {
00212 return protocol;
00213 }
00214
00215 string get_location() {
00216 return location;
00217 }
00218 };
00219
00236 static size_t
00237 save_raw_http_headers(void *ptr, size_t size, size_t nmemb, void *resp_hdrs)
00238 {
00239 DBG2(cerr << "Inside the header parser." << endl);
00240 vector<string> *hdrs = static_cast<vector<string> * >(resp_hdrs);
00241
00242
00243 string complete_line;
00244 if (nmemb > 1 && *(static_cast<char*>(ptr) + size * (nmemb - 2)) == '\r')
00245 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 2));
00246 else
00247 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 1));
00248
00249
00250 if (complete_line != "" && complete_line.find("HTTP") == string::npos) {
00251 DBG(cerr << "Header line: " << complete_line << endl);
00252 hdrs->push_back(complete_line);
00253 }
00254
00255 return size * nmemb;
00256 }
00257
00259 static int
00260 curl_debug(CURL *, curl_infotype info, char *msg, size_t size, void *)
00261 {
00262 string message(msg, size);
00263
00264 switch (info) {
00265 case CURLINFO_TEXT:
00266 cerr << "Text: " << message; break;
00267 case CURLINFO_HEADER_IN:
00268 cerr << "Header in: " << message; break;
00269 case CURLINFO_HEADER_OUT:
00270 cerr << "Header out: " << message; break;
00271 case CURLINFO_DATA_IN:
00272 cerr << "Data in: " << message; break;
00273 case CURLINFO_DATA_OUT:
00274 cerr << "Data out: " << message; break;
00275 case CURLINFO_END:
00276 cerr << "End: " << message; break;
00277 #ifdef CURLINFO_SSL_DATA_IN
00278 case CURLINFO_SSL_DATA_IN:
00279 cerr << "SSL Data in: " << message; break;
00280 #endif
00281 #ifdef CURLINFO_SSL_DATA_OUT
00282 case CURLINFO_SSL_DATA_OUT:
00283 cerr << "SSL Data out: " << message; break;
00284 #endif
00285 default:
00286 cerr << "Curl info: " << message; break;
00287 }
00288 return 0;
00289 }
00290
00294 void
00295 HTTPConnect::www_lib_init()
00296 {
00297 d_curl = curl_easy_init();
00298 if (!d_curl)
00299 throw InternalErr(__FILE__, __LINE__, "Could not initialize libcurl.");
00300
00301
00302
00303
00304
00305 if (!d_rcr->get_proxy_server_host().empty()) {
00306 DBG(cerr << "Setting up a proxy server." << endl);
00307 DBG(cerr << "Proxy host: " << d_rcr->get_proxy_server_host()
00308 << endl);
00309 DBG(cerr << "Proxy port: " << d_rcr->get_proxy_server_port()
00310 << endl);
00311 DBG(cerr << "Proxy pwd : " << d_rcr->get_proxy_server_userpw()
00312 << endl);
00313 curl_easy_setopt(d_curl, CURLOPT_PROXY,
00314 d_rcr->get_proxy_server_host().c_str());
00315 curl_easy_setopt(d_curl, CURLOPT_PROXYPORT,
00316 d_rcr->get_proxy_server_port());
00317
00318
00319 #ifdef CURLOPT_PROXYAUTH
00320 curl_easy_setopt(d_curl, CURLOPT_PROXYAUTH, (long)CURLAUTH_ANY);
00321 #endif
00322
00323
00324 if (!d_rcr->get_proxy_server_userpw().empty())
00325 curl_easy_setopt(d_curl, CURLOPT_PROXYUSERPWD,
00326 d_rcr->get_proxy_server_userpw().c_str());
00327 }
00328
00329 curl_easy_setopt(d_curl, CURLOPT_ERRORBUFFER, d_error_buffer);
00330
00331
00332 curl_easy_setopt(d_curl, CURLOPT_FAILONERROR, 0);
00333
00334
00335
00336
00337 curl_easy_setopt(d_curl, CURLOPT_HTTPAUTH, (long)CURLAUTH_ANY);
00338
00339 curl_easy_setopt(d_curl, CURLOPT_NOPROGRESS, 1);
00340 curl_easy_setopt(d_curl, CURLOPT_NOSIGNAL, 1);
00341 curl_easy_setopt(d_curl, CURLOPT_HEADERFUNCTION, save_raw_http_headers);
00342
00343
00344
00345
00346 curl_easy_setopt(d_curl, CURLOPT_FOLLOWLOCATION, 1);
00347 curl_easy_setopt(d_curl, CURLOPT_MAXREDIRS, 5);
00348
00349
00350 if (!d_rcr->get_validate_ssl() == 0) {
00351 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYPEER, 0);
00352 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYHOST, 0);
00353 }
00354
00355 if (www_trace) {
00356 cerr << "Curl version: " << curl_version() << endl;
00357 curl_easy_setopt(d_curl, CURLOPT_VERBOSE, 1);
00358 curl_easy_setopt(d_curl, CURLOPT_DEBUGFUNCTION, curl_debug);
00359 }
00360 }
00361
00365 class BuildHeaders : public unary_function<const string &, void>
00366 {
00367 struct curl_slist *d_cl;
00368
00369 public:
00370 BuildHeaders() : d_cl(0)
00371 {}
00372
00373 void operator()(const string &header)
00374 {
00375 DBG(cerr << "Adding '" << header.c_str() << "' to the header list."
00376 << endl);
00377 d_cl = curl_slist_append(d_cl, header.c_str());
00378 }
00379
00380 struct curl_slist *get_headers()
00381 {
00382 return d_cl;
00383 }
00384 };
00385
00400 long
00401 HTTPConnect::read_url(const string &url, FILE *stream,
00402 vector<string> *resp_hdrs,
00403 const vector<string> *headers)
00404 {
00405 curl_easy_setopt(d_curl, CURLOPT_URL, url.c_str());
00406
00407 #ifdef WIN32
00408
00409
00410
00411
00412
00413
00414
00415 curl_easy_setopt(d_curl, CURLOPT_FILE, stream);
00416 curl_easy_setopt(d_curl, CURLOPT_WRITEFUNCTION, &fwrite);
00417 #else
00418 curl_easy_setopt(d_curl, CURLOPT_FILE, stream);
00419 #endif
00420
00421 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
00422 ostream_iterator<string>(cerr, "\n")));
00423
00424 BuildHeaders req_hdrs;
00425 req_hdrs = for_each(d_request_headers.begin(), d_request_headers.end(),
00426 req_hdrs);
00427 if (headers)
00428 req_hdrs = for_each(headers->begin(), headers->end(), req_hdrs);
00429 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, req_hdrs.get_headers());
00430
00431 if (d_accept_deflate)
00432 curl_easy_setopt(d_curl, CURLOPT_ENCODING, "deflate");
00433
00434
00435 bool temporary_proxy = false;
00436 if ((temporary_proxy = url_uses_no_proxy_for(url))) {
00437 DBG(cerr << "Suppress proxy for url: " << url << endl);
00438 curl_easy_setopt(d_curl, CURLOPT_PROXY, 0);
00439 }
00440
00441 string::size_type at_sign = url.find('@');
00442
00443
00444
00445 if (at_sign != url.npos)
00446 d_upstring = url.substr(7, at_sign - 7);
00447
00448 if (!d_upstring.empty())
00449 curl_easy_setopt(d_curl, CURLOPT_USERPWD, d_upstring.c_str());
00450
00451
00452
00453
00454 curl_easy_setopt(d_curl, CURLOPT_WRITEHEADER, resp_hdrs);
00455
00456 CURLcode res = curl_easy_perform(d_curl);
00457
00458
00459 curl_slist_free_all(req_hdrs.get_headers());
00460 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, 0);
00461
00462
00463 if (temporary_proxy && !d_rcr->get_proxy_server_host().empty())
00464 curl_easy_setopt(d_curl, CURLOPT_PROXY,
00465 d_rcr->get_proxy_server_host().c_str());
00466
00467 if (res != 0)
00468 throw Error(d_error_buffer);
00469
00470 long status;
00471 res = curl_easy_getinfo(d_curl, CURLINFO_HTTP_CODE, &status);
00472 if (res != 0)
00473 throw Error(d_error_buffer);
00474
00475 return status;
00476 }
00477
00481 bool
00482 HTTPConnect::url_uses_proxy_for(const string &url) throw()
00483 {
00484 if (d_rcr->is_proxy_for_used()) {
00485 Regex host_regex(d_rcr->get_proxy_for_regexp().c_str());
00486 int index = 0, matchlen;
00487 return host_regex.search(url.c_str(), url.size(), matchlen, index)
00488 != -1;
00489 }
00490
00491 return false;
00492 }
00493
00497 bool
00498 HTTPConnect::url_uses_no_proxy_for(const string &url) throw()
00499 {
00500 return d_rcr->is_no_proxy_for_used()
00501 && url.find(d_rcr->get_no_proxy_for_host()) != string::npos;
00502 }
00503
00504
00505
00512 HTTPConnect::HTTPConnect(RCReader *rcr) : d_username(""), d_password("")
00513 {
00514 d_accept_deflate = rcr->get_deflate();
00515 d_rcr = rcr;
00516
00517
00518
00519
00520
00521 d_request_headers.push_back(string("Pragma:"));
00522 string user_agent = string("User-Agent: ") + string(CNAME)
00523 + string("/") + string(CVER);
00524 d_request_headers.push_back(user_agent);
00525 if (d_accept_deflate)
00526 d_request_headers.push_back(string("Accept-Encoding: deflate, gzip, compress"));
00527
00528
00529 if (d_rcr->get_use_cache())
00530 d_http_cache = HTTPCache::instance(d_rcr->get_dods_cache_root(),
00531 true);
00532 else
00533 d_http_cache = 0;
00534
00535 DBG2(cerr << "Cache object created (" << hex << d_http_cache << dec
00536 << ")" << endl);
00537
00538 if (d_http_cache) {
00539 d_http_cache->set_cache_enabled(d_rcr->get_use_cache());
00540 d_http_cache->set_expire_ignored(d_rcr->get_ignore_expires() != 0);
00541 d_http_cache->set_max_size(d_rcr->get_max_cache_size());
00542 d_http_cache->set_max_entry_size(d_rcr->get_max_cached_obj());
00543 d_http_cache->set_default_expiration(d_rcr->get_default_expires());
00544 d_http_cache->set_always_validate(d_rcr->get_always_validate() != 0);
00545 }
00546
00547 www_lib_init();
00548 }
00549
00550 HTTPConnect::~HTTPConnect()
00551 {
00552 DBG2(cerr << "Entering the HTTPConnect dtor" << endl);
00553
00554 curl_easy_cleanup(d_curl);
00555
00556 DBG2(cerr << "Leaving the HTTPConnect dtor" << endl);
00557 }
00558
00571 HTTPResponse *
00572 HTTPConnect::fetch_url(const string &url)
00573 {
00574 #ifdef HTTP_TRACE
00575 cout << "GET " << url << " HTTP/1.0" << endl;
00576 #endif
00577
00578 HTTPResponse *stream;
00579
00580 if (d_http_cache && d_http_cache->is_cache_enabled()) {
00581 stream = caching_fetch_url(url);
00582 }
00583 else {
00584 stream = plain_fetch_url(url);
00585 }
00586
00587 #ifdef HTTP_TRACE
00588 stringstream ss;
00589 ss << "HTTP/1.0 " << stream->get_status() << " -" << endl;
00590 for (size_t i = 0; i < stream->get_headers()->size(); i++) {
00591 ss << stream->get_headers()->at(i) << endl;
00592 }
00593 cout << ss.str();
00594 #endif
00595
00596 ParseHeader parser;
00597
00598 parser = for_each(stream->get_headers()->begin(),
00599 stream->get_headers()->end(), ParseHeader());
00600
00601 #ifdef HTTP_TRACE
00602 cout << endl << endl;
00603 #endif
00604
00605
00606 if (parser.get_location() != "" &&
00607 url.substr(0,url.find("?",0)).compare(parser.get_location().substr(0,url.find("?",0))) != 0) {
00608 return fetch_url(parser.get_location());
00609 }
00610
00611 stream->set_type(parser.get_object_type());
00612 stream->set_version(parser.get_server());
00613 stream->set_protocol(parser.get_protocol());
00614
00615 return stream;
00616 }
00617
00618
00619
00620
00621
00622
00623
00624
00625
00626
00627
00628
00629
00630
00631 static char *
00632 get_tempfile_template(char *file_template)
00633 {
00634 char *c;
00635
00636 #ifdef WIN32
00637
00638 Regex directory("[-a-zA-Z0-9_\\]*");
00639
00640 c = getenv("TEMP");
00641 if (c && directory.match(c, strlen(c)) && (access(getenv("TEMP"), 6) == 0))
00642 goto valid_temp_directory;
00643
00644 c= getenv("TMP");
00645 if (c && directory.match(c, strlen(c)) && (access(getenv("TEMP"), 6) == 0))
00646 goto valid_temp_directory;
00647 #else
00648
00649 Regex directory("[-a-zA-Z0-9_/]*");
00650
00651 c = getenv("TMPDIR");
00652 if (c && directory.match(c, strlen(c)) && (access(c, W_OK | R_OK) == 0))
00653 goto valid_temp_directory;
00654
00655 #ifdef P_tmpdir
00656 if (access(P_tmpdir, W_OK | R_OK) == 0) {
00657 c = P_tmpdir;
00658 goto valid_temp_directory;
00659 }
00660 #endif
00661
00662 #endif // WIN32
00663
00664 c = ".";
00665
00666 valid_temp_directory:
00667
00668 int size = strlen(c) + strlen(file_template) + 2;
00669 if (!size_ok(1, size))
00670 throw Error("Bad temporary file name.");
00671
00672 char *temp = new char[size];
00673 strncpy(temp, c, size-2);
00674 strcat(temp, "/");
00675
00676 strcat(temp, file_template);
00677
00678 return temp;
00679 }
00680
00699 string
00700 get_temp_file(FILE *&stream) throw(InternalErr)
00701 {
00702
00703 char *dods_temp = get_tempfile_template("dodsXXXXXX");
00704
00705
00706 #if defined(WIN32) || defined(TEST_WIN32_TEMPS)
00707 stream = fopen(_mktemp(dods_temp), "w+b");
00708 #else
00709 stream = fdopen(mkstemp(dods_temp), "w+");
00710 #endif
00711
00712 if (!stream)
00713 throw InternalErr("I/O Error: Failed to open a temporary file for the data values.");
00714
00715 string dods_temp_s = dods_temp;
00716 delete[] dods_temp; dods_temp = 0;
00717
00718 return dods_temp_s;
00719 }
00720
00722 void
00723 close_temp(FILE *s, const string &name)
00724 {
00725 int res = fclose(s);
00726 if (res)
00727 DBG(cerr << "Failed to close " << (void *)s << endl);
00728
00729 unlink(name.c_str());
00730 }
00731
00753 HTTPResponse * HTTPConnect::caching_fetch_url(const string &url) {
00754 DBG(cerr << "Is this URL (" << url << ") in the cache?... ");
00755
00756 vector<string> *headers = new vector<string>;
00757 FILE *s = d_http_cache->get_cached_response(url, *headers);
00758 if (!s) {
00759
00760 DBGN(cerr << "no; getting response and caching." << endl);
00761 time_t now = time(0);
00762 HTTPResponse *rs = plain_fetch_url(url);
00763 d_http_cache->cache_response(url, now, *(rs->get_headers()),
00764 rs->get_stream());
00765
00766 return rs;
00767 }
00768 else {
00769 DBGN(cerr << "yes... ");
00770
00771 if (d_http_cache->is_url_valid(url)) {
00772 DBGN(cerr << "and it's valid; using cached response." << endl);
00773 HTTPCacheResponse *crs = new HTTPCacheResponse(s, 200, headers, d_http_cache);
00774 return crs;
00775 }
00776 else {
00777 DBGN(cerr << "but it's not valid; validating... ");
00778
00779 d_http_cache->release_cached_response(s);
00780
00781 vector<string> *resp_hdrs = new vector<string>;
00782 vector<string> cond_hdrs =
00783 d_http_cache->get_conditional_request_headers(url);
00784 FILE *body = 0;
00785 string dods_temp = get_temp_file(body);
00786 time_t now = time(0);
00787 long http_status;
00788
00789 try {
00790 http_status = read_url(url, body, resp_hdrs, &cond_hdrs);
00791 rewind(body);
00792 }
00793 catch (Error &e) {
00794 close_temp(body, dods_temp);
00795 throw;
00796 }
00797
00798 switch (http_status) {
00799 case 200: {
00800 DBGN(cerr << "read a new response; caching." << endl);
00801
00802 d_http_cache->cache_response(url, now, *resp_hdrs, body);
00803 HTTPResponse *rs = new HTTPResponse(body, http_status, resp_hdrs,
00804 dods_temp);
00805
00806 return rs;
00807 }
00808
00809 case 304: {
00810 DBGN(cerr << "cached response valid; updating." << endl);
00811
00812 close_temp(body, dods_temp);
00813 d_http_cache->update_response(url, now, *resp_hdrs);
00814
00815 vector<string> *headers = new vector<string>;
00816 FILE *hs = d_http_cache->get_cached_response(url, *headers);
00817 HTTPCacheResponse *crs = new HTTPCacheResponse(hs, 304, headers, d_http_cache);
00818 return crs;
00819 }
00820
00821 default: {
00822 close_temp(body, dods_temp);
00823 if (http_status >= 400) {
00824 string msg = "Error while reading the URL: ";
00825 msg += url;
00826 msg
00827 += ".\nThe OPeNDAP server returned the following message:\n";
00828 msg += http_status_to_string(http_status);
00829 throw Error(msg);
00830 } else {
00831 throw InternalErr(__FILE__, __LINE__,
00832 "Bad response from the HTTP server: " + long_to_string(http_status));
00833 }
00834 }
00835
00836 }
00837 }
00838 }
00839
00840 throw InternalErr(__FILE__, __LINE__, "Should never get here");
00841 }
00842
00843 #if 0
00844 HTTPResponse * HTTPConnect::caching_fetch_url(const string &url) {
00845 DBG(cerr << "Is this URL (" << url << ") in the cache?... ");
00846
00847 if (d_http_cache->is_url_in_cache(url)) {
00848 DBGN(cerr << "yes... ");
00849
00850 if (d_http_cache->is_url_valid(url)) {
00851 DBGN(cerr << "and it's valid; using cached response." << endl);
00852
00853 vector<string> *headers = new vector<string>;
00854 FILE *s = d_http_cache->get_cached_response(url, *headers);
00855 HTTPCacheResponse *crs = new HTTPCacheResponse(s, 200, headers, d_http_cache);
00856
00857 return crs;
00858 } else {
00859 DBGN(cerr << "but it's not valid; validating... ");
00860
00861
00862 vector<string> *resp_hdrs = new vector<string>;
00863 vector<string> cond_hdrs =
00864 d_http_cache->get_conditional_request_headers(url);
00865 FILE *body = 0;
00866 string dods_temp = get_temp_file(body);
00867 time_t now = time(0);
00868 long http_status;
00869
00870 try {
00871 http_status = read_url(url, body, resp_hdrs, &cond_hdrs);
00872 rewind(body);
00873 }
00874 catch (Error &e) {
00875 close_temp(body, dods_temp);
00876 throw;
00877 }
00878
00879 switch (http_status) {
00880 case 200: {
00881 DBGN(cerr << "read a new response; caching." << endl);
00882
00883 d_http_cache->cache_response(url, now, *resp_hdrs, body);
00884 HTTPResponse *rs = new HTTPResponse(body, http_status, resp_hdrs,
00885 dods_temp);
00886
00887 return rs;
00888 }
00889 break;
00890
00891 case 304: {
00892 DBGN(cerr << "cached response valid; updating." << endl);
00893
00894 close_temp(body, dods_temp);
00895 d_http_cache->update_response(url, now, *resp_hdrs);
00896
00897 vector<string> *headers = new vector<string>;
00898 FILE *s = d_http_cache->get_cached_response(url, *headers);
00899 HTTPCacheResponse *crs = new HTTPCacheResponse(s, 304, headers, d_http_cache);
00900 return crs;
00901 }
00902 break;
00903
00904 default: {
00905 close_temp(body, dods_temp);
00906 if (http_status >= 400) {
00907 string msg = "Error while reading the URL: ";
00908 msg += url;
00909 msg += ".\nThe OPeNDAP server returned the following message:\n";
00910 msg += http_status_to_string(http_status);
00911 throw Error(msg);
00912 } else {
00913 throw InternalErr(__FILE__, __LINE__,
00914 "Bad response from the HTTP server: " + long_to_string(http_status));
00915 }
00916 }
00917 break;
00918 }
00919 }
00920 } else {
00921 DBGN(cerr << "no; getting response and caching." << endl);
00922 time_t now = time(0);
00923 HTTPResponse *rs = plain_fetch_url(url);
00924 d_http_cache->cache_response(url, now, *(rs->get_headers()),
00925 rs->get_stream());
00926
00927 return rs;
00928 }
00929
00930 throw InternalErr(__FILE__, __LINE__, "Unexpected cache response.");
00931 }
00932 #endif
00933
00945 HTTPResponse *
00946 HTTPConnect::plain_fetch_url(const string &url)
00947 {
00948 DBG(cerr << "Getting URL: " << url << endl);
00949 FILE *stream = 0;
00950 string dods_temp = get_temp_file(stream);
00951 vector<string> *resp_hdrs = new vector<string>;
00952
00953 int status = -1;
00954 try {
00955 status = read_url(url, stream, resp_hdrs);
00956 if (status >= 400) {
00957 string msg = "Error while reading the URL: ";
00958 msg += url;
00959 msg += ".\nThe OPeNDAP server returned the following message:\n";
00960 msg += http_status_to_string(status);
00961 throw Error(msg);
00962 }
00963 }
00964
00965 catch (Error &e) {
00966 close_temp(stream, dods_temp);
00967 throw e;
00968 }
00969
00970 rewind(stream);
00971
00972 return new HTTPResponse(stream, status, resp_hdrs, dods_temp);
00973 }
00974
00986 void
00987 HTTPConnect::set_accept_deflate(bool deflate)
00988 {
00989 d_accept_deflate = deflate;
00990
00991 if (d_accept_deflate) {
00992 if (find(d_request_headers.begin(), d_request_headers.end(),
00993 "Accept-Encoding: deflate, gzip, compress") == d_request_headers.end())
00994 d_request_headers.push_back(string("Accept-Encoding: deflate, gzip, compress"));
00995 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
00996 ostream_iterator<string>(cerr, "\n")));
00997 }
00998 else {
00999 vector<string>::iterator i;
01000 i = remove_if(d_request_headers.begin(), d_request_headers.end(),
01001 bind2nd(equal_to<string>(),
01002 string("Accept-Encoding: deflate, gzip, compress")));
01003 d_request_headers.erase(i, d_request_headers.end());
01004 }
01005 }
01006
01022 void
01023 HTTPConnect::set_credentials(const string &u, const string &p)
01024 {
01025 if (u.empty())
01026 return;
01027
01028
01029 d_username = u;
01030 d_password = p;
01031
01032 d_upstring = u + ":" + p;
01033 }
01034
01035 }