81 #define CLIENT_ERR_MIN 400
82 #define CLIENT_ERR_MAX 417
86 "Unauthorized: Contact the server administrator.",
88 "Forbidden: Contact the server administrator.",
89 "Not Found: The data source or server could not be found.\n\
90 Often this means that the OPeNDAP server is missing or needs attention;\n\
91 Please contact the server administrator.",
92 "Method Not Allowed.",
94 "Proxy Authentication Required.",
99 "Precondition Failed.",
100 "Request Entity Too Large.",
101 "Request URI Too Large.",
102 "Unsupported Media Type.",
103 "Requested Range Not Satisfiable.",
104 "Expectation Failed."
107 #define SERVER_ERR_MIN 500
108 #define SERVER_ERR_MAX 505
111 "Internal Server Error.",
114 "Service Unavailable.",
116 "HTTP Version Not Supported."
122 http_status_to_string(
int status)
129 return string(
"Unknown Error: This indicates a problem with libdap++.\nPlease report this to support@opendap.org.");
133 determine_object_type(
const string &header_value)
138 string::size_type plus = header_value.find(
'+');
140 string type_extension =
"";
141 if (plus != string::npos) {
142 base_type= header_value.substr(0, plus);
143 type_extension = header_value.substr(plus+1);
146 base_type = header_value;
148 if (base_type == DMR_Content_Type
149 || (base_type.find(
"application/") != string::npos
150 && base_type.find(
"dap4.dataset-metadata") != string::npos)) {
151 if (type_extension ==
"xml")
156 else if (base_type == DAP4_DATA_Content_Type
157 || (base_type.find(
"application/") != string::npos
158 && base_type.find(
"dap4.data") != string::npos)) {
161 else if (header_value.find(
"text/html") != string::npos) {
172 class ParseHeader :
public unary_function<const string &, void>
180 ParseHeader() : type(
unknown_type), server(
"dods/0.0"), protocol(
"2.0")
183 void operator()(
const string &line)
188 DBG2(cerr << name <<
": " << value << endl);
194 type = determine_object_type(value);
202 else if (name ==
"xdods-server" && server ==
"dods/0.0") {
205 else if (name ==
"xopendap-server") {
208 else if (name ==
"xdap") {
211 else if (server ==
"dods/0.0" && name ==
"server") {
214 else if (name ==
"location") {
229 string get_protocol()
234 string get_location() {
255 save_raw_http_headers(
void *ptr,
size_t size,
size_t nmemb,
void *resp_hdrs)
257 DBG2(cerr <<
"Inside the header parser." << endl);
258 vector<string> *hdrs =
static_cast<vector<string> *
>(resp_hdrs);
261 string complete_line;
262 if (nmemb > 1 && *(static_cast<char*>(ptr) + size * (nmemb - 2)) ==
'\r')
263 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 2));
265 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 1));
268 if (complete_line !=
"" && complete_line.find(
"HTTP") == string::npos) {
269 DBG(cerr <<
"Header line: " << complete_line << endl);
270 hdrs->push_back(complete_line);
278 curl_debug(CURL *, curl_infotype info,
char *msg,
size_t size,
void *)
280 string message(msg, size);
284 cerr <<
"Text: " << message;
break;
285 case CURLINFO_HEADER_IN:
286 cerr <<
"Header in: " << message;
break;
287 case CURLINFO_HEADER_OUT:
288 cerr <<
"Header out: " << message;
break;
289 case CURLINFO_DATA_IN:
290 cerr <<
"Data in: " << message;
break;
291 case CURLINFO_DATA_OUT:
292 cerr <<
"Data out: " << message;
break;
294 cerr <<
"End: " << message;
break;
295 #ifdef CURLINFO_SSL_DATA_IN
296 case CURLINFO_SSL_DATA_IN:
297 cerr <<
"SSL Data in: " << message;
break;
299 #ifdef CURLINFO_SSL_DATA_OUT
300 case CURLINFO_SSL_DATA_OUT:
301 cerr <<
"SSL Data out: " << message;
break;
304 cerr <<
"Curl info: " << message;
break;
313 HTTPConnect::www_lib_init()
315 d_curl = curl_easy_init();
317 throw InternalErr(__FILE__, __LINE__,
"Could not initialize libcurl.");
323 if (!d_rcr->get_proxy_server_host().empty()) {
324 DBG(cerr <<
"Setting up a proxy server." << endl);
325 DBG(cerr <<
"Proxy host: " << d_rcr->get_proxy_server_host()
327 DBG(cerr <<
"Proxy port: " << d_rcr->get_proxy_server_port()
329 DBG(cerr <<
"Proxy pwd : " << d_rcr->get_proxy_server_userpw()
331 curl_easy_setopt(d_curl, CURLOPT_PROXY,
332 d_rcr->get_proxy_server_host().c_str());
333 curl_easy_setopt(d_curl, CURLOPT_PROXYPORT,
334 d_rcr->get_proxy_server_port());
337 #ifdef CURLOPT_PROXYAUTH
338 curl_easy_setopt(d_curl, CURLOPT_PROXYAUTH, (
long)CURLAUTH_ANY);
342 if (!d_rcr->get_proxy_server_userpw().empty())
343 curl_easy_setopt(d_curl, CURLOPT_PROXYUSERPWD,
344 d_rcr->get_proxy_server_userpw().c_str());
347 curl_easy_setopt(d_curl, CURLOPT_ERRORBUFFER, d_error_buffer);
350 curl_easy_setopt(d_curl, CURLOPT_FAILONERROR, 0);
355 curl_easy_setopt(d_curl, CURLOPT_HTTPAUTH, (
long)CURLAUTH_ANY);
357 curl_easy_setopt(d_curl, CURLOPT_NOPROGRESS, 1);
358 curl_easy_setopt(d_curl, CURLOPT_NOSIGNAL, 1);
359 curl_easy_setopt(d_curl, CURLOPT_HEADERFUNCTION, save_raw_http_headers);
364 curl_easy_setopt(d_curl, CURLOPT_FOLLOWLOCATION, 1);
365 curl_easy_setopt(d_curl, CURLOPT_MAXREDIRS, 5);
368 if (d_rcr->get_validate_ssl() == 0) {
369 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYPEER, 0);
370 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYHOST, 0);
377 if (!d_cookie_jar.empty()) {
378 DBG(cerr <<
"Setting the cookie jar to: " << d_cookie_jar << endl);
379 curl_easy_setopt(d_curl, CURLOPT_COOKIEJAR, d_cookie_jar.c_str());
380 curl_easy_setopt(d_curl, CURLOPT_COOKIESESSION, 1);
384 cerr <<
"Curl version: " << curl_version() << endl;
385 curl_easy_setopt(d_curl, CURLOPT_VERBOSE, 1);
386 curl_easy_setopt(d_curl, CURLOPT_DEBUGFUNCTION, curl_debug);
393 class BuildHeaders :
public unary_function<const string &, void>
395 struct curl_slist *d_cl;
398 BuildHeaders() : d_cl(0)
401 void operator()(
const string &header)
403 DBG(cerr <<
"Adding '" << header.c_str() <<
"' to the header list."
405 d_cl = curl_slist_append(d_cl, header.c_str());
408 struct curl_slist *get_headers()
429 HTTPConnect::read_url(
const string &url, FILE *stream, vector<string> *resp_hdrs,
const vector<string> *headers)
431 curl_easy_setopt(d_curl, CURLOPT_URL, url.c_str());
441 curl_easy_setopt(d_curl, CURLOPT_WRITEDATA, stream);
442 curl_easy_setopt(d_curl, CURLOPT_WRITEFUNCTION, &fwrite);
444 curl_easy_setopt(d_curl, CURLOPT_WRITEDATA, stream);
447 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
448 ostream_iterator<string>(cerr,
"\n")));
450 BuildHeaders req_hdrs;
451 req_hdrs = for_each(d_request_headers.begin(), d_request_headers.end(),
454 req_hdrs = for_each(headers->begin(), headers->end(), req_hdrs);
455 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, req_hdrs.get_headers());
458 bool temporary_proxy =
false;
459 if ((temporary_proxy = url_uses_no_proxy_for(url))) {
460 DBG(cerr <<
"Suppress proxy for url: " << url << endl);
461 curl_easy_setopt(d_curl, CURLOPT_PROXY, 0);
464 string::size_type at_sign = url.find(
'@');
468 if (at_sign != url.npos)
469 d_upstring = url.substr(7, at_sign - 7);
471 if (!d_upstring.empty())
472 curl_easy_setopt(d_curl, CURLOPT_USERPWD, d_upstring.c_str());
477 curl_easy_setopt(d_curl, CURLOPT_WRITEHEADER, resp_hdrs);
482 CURLcode res = curl_easy_perform(d_curl);
485 curl_slist_free_all(req_hdrs.get_headers());
486 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, 0);
489 if (temporary_proxy && !d_rcr->get_proxy_server_host().empty())
490 curl_easy_setopt(d_curl, CURLOPT_PROXY,
491 d_rcr->get_proxy_server_host().c_str());
494 throw Error(d_error_buffer);
497 res = curl_easy_getinfo(d_curl, CURLINFO_HTTP_CODE, &status);
499 throw Error(d_error_buffer);
502 res = curl_easy_getinfo(d_curl, CURLINFO_CONTENT_TYPE, &ct_ptr);
503 if (res == CURLE_OK && ct_ptr)
504 d_content_type = ct_ptr;
515 HTTPConnect::url_uses_proxy_for(
const string &url)
throw()
517 if (d_rcr->is_proxy_for_used()) {
518 Regex host_regex(d_rcr->get_proxy_for_regexp().c_str());
519 int index = 0, matchlen;
520 return host_regex.search(url.c_str(), url.size(), matchlen, index) != -1;
530 HTTPConnect::url_uses_no_proxy_for(
const string &url)
throw()
532 return d_rcr->is_no_proxy_for_used()
533 && url.find(d_rcr->get_no_proxy_for_host()) != string::npos;
544 HTTPConnect::HTTPConnect(
RCReader *rcr,
bool use_cpp) : d_username(
""), d_password(
""), d_cookie_jar(
""),
545 d_dap_client_protocol_major(2), d_dap_client_protocol_minor(0), d_use_cpp_streams(use_cpp)
555 d_request_headers.push_back(
string(
"Pragma:"));
556 string user_agent = string(
"User-Agent: ") + string(
CNAME)
557 + string(
"/") + string(
CVER);
558 d_request_headers.push_back(user_agent);
559 if (d_accept_deflate)
560 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
568 DBG2(cerr <<
"Cache object created (" << hex << d_http_cache << dec
587 DBG2(cerr <<
"Entering the HTTPConnect dtor" << endl);
589 curl_easy_cleanup(d_curl);
591 DBG2(cerr <<
"Leaving the HTTPConnect dtor" << endl);
595 class HeaderMatch :
public unary_function<const string &, bool> {
596 const string &d_header;
598 HeaderMatch(
const string &header) : d_header(header) {}
599 bool operator()(
const string &arg) {
return arg.find(d_header) == 0; }
618 cout <<
"GET " << url <<
" HTTP/1.0" << endl;
624 stream = caching_fetch_url(url);
627 stream = plain_fetch_url(url);
632 ss <<
"HTTP/1.0 " << stream->
get_status() <<
" -" << endl;
633 for (
size_t i = 0; i < stream->
get_headers()->size(); i++) {
646 HeaderMatch(
"Content-Type:")) == stream->
get_headers()->end())
647 stream->
get_headers()->push_back(
"Content-Type: " + d_content_type);
651 cout << endl << endl;
655 if (parser.get_location() !=
"" &&
656 url.substr(0,url.find(
"?",0)).compare(parser.get_location().substr(0,url.find(
"?",0))) != 0) {
661 stream->
set_type(parser.get_object_type());
666 if (d_use_cpp_streams) {
684 get_tempfile_template(
const string &file_template)
691 Regex directory(
"[-a-zA-Z0-9_:\\]*");
696 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
697 goto valid_temp_directory;
700 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
701 goto valid_temp_directory;
706 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
707 goto valid_temp_directory;
709 #else // Unix/Linux/OSX has another...
711 Regex directory(
"[-a-zA-Z0-9_/]*");
713 c = getenv(
"TMPDIR");
714 if (directory.match(c.c_str(), c.length()) && (access(c.c_str(), W_OK | R_OK) == 0))
715 goto valid_temp_directory;
720 if (access(P_tmpdir, W_OK | R_OK) == 0) {
722 goto valid_temp_directory;
728 if (directory.match(c.c_str(), c.length()) && (access(c.c_str(), W_OK | R_OK) == 0))
729 goto valid_temp_directory;
736 valid_temp_directory:
739 c +=
"\\" + file_template;
741 c +=
"/" + file_template;
768 string dods_temp = get_tempfile_template((
string)
"dodsXXXXXX");
770 vector<char> pathname(dods_temp.length() + 1);
772 strncpy(&pathname[0], dods_temp.c_str(), dods_temp.length());
774 DBG(cerr <<
"pathanme: " << &pathname[0] <<
" (" << dods_temp.length() + 1 <<
")" << endl);
777 #if defined(WIN32) || defined(TEST_WIN32_TEMPS)
778 stream = fopen(_mktemp(&pathname[0]),
"w+b");
782 stream = fdopen(mkstemp(&pathname[0]),
"w+");
786 throw Error(
"Failed to open a temporary file for the data values (" + dods_temp +
")");
788 dods_temp = &pathname[0];
805 res = unlink(name.c_str());
832 HTTPConnect::caching_fetch_url(
const string &url)
834 DBG(cerr <<
"Is this URL (" << url <<
") in the cache?... ");
836 vector<string> *headers =
new vector<string>;
841 DBGN(cerr <<
"no; getting response and caching." << endl);
842 delete headers; headers = 0;
843 time_t now = time(0);
844 HTTPResponse *rs = plain_fetch_url(url);
845 d_http_cache->
cache_response(url, now, *(rs->get_headers()), rs->get_stream());
850 DBGN(cerr <<
"yes... ");
853 DBGN(cerr <<
"and it's valid; using cached response." << endl);
854 HTTPCacheResponse *crs =
new HTTPCacheResponse(s, 200, headers, file_name, d_http_cache);
858 DBGN(cerr <<
"but it's not valid; validating... ");
865 time_t now = time(0);
869 http_status = read_url(url, body, headers, &cond_hdrs);
878 switch (http_status) {
880 DBGN(cerr <<
"read a new response; caching." << endl);
883 HTTPResponse *rs =
new HTTPResponse(body, http_status, headers, dods_temp);
889 DBGN(cerr <<
"cached response valid; updating." << endl);
895 HTTPCacheResponse *crs =
new HTTPCacheResponse(hs, 304, headers, file_name, d_http_cache);
901 if (http_status >= 400) {
902 delete headers; headers = 0;
903 string msg =
"Error while reading the URL: ";
906 +=
".\nThe OPeNDAP server returned the following message:\n";
907 msg += http_status_to_string(http_status);
911 delete headers; headers = 0;
912 throw InternalErr(__FILE__, __LINE__,
913 "Bad response from the HTTP server: " +
long_to_string(http_status));
920 throw InternalErr(__FILE__, __LINE__,
"Should never get here");
935 HTTPConnect::plain_fetch_url(
const string &url)
937 DBG(cerr <<
"Getting URL: " << url << endl);
940 vector<string> *resp_hdrs =
new vector<string>;
944 status = read_url(url, stream, resp_hdrs);
947 string msg =
"Error while reading the URL: ";
949 msg +=
".\nThe OPeNDAP server returned the following message:\n";
950 msg += http_status_to_string(status);
962 if (d_use_cpp_streams) {
964 fstream *in =
new fstream(dods_temp.c_str(), ios::in|
ios::binary);
965 return new HTTPResponse(in, status, resp_hdrs, dods_temp);
970 return new HTTPResponse(stream, status, resp_hdrs, dods_temp);
992 if (d_accept_deflate) {
993 if (find(d_request_headers.begin(), d_request_headers.end(),
994 "Accept-Encoding: deflate, gzip, compress") == d_request_headers.end())
995 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
996 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
997 ostream_iterator<string>(cerr,
"\n")));
1000 vector<string>::iterator i;
1001 i = remove_if(d_request_headers.begin(), d_request_headers.end(),
1002 bind2nd(equal_to<string>(),
1003 string(
"Accept-Encoding: deflate, gzip, compress")));
1004 d_request_headers.erase(i, d_request_headers.end());
1020 vector<string>::iterator i;
1021 i = find_if(d_request_headers.begin(), d_request_headers.end(),
1022 HeaderMatch(
"XDAP-Accept:"));
1023 if (i != d_request_headers.end())
1024 d_request_headers.erase(i);
1027 d_dap_client_protocol_major = major;
1028 d_dap_client_protocol_minor = minor;
1029 ostringstream xdap_accept;
1030 xdap_accept <<
"XDAP-Accept: " << major <<
"." << minor;
1032 d_request_headers.push_back(xdap_accept.str());
1034 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
1035 ostream_iterator<string>(cerr,
"\n")));
1063 d_upstring = u +
":" + p;
vector< string > get_conditional_request_headers(const string &url)
virtual int get_status() const
void set_cache_enabled(bool mode)
bool is_url_valid(const string &url)
void set_credentials(const string &u, const string &p)
static HTTPCache * instance(const string &cache_root, bool force=false)
void set_max_size(unsigned long size)
int get_ignore_expires() const
virtual void set_type(ObjectType o)
FILE * get_cached_response(const string &url, vector< string > &headers, string &cacheName)
string get_cookie_jar() const
ObjectType
The type of object in the stream coming from the data server.
HTTPResponse * fetch_url(const string &url)
int get_default_expires() const
virtual void set_version(const std::string &v)
A class for software fault reporting.
void parse_mime_header(const string &header, string &name, string &value)
unsigned int get_max_cached_obj() const
int get_max_cache_size() const
ObjectType get_description_type(const string &value)
void update_response(const string &url, time_t request_time, const vector< string > &headers)
void close_temp(FILE *s, const string &name)
string get_temp_file(FILE *&stream)
bool cache_response(const string &url, time_t request_time, const vector< string > &headers, const FILE *body)
void set_accept_deflate(bool defalte)
string long_to_string(long val, int base)
void set_always_validate(bool validate)
void set_xdap_protocol(int major, int minor)
void set_default_expiration(int exp_time)
bool get_use_cache() const
int get_always_validate() const
virtual void set_protocol(const std::string &p)
void release_cached_response(FILE *response)
A class for error processing.
void set_expire_ignored(bool mode)
string get_dods_cache_root() const
void set_max_entry_size(unsigned long size)
virtual std::vector< std::string > * get_headers() const