32 #include "BESSyntaxUserError.h"
34 #include "WhiteList.h"
36 #include "curl_utils.h"
47 #define CLIENT_ERR_MIN 400
48 #define CLIENT_ERR_MAX 417
49 const char *http_client_errors[CLIENT_ERR_MAX - CLIENT_ERR_MIN +1] =
52 "Unauthorized: Contact the server administrator.",
54 "Forbidden: Contact the server administrator.",
55 "Not Found: The data source or server could not be found.\n"
56 "Often this means that the OPeNDAP server is missing or needs attention.\n"
57 "Please contact the server administrator.",
58 "Method Not Allowed.",
60 "Proxy Authentication Required.",
65 "Precondition Failed.",
66 "Request Entity Too Large.",
67 "Request URI Too Large.",
68 "Unsupported Media Type.",
69 "Requested Range Not Satisfiable.",
73 #define SERVER_ERR_MIN 500
74 #define SERVER_ERR_MAX 505
75 const char *http_server_errors[SERVER_ERR_MAX - SERVER_ERR_MIN + 1] =
77 "Internal Server Error.",
80 "Service Unavailable.",
82 "HTTP Version Not Supported."
88 string http_status_to_string(
int status)
90 if (status >= CLIENT_ERR_MIN && status <= CLIENT_ERR_MAX)
91 return string(http_client_errors[status - CLIENT_ERR_MIN]);
92 else if (status >= SERVER_ERR_MIN && status <= SERVER_ERR_MAX)
93 return string(http_server_errors[status - SERVER_ERR_MIN]);
95 return string(
"Unknown Error: This indicates a problem with libdap++.\nPlease report this to support@opendap.org.");
98 static string getCurlAuthTypeName(
const int authType){
100 string authTypeString;
103 match = authType & CURLAUTH_BASIC;
105 authTypeString +=
"CURLAUTH_BASIC";
108 match = authType & CURLAUTH_DIGEST;
110 if(!authTypeString.empty())
111 authTypeString +=
" ";
112 authTypeString +=
"CURLAUTH_DIGEST";
115 match = authType & CURLAUTH_DIGEST_IE;
117 if(!authTypeString.empty())
118 authTypeString +=
" ";
119 authTypeString +=
"CURLAUTH_DIGEST_IE";
122 match = authType & CURLAUTH_GSSNEGOTIATE;
124 if(!authTypeString.empty())
125 authTypeString +=
" ";
126 authTypeString +=
"CURLAUTH_GSSNEGOTIATE";
129 match = authType & CURLAUTH_NTLM;
131 if(!authTypeString.empty())
132 authTypeString +=
" ";
133 authTypeString +=
"CURLAUTH_NTLM";
137 match = authType & CURLAUTH_ANY;
139 if(!authTypeString.empty())
140 authTypeString +=
" ";
141 authTypeString +=
"CURLAUTH_ANY";
145 match = authType & CURLAUTH_ANY;
147 if(!authTypeString.empty())
148 authTypeString +=
" ";
149 authTypeString +=
"CURLAUTH_ANYSAFE";
153 match = authType & CURLAUTH_ANY;
155 if(!authTypeString.empty())
156 authTypeString +=
" ";
157 authTypeString +=
"CURLAUTH_ONLY";
161 return authTypeString;
169 static size_t writeToOpenfileDescriptor(
char *data,
size_t ,
size_t nmemb,
void *userdata){
171 int *fd = (
int *) userdata;
173 BESDEBUG(MODULE,
"curl_utils::writeToOpenfileDescriptor() - Bytes received " << libdap::long_to_string(nmemb) << endl);
174 int wrote = write(*fd, data, nmemb);
175 BESDEBUG(MODULE,
"curl_utils::writeToOpenfileDescriptor() - Bytes written " << libdap::long_to_string(wrote) << endl);
204 static size_t save_raw_http_headers(
void *ptr,
size_t size,
size_t nmemb,
void *resp_hdrs)
206 BESDEBUG(MODULE,
"curl_utils::save_raw_http_headers() - Inside the header parser." << endl);
207 vector<string> *hdrs =
static_cast<vector<string> *
>(resp_hdrs);
210 string complete_line;
211 if (nmemb > 1 && *(
static_cast<char*
>(ptr) + size * (nmemb - 2)) ==
'\r')
212 complete_line.assign(
static_cast<char *
>(ptr), size * (nmemb - 2));
214 complete_line.assign(
static_cast<char *
>(ptr), size * (nmemb - 1));
217 if (complete_line !=
"" && complete_line.find(
"HTTP") == string::npos) {
218 BESDEBUG(MODULE,
"curl_utils::save_raw_http_headers() - Header line: " << complete_line << endl);
219 hdrs->push_back(complete_line);
230 static int curl_debug(CURL *, curl_infotype info,
char *msg,
size_t size,
void *)
232 string message(msg, size);
236 BESDEBUG(MODULE,
"curl_utils::curl_debug() - Text: " << message << endl );
break;
237 case CURLINFO_HEADER_IN:
238 BESDEBUG(MODULE,
"curl_utils::curl_debug() - Header in: " << message << endl );
break;
239 case CURLINFO_HEADER_OUT:
240 BESDEBUG(MODULE,
"curl_utils::curl_debug() - Header out: " << endl << message << endl );
break;
241 case CURLINFO_DATA_IN:
242 BESDEBUG(MODULE,
"curl_utils::curl_debug() - Data in: " << message << endl );
break;
243 case CURLINFO_DATA_OUT:
244 BESDEBUG(MODULE,
"curl_utils::curl_debug() - Data out: " << message << endl );
break;
246 BESDEBUG(MODULE,
"curl_utils::curl_debug() - End: " << message << endl );
break;
247 #ifdef CURLINFO_SSL_DATA_IN
248 case CURLINFO_SSL_DATA_IN:
249 BESDEBUG(MODULE,
"curl_utils::curl_debug() - SSL Data in: " << message << endl );
break;
251 #ifdef CURLINFO_SSL_DATA_OUT
252 case CURLINFO_SSL_DATA_OUT:
253 BESDEBUG(MODULE,
"curl_utils::curl_debug() - SSL Data out: " << message << endl );
break;
256 BESDEBUG(MODULE,
"curl_utils::curl_debug() - Curl info: " << message << endl );
break;
269 class BuildHeaders :
public std::unary_function<const string &, void>
271 struct curl_slist *d_cl;
274 BuildHeaders() : d_cl(0)
277 void operator()(
const string &header)
279 BESDEBUG(MODULE,
"BuildHeaders::operator() - Adding '" << header.c_str() <<
"' to the header list." << endl);
280 d_cl = curl_slist_append(d_cl, header.c_str());
283 struct curl_slist *get_headers()
308 bool configureProxy(CURL *curl,
const string &url) {
309 BESDEBUG( MODULE,
"curl_utils::configureProxy() - BEGIN." << endl);
311 bool using_proxy =
false;
317 string proxyHost = cmr::CmrUtils::ProxyHost;
318 int proxyPort = cmr::CmrUtils::ProxyPort;
319 string proxyPassword = cmr::CmrUtils::ProxyPassword;
320 string proxyUser = cmr::CmrUtils::ProxyUser;
321 string proxyUserPW = cmr::CmrUtils::ProxyUserPW;
322 int proxyAuthType = cmr::CmrUtils::ProxyAuthType;
324 if (!proxyHost.empty()) {
335 BESDEBUG( MODULE,
"curl_utils::configureProxy() - Found proxy configuration." << endl);
341 if (!CmrUtils::NoProxyRegex.empty()) {
342 BESDEBUG( MODULE,
"curl_utils::configureProxy() - Found NoProxyRegex." << endl);
343 libdap::Regex r(CmrUtils::NoProxyRegex.c_str());
344 if (r.match(url.c_str(), url.length()) != -1) {
345 BESDEBUG( MODULE,
"curl_utils::configureProxy() - Found NoProxy match. Regex: " << CmrUtils::NoProxyRegex <<
"; Url: " << url << endl);
352 BESDEBUG(MODULE,
"curl_utils::configureProxy() - Setting up a proxy server." << endl);
353 BESDEBUG(MODULE,
"curl_utils::configureProxy() - Proxy host: " << proxyHost << endl);
354 BESDEBUG(MODULE,
"curl_utils::configureProxy() - Proxy port: " << proxyPort << endl);
356 curl_easy_setopt(curl, CURLOPT_PROXY, proxyHost.data());
357 curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxyPort);
368 BESDEBUG(MODULE,
"curl_utils::configureProxy() - CURLOPT_PROXYAUTH = " << CURLOPT_PROXYAUTH << endl);
369 BESDEBUG(MODULE,
"curl_utils::configureProxy() - CURLAUTH_BASIC = " << CURLAUTH_BASIC << endl);
370 BESDEBUG(MODULE,
"curl_utils::configureProxy() - CURLAUTH_DIGEST = " << CURLAUTH_DIGEST << endl);
371 BESDEBUG(MODULE,
"curl_utils::configureProxy() - CURLAUTH_DIGEST_IE = " << CURLAUTH_DIGEST_IE << endl);
372 BESDEBUG(MODULE,
"curl_utils::configureProxy() - CURLAUTH_GSSNEGOTIATE = " << CURLAUTH_GSSNEGOTIATE << endl);
373 BESDEBUG(MODULE,
"curl_utils::configureProxy() - CURLAUTH_NTLM = " << CURLAUTH_NTLM << endl);
374 BESDEBUG(MODULE,
"curl_utils::configureProxy() - CURLAUTH_ANY = " << CURLAUTH_ANY << endl);
375 BESDEBUG(MODULE,
"curl_utils::configureProxy() - CURLAUTH_ANYSAFE = " << CURLAUTH_ANYSAFE << endl);
376 BESDEBUG(MODULE,
"curl_utils::configureProxy() - CURLAUTH_ONLY = " << CURLAUTH_ONLY << endl);
377 BESDEBUG(MODULE,
"curl_utils::configureProxy() - Using CURLOPT_PROXYAUTH = " << proxyAuthType << endl);
380 BESDEBUG(MODULE,
"curl_utils::configureProxy() - Using CURLOPT_PROXYAUTH = " << getCurlAuthTypeName(proxyAuthType) << endl);
381 curl_easy_setopt(curl, CURLOPT_PROXYAUTH, proxyAuthType);
386 if (!proxyUser.empty()){
387 curl_easy_setopt(curl, CURLOPT_PROXYUSERNAME, proxyUser.data());
388 BESDEBUG(MODULE,
"curl_utils::configureProxy() - CURLOPT_PROXYUSER : " << proxyUser << endl);
390 if (!proxyPassword.empty()){
391 curl_easy_setopt(curl, CURLOPT_PROXYPASSWORD, proxyPassword.data());
392 BESDEBUG(MODULE,
"curl_utils::configureProxy() - CURLOPT_PROXYPASSWORD: " << proxyPassword << endl);
395 else if (!proxyUserPW.empty()){
397 "curl_utils::configureProxy() - CURLOPT_PROXYUSERPWD : " << proxyUserPW << endl);
398 curl_easy_setopt(curl, CURLOPT_PROXYUSERPWD, proxyUserPW.data());
403 BESDEBUG( MODULE,
"curl_utils::configureProxy() - END." << endl);
430 CURL *init(
char *error_buffer)
433 CURL *curl = curl_easy_init();
435 throw libdap::InternalErr(__FILE__, __LINE__,
"Could not initialize libcurl.");
447 #ifndef CURLOPT_ACCEPT_ENCODING
448 curl_easy_setopt(curl, CURLOPT_ENCODING,
"");
450 curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING,
"");
453 curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, error_buffer);
456 curl_easy_setopt(curl, CURLOPT_FAILONERROR, 0);
461 curl_easy_setopt(curl, CURLOPT_HTTPAUTH, (
long)CURLAUTH_ANY);
464 curl_easy_setopt(curl, CURLOPT_NETRC, 1);
467 curl_easy_setopt(curl, CURLOPT_COOKIEFILE,
"/tmp/.hyrax_cookies");
468 curl_easy_setopt(curl, CURLOPT_COOKIEJAR,
"/tmp/.hyrax_cookies");
470 curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
471 curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
472 curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, save_raw_http_headers);
477 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
478 curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5);
484 curl_easy_setopt(curl, CURLOPT_USERAGENT, curl_version());
489 if (!d_rcr->get_validate_ssl() == 0) {
490 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
491 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
498 if (!d_cookie_jar.empty()) {
499 BESDEBUG(cerr <<
"Setting the cookie jar to: " << d_cookie_jar << endl);
500 curl_easy_setopt(curl, CURLOPT_COOKIEJAR, d_cookie_jar.c_str());
501 curl_easy_setopt(curl, CURLOPT_COOKIESESSION, 1);
507 BESDEBUG(MODULE,
"curl_utils::www_lib_init() - Curl version: " << curl_version() << endl);
508 curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
509 BESDEBUG(MODULE,
"curl_utils::www_lib_init() - Curl in verbose mode."<< endl);
510 curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, curl_debug);
511 BESDEBUG(MODULE,
"curl_utils::www_lib_init() - Curl debugging function installed."<< endl);
515 BESDEBUG(MODULE,
"curl_utils::www_lib_init() - curl: " << curl << endl);
540 long read_url(CURL *curl,
543 vector<string> *resp_hdrs,
544 const vector<string> *request_headers,
547 string prolog = string(
"curl_utils.cc ") + __func__ +
"() - ";
549 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
553 string err = (string)
"The specified URL " + url
554 +
" does not match any of the accessible services in"
555 +
" the white list.";
556 BESDEBUG(MODULE, prolog << err << endl);
560 curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
561 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeToOpenfileDescriptor);
563 #ifdef CURLOPT_WRITEDATA
564 curl_easy_setopt(curl, CURLOPT_WRITEDATA, &fd);
566 curl_easy_setopt(curl, CURLOPT_FILE, &fd);
569 BuildHeaders req_hdrs;
572 req_hdrs = for_each(request_headers->begin(), request_headers->end(), req_hdrs);
573 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, req_hdrs.get_headers());
578 curl_easy_setopt(curl, CURLOPT_WRITEHEADER, resp_hdrs);
581 CURLcode res = curl_easy_perform(curl);
584 curl_slist_free_all(req_hdrs.get_headers());
585 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, 0);
588 BESDEBUG(MODULE, prolog <<
"OUCH! CURL returned an error! curl msg: " << curl_easy_strerror(res) << endl);
589 BESDEBUG(MODULE, prolog <<
"OUCH! CURL returned an error! error_buffer: " << error_buffer << endl);
590 throw libdap::Error(error_buffer);
593 res = curl_easy_getinfo(curl, CURLINFO_HTTP_CODE, &status);
594 BESDEBUG(MODULE, prolog <<
"HTTP Status " << status << endl);
596 throw libdap::Error(error_buffer);
597 BESDEBUG(MODULE, prolog <<
"END" << endl);