34 #include "BESSyntaxUserError.h"
35 #include "BESInternalError.h"
36 #include "BESInternalFatalError.h"
37 #include "WhiteList.h"
39 #include "curl_utils.h"
40 #include "HttpdCatalogUtils.h"
41 #include "HttpdCatalogNames.h"
43 #define prolog string("curl_utils.cc: ").append(__func__).append("() - ")
47 namespace httpd_catalog {
52 #define CLIENT_ERR_MIN 400
53 #define CLIENT_ERR_MAX 417
54 const char *http_client_errors[CLIENT_ERR_MAX - CLIENT_ERR_MIN + 1] = {
56 "Unauthorized: Contact the server administrator.",
58 "Forbidden: Contact the server administrator.",
59 "Not Found: The data source or server could not be found.\n"
60 "Often this means that the OPeNDAP server is missing or needs attention.\n"
61 "Please contact the server administrator.",
62 "Method Not Allowed.",
64 "Proxy Authentication Required.",
69 "Precondition Failed.",
70 "Request Entity Too Large.",
71 "Request URI Too Large.",
72 "Unsupported Media Type.",
73 "Requested Range Not Satisfiable.",
77 #define SERVER_ERR_MIN 500
78 #define SERVER_ERR_MAX 505
79 const char *http_server_errors[SERVER_ERR_MAX - SERVER_ERR_MIN + 1] = {
"Internal Server Error.",
"Not Implemented.",
"Bad Gateway.",
80 "Service Unavailable.",
"Gateway Time-out.",
"HTTP Version Not Supported." };
84 string http_status_to_string(
int status)
86 if (status >= CLIENT_ERR_MIN && status <= CLIENT_ERR_MAX)
87 return string(http_client_errors[status - CLIENT_ERR_MIN]);
88 else if (status >= SERVER_ERR_MIN && status <= SERVER_ERR_MAX)
89 return string(http_server_errors[status - SERVER_ERR_MIN]);
91 return string(
"Unknown Error: This indicates a problem with libdap++.\nPlease report this to support@opendap.org.");
94 static string get_curl_auth_type_name(
const int authType)
96 string authTypeString;
99 match = authType & CURLAUTH_BASIC;
101 authTypeString +=
"CURLAUTH_BASIC";
104 match = authType & CURLAUTH_DIGEST;
106 if (!authTypeString.empty()) authTypeString +=
" ";
107 authTypeString +=
"CURLAUTH_DIGEST";
110 match = authType & CURLAUTH_DIGEST_IE;
112 if (!authTypeString.empty()) authTypeString +=
" ";
113 authTypeString +=
"CURLAUTH_DIGEST_IE";
116 match = authType & CURLAUTH_GSSNEGOTIATE;
118 if (!authTypeString.empty()) authTypeString +=
" ";
119 authTypeString +=
"CURLAUTH_GSSNEGOTIATE";
122 match = authType & CURLAUTH_NTLM;
124 if (!authTypeString.empty()) authTypeString +=
" ";
125 authTypeString +=
"CURLAUTH_NTLM";
129 match = authType & CURLAUTH_ANY;
131 if(!authTypeString.empty())
132 authTypeString +=
" ";
133 authTypeString +=
"CURLAUTH_ANY";
136 match = authType & CURLAUTH_ANY;
138 if(!authTypeString.empty())
139 authTypeString +=
" ";
140 authTypeString +=
"CURLAUTH_ANYSAFE";
143 match = authType & CURLAUTH_ANY;
145 if(!authTypeString.empty())
146 authTypeString +=
" ";
147 authTypeString +=
"CURLAUTH_ONLY";
151 return authTypeString;
158 static size_t writeToOpenfileDescriptor(
char *data,
size_t ,
size_t nmemb,
void *userdata)
160 int *fd = (
int *) userdata;
162 BESDEBUG(MODULE, prolog <<
"Bytes received " << libdap::long_to_string(nmemb) << endl);
163 int wrote = write(*fd, data, nmemb);
164 BESDEBUG(MODULE, prolog <<
"Bytes written " << libdap::long_to_string(wrote) << endl);
192 static size_t save_raw_http_headers(
void *ptr,
size_t size,
size_t nmemb,
void *resp_hdrs)
194 BESDEBUG(MODULE, prolog <<
"Inside the header parser." << endl);
195 vector<string> *hdrs =
static_cast<vector<string> *
>(resp_hdrs);
198 string complete_line;
199 if (nmemb > 1 && *(
static_cast<char*
>(ptr) + size * (nmemb - 2)) ==
'\r')
200 complete_line.assign(
static_cast<char *
>(ptr), size * (nmemb - 2));
202 complete_line.assign(
static_cast<char *
>(ptr), size * (nmemb - 1));
205 if (complete_line !=
"" && complete_line.find(
"HTTP") == string::npos) {
206 BESDEBUG(MODULE, prolog <<
"Header line: " << complete_line << endl);
207 hdrs->push_back(complete_line);
214 static int curl_debug(CURL *, curl_infotype info,
char *msg,
size_t size,
void *)
216 string message(msg, size);
220 BESDEBUG(MODULE, prolog <<
"Text: " << message << endl);
222 case CURLINFO_HEADER_IN:
223 BESDEBUG(MODULE, prolog <<
"Header in: " << message << endl);
225 case CURLINFO_HEADER_OUT:
226 BESDEBUG(MODULE, prolog <<
"Header out: " << endl << message << endl);
228 case CURLINFO_DATA_IN:
229 BESDEBUG(MODULE, prolog <<
"Data in: " << message << endl);
231 case CURLINFO_DATA_OUT:
232 BESDEBUG(MODULE, prolog <<
"Data out: " << message << endl);
235 BESDEBUG(MODULE, prolog <<
"End: " << message << endl);
237 #ifdef CURLINFO_SSL_DATA_IN
238 case CURLINFO_SSL_DATA_IN:
239 BESDEBUG(MODULE, prolog <<
"SSL Data in: " << message << endl );
break;
241 #ifdef CURLINFO_SSL_DATA_OUT
242 case CURLINFO_SSL_DATA_OUT:
243 BESDEBUG(MODULE, prolog <<
"SSL Data out: " << message << endl );
break;
246 BESDEBUG(MODULE, prolog <<
"Curl info: " << message << endl);
255 class BuildHeaders:
public std::unary_function<const string &, void> {
256 struct curl_slist *d_cl;
264 void operator()(
const string &header)
266 BESDEBUG(MODULE,
"BuildHeaders::operator() - Adding '" << header.c_str() <<
"' to the header list." << endl);
267 d_cl = curl_slist_append(d_cl, header.c_str());
270 struct curl_slist *get_headers()
289 bool configureProxy(CURL *curl,
const string &url)
291 BESDEBUG(MODULE, prolog <<
" BEGIN." << endl);
293 bool using_proxy =
false;
295 string proxyHost = HttpdCatalogUtils::ProxyHost;
296 int proxyPort = HttpdCatalogUtils::ProxyPort;
297 string proxyPassword = HttpdCatalogUtils::ProxyPassword;
298 string proxyUser = HttpdCatalogUtils::ProxyUser;
299 string proxyUserPW = HttpdCatalogUtils::ProxyUserPW;
300 int proxyAuthType = HttpdCatalogUtils::ProxyAuthType;
302 if (!proxyHost.empty()) {
304 if (proxyPort == 0) proxyPort = 8080;
308 BESDEBUG(MODULE, prolog <<
"Found proxy configuration." << endl);
314 if (!HttpdCatalogUtils::NoProxyRegex.empty()) {
315 BESDEBUG(MODULE, prolog <<
"Found NoProxyRegex." << endl);
316 libdap::Regex r(HttpdCatalogUtils::NoProxyRegex.c_str());
317 if (r.match(url.c_str(), url.length()) != -1) {
318 BESDEBUG(MODULE, prolog <<
"Found NoProxy match. Regex: " << HttpdCatalogUtils::NoProxyRegex <<
"; Url: " << url << endl);
325 BESDEBUG(MODULE, prolog <<
"Setting up a proxy server." << endl);
326 BESDEBUG(MODULE, prolog <<
"Proxy host: " << proxyHost << endl);
327 BESDEBUG(MODULE, prolog <<
"Proxy port: " << proxyPort << endl);
329 curl_easy_setopt(curl, CURLOPT_PROXY, proxyHost.data());
330 curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxyPort);
335 BESDEBUG(MODULE, prolog <<
"CURLOPT_PROXYAUTH = " << CURLOPT_PROXYAUTH << endl);
336 BESDEBUG(MODULE, prolog <<
"CURLAUTH_BASIC = " << CURLAUTH_BASIC << endl);
337 BESDEBUG(MODULE, prolog <<
"CURLAUTH_DIGEST = " << CURLAUTH_DIGEST << endl);
338 BESDEBUG(MODULE, prolog <<
"CURLAUTH_DIGEST_IE = " << CURLAUTH_DIGEST_IE << endl);
339 BESDEBUG(MODULE, prolog <<
"CURLAUTH_GSSNEGOTIATE = " << CURLAUTH_GSSNEGOTIATE << endl);
340 BESDEBUG(MODULE, prolog <<
"CURLAUTH_NTLM = " << CURLAUTH_NTLM << endl);
341 BESDEBUG(MODULE, prolog <<
"CURLAUTH_ANY = " << CURLAUTH_ANY << endl);
342 BESDEBUG(MODULE, prolog <<
"CURLAUTH_ANYSAFE = " << CURLAUTH_ANYSAFE << endl);
343 BESDEBUG(MODULE, prolog <<
"CURLAUTH_ONLY = " << CURLAUTH_ONLY << endl);
344 BESDEBUG(MODULE, prolog <<
"Using CURLOPT_PROXYAUTH = " << proxyAuthType << endl);
347 BESDEBUG(MODULE, prolog <<
"Using CURLOPT_PROXYAUTH = " << get_curl_auth_type_name(proxyAuthType) << endl);
348 curl_easy_setopt(curl, CURLOPT_PROXYAUTH, proxyAuthType);
350 if (!proxyUser.empty()) {
351 curl_easy_setopt(curl, CURLOPT_PROXYUSERNAME, proxyUser.data());
352 BESDEBUG(MODULE, prolog <<
"CURLOPT_PROXYUSER : " << proxyUser << endl);
354 if (!proxyPassword.empty()) {
355 curl_easy_setopt(curl, CURLOPT_PROXYPASSWORD, proxyPassword.data());
356 BESDEBUG(MODULE, prolog <<
"CURLOPT_PROXYPASSWORD: " << proxyPassword << endl);
359 else if (!proxyUserPW.empty()) {
360 BESDEBUG(MODULE, prolog <<
"CURLOPT_PROXYUSERPWD : " << proxyUserPW << endl);
361 curl_easy_setopt(curl, CURLOPT_PROXYUSERPWD, proxyUserPW.data());
366 BESDEBUG(MODULE, prolog <<
"END." << endl);
380 CURL *init(
char *error_buffer)
382 CURL *curl = curl_easy_init();
394 #ifndef CURLOPT_ACCEPT_ENCODING
395 curl_easy_setopt(curl, CURLOPT_ENCODING,
"");
397 curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING,
"");
400 curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, error_buffer);
403 curl_easy_setopt(curl, CURLOPT_FAILONERROR, 0);
408 curl_easy_setopt(curl, CURLOPT_HTTPAUTH, (
long)CURLAUTH_ANY);
411 curl_easy_setopt(curl, CURLOPT_NETRC, 1);
414 curl_easy_setopt(curl, CURLOPT_COOKIEFILE,
"/tmp/.hyrax_cookies");
415 curl_easy_setopt(curl, CURLOPT_COOKIEJAR,
"/tmp/.hyrax_cookies");
417 curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
418 curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
419 curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, save_raw_http_headers);
424 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
425 curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5);
428 curl_easy_setopt(curl, CURLOPT_USERAGENT, curl_version());
432 if (!d_rcr->get_validate_ssl() == 0) {
433 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
434 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
441 if (!d_cookie_jar.empty()) {
442 BESDEBUG(cerr <<
"Setting the cookie jar to: " << d_cookie_jar << endl);
443 curl_easy_setopt(curl, CURLOPT_COOKIEJAR, d_cookie_jar.c_str());
444 curl_easy_setopt(curl, CURLOPT_COOKIESESSION, 1);
449 BESDEBUG(MODULE, prolog <<
"Curl version: " << curl_version() << endl);
450 curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
451 BESDEBUG(MODULE, prolog <<
"Curl in verbose mode."<< endl);
452 curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, curl_debug);
453 BESDEBUG(MODULE, prolog <<
"Curl debugging function installed."<< endl);
456 BESDEBUG(MODULE, prolog <<
"curl: " << curl << endl);
476 long read_url(CURL *curl,
const string &url,
int fd, vector<string> *resp_hdrs,
const vector<string> *request_headers,
char error_buffer[])
478 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
479 BESDEBUG(MODULE, prolog <<
"url: " << url << endl);
483 string err = string(
"The specified URL ") + url +
" does not match any of the accessible services in the white list.";
484 BESDEBUG(MODULE, prolog << err << endl);
488 curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
489 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeToOpenfileDescriptor);
491 #ifdef CURLOPT_WRITEDATA
492 curl_easy_setopt(curl, CURLOPT_WRITEDATA, &fd);
494 curl_easy_setopt(curl, CURLOPT_FILE, &fd);
497 BuildHeaders req_hdrs;
498 if (request_headers) req_hdrs = for_each(request_headers->begin(), request_headers->end(), req_hdrs);
499 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, req_hdrs.get_headers());
504 curl_easy_setopt(curl, CURLOPT_WRITEHEADER, resp_hdrs);
508 curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &urlp);
509 BESDEBUG(MODULE, prolog <<
"url in curl object: " << urlp << endl);
512 CURLcode res = curl_easy_perform(curl);
514 if (res != CURLE_OK) {
515 BESDEBUG(MODULE, prolog <<
"OUCH! CURL returned an error! curl msg: " << curl_easy_strerror(res) << endl);
516 throw BESInternalError(
string(
"CURL returned an error! curl msg: ").append(curl_easy_strerror(res)), __FILE__, __LINE__);
520 curl_slist_free_all(req_hdrs.get_headers());
521 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, 0);
524 res = curl_easy_getinfo(curl, CURLINFO_HTTP_CODE, &status);
525 BESDEBUG(MODULE, prolog <<
"HTTP Status " << status << endl);
527 if (res != CURLE_OK) {
529 oss <<
"HTTP Status: " << status;
530 throw BESInternalError(oss.str().append(
"; ").append(curl_easy_strerror(res)), __FILE__, __LINE__);
533 BESDEBUG(MODULE, prolog <<
"END" << endl);