bes  Updated for version 3.20.5
DmrppRequestHandler.cc
1 // DmrppRequestHandler.cc
2 
3 // Copyright (c) 2016 OPeNDAP, Inc. Author: James Gallagher
4 // <jgallagher@opendap.org>, Patrick West <pwest@opendap.org>
5 // Nathan Potter <npotter@opendap.org>
6 //
7 // modify it under the terms of the GNU Lesser General Public License
8 // as published by the Free Software Foundation; either version 2.1 of
9 // the License, or (at your option) any later version.
10 //
11 // This library is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 // Lesser General Public License for more details.
15 //
16 // License along with this library; if not, write to the Free Software
17 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18 // 02110-1301 U\ SA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI.
21 // 02874-0112.
22 
23 #include "config.h"
24 
25 #include <string>
26 #include <memory>
27 #include <sstream>
28 
29 #include <curl/curl.h>
30 #include <stdlib.h>
31 
32 #include <Ancillary.h>
33 #include <ObjMemCache.h>
34 #include <DMR.h>
35 #include <D4Group.h>
36 #include <DAS.h>
37 
38 #include <InternalErr.h>
39 #include <mime_util.h> // for name_path
40 
41 #include <BESResponseHandler.h>
42 #include <BESResponseNames.h>
43 #include <BESDapNames.h>
44 #include <BESDataNames.h>
45 #include <BESDASResponse.h>
46 #include <BESDDSResponse.h>
47 #include <BESDataDDSResponse.h>
48 #include <BESVersionInfo.h>
49 #include <BESTextInfo.h>
50 #include <BESContainer.h>
51 
52 #include <BESDMRResponse.h>
53 
54 #include <BESConstraintFuncs.h>
55 #include <BESServiceRegistry.h>
56 #include <BESUtil.h>
57 #include <TheBESKeys.h>
58 
59 #include <BESDapError.h>
60 #include <BESInternalFatalError.h>
61 #include <BESDebug.h>
62 #include <BESStopWatch.h>
63 
64 #include "DMRpp.h"
65 #include "DmrppTypeFactory.h"
66 #include "DmrppParserSax2.h"
67 #include "DmrppRequestHandler.h"
68 #include "CurlHandlePool.h"
69 #include "DmrppMetadataStore.h"
70 
71 using namespace bes;
72 using namespace libdap;
73 using namespace std;
74 
75 namespace dmrpp {
76 
77 const string module = "dmrpp";
78 
79 ObjMemCache *DmrppRequestHandler::das_cache = 0;
80 ObjMemCache *DmrppRequestHandler::dds_cache = 0;
81 ObjMemCache *DmrppRequestHandler::dmr_cache = 0;
82 
83 // This is used to maintain a pool of reusable curl handles that enable connection
84 // reuse. jhrg
85 CurlHandlePool *DmrppRequestHandler::curl_handle_pool = 0;
86 
87 bool DmrppRequestHandler::d_use_parallel_transfers = true;
88 int DmrppRequestHandler::d_max_parallel_transfers = 8;
89 
90 static void read_key_value(const std::string &key_name, bool &key_value)
91 {
92  bool key_found = false;
93  string value;
94  TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
95  if (key_found) {
96  value = BESUtil::lowercase(value);
97  key_value = (value == "true" || value == "yes");
98  }
99 }
100 
101 static void read_key_value(const std::string &key_name, int &key_value)
102 {
103  bool key_found = false;
104  string value;
105  TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
106  if (key_found) {
107  istringstream iss(value);
108  iss >> key_value;
109  }
110 }
111 
116 DmrppRequestHandler::DmrppRequestHandler(const string &name) :
117  BESRequestHandler(name)
118 {
119  add_method(DMR_RESPONSE, dap_build_dmr);
120  add_method(DAP4DATA_RESPONSE, dap_build_dap4data);
121  add_method(DAS_RESPONSE, dap_build_das);
122  add_method(DDS_RESPONSE, dap_build_dds);
123  add_method(DATA_RESPONSE, dap_build_dap2data);
124 
125  add_method(VERS_RESPONSE, dap_build_vers);
126  add_method(HELP_RESPONSE, dap_build_help);
127 
128  read_key_value("DMRPP.UseParallelTransfers", d_use_parallel_transfers);
129  read_key_value("DMRPP.MaxParallelTransfers", d_max_parallel_transfers);
130 
131  if (!curl_handle_pool)
132  curl_handle_pool = new CurlHandlePool();
133 
134  curl_global_init(CURL_GLOBAL_DEFAULT);
135 }
136 
137 DmrppRequestHandler::~DmrppRequestHandler()
138 {
139  delete curl_handle_pool;
140  curl_global_cleanup();
141 }
142 
143 void DmrppRequestHandler::build_dmr_from_file(BESContainer *container, DMR* dmr)
144 {
145  string data_pathname = container->access();
146 
147  dmr->set_filename(data_pathname);
148  dmr->set_name(name_path(data_pathname));
149 
150  DmrppTypeFactory BaseFactory; // Use the factory for this handler's types
151  dmr->set_factory(&BaseFactory);
152 
153  DmrppParserSax2 parser;
154  ifstream in(data_pathname.c_str(), ios::in);
155 
156  parser.intern(in, dmr, BESDebug::IsSet(module));
157 
158  dmr->set_factory(0);
159 }
160 
174 {
175  BESDEBUG(module, "Entering dap_build_dmr..." << endl);
176 
177  BESResponseObject *response = dhi.response_handler->get_response_object();
178  BESDMRResponse *bdmr = dynamic_cast<BESDMRResponse *>(response);
179  if (!bdmr) throw BESInternalError("Cast error, expected a BESDDSResponse object.", __FILE__, __LINE__);
180 
181  try {
182  build_dmr_from_file(dhi.container, bdmr->get_dmr());
183 
184  bdmr->set_dap4_constraint(dhi);
185  bdmr->set_dap4_function(dhi);
186  }
187  catch (BESError &e) {
188  throw e;
189  }
190  catch (InternalErr & e) {
191  throw BESDapError(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
192  }
193  catch (Error & e) {
194  throw BESDapError(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
195  }
196  catch (...) {
197  throw BESInternalFatalError("Unknown exception caught building a DMR", __FILE__, __LINE__);
198  }
199 
200  BESDEBUG(module, "Leaving dap_build_dmr..." << endl);
201 
202  return true;
203 }
204 
205 bool DmrppRequestHandler::dap_build_dap4data(BESDataHandlerInterface &dhi)
206 {
207  BESDEBUG(module, "Entering dap_build_dap4data..." << endl);
208 
209  BESResponseObject *response = dhi.response_handler->get_response_object();
210  BESDMRResponse *bdmr = dynamic_cast<BESDMRResponse *>(response);
211  if (!bdmr) throw BESInternalError("Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
212 
213  try {
214  // Check the Container to see if the handler should get the response from the MDS.
215  if (dhi.container->get_attributes().find(MDS_HAS_DMRPP) != string::npos) {
216  DmrppMetadataStore *mds = DmrppMetadataStore::get_instance();
217  if (!mds)
218  throw BESInternalError("MDS configuration error: The DMR++ module could not find the MDS", __FILE__, __LINE__);
219 
221  if (!dmrpp)
222  throw BESInternalError("DMR++ module error: Null DMR++ object read from the MDS", __FILE__, __LINE__);
223 
224  delete bdmr->get_dmr();
225  bdmr->set_dmr(dmrpp);
226  }
227  else {
228  build_dmr_from_file(dhi.container, bdmr->get_dmr());
229  }
230 
231  bdmr->set_dap4_constraint(dhi);
232  bdmr->set_dap4_function(dhi);
233  }
234  catch (BESError &e) {
235  throw e;
236  }
237  catch (InternalErr & e) {
238  throw BESDapError(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
239  }
240  catch (Error & e) {
241  throw BESDapError(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
242  }
243  catch (...) {
244  throw BESInternalFatalError("Unknown exception caught building DAP4 Data response", __FILE__, __LINE__);
245  }
246 
247  BESDEBUG(module, "Leaving dap_build_dap4data..." << endl);
248 
249  return false;
250 }
251 
256 {
257  BESStopWatch sw;
258  if (BESISDEBUG(TIMING_LOG)) sw.start("DmrppRequestHandler::dap_build_dap2data()", dhi.data[REQUEST_ID]);
259 
260  BESDEBUG(module, __func__ << "() - BEGIN" << endl);
261 
262  BESResponseObject *response = dhi.response_handler->get_response_object();
263  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(response);
264  if (!bdds) throw BESInternalError("Cast error, expected a BESDataDDSResponse object.", __FILE__, __LINE__);
265 
266  try {
267  string container_name_str = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
268 
269  DDS *dds = bdds->get_dds();
270  if (!container_name_str.empty()) dds->container_name(container_name_str);
271  string accessed = dhi.container->access();
272 
273  // Look in memory cache, if it's initialized
274  DDS *cached_dds_ptr = 0;
275  if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(accessed)))) {
276  // copy the cached DAS into the BES response object
277  BESDEBUG(module, "DDS Cached hit for : " << accessed << endl);
278  *dds = *cached_dds_ptr;
279  bdds->set_constraint(dhi);
280  }
281  else {
282  // Not in the local binary cache, make one...
283  DMR *dmr = NULL;
284 
285  // Check the Container to see if the handler should get the response from the MDS.
286  if (dhi.container->get_attributes().find(MDS_HAS_DMRPP) != string::npos) {
287  DmrppMetadataStore *mds = DmrppMetadataStore::get_instance();
288  if (!mds)
289  throw BESInternalError("MDS configuration error: The DMR++ module could not find the MDS", __FILE__, __LINE__);
290 
291  dmr = mds->get_dmrpp_object(dhi.container->get_relative_name());
292  if (!dmr)
293  throw BESInternalError("DMR++ module error: Null DMR++ object read from the MDS", __FILE__, __LINE__);
294  }
295  else {
296  dmr = new DMR();
297  build_dmr_from_file(dhi.container, dmr);
298  }
299 
300  // delete the current one;
301  delete dds;
302  // assign the new one.
303  dds = dmr->getDDS();
304 
305  // Stuff it into the response.
306  bdds->set_dds(dds);
307  bdds->set_constraint(dhi);
308 
309  delete dmr;
310 
311  // Cache it, if the cache is active.
312  if (dds_cache) {
313  dds_cache->add(new DDS(*dds), accessed);
314  }
315  }
316 
317  bdds->clear_container();
318  }
319  catch (BESError &e) {
320  throw;
321  }
322  catch (InternalErr & e) {
323  BESDapError ex(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
324  throw ex;
325  }
326  catch (Error & e) {
327  BESDapError ex(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
328  throw ex;
329  }
330  catch (std::exception &e) {
331  string s = string("C++ Exception: ") + e.what();
332  BESInternalFatalError ex(s, __FILE__, __LINE__);
333  throw ex;
334  }
335  catch (...) {
336  string s = "unknown exception caught building DDS";
337  BESInternalFatalError ex(s, __FILE__, __LINE__);
338  throw ex;
339  }
340 
341  BESDEBUG(module, "DmrppRequestHandler::dap_build_dds() - END" << endl);
342  return true;
343 }
344 
349 {
350  BESStopWatch sw;
351  if (BESISDEBUG(TIMING_LOG)) sw.start("DmrppRequestHandler::dap_build_dds()", dhi.data[REQUEST_ID]);
352 
353  BESDEBUG(module, __func__ << "() - BEGIN" << endl);
354 
355  BESResponseObject *response = dhi.response_handler->get_response_object();
356  BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *>(response);
357  if (!bdds) throw BESInternalError("Cast error, expected a BESDDSResponse object.", __FILE__, __LINE__);
358 
359  try {
360  string container_name_str = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
361 
362  DDS *dds = bdds->get_dds();
363  if (!container_name_str.empty()) dds->container_name(container_name_str);
364  string accessed = dhi.container->access();
365 
366  // Look in memory cache, if it's initialized
367  DDS *cached_dds_ptr = 0;
368  if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(accessed)))) {
369  // copy the cached DAS into the BES response object
370  BESDEBUG(module, "DDS Cached hit for : " << accessed << endl);
371  *dds = *cached_dds_ptr;
372  }
373  else {
374  // Not in cache, make one...
375  DMR *dmr = new DMR(); // FIXME is this leaked? jhrg 6/1/18
376  build_dmr_from_file(dhi.container, dmr);
377 
378  // delete the current one;
379  delete dds;
380  dds = 0;
381 
382  // assign the new one.
383  dds = dmr->getDDS();
384 
385  // Stuff it into the response.
386  bdds->set_dds(dds);
387  bdds->set_constraint(dhi);
388 
389  // Cache it, if the cache is active.
390  if (dds_cache) {
391  dds_cache->add(new DDS(*dds), accessed);
392  }
393  }
394 
395  bdds->clear_container();
396  }
397  catch (BESError &e) {
398  throw;
399  }
400  catch (InternalErr & e) {
401  BESDapError ex(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
402  throw ex;
403  }
404  catch (Error & e) {
405  BESDapError ex(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
406  throw ex;
407  }
408  catch (std::exception &e) {
409  string s = string("C++ Exception: ") + e.what();
410  BESInternalFatalError ex(s, __FILE__, __LINE__);
411  throw ex;
412  }
413  catch (...) {
414  string s = "unknown exception caught building DDS";
415  BESInternalFatalError ex(s, __FILE__, __LINE__);
416  throw ex;
417  }
418 
419  BESDEBUG(module, "DmrppRequestHandler::dap_build_dds() - END" << endl);
420  return true;
421 }
422 
428 {
429  BESStopWatch sw;
430  if (BESISDEBUG(TIMING_LOG)) sw.start("DmrppRequestHandler::dap_build_das()", dhi.data[REQUEST_ID]);
431 
432  BESResponseObject *response = dhi.response_handler->get_response_object();
433  BESDASResponse *bdas = dynamic_cast<BESDASResponse *>(response);
434  if (!bdas) throw BESInternalError("Cast error, expected a BESDASResponse object.", __FILE__, __LINE__);
435 
436  try {
437  string container_name_str = bdas->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
438 
439  DAS *das = bdas->get_das();
440  if (!container_name_str.empty()) das->container_name(container_name_str);
441  string accessed = dhi.container->access();
442 
443  // Look in memory cache (if it's initialized)
444  DAS *cached_das_ptr = 0;
445  if (das_cache && (cached_das_ptr = static_cast<DAS*>(das_cache->get(accessed)))) {
446  // copy the cached DAS into the BES response object
447  *das = *cached_das_ptr;
448  }
449  else {
450  // Not in cache, better make one!
451  // 1) Build a DMR
452  DMR *dmr = new DMR();
453  build_dmr_from_file(dhi.container, dmr);
454 
455  // Get a DDS from the DMR
456  DDS *dds = dmr->getDDS();
457 
458  // Load the BESDASResponse DAS from the DDS
459  dds->get_das(das);
460 
461  delete dds;
462  delete dmr;
463 
464  Ancillary::read_ancillary_das(*das, accessed);
465  // Add to cache if cache is active
466  if (das_cache) {
467  das_cache->add(new DAS(*das), accessed);
468  }
469  }
470 
471  bdas->clear_container();
472  }
473  catch (BESError &e) {
474  throw;
475  }
476  catch (InternalErr & e) {
477  BESDapError ex(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
478  throw ex;
479  }
480  catch (Error & e) {
481  BESDapError ex(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
482  throw ex;
483  }
484  catch (std::exception &e) {
485  string s = string("C++ Exception: ") + e.what();
486  BESInternalFatalError ex(s, __FILE__, __LINE__);
487  throw ex;
488  }
489  catch (...) {
490  string s = "unknown exception caught building DAS";
491  BESInternalFatalError ex(s, __FILE__, __LINE__);
492  throw ex;
493  }
494 
495  BESDEBUG(module, __func__ << "() - END" << endl);
496  return true;
497 }
498 
499 
500 bool DmrppRequestHandler::dap_build_vers(BESDataHandlerInterface &dhi)
501 {
502  BESVersionInfo *info = dynamic_cast<BESVersionInfo *>(dhi.response_handler->get_response_object());
503  if (!info) throw BESInternalFatalError("Expected a BESVersionInfo instance.", __FILE__, __LINE__);
504 
505  info->add_module(MODULE_NAME, MODULE_VERSION);
506  return true;
507 }
508 
509 bool DmrppRequestHandler::dap_build_help(BESDataHandlerInterface &dhi)
510 {
511  BESInfo *info = dynamic_cast<BESInfo *>(dhi.response_handler->get_response_object());
512  if (!info) throw BESInternalFatalError("Expected a BESVersionInfo instance.", __FILE__, __LINE__);
513 
514  // This is an example. If you had a help file you could load it like
515  // this and if your handler handled the following responses.
516  map<string, string> attrs;
517  attrs["name"] = MODULE_NAME /* PACKAGE_NAME */;
518  attrs["version"] = MODULE_VERSION /* PACKAGE_VERSION */;
519  list<string> services;
520  BESServiceRegistry::TheRegistry()->services_handled(module, services);
521  if (services.size() > 0) {
522  string handles = BESUtil::implode(services, ',');
523  attrs["handles"] = handles;
524  }
525  info->begin_tag("module", &attrs);
526  info->end_tag("module");
527 
528  return true;
529 }
530 
531 void DmrppRequestHandler::dump(ostream &strm) const
532 {
533  strm << BESIndent::LMarg << "DmrppRequestHandler::dump - (" << (void *) this << ")" << endl;
534  BESIndent::Indent();
536  BESIndent::UnIndent();
537 }
538 
539 } // namespace dmrpp
void set_dds(libdap::DDS *ddsIn)
static bool dap_build_dap2data(BESDataHandlerInterface &dhi)
exception thrown if an internal error is found and is fatal to the BES
exception thrown if inernal error encountered
std::string get_relative_name() const
Get the relative name of the object in this container.
Definition: BESContainer.h:189
Provide a way to print the DMR++ response.
Definition: DMRpp.h:42
static string lowercase(const string &s)
Definition: BESUtil.cc:197
Holds a DDS object within the BES.
virtual void clear_container()
clear the container in the DAP response object
void set_dds(libdap::DDS *ddsIn)
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105
libdap::DDS * get_dds()
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:420
Store the DAP DMR++ metadata responses.
virtual bool start(string name)
Definition: BESStopWatch.cc:57
virtual bool add_method(const string &name, p_request_handler_method method)
add a handler method to the request handler that knows how to fill in a specific response object
virtual string access()=0
returns the true name of this container
virtual void clear_container()
clear the container in the DAP response object
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
informational response object
Definition: BESInfo.h:68
static string implode(const list< string > &values, char delim)
Definition: BESUtil.cc:635
static bool dap_build_das(BESDataHandlerInterface &dhi)
virtual dmrpp::DMRpp * get_dmrpp_object(const std::string &name)
Build a DMR++ object from the cached Response.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:61
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void dump(std::ostream &strm) const
dumps information about this object
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Represents a specific data type request handler.
static bool dap_build_dds(BESDataHandlerInterface &dhi)
virtual void dump(ostream &strm) const
dumps information about this object
Structure storing information used by the BES to handle the request.
map< string, string > data
the map of string data that will be required for the current request.
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
Represents an OPeNDAP DAS DAP2 data object within the BES.
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:68
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:86
string get_attributes() const
retrieve the attributes desired from this container
Definition: BESContainer.h:245
Abstract base class representing a specific set of information in response to a request to the BES.
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:157
BESContainer * container
pointer to current container in this interface
static bool dap_build_dmr(BESDataHandlerInterface &dhi)
virtual void services_handled(const string &handler, list< string > &services)
returns the list of servies provided by the handler in question
string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:224