bes  Updated for version 3.20.6
DDSLoader.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2009 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 
30 #include "config.h"
31 
32 #include <sstream>
33 #include <algorithm>
34 
35 #include <DataDDS.h>
36 
37 #include <BESConstraintFuncs.h>
38 #include <BESContainerStorage.h>
39 #include <BESContainerStorageList.h>
40 #include <BESDapNames.h>
41 #include <BESDapResponse.h>
42 #include <BESDataDDSResponse.h>
43 #include <BESDataHandlerInterface.h>
44 #include <BESDDSResponse.h>
45 #include <BESStopWatch.h>
46 #include <BESInternalError.h>
47 #include <BESResponseHandler.h>
48 #include <BESResponseNames.h>
49 #include <BESRequestHandlerList.h>
50 #include <BESServiceRegistry.h>
51 #include <BESTextInfo.h>
52 #include <BESUtil.h>
53 #include <BESVersionInfo.h>
54 
55 #include <BESDebug.h>
56 #include <BESLog.h>
57 
58 #include "DDSLoader.h"
59 #include "NCMLDebug.h"
60 #include "NCMLUtil.h"
61 
62 using namespace std;
63 using namespace agg_util;
64 using namespace libdap;
65 
66 // Rep Init
67 
68 /* static */
69 long DDSLoader::_gensymID = 0L;
70 
71 // Impl
72 
73 DDSLoader::DDSLoader(BESDataHandlerInterface& dhi) :
74  _dhi(dhi), /*d_saved_dhi(0),*/_hijacked(false), _filename(""), _store(0), _containerSymbol(""), _origAction(""), _origActionName(
75  ""), _origContainer(0), _origResponse(0)
76 {
77 }
78 
79 // WE ONLY COPY THE DHI! I got forced to impl this.
80 DDSLoader::DDSLoader(const DDSLoader& proto) :
81  _dhi(proto._dhi), /*d_saved_dhi(0),*/_hijacked(false), _filename(""), _store(0), _containerSymbol(""), _origAction(
82  ""), _origActionName(""), _origContainer(0), _origResponse(0)
83 {
84 }
85 
86 DDSLoader&
87 DDSLoader::operator=(const DDSLoader& rhs)
88 {
89  BESDEBUG("ncml", "DDSLoader::operator=: " << endl);
90 
91  if (&rhs == this) {
92  return *this;
93  }
94 
95  // First cleanup any state
96 
97  // Old comment, written in the midst of fixing bug #2176...
98  // I removed this call because ensureClean() will call restoreDHI()
99  // and then we will call the BESDataHandlerInterface::clone() method
100  // which will take the DHI from the DDSLoader passed in and clone it.
101  // ...no sense setting it twice.
102  //
103  // After the fix...
104  // Added it back in because I think it might be used after all. In many
105  // (all?) cases, the rhs._dhi is the same object as this->_dhi, so the
106  // clone() method will never get called. However, the 'saved state' of
107  // the DHI instance might be needed. It's not needed for the current
108  // operations (no tests fail when it is removed), but future versions
109  // might make use of the saved state.
110  // jhrg 4/18/14
111  ensureClean();
112 
113  // Now copy the dhi only, since
114  // we assume we'll be using this fresh.
115  // Normally we don't want to copy these
116  // but I got forced into it.
117  //
118  // Update. Fix for bug #2176. With clang-503.0.40 calling
119  // BESDataHandlerInterface::make_copy() was inexplicably nulling the 'data'
120  // map member of the DHI. This was happening because the two maps were the same
121  // because the two DHI instances were the same - that is the 'rhs._dhi' field
122  // and this' _dhi field were/are one and the same. I'm going to leave this
123  // test here even though the code in BESDataHandlerInterface has been fixed to
124  // test for this case - and new copy ctor and operator=() methods added.
125  // jhrg 4/18/14
126  if (&_dhi != &rhs._dhi) _dhi.make_copy(rhs._dhi);
127 
128  return *this;
129 }
130 
132 {
133  ensureClean();
134 }
135 
136 #if 0
137 // Never used. 10/16/15 jhrg
138 auto_ptr<BESDapResponse> DDSLoader::load(const string& location, ResponseType type)
139 {
140  // We need to make the proper response object as well, since in this call the dhi is coming in with the
141  // response object for the original ncml request.
142  std::auto_ptr<BESDapResponse> response = makeResponseForType(type);
143  loadInto(location, type, response.get());
144  return response; // relinquish
145 }
146 #endif
147 
148 void DDSLoader::loadInto(const std::string& location, ResponseType type, BESDapResponse* pResponse)
149 {
150  VALID_PTR(pResponse);
151  VALID_PTR(_dhi.response_handler);
152 
153  // Just be sure we're cleaned up before doing anything, in case the caller calls load again after exception
154  // and before dtor.
155  ensureClean();
156 
157  _filename = location;
158 
159  // Remember current state of dhi before we touch it -- _hijacked is now true!!
160  snapshotDHI();
161 
162 #if 0
163  BESContainer *container = nullptr;
164 
165  try {
166  // Add a new symbol to the storage list and return container for it.
167  // We will remove this new container on the way out.
168  //
169  // HK-474: If there is no handler configured to read the dataset (based on the values
170  // of the various TypeMatch parameters), code nested inside this call with throw a
171  // BESInternalError and the 'unwinding' of the containers will fail with some odd
172  // ramifications, including segfaults and no message to the user/client. jhrg 11/13/19
173  container = addNewContainerToStorage();
174  }
175  catch (BESError &e) {
176  *(BESLog::TheLog()) << "WARNING - " << string(__PRETTY_FUNCTION__) << ": " << e.get_file() << ":" << e.get_line() << ": "
177  << e.get_message() << " (the exception was re-thrown)."<< endl;
178 
179  // Get rid of the container we added.
180  removeContainerFromStorage();
181 
182  // Put back the dhi state we hijacked
183  restoreDHI();
184 
185  throw e;
186  }
187 #endif
188 
189  try {
190  // Add a new symbol to the storage list and return container for it.
191  // We will remove this new container on the way out.
192  //
193  // HK-474: If there is no handler configured to read the dataset (based on the values
194  // of the various TypeMatch parameters), code nested inside this call with throw a
195  // BESInternalError and the 'unwinding' of the containers will fail with some odd
196  // ramifications, including segfaults and no message to the user/client. jhrg 11/13/19
197  BESContainer *container = addNewContainerToStorage();
198 
199  // Take over the dhi
200  // Container is allocated using ptr_duplicate. Must free existing container. See RestoreDHI. jhrg 6/19/19
201  _dhi.container = container;
202  _dhi.response_handler->set_response_object(pResponse);
203 
204  // Choose the proper request type...
205  _dhi.action = getActionForType(type);
206  _dhi.action_name = getActionNameForType(type);
207 
208  // Figure out which underlying type of response it is to get the DDS (or DataDDS via DDS super).
210  if (!pDDS) {
211  THROW_NCML_INTERNAL_ERROR("DDSLoader::load expected BESDDSResponse or BESDataDDSResponse but got neither!");
212  }
213  pDDS->set_request_xml_base(pResponse->get_request_xml_base());
214 
215  // DO IT!
216 
217  BESDEBUG("ncml", "Before BESRequestHandlerList::TheList()->execute_current" << endl);
218  BESDEBUG("ncml", "Handler name: " << BESRequestHandlerList::TheList()->get_handler_names() << endl);
219 
220  BESRequestHandlerList::TheList()->execute_current(_dhi);
221 
222  // Some NcML operations like rename/add attributes need to have attributes in the data access.
223  // So we need to check if the attributes are added by the underneath handlers.
224  // If not, it will be added here. KY 10/30/19
225  if(type == eRT_RequestDataDDS) {
226 
227  BESResponseObject *response = _dhi.response_handler->get_response_object();
228  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *> (response);
229  if (!bdds)
230  throw BESInternalError("cast error", __FILE__, __LINE__);
231 
232  if(bdds->get_ia_flag() == false) {
233  BESDEBUG("ncml", "Underneath handler "<< _dhi.container->get_container_type() << " call add_attributes() " << endl);
234  BESRequestHandler *besRH = BESRequestHandlerList::TheList()->find_handler(_dhi.container->get_container_type());
235  besRH->add_attributes(_dhi);
236  }
237  }
238 
239  BESDEBUG("ncml", "After BESRequestHandlerList::TheList()->execute_current" << endl);
240 
241  _filename = "";
242 
243  ensureClean();
244  }
245  catch (BESError &e) {
246  *(BESLog::TheLog()) << "WARNING - " << string(__PRETTY_FUNCTION__) << ": " << e.get_file() << ":" << e.get_line() << ": "
247  << e.get_message() << " (the exception was re-thrown)."<< endl;
248 
249  // We should be clean here too.
250  ensureClean();
251 
252  throw e;
253  }
254 }
255 
257 {
258  ensureClean();
259 }
260 
261 bool is_url(std::string location){
262  std::string http("http://");
263  std::string https("https://");
264 
265  // case insensitive check
266  std::string tip = location.substr(0,http.size());
267  std::transform(tip.begin(), tip.end(), tip.begin(), ::tolower);
268  bool result = http.compare(tip)==0;
269 
270  // case insensitive check
271  tip = location.substr(0,https.size());
272  std::transform(tip.begin(), tip.end(), tip.begin(), ::tolower);
273 
274  result = result || http.compare(tip)==0;
275 
276  return result;
277 }
278 
280 DDSLoader::addNewContainerToStorage()
281 {
282  // Make sure we can find the storage
283  BESContainerStorageList *store_list = BESContainerStorageList::TheList();
284  VALID_PTR(store_list);
285 
286  // Check for URL in the _filename, 'cause that means gateway!
287  BESContainerStorage* store;
288  if(is_url(_filename)){
289  BESDEBUG("ncml", __func__ << "() - GATEWAY CONTAINER!" << endl);
290  store = store_list->find_persistence("gateway");
291  }
292  else {
293  store = store_list->find_persistence("catalog");
294  }
295  if (!store) {
296  throw BESInternalError("couldn't find the catalog storage", __FILE__, __LINE__);
297  }
298 
299  // Make a new symbol from the ncml filename
300  // NCML_ASSERT_MSG(_dhi.container, "DDSLoader::addNewContainerToStorage(): null container!");
301  // string newSymbol = _dhi.container->get_symbolic_name() + "_location_" + _filename;
302  string newSymbol = getNextContainerName() + "__" + _filename;
303 
304  // this will throw an exception if the location isn't found in the
305  // catalog. Might want to catch this. Wish the add returned the
306  // container object created. Might want to change it.
307  //
308  // HK-474 This is the crux of the problem.
309  store->add_container(newSymbol, _filename, "");
310 
311  // If we were successful, note the store location and symbol we added for removal later.
312  _store = store;
313  _containerSymbol = newSymbol;
314 
315  // Now look up the symbol we added
316  BESContainer *container = store->look_for(_containerSymbol);
317  if (!container) {
318  throw BESInternalError("couldn't find the container we just added:" + newSymbol, __FILE__, __LINE__);
319  }
320 
321  return container;
322 }
323 
324 void DDSLoader::removeContainerFromStorage()
325 {
326  // If we have non-null _store, we added the container symbol,
327  // so get rid of it
328  if (_store) {
329  try {
330  // This should go through, but if there's an exception, we could unwind through the dtor,
331  // so make sure we don't.
332  _store->del_container(_containerSymbol);
333  }
334  catch (BESError& besErr) {
335  *(BESLog::TheLog()) << "WARNING: tried to remove symbol " << _containerSymbol
336  << " from singleton but unexpectedly it was not there." << endl;
337  }
338  _containerSymbol = "";
339  _store = 0;
340  }
341 }
342 
343 void DDSLoader::snapshotDHI()
344 {
345  VALID_PTR(_dhi.response_handler);
346 
347  BESDEBUG( "ncml", "DDSLoader::snapshotDHI() - Taking snapshot of DataHAndlerInterface for (action: " << _dhi.action << " action_name: " << _dhi.action_name << ")" << endl );
348  BESDEBUG( "ncml_verbose", "original dhi = " << _dhi << endl );
349 
350  // Store off the container for the original ncml file call and replace with the new one
351  _origContainer = _dhi.container;
352  _dhi.container = 0; // Mark the container so that we do not call release() more than once. jhrg 11/14/19
353  _origAction = _dhi.action;
354  _origActionName = _dhi.action_name;
355 
356  _origResponse = _dhi.response_handler->get_response_object();
357 
358  BESDEBUG( "ncml", "DDSLoader::snapshotDHI() - Replaced with DataHAndlerInterface for (action: " << _dhi.action << " action_name: " << _dhi.action_name << ")" << endl );
359 
360  _hijacked = true;
361 }
362 
363 void DDSLoader::restoreDHI()
364 {
365  VALID_PTR(_dhi.response_handler);
366 
367  // Make sure we have state before we go mucking
368  if (!_hijacked) {
369  return;
370  }
371 
372  // Before we overwrite the 'high jacked' DHI's container, call
373  // release(). If this is a simple file, it's no big deal to
374  // skip this (because the file is closed elsewhere). But, if the
375  // DHI is working with a _compressed_ file, it is a big deal
376  // because this is the call that closes the cached uncompressed
377  // file and frees the lock. This was the bug associated with
378  // ticket HR-64. jhrg 10/16/15
379  // Only release the container if it is not null. This was happening
380  // because DDSLoader:loadInto() was failing in mid-process and the
381  // DHI had been hijacked but the new container was not set. See HK-474.
382  // jhrg 11/14/19
383  if (_dhi.container) _dhi.container->release();
384  // Leak. Allocated locally by addNewContainerToStorage() in loadInto(). jhrg 6/19/19
385  delete _dhi.container;
386 
387  // Restore saved state
388  _dhi.container = _origContainer;
389  _dhi.action = _origAction;
390  _dhi.action_name = _origActionName;
391 
392  _dhi.response_handler->set_response_object(_origResponse);
393 
394  BESDEBUG( "ncml", "DDSLoader::restoreDHI() - Restored of DataHAndlerInterface for (action: " << _dhi.action << " action_name: " << _dhi.action_name << ")" << endl );
395 
396  BESDEBUG( "ncml_verbose", "restored dhi = " << _dhi << endl );
397 
398  // clear our copy of saved state
399  _origAction = "";
400  _origActionName = "";
401  _origResponse = 0;
402  _origContainer = 0;
403  _filename = "";
404 
405  _hijacked = false;
406 }
407 
411 void DDSLoader::ensureClean()
412 {
413  // If we're still hijacked here, there was an exception in load, so clean
414  // up if needed.
415  if (_hijacked) {
416  restoreDHI();
417  }
418 
419  // Make sure we've removed the new symbol from the container list as well.
420  removeContainerFromStorage();
421 }
422 
423 /* static */
424 std::string DDSLoader::getNextContainerName()
425 {
426  static const string _sPrefix = "__DDSLoader_Container_ID_";
427  _gensymID++;
428  std::ostringstream oss;
429  oss << _sPrefix << (_gensymID);
430  return oss.str();
431 }
432 
433 std::auto_ptr<BESDapResponse> DDSLoader::makeResponseForType(ResponseType type)
434 {
435  if (type == eRT_RequestDDX) {
436  // The BaseTypeFactory is leaked. jhrg 6/19/19
437  return auto_ptr<BESDapResponse>(new BESDDSResponse(new DDS(0 /*new BaseTypeFactory()*/, "virtual")));
438  }
439  else if (type == eRT_RequestDataDDS) {
440  // Leak fix jhrg 6/19/19
441  return auto_ptr<BESDapResponse>(new BESDataDDSResponse(new DDS(0 /*new BaseTypeFactory()*/, "virtual")));
442  }
443  else {
444  THROW_NCML_INTERNAL_ERROR("DDSLoader::makeResponseForType() got unknown type!");
445  }
446 }
447 
449 {
450  if (type == eRT_RequestDDX) {
451  return DDS_RESPONSE;
452  }
453  else if (type == eRT_RequestDataDDS) {
454  return DATA_RESPONSE;
455  }
456 
457  THROW_NCML_INTERNAL_ERROR("DDSLoader::getActionForType(): unknown type!");
458 }
459 
461 {
462  if (type == eRT_RequestDDX) {
463  return DDX_RESPONSE_STR;
464  }
465  else if (type == eRT_RequestDataDDS) {
466  return DATA_RESPONSE_STR;
467  }
468 
469  THROW_NCML_INTERNAL_ERROR("DDSLoader::getActionNameForType(): unknown type!");
470 }
471 
473 {
474  if (type == eRT_RequestDDX) {
475  return dynamic_cast<BESDDSResponse*>(pResponse);
476  }
477  else if (type == eRT_RequestDataDDS) {
478  return dynamic_cast<BESDataDDSResponse*>(pResponse);
479  }
480  else {
481  return false;
482  }
483 }
484 
BESRequestHandler
Represents a specific data type request handler.
Definition: BESRequestHandler.h:74
BESDataHandlerInterface::container
BESContainer * container
pointer to current container in this interface
Definition: BESDataHandlerInterface.h:75
BESError::get_line
virtual int get_line()
get the line number where the exception was thrown
Definition: BESError.h:115
agg_util::DDSLoader
Definition: DDSLoader.h:62
BESContainerStorage
provides persistent storage for data storage information represented by a container.
Definition: BESContainerStorage.h:67
BESRequestHandlerList::execute_current
virtual void execute_current(BESDataHandlerInterface &dhi)
Execute a single method for the current container that will fill in the response object rather than i...
Definition: BESRequestHandlerList.cc:251
BESDataHandlerInterface::action
std::string action
the response object requested, e.g. das, dds
Definition: BESDataHandlerInterface.h:79
agg_util::DDSLoader::~DDSLoader
virtual ~DDSLoader()
Dtor restores the state of dhi Restores the state of the dhi to what it was when object if it is stil...
Definition: DDSLoader.cc:131
BESDapResponse
Represents an OPeNDAP DAP response object within the BES.
Definition: BESDapResponse.h:41
BESError::get_message
virtual std::string get_message()
get the error message for this exception
Definition: BESError.h:99
BESError::get_file
virtual std::string get_file()
get the file name where the exception was thrown
Definition: BESError.h:107
BESContainer::get_container_type
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Definition: BESContainer.h:232
agg_util::DDSLoader::getActionNameForType
static std::string getActionNameForType(ResponseType type)
Definition: DDSLoader.cc:460
BESDataHandlerInterface::make_copy
void make_copy(const BESDataHandlerInterface &copy_from)
deprecated
Definition: BESDataHandlerInterface.cc:58
agg_util::DDSLoader::DDSLoader
DDSLoader(BESDataHandlerInterface &dhi)
Create a loader that will hijack dhi on a load call, then restore it's state.
Definition: DDSLoader.cc:73
BESContainerStorage::look_for
virtual BESContainer * look_for(const std::string &sym_name)=0
looks for a container in this persistent store
agg_util
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
Definition: AggMemberDataset.cc:38
agg_util::DDSLoader::makeResponseForType
static std::auto_ptr< BESDapResponse > makeResponseForType(ResponseType type)
Definition: DDSLoader.cc:433
libdap
Definition: BESDapFunctionResponseCache.h:35
agg_util::DDSLoader::cleanup
void cleanup()
restore dhi to clean state
Definition: DDSLoader.cc:256
BESResponseHandler::set_response_object
virtual BESResponseObject * set_response_object(BESResponseObject *o)
replaces the current response object with the specified one, returning the current response object
Definition: BESResponseHandler.cc:88
ncml_module::NCMLUtil::getDDSFromEitherResponse
static libdap::DDS * getDDSFromEitherResponse(BESDapResponse *response)
Definition: NCMLUtil.cc:354
BESResponseHandler::get_response_object
virtual BESResponseObject * get_response_object()
return the current response object
Definition: BESResponseHandler.cc:82
BESContainerStorageList::find_persistence
virtual BESContainerStorage * find_persistence(const std::string &persist_name)
find the persistence store with the given name
Definition: BESContainerStorageList.cc:212
agg_util::DDSLoader::ResponseType
ResponseType
Definition: DDSLoader.h:94
BESContainerStorage::add_container
virtual void add_container(const std::string &sym_name, const std::string &real_name, const std::string &type)=0
adds a container with the provided information
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
BESDDSResponse
Holds a DDS object within the BES.
Definition: BESDDSResponse.h:50
BESDataDDSResponse
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Definition: BESDataDDSResponse.h:46
BESContainer
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
agg_util::DDSLoader::loadInto
void loadInto(const std::string &location, ResponseType type, BESDapResponse *pResponse)
Load a DDX or DataDDS response into the given pResponse object, which must be non-null.
Definition: DDSLoader.cc:148
BESContainerStorage::del_container
virtual bool del_container(const std::string &s_name)=0
removes a container with the given symbolic name
agg_util::DDSLoader::checkResponseIsValidType
static bool checkResponseIsValidType(ResponseType type, BESDapResponse *pResponse)
Definition: DDSLoader.cc:472
BESContainerStorageList
Provides a mechanism for accessing container information from different container stores registered w...
Definition: BESContainerStorageList.h:71
BESDapResponse::get_request_xml_base
std::string get_request_xml_base() const
Return the xml:base URL for this request.
Definition: BESDapResponse.h:76
BESDataHandlerInterface
Structure storing information used by the BES to handle the request.
Definition: BESDataHandlerInterface.h:56
BESError
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
agg_util::DDSLoader::getActionForType
static std::string getActionForType(ResponseType type)
Definition: DDSLoader.cc:448
BESRequestHandlerList::find_handler
virtual BESRequestHandler * find_handler(const std::string &handler_name)
find and return the specified request handler
Definition: BESRequestHandlerList.cc:95
BESResponseObject
Abstract base class representing a specific set of information in response to a request to the BES.
Definition: BESResponseObject.h:45