bes  Updated for version 3.20.6
NCMLRequestHandler.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2009 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 #include "config.h"
30 
31 #include <memory>
32 
33 #include <DMR.h>
34 #include <DataDDS.h>
35 
36 #include <mime_util.h>
37 #include <D4BaseTypeFactory.h>
38 
39 #include "NCMLRequestHandler.h"
40 
41 #include <BESConstraintFuncs.h>
42 #include <BESContainerStorage.h>
43 #include <BESContainerStorageList.h>
44 #include <BESDapNames.h>
45 #include "BESDataDDSResponse.h"
46 #include <BESDataNames.h>
47 #include <BESDASResponse.h>
48 #include <BESDDSResponse.h>
49 #include <BESDMRResponse.h>
50 
51 #include <BESDebug.h>
52 #include "BESStopWatch.h"
53 #include <BESInternalError.h>
54 #include <BESDapError.h>
55 #include <BESError.h>
56 #include <BESRequestHandlerList.h>
57 #include <BESResponseHandler.h>
58 #include <BESResponseNames.h>
59 #include <BESServiceRegistry.h>
60 #include <BESTextInfo.h>
61 #include <BESUtil.h>
62 #include <BESVersionInfo.h>
63 #include <TheBESKeys.h>
64 
65 #include "DDSLoader.h"
66 
67 #include "NCMLDebug.h"
68 #include "NCMLUtil.h"
69 #include "NCMLParser.h"
70 #include "NCMLResponseNames.h"
71 #include "SimpleLocationParser.h"
72 
73 using namespace agg_util;
74 using namespace ncml_module;
75 using namespace libdap;
76 
77 bool NCMLRequestHandler::_global_attributes_container_name_set = false;
78 string NCMLRequestHandler::_global_attributes_container_name = "";
79 
80 NCMLRequestHandler::NCMLRequestHandler(const string &name) :
81  BESRequestHandler(name)
82 {
83  add_method(DAS_RESPONSE, NCMLRequestHandler::ncml_build_das);
84  add_method(DDS_RESPONSE, NCMLRequestHandler::ncml_build_dds);
85  add_method(DATA_RESPONSE, NCMLRequestHandler::ncml_build_data);
86 
87  add_method(DMR_RESPONSE, NCMLRequestHandler::ncml_build_dmr);
88  add_method(DAP4DATA_RESPONSE, NCMLRequestHandler::ncml_build_dmr);
89 
90  add_method(VERS_RESPONSE, NCMLRequestHandler::ncml_build_vers);
91  add_method(HELP_RESPONSE, NCMLRequestHandler::ncml_build_help);
92 
93  if (NCMLRequestHandler::_global_attributes_container_name_set == false) {
94  bool key_found = false;
95  string value;
96  TheBESKeys::TheKeys()->get_value("NCML.GlobalAttributesContainerName", value, key_found);
97  if (key_found) {
98  // It was set in the conf file
99  NCMLRequestHandler::_global_attributes_container_name_set = true;
100 
101  NCMLRequestHandler::_global_attributes_container_name = value;
102  }
103  }
104 }
105 
106 NCMLRequestHandler::~NCMLRequestHandler()
107 {
108 }
109 
110 #if 0
111 // Not used. jhrg 4/16/14
112 
113 // This is the original example from Patrick or James for loading local file within the BES...
114 // Still used by DataDDS call, but the other callbacks use DDSLoader
115 // to get a brandy new DDX response.
116 // @see DDSLoader
117 bool NCMLRequestHandler::ncml_build_redirect(BESDataHandlerInterface &dhi, const string& location)
118 {
119  // The current container in dhi is a reference to the ncml file.
120  // Need to parse the ncml file here and get the list of locations
121  // that we will be using. Any constraints defined?
122 
123  // do this for each of the locations retrieved from the ncml file.
124  // If there are more than one locations in the ncml then we can't
125  // set the context for dap_format to dap2. This will create a
126  // structure for each of the locations in the resulting dap object.
127  string sym_name = dhi.container->get_symbolic_name();
128  BESContainerStorageList *store_list = BESContainerStorageList::TheList();
129  BESContainerStorage *store = store_list->find_persistence("catalog");
130  if (!store) {
131  throw BESInternalError("couldn't find the catalog storage", __FILE__, __LINE__);
132  }
133  // this will throw an exception if the location isn't found in the
134  // catalog. Might want to catch this. Wish the add returned the
135  // container object created. Might want to change it.
136  string new_sym = sym_name + "_location1";
137  store->add_container(new_sym, location, "");
138 
139  BESContainer *container = store->look_for(new_sym);
140  if (!container) {
141  throw BESInternalError("couldn't find the container" + sym_name, __FILE__, __LINE__);
142  }
143  BESContainer *ncml_container = dhi.container;
144  dhi.container = container;
145 
146  // this will throw an exception if there is a problem building the
147  // response for this container. Might want to catch this
148  BESRequestHandlerList::TheList()->execute_current(dhi);
149 
150  // clean up
151  dhi.container = ncml_container;
152  store->del_container(new_sym);
153 
154  return true;
155 }
156 #endif
157 
158 // Here we load the DDX response with by hijacking the current dhi via DDSLoader
159 // and hand it to our parser to load the ncml, load the DDX for the location,
160 // apply ncml transformations to it, then return the modified DDS.
161 bool NCMLRequestHandler::ncml_build_das(BESDataHandlerInterface &dhi)
162 {
163  BESStopWatch sw;
164  if (BESISDEBUG(TIMING_LOG)) sw.start("NCMLRequestHandler::ncml_build_das", dhi.data[REQUEST_ID]);
165 
166  string filename = dhi.container->access();
167 
168  // Any exceptions winding through here will cause the loader and parser dtors
169  // to clean up dhi state, etc.
170  DDSLoader loader(dhi);
171  NCMLParser parser(loader);
172  auto_ptr<BESDapResponse> loaded_bdds = parser.parse(filename, DDSLoader::eRT_RequestDDX);
173 
174  // Now fill in the desired DAS response object from the DDS
175  DDS* dds = NCMLUtil::getDDSFromEitherResponse(loaded_bdds.get());
176  VALID_PTR(dds);
177 
178  BESDASResponse *bdas = dynamic_cast<BESDASResponse *>(dhi.response_handler->get_response_object());
179  VALID_PTR(bdas);
180 
181  // Copy the modified DDS attributes into the DAS response object!
182  DAS *das = bdas->get_das();
183 
184  if (dds->get_dap_major() < 4)
185  NCMLUtil::hackGlobalAttributesForDAP2(dds->get_attr_table(),
186  NCMLRequestHandler::get_global_attributes_container_name());
187 
188  NCMLUtil::populateDASFromDDS(das, *dds);
189 
190  // loaded_bdds destroys itself.
191  return true;
192 }
193 
194 bool NCMLRequestHandler::ncml_build_dds(BESDataHandlerInterface &dhi)
195 {
196 #if 0
197  // original version 8/13/15
198  BESStopWatch sw;
199  if (BESISDEBUG(TIMING_LOG)) sw.start("NCMLRequestHandler::ncml_build_dds", dhi.data[REQUEST_ID]);
200 
201  string filename = dhi.container->access();
202 
203  // Any exceptions winding through here will cause the loader and parser dtors
204  // to clean up dhi state, etc.
205  auto_ptr<BESDapResponse> loaded_bdds(0);
206  {
207  DDSLoader loader(dhi);
208  NCMLParser parser(loader);
209  loaded_bdds = parser.parse(filename, DDSLoader::eRT_RequestDDX);
210  }
211  if (!loaded_bdds.get()) {
212  throw BESInternalError("Null BESDDSResonse in ncml DDS handler.", __FILE__, __LINE__);
213  }
214 
215  // Poke the handed down original response object with the loaded and modified one.
216  DDS* dds = NCMLUtil::getDDSFromEitherResponse(loaded_bdds.get());
217  VALID_PTR(dds);
218  BESResponseObject *response = dhi.response_handler->get_response_object();
219  BESDDSResponse *bdds_out = dynamic_cast<BESDDSResponse *>(response);
220  VALID_PTR(bdds_out);
221  DDS *dds = bdds_out->get_dds();
222  VALID_PTR(dds);
223 
224  if (dds->get_dap_major() < 4)
225  NCMLUtil::hackGlobalAttributesForDAP2(dds->get_attr_table(),
226  NCMLRequestHandler::get_global_attributes_container_name());
227 
228  // If we just use DDS::operator=, we get into trouble with copied
229  // pointers, bashing of the dataset name, etc etc so I specialize the copy for now.
231 
232  // Apply constraints to the result
233  // See comment below. jhrg 8/12/15 dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();
234  bdds_out->set_constraint(dhi);
235 
236  // Also copy in the name of the original ncml request
237  // TODO @HACK Not sure I want just the basename for the filename,
238  // but since the DDS/DataDDS response fills the dataset name with it,
239  // Our bes-testsuite fails since we get local path info in the dataset name.
240  dds->filename(name_path(filename));
241  dds->set_dataset_name(name_path(filename));
242 
243  return true;
244 #endif
245 
246  BESStopWatch sw;
247  if (BESISDEBUG(TIMING_LOG)) sw.start("NCMLRequestHandler::ncml_build_dds", dhi.data[REQUEST_ID]);
248 
249  string filename = dhi.container->access();
250 
251  // it better be a data response!
252  BESDDSResponse* ddsResponse = dynamic_cast<BESDDSResponse *>(dhi.response_handler->get_response_object());
253  NCML_ASSERT_MSG(ddsResponse,
254  "NCMLRequestHandler::ncml_build_data(): expected BESDDSResponse* but didn't get it!!");
255 
256  // Block it up to force cleanup of DHI.
257  {
258  DDSLoader loader(dhi);
259  NCMLParser parser(loader);
260  parser.parseInto(filename, DDSLoader::eRT_RequestDDX, ddsResponse);
261  }
262 
263  DDS *dds = ddsResponse->get_dds();
264  VALID_PTR(dds);
265 
266  if (dds->get_dap_major() < 4)
267  NCMLUtil::hackGlobalAttributesForDAP2(dds->get_attr_table(),
268  NCMLRequestHandler::get_global_attributes_container_name());
269 
270  // Apply constraints to the result
271  // See comment below. jhrg 8/12/15 dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();
272  ddsResponse->set_constraint(dhi);
273 
274  // Also copy in the name of the original ncml request
275  dds->filename(name_path(filename));
276  dds->set_dataset_name(name_path(filename));
277 
278  return true;
279 }
280 
281 bool NCMLRequestHandler::ncml_build_data(BESDataHandlerInterface &dhi)
282 {
283  BESStopWatch sw;
284  if (BESISDEBUG(TIMING_LOG)) sw.start("NCMLRequestHandler::ncml_build_data", dhi.data[REQUEST_ID]);
285 
286  string filename = dhi.container->access();
287 
288  // it better be a data response!
289  BESDataDDSResponse* dataResponse = dynamic_cast<BESDataDDSResponse *>(dhi.response_handler->get_response_object());
290  NCML_ASSERT_MSG(dataResponse,
291  "NCMLRequestHandler::ncml_build_data(): expected BESDataDDSResponse* but didn't get it!!");
292 
293  // Block it up to force cleanup of DHI.
294  {
295  DDSLoader loader(dhi);
296  NCMLParser parser(loader);
297  parser.parseInto(filename, DDSLoader::eRT_RequestDataDDS, dataResponse);
298  }
299 
300  // Apply constraints to the result
301 
302  // dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();
303  // Replaced the above with the code below. P West said, a while ago, that using set_constraint
304  // was better because BES containers would be supported. Not sure if that's a factor in this
305  // code... jhrg 8/12/15
306  dataResponse->set_constraint(dhi);
307 
308  // Also copy in the name of the original ncml request
309  DDS* dds = NCMLUtil::getDDSFromEitherResponse(dataResponse);
310  VALID_PTR(dds);
311 
312  dds->filename(name_path(filename));
313  dds->set_dataset_name(name_path(filename));
314 
315  return true;
316 }
317 
318 bool NCMLRequestHandler::ncml_build_dmr(BESDataHandlerInterface &dhi)
319 {
320  BESStopWatch sw;
321  if (BESISDEBUG(TIMING_LOG)) sw.start("NCMLRequestHandler::ncml_build_dmr", dhi.data[REQUEST_ID]);
322 
323  // Because this code does not yet know how to build a DMR directly, use
324  // the DMR ctor that builds a DMR using a 'full DDS' (a DDS with attributes).
325  // First step, build the 'full DDS'
326  string data_path = dhi.container->access();
327 
328  DDS *dds = 0; // This will be deleted when loaded_bdds goes out of scope.
329  auto_ptr<BESDapResponse> loaded_bdds(0);
330  try {
331  DDSLoader loader(dhi);
332  NCMLParser parser(loader);
333  loaded_bdds = parser.parse(data_path, DDSLoader::eRT_RequestDDX);
334  if (!loaded_bdds.get()) throw BESInternalError("Null BESDDSResonse in ncml DDS handler.", __FILE__, __LINE__);
335  dds = NCMLUtil::getDDSFromEitherResponse(loaded_bdds.get());
336  VALID_PTR(dds);
337  dds->filename(data_path);
338  dds->set_dataset_name(data_path);
339  }
340  catch (InternalErr &e) {
341  throw BESDapError(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
342  }
343  catch (Error &e) {
344  throw BESDapError(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
345  }
346  catch (BESError &e){
347  throw;
348  }
349  catch (...) {
350  throw BESDapError("Caught unknown error build ** DMR response", true, unknown_error, __FILE__, __LINE__);
351  }
352 
353  // Extract the DMR Response object - this holds the DMR used by the
354  // other parts of the framework.
355  BESResponseObject *response = dhi.response_handler->get_response_object();
356  BESDMRResponse &bdmr = dynamic_cast<BESDMRResponse &>(*response);
357 
358  // Get the DMR made by the BES in the BES/dap/BESDMRResponseHandler, make sure there's a
359  // factory we can use and then dump the DAP2 variables and attributes in using the
360  // BaseType::transform_to_dap4() method that transforms individual variables
361  DMR *dmr = bdmr.get_dmr();
362  dmr->set_factory(new D4BaseTypeFactory);
363  dmr->build_using_dds(*dds);
364 
365  // Instead of fiddling with the internal storage of the DHI object,
366  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
367  // methods to set the constraints. But, why? Ans: from Patrick is that
368  // in the 'container' mode of BES each container can have a different
369  // CE.
370  bdmr.set_dap4_constraint(dhi);
371  bdmr.set_dap4_function(dhi);
372 
373  return true;
374 }
375 
376 bool NCMLRequestHandler::ncml_build_vers(BESDataHandlerInterface &dhi)
377 {
378  BESVersionInfo *info = dynamic_cast<BESVersionInfo *>(dhi.response_handler->get_response_object());
379  if (!info) throw InternalErr(__FILE__, __LINE__, "Expected a BESVersionInfo instance");
380 
381  info->add_module(MODULE_NAME, MODULE_VERSION);
382  return true;
383 }
384 
385 bool NCMLRequestHandler::ncml_build_help(BESDataHandlerInterface &dhi)
386 {
387  BESInfo *info = dynamic_cast<BESInfo *>(dhi.response_handler->get_response_object());
388  if (!info) throw InternalErr(__FILE__, __LINE__, "Expected a BESVersionInfo instance");
389 
390  // This is an example. If you had a help file you could load it like
391  // this and if your handler handled the following responses.
392  map<string, string> attrs;
393  attrs["name"] = MODULE_NAME;
394  attrs["version"] = MODULE_VERSION;
395 
396  list<string> services;
397  BESServiceRegistry::TheRegistry()->services_handled(ncml_module::ModuleConstants::NCML_NAME, services);
398  if (services.size() > 0) {
399  string handles = BESUtil::implode(services, ',');
400  attrs["handles"] = handles;
401  }
402  info->begin_tag("module", &attrs);
403  //info->add_data_from_file( "NCML.Help", "NCML Help" ) ;
404  info->add_data("Please consult the online documentation at " + ncml_module::ModuleConstants::DOC_WIKI_URL);
405  info->end_tag("module");
406 
407  return true;
408 }
409 
410 void NCMLRequestHandler::dump(ostream &strm) const
411 {
412  strm << BESIndent::LMarg << "NCMLRequestHandler::dump - (" << (void *) this << ")" << endl;
413  BESIndent::Indent();
415  BESIndent::UnIndent();
416 }
417 
BESRequestHandler
Represents a specific data type request handler.
Definition: BESRequestHandler.h:74
BESDataHandlerInterface::container
BESContainer * container
pointer to current container in this interface
Definition: BESDataHandlerInterface.h:75
BESServiceRegistry::services_handled
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
Definition: BESServiceRegistry.cc:334
agg_util::DDSLoader
Definition: DDSLoader.h:62
BESDapResponse::set_dap4_constraint
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:137
BESStopWatch::start
virtual bool start(std::string name)
Definition: BESStopWatch.cc:58
BESContainerStorage
provides persistent storage for data storage information represented by a container.
Definition: BESContainerStorage.h:67
BESRequestHandlerList::execute_current
virtual void execute_current(BESDataHandlerInterface &dhi)
Execute a single method for the current container that will fill in the response object rather than i...
Definition: BESRequestHandlerList.cc:251
BESDapResponse::set_dap4_function
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:154
BESContainer::get_symbolic_name
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
BESDDSResponse::get_dds
libdap::DDS * get_dds()
Definition: BESDDSResponse.h:80
ncml_module::NCMLParser
Definition: NCMLParser.h:158
BESDASResponse
Represents an OPeNDAP DAS DAP2 data object within the BES.
Definition: BESDASResponse.h:44
BESInfo
informational response object
Definition: BESInfo.h:63
BESContainerStorage::look_for
virtual BESContainer * look_for(const std::string &sym_name)=0
looks for a container in this persistent store
agg_util
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
Definition: AggMemberDataset.cc:38
libdap
Definition: BESDapFunctionResponseCache.h:35
ncml_module::ModuleConstants::NCML_NAME
static const std::string NCML_NAME
Definition: NCMLResponseNames.h:38
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:62
ncml_module::NCMLUtil::getDDSFromEitherResponse
static libdap::DDS * getDDSFromEitherResponse(BESDapResponse *response)
Definition: NCMLUtil.cc:354
BESResponseHandler::get_response_object
virtual BESResponseObject * get_response_object()
return the current response object
Definition: BESResponseHandler.cc:82
BESVersionInfo
Definition: BESVersionInfo.h:47
BESContainerStorageList::find_persistence
virtual BESContainerStorage * find_persistence(const std::string &persist_name)
find the persistence store with the given name
Definition: BESContainerStorageList.cc:212
BESContainer::access
virtual std::string access()=0
returns the true name of this container
BESDataHandlerInterface::data
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
Definition: BESDataHandlerInterface.h:90
ncml_module::NCMLUtil::copyVariablesAndAttributesInto
static void copyVariablesAndAttributesInto(libdap::DDS *dds_out, const libdap::DDS &dds_in)
Definition: NCMLUtil.cc:330
BESContainerStorage::add_container
virtual void add_container(const std::string &sym_name, const std::string &real_name, const std::string &type)=0
adds a container with the provided information
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
BESStopWatch
Definition: BESStopWatch.h:55
TheBESKeys::get_value
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:272
BESDDSResponse
Holds a DDS object within the BES.
Definition: BESDDSResponse.h:50
BESInfo::add_data
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
Definition: BESInfo.cc:160
ncml_module::ModuleConstants::DOC_WIKI_URL
static const std::string DOC_WIKI_URL
Definition: NCMLResponseNames.h:41
BESDapResponse::set_constraint
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:115
BESDataDDSResponse
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Definition: BESDataDDSResponse.h:46
BESContainer
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
ncml_module
NcML Parser for adding/modifying/removing metadata (attributes) to existing local datasets using NcML...
Definition: AggregationElement.cc:72
BESRequestHandler::dump
virtual void dump(std::ostream &strm) const
dumps information about this object
Definition: BESRequestHandler.cc:163
Error
BESContainerStorage::del_container
virtual bool del_container(const std::string &s_name)=0
removes a container with the given symbolic name
BESUtil::implode
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:638
BESContainerStorageList
Provides a mechanism for accessing container information from different container stores registered w...
Definition: BESContainerStorageList.h:71
BESDapError
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
BESDataHandlerInterface
Structure storing information used by the BES to handle the request.
Definition: BESDataHandlerInterface.h:56
ncml_module::NCMLUtil::populateDASFromDDS
static void populateDASFromDDS(libdap::DAS *das, const libdap::DDS &dds_const)
Definition: NCMLUtil.cc:274
BESError
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
ncml_module::NCMLRequestHandler::dump
virtual void dump(std::ostream &strm) const
dumps information about this object
Definition: NCMLRequestHandler.cc:410
BESResponseObject
Abstract base class representing a specific set of information in response to a request to the BES.
Definition: BESResponseObject.h:45
BESDMRResponse
Represents an OPeNDAP DMR DAP4 data object within the BES.
Definition: BESDMRResponse.h:39