bes  Updated for version 3.20.6
NCMLUtil.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2009 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 
30 #include <ctype.h>
31 
32 #include <Array.h>
33 #include "Constructor.h"
34 #include "DAS.h"
35 #include "DDS.h"
36 #include "Grid.h"
37 #include <DataDDS.h>
38 #include <AttrTable.h>
39 
40 #include "BESDapResponse.h"
41 #include "BESDataDDSResponse.h"
42 #include "BESDDSResponse.h"
43 #include "BESDebug.h"
44 #include "BESInternalError.h"
45 
46 #include "NCMLUtil.h"
47 #include "NCMLDebug.h"
48 
49 using namespace libdap;
50 using namespace std;
51 
52 namespace ncml_module {
53 
54 const std::string NCMLUtil::WHITESPACE = " \t\n";
55 
56 int NCMLUtil::tokenize(const string& str, vector<string>& tokens, const string& delimiters)
57 {
58  BESDEBUG("ncml", "NCMLUtil::tokenize value of str:" << str << endl);
59 
60  // start empty
61  tokens.resize(0);
62  // Skip delimiters at beginning.
63  string::size_type lastPos = str.find_first_not_of(delimiters, 0);
64  // Find first "non-delimiter".
65  string::size_type pos = str.find_first_of(delimiters, lastPos);
66 
67  int count = 0; // how many we added.
68  while (string::npos != pos || string::npos != lastPos) {
69  // Found a token, add it to the vector.
70  tokens.push_back(str.substr(lastPos, pos - lastPos));
71  count++;
72  // Skip delimiters. Note the "not_of"
73  lastPos = str.find_first_not_of(delimiters, pos);
74  // Find next "non-delimiter"
75  pos = str.find_first_of(delimiters, lastPos);
76  }
77  return count;
78 }
79 
80 int NCMLUtil::tokenizeChars(const string& str, vector<string>& tokens)
81 {
82  tokens.resize(0);
83  // push each char as a token
84  for (unsigned int i = 0; i < str.size(); ++i) {
85  string val = "";
86  val += str[i];
87  tokens.push_back(val);
88  }
89  return str.size();
90 }
91 
92 bool NCMLUtil::isAscii(const string& str)
93 {
94  string::const_iterator endIt = str.end();
95  for (string::const_iterator it = str.begin(); it != endIt; ++it) {
96  if (!isascii(*it)) {
97  return false;
98  }
99  }
100  return true;
101 }
102 
103 bool NCMLUtil::isAllWhitespace(const string& str)
104 {
105  return (str.find_first_not_of(" \t\n") == string::npos);
106 }
107 
108 void NCMLUtil::trimLeft(std::string& input, const std::string& trimChars /* = WHITESPACE */)
109 {
110  size_t firstValid = input.find_first_not_of(trimChars);
111  input.erase(0, firstValid);
112 }
113 
117 void NCMLUtil::trimRight(std::string& input, const std::string& trimChars /* = WHITESPACE */)
118 {
119  size_t lastValid = input.find_last_not_of(trimChars);
120  if (lastValid != string::npos) {
121  input.erase(lastValid + 1, string::npos);
122  }
123 }
124 
125 void NCMLUtil::trimAll(std::vector<std::string>& tokens, const std::string& trimChars /* = WHITESPACE */)
126 {
127  unsigned int num = tokens.size();
128  for (unsigned int i = 0; i < num; ++i) {
129  trim(tokens[i], trimChars);
130  }
131 }
132 
133 bool NCMLUtil::toUnsignedInt(const std::string& stringVal, unsigned int& oVal)
134 {
135  bool success = true;
136  oVal = 0;
137  istringstream iss(stringVal);
138  iss >> oVal;
139  if (iss.fail() || (stringVal[0] == '-') // parsing negatives is locale-dependent, but we DO NOT want them allowed.
140  ) {
141  success = false;
142  }
143  return success;
144 }
145 
146 #if 0
147 
157 static bool
158 has_dap2_attributes(AttrTable &a)
159 {
160  for (AttrTable::Attr_iter i = a.attr_begin(), e = a.attr_end(); i != e; ++i) {
161  if (a.get_attr_type(i) != Attr_container) {
162  return true;
163  }
164  else if (has_dap2_attributes(*a.get_attr_table(i))) {
165  return true;
166  }
167  }
168 
169  return false;
170 }
171 
181 static bool
182 has_dap2_attributes(BaseType *btp)
183 {
184  if (btp->get_attr_table().get_size() && has_dap2_attributes(btp->get_attr_table())) {
185  return true;
186  }
187 
188  Constructor *cons = dynamic_cast<Constructor *>(btp);
189  if (cons) {
190  Grid* grid = dynamic_cast<Grid*>(btp);
191  if(grid) {
192  return has_dap2_attributes(grid->get_array());
193  }
194  else {
195  for (Constructor::Vars_iter i = cons->var_begin(), e = cons->var_end(); i != e; i++) {
196  if (has_dap2_attributes(*i)) return true;
197  }
198  }
199  }
200  return false;
201 }
202 #endif
203 
204 
209 static void populateAttrTableForContainerVariableRecursive(AttrTable* dasTable, Constructor* consVar)
210 {
211  VALID_PTR(dasTable);
212  VALID_PTR(consVar);
213 
214  if(!has_dap2_attributes(consVar))
215  return;
216 
217 
218  Grid* grid = dynamic_cast<Grid*>(consVar);
219  if(grid){
220  // Here we take the Attributes from the Grid Array variable and copy them into the DAS container for the Grid.
221  // This essentially flattens the Grid in the DAS. Note too that we do now pursue the MAP vectors so they
222  // do not appear in the DAS container for the Grid.
223  BESDEBUG("ncml", __func__ << "() The variable " << grid->name() << " is a Grid. So, we promote the Grid Array AttrTable content to the DAS container for Grid " << grid->name() << endl);
224  Array *gArray = grid->get_array();
225  AttrTable arrayAT = gArray->get_attr_table();
226  for( AttrTable::Attr_iter atIter = arrayAT.attr_begin(); atIter!=arrayAT.attr_end(); ++atIter){
227  AttrType type = arrayAT.get_attr_type(atIter);
228  string childName = arrayAT.get_name(atIter);
229  if (type == Attr_container){
230  BESDEBUG("ncml", __func__ << "() Adding child AttrTable '" << childName << "' to Grid " << grid->name() << endl);
231  dasTable->append_container( new AttrTable(*arrayAT.get_attr_table(atIter)), childName);
232  }
233  else {
234  vector<string>* pAttrTokens = arrayAT.get_attr_vector(atIter);
235  // append_attr makes a copy of the vector, so we don't have to do so here.
236  BESDEBUG("ncml", __func__ << "() Adding child Attrbute '" << childName << "' to Grid " << grid->name() << endl);
237  dasTable->append_attr(childName, AttrType_to_String(type), pAttrTokens);
238  }
239  }
240  }
241  else {
242  // It's not a Grid but it's still a Constructor.
243  BESDEBUG("ncml", __func__ << "() Adding attribute tables for children of a Constructor type variable " << consVar->name() << endl);
244  Constructor::Vars_iter endIt = consVar->var_end();
245  for (Constructor::Vars_iter it = consVar->var_begin(); it != endIt; ++it) {
246  BaseType* var = *it;
247  VALID_PTR(var);
248 
249  if(has_dap2_attributes(var)){
250  BESDEBUG("ncml", __func__ << "() Adding attribute table for var: " << var->name() << endl);
251  // Make a new table for the child variable
252  AttrTable* newTable = new AttrTable(var->get_attr_table());
253  // Add it to the DAS's attribute table for the consVar scope.
254  dasTable->append_container(newTable, var->name());
255 
256  // If it's a container type, we need to recurse.
257  if (var->is_constructor_type()) {
258  Constructor* child = dynamic_cast<Constructor*>(var);
259  if (!child) {
260  throw BESInternalError("Type cast error", __FILE__, __LINE__);
261  }
262  BESDEBUG("ncml", __func__ << "() Var " << child->name() << " is Constructor type, recursively adding attribute tables" << endl);
263  populateAttrTableForContainerVariableRecursive(newTable, child);
264  }
265  }
266  else {
267  BESDEBUG("ncml", __func__ << "() Variable '" << var->name() << "' has no dap2 attributes,. Skipping."<< endl);
268  }
269  }
270  }
271 }
272 
273 // This is basically the opposite of transfer_attributes.
274 void NCMLUtil::populateDASFromDDS(DAS* das, const DDS& dds_const)
275 {
276  BESDEBUG("ncml", "Populating a DAS from a DDS...." << endl);
277 
278  VALID_PTR(das);
279 
280  // Make sure the DAS is empty to start.
281  das->erase();
282 
283  // dds is semantically const in this function, but the calls to it aren't...
284  DDS& dds = const_cast<DDS&>(dds_const);
285 
286  // First, make sure we don't have a container at top level since we're assuming for now
287  // that we only have one dataset per call (right?)
288  if (dds.container()) {
289  BESDEBUG("ncml", __func__ << "() Got unexpected container " << dds.container_name() << " and is failing." << endl);
290  throw BESInternalError("Unexpected Container Error creating DAS from DDS in NCMLHandler", __FILE__, __LINE__);
291  }
292 
293  // Copy over the global attributes table
294  //BESDEBUG("ncml", "Coping global attribute tables from DDS to DAS..." << endl);
295  *(das->get_top_level_attributes()) = dds.get_attr_table();
296 
297  // For each variable in the DDS, make a table in the DAS.
298  // If the variable in composite, then recurse
299  // BESDEBUG("ncml", "Adding attribute tables for all DDS variables into DAS recursively..." << endl);
300  DDS::Vars_iter endIt = dds.var_end();
301  for (DDS::Vars_iter it = dds.var_begin(); it != endIt; ++it) {
302  // For each BaseType*, copy its table and add to DAS under its name.
303  BaseType* var = *it;
304  VALID_PTR(var);
305 
306  // By adding this test we stop adding empty top=level containers to the DAS.
307  if(has_dap2_attributes(var)){
308  BESDEBUG("ncml", "Adding attribute table for variable: " << var->name() << endl);
309  AttrTable* clonedVarTable = new AttrTable(var->get_attr_table());
310  VALID_PTR(clonedVarTable);
311  das->add_table(var->name(), clonedVarTable);
312 
313  // If it's a container type, we need to recurse.
314  if (var->is_constructor_type()) {
315  Constructor* consVar = dynamic_cast<Constructor*>(var);
316  if (!consVar) {
317  throw BESInternalError("Type cast error", __FILE__, __LINE__);
318  }
319  populateAttrTableForContainerVariableRecursive(clonedVarTable, consVar);
320  }
321  }
322  else {
323  BESDEBUG("ncml", __func__ << "() Variable '" << var->name() << "' has no dap2 attributes,. Skipping."<< endl);
324  }
325  }
326 }
327 
328 // This function was added since DDS::operator= had some bugs we need to fix.
329 // At that point, we can just use that function, probably.
330 void NCMLUtil::copyVariablesAndAttributesInto(DDS* dds_out, const DDS& dds_in)
331 {
332  VALID_PTR(dds_out);
333 
334  // Avoid obvious bugs
335  if (dds_out == &dds_in) {
336  return;
337  }
338 
339  // handle semantic constness
340  DDS& dds = const_cast<DDS&>(dds_in);
341 
342  // Copy the global attribute table
343  dds_out->get_attr_table() = dds.get_attr_table();
344 
345  // copy the things pointed to by the variable list, not just the pointers
346  // add_var is designed to deepcopy *i, so this should get all the children
347  // as well.
348  for (DDS::Vars_iter i = dds.var_begin(); i != dds.var_end(); ++i) {
349  dds_out->add_var(*i); // add_var() dups the BaseType.
350  }
351 }
352 
353 libdap::DDS*
354 NCMLUtil::getDDSFromEitherResponse(BESDapResponse* response)
355 {
356  DDS* pDDS = 0;
357  BESDDSResponse* pDDXResponse = dynamic_cast<BESDDSResponse*>(response);
358  BESDataDDSResponse* pDataDDSResponse = dynamic_cast<BESDataDDSResponse*>(response);
359 
360  if (pDDXResponse) {
361  pDDS = pDDXResponse->get_dds();
362  }
363  else if (pDataDDSResponse) {
364  pDDS = pDataDDSResponse->get_dds(); // return as superclass ptr
365  }
366  else {
367  pDDS = 0; // return null on error
368  }
369 
370  return pDDS;
371 }
372 
373 // This little gem takes attributes that have been added to the top level
374 // attribute table (which is allowed in DAP4) and moves them all to a single
375 // container. In DAP2, only containers are allowed at the top level of the
376 // DAS. By _convention_ the name of the global attributes is NC_GLOBAL although
377 // other names are equally valid...
378 //
379 // How this works: The top-level attribute table is filled with various global
380 // attributes. To follow the spec for DAP2 that top-level container must contain
381 // _only_ other containers, each of which must be named. There are four cases...
382 //
383 // jhrg 12/15/11
384 void NCMLUtil::hackGlobalAttributesForDAP2(libdap::AttrTable &global_attributes,
385  const std::string &global_container_name)
386 {
387  if (global_container_name.empty()) return;
388 
389  // Cases: 1. only containers at the top --> return
390  // 2. only simple attrs at the top --> move them into one container
391  // 3. mixture of simple and containers --> move the simples into a new container
392  // 4. mixture ... and global_container_name exists --> move simples into that container
393 
394  // Look at the top-level container and see if it has any simple attributes.
395  // If it is empty or has only containers, do nothing.
396  bool simple_attribute_found = false;
397  AttrTable::Attr_iter i = global_attributes.attr_begin();
398  while (!simple_attribute_found && i != global_attributes.attr_end()) {
399  if (!global_attributes.is_container(i)) simple_attribute_found = true;
400  ++i;
401  }
402 
403  // Case 1
404  if (!simple_attribute_found) return;
405 #if 0
406  // Now determine if there are _only_ simple attributes
407  bool only_simple_attributes = true;
408  i = global_attributes.attr_begin();
409  while (only_simple_attributes && i != global_attributes.attr_end()) {
410  if (global_attributes.is_container(i))
411  only_simple_attributes = false;
412  ++i;
413  }
414 
415  // Case 2
416  // Note that the assignment operator first clears the destination and
417  // then performs a deep copy, so the 'new_global_attr_container' will completely
418  // replace the existing collection of attributes at teh top-level.
419  if (only_simple_attributes)
420  {
421  AttrTable *new_global_attr_container = new AttrTable();
422  AttrTable *new_attr_container = new_global_attr_container->append_container(global_container_name);
423  *new_attr_container = global_attributes;
424  global_attributes = *new_global_attr_container;
425 
426  return;
427  }
428 #endif
429  // Cases 2, 3 & 4
430  AttrTable *new_attr_container = global_attributes.find_container(global_container_name);
431  if (!new_attr_container) new_attr_container = global_attributes.append_container(global_container_name);
432 
433  // Now we have a destination for all the simple attributes
434  i = global_attributes.attr_begin();
435  while (i != global_attributes.attr_end()) {
436  if (!global_attributes.is_container(i)) {
437  new_attr_container->append_attr(global_attributes.get_name(i), global_attributes.get_type(i),
438  global_attributes.get_attr_vector(i));
439  }
440  ++i;
441  }
442 
443  // Now delete the simple attributes we just moved; they are not deleted in the
444  // above loop because deleting things in a container invalidates iterators
445  i = global_attributes.attr_begin();
446  while (i != global_attributes.attr_end()) {
447  if (!global_attributes.is_container(i)) {
448  global_attributes.del_attr(global_attributes.get_name(i));
449  // delete invalidates iterators; must restart the loop
450  i = global_attributes.attr_begin();
451  }
452  else {
453  ++i;
454  }
455  }
456 
457  return;
458 }
459 
460 void NCMLUtil::setVariableNameProperly(libdap::BaseType* pVar, const std::string& name)
461 {
462  VALID_PTR(pVar);
463  pVar->set_name(name);
464  // if template, set it too since it's used to print dds...
465  BaseType* pTemplate = pVar->var();
466  if (pTemplate) {
467  pTemplate->set_name(name);
468  }
469 }
470 } // namespace ncml_module
BESDapResponse
Represents an OPeNDAP DAP response object within the BES.
Definition: BESDapResponse.h:41
BESDDSResponse::get_dds
libdap::DDS * get_dds()
Definition: BESDDSResponse.h:80
libdap
Definition: BESDapFunctionResponseCache.h:35
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
BESDDSResponse
Holds a DDS object within the BES.
Definition: BESDDSResponse.h:50
BESDataDDSResponse
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Definition: BESDataDDSResponse.h:46
ncml_module
NcML Parser for adding/modifying/removing metadata (attributes) to existing local datasets using NcML...
Definition: AggregationElement.cc:72