bes  Updated for version 3.20.6
VariableElement.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2009 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 #include <ctype.h>
30 #include <memory>
31 #include <sstream>
32 
33 #include <Array.h>
34 #include <BaseType.h>
35 #include <Structure.h>
36 #include <Grid.h>
37 #include <dods-limits.h>
38 
39 
40 #include "MyBaseTypeFactory.h"
41 #include "NCMLBaseArray.h"
42 #include "NCMLDebug.h"
43 #include "NCMLParser.h"
44 #include "NCMLUtil.h"
45 #include "NetcdfElement.h"
46 #include "RenamedArrayWrapper.h"
47 #include "AggregationElement.h"
48 #include "DimensionElement.h"
49 
50 #include "VariableElement.h"
51 
52 using namespace libdap;
53 using std::vector;
54 using std::auto_ptr;
55 
56 namespace ncml_module {
57 
58 const string VariableElement::_sTypeName = "variable";
59 const vector<string> VariableElement::_sValidAttributes = getValidAttributes();
60 
61 VariableElement::VariableElement() :
62  RCObjectInterface(), NCMLElement(0), _name(""), _type(""), _shape(""), _orgName(""), _shapeTokens(), _pNewlyCreatedVar(
63  0), _gotValues(false)
64 {
65 }
66 
67 VariableElement::VariableElement(const VariableElement& proto) :
68  RCObjectInterface(), NCMLElement(proto)
69 {
70  _name = proto._name;
71  _type = proto._type;
72  _shape = proto._shape;
73  _orgName = proto._orgName;
74  _shapeTokens = proto._shapeTokens;
75  _pNewlyCreatedVar = 0; // not safe to copy the proto one, so pretend we didn't.
76  _gotValues = false; // to be consistent with previosu line
77 }
78 
79 VariableElement::~VariableElement()
80 {
81  // help clear up memory
82  _shapeTokens.resize(0);
83  _shapeTokens.clear();
84 }
85 
86 const string&
88 {
89  return _sTypeName;
90 }
91 
94 {
95  return new VariableElement(*this);
96 }
97 
99 {
100  validateAttributes(attrs, _sValidAttributes);
101 
102  _name = attrs.getValueForLocalNameOrDefault("name", "");
103  _type = attrs.getValueForLocalNameOrDefault("type", "");
104  _shape = attrs.getValueForLocalNameOrDefault("shape", "");
105  _orgName = attrs.getValueForLocalNameOrDefault("orgName", "");
106 }
107 
109 {
110  VALID_PTR(_parser);
111  processBegin(*_parser);
112 }
113 
114 void VariableElement::handleContent(const string& content)
115 {
116  // Variables cannot have content like attribute. It must be within a <values> element.
117  if (!NCMLUtil::isAllWhitespace(content)) {
118  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
119  "Got non-whitespace for element content and didn't expect it. "
120  "Element='" + toString() + "' content=\"" + content + "\"");
121  }
122 }
123 
125 {
126  processEnd(*_parser);
127 }
128 
130 {
131  return "<" + _sTypeName + " name=\"" + _name + "\"" + " type=\"" + _type + "\""
132  + ((!_shape.empty()) ? (" shape=\"" + _shape + "\"") : (""))
133  + ((!_orgName.empty()) ? (" orgName=\"" + _orgName + "\"") : ("")) + ">";
134 }
135 
137 {
138  return _pNewlyCreatedVar;
139 }
140 
142 {
143  return _gotValues;
144 }
145 
147 {
148  _gotValues = true;
149 }
150 
152 
153 void VariableElement::processBegin(NCMLParser& p)
154 {
155  BESDEBUG("ncml", "VariableElement::handleBegin called for " << toString() << endl);
156 
157  if (!p.withinNetcdf()) {
158  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
159  "Got element '" + toString() + "' while not in <netcdf> node!");
160  }
161 
162  // Can only specify variable globally or within a composite variable now.
163  if (!(p.isScopeGlobal() || p.isScopeCompositeVariable())) {
164  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
165  "Got <variable> element while not within a <netcdf> or within variable container. scope='"
166  + p.getScopeString() + "'");
167  }
168 
169  // If a request to rename the variable
170  if (!_orgName.empty()) {
171  processRenameVariable(p);
172  }
173  else // Otherwise see if it's an existing or new variable _name at scope of p
174  {
175  // Lookup the variable in whatever the parser's current variable container is
176  // this could be the DDS top level or a container (constructor) variable.
177  BaseType* pVar = p.getVariableInCurrentVariableContainer(_name);
178  if (!pVar) {
179  processNewVariable(p);
180  }
181  else {
182  processExistingVariable(p, pVar);
183  }
184  }
185 }
186 
187 void VariableElement::processEnd(NCMLParser& p)
188 {
189  BESDEBUG("ncml", "VariableElement::handleEnd called at scope: '" << p.getScopeString() << "'" << endl);
190  if (!p.isScopeVariable()) {
191  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
192  "VariableElement::handleEnd called when not parsing a variable element! "
193  "Scope='" + p.getTypedScopeString()+"'");
194  }
195 
196  // We need to defer the setting of values until the dataset
197  // is closed since that's when a joinNew aggregation new map will have all its
198  // values to set, not before.
199  // But we need to allow the user to specify
200  // the metadata for the variable PRIOR to this happening, without specifying
201  // values. We need to catch the case of values not being set later...
202  // To handle this, we push new variables WITHOUT values onto a request queue
203  // in the containing dataset which will validate that they have gotten their values
204  // set at the point where the containing dataset is closed, but after the aggregations
205  // have run.
206  if (isNewVariable() && !checkGotValues()) {
207  BESDEBUG("ncml",
208  "WARNING: at parse line: " << _parser->getParseLineNumber() << " the newly created variable='" << toString() << "' did not have its values set! We will assume this is a placeholder variable"
209  " for an aggregation (such as the new outer dimension of a joinNew) and will"
210  " defer checking that required values are set until the point when this "
211  " netcdf element is closed..." " Scope='" << p.getScopeString() << "'" << endl);
212  BESDEBUG("ncml",
213  "Adding new variable name='" << _pNewlyCreatedVar->name() << "' to the validation watch list for the closing of this netcdf." << endl);
214  _parser->getCurrentDataset()->addVariableToValidateOnClose(_pNewlyCreatedVar, this);
215  }
216 
217  NCML_ASSERT_MSG(p.getCurrentVariable(),
218  "Error: VariableElement::handleEnd(): Expected non-null parser.getCurrentVariable()!");
219 
220  // Pop up the stack from this variable
221  exitScope(p);
222 
223  BaseType* pVar = p.getCurrentVariable();
224  BESDEBUG("ncml", "Variable scope now with name: " << ((pVar)?(pVar->name()):("<NULL>")) << endl);
225 }
226 
227 void VariableElement::processExistingVariable(NCMLParser& p, BaseType* pVar)
228 {
229  BESDEBUG("ncml",
230  "VariableElement::processExistingVariable() called with name='" << _name << "' at scope='" << p.getTypedScopeString() << "'" << endl);
231 
232  // If undefined, look it up
233  if (!pVar) {
234  pVar = p.getVariableInCurrentVariableContainer(_name);
235  }
236 
237  // It better exist by now
238  VALID_PTR(pVar);
239 
240  // Make sure the type matches. NOTE:
241  // We use "Structure" to mean Grid, Structure, Sequence!
242  // Also type="" will match ANY type.
243  // TODO This fails on Array as well due to NcML making arrays be a basic type with a non-zero dimension.
244  // We're gonna ignore that until we allow addition of variables, but let's leave this check here for now
245  if (!_type.empty() && !p.typeCheckDAPVariable(*pVar, p.convertNcmlTypeToCanonicalType(_type))) {
246  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
247  "Type Mismatch in variable element with name='" + _name + "' at scope='" + p.getScopeString()
248  + "' Expected type='" + _type + "' but found variable with type='" + pVar->type_name()
249  + "' In order to match a variable of any type, please do not specify variable@type attribute in your NcML file.");
250  }
251 
252  // Use this variable as the new scope until we get a handleEnd()
253  enterScope(p, pVar);
254 }
255 
256 #if 0
257 
264 void VariableElement::processRenameVariableDataWorker(NCMLParser& p, BaseType* pOrgVar)
265 {
266 
267  if (pOrgVar->is_vector_type()) {
268  // If the variable is an Array, we need to wrap it in a RenamedArrayWrapper
269  // so that it finds its data correctly.
270  // This will remove the old one and replace our wrapper under the new _name if it's an Array subclass!
271  pOrgVar = replaceArrayIfNeeded(p, pOrgVar, _name);
272  }
273  else if (pOrgVar->is_constructor_type()) {
274  // If the variable is a constructor then we are going to have to some special things so that any child variables
275  // can be read after renaming
276 
277  }
278  else if (pOrgVar->is_simple_type()) {
279  // If it's a simple type then force it to read or we won't find the new name in the source
280  // dataset when it comes time to serialize.
281  pOrgVar->read();
282  }
283 
284  // This is safe whether we converted it or not. Rename!
285  NCMLUtil::setVariableNameProperly(pOrgVar, _name);
286 }
287 #endif
288 
289 void VariableElement::processRenameVariable(NCMLParser& p)
290 {
291  BESDEBUG("ncml",
292  "VariableElement::processRenameVariable() called on '" + toString() << "' at scope='" << p.getTypedScopeString() << "'"<< endl);
293 
294  // First, make sure the data is valid
295  NCML_ASSERT_MSG(!_name.empty(), "Can't have an empty variable@name if variable@orgName is specified!");
296  NCML_ASSERT(!_orgName.empty()); // we shouldn't even call this if this is the case, but...
297 
298  // Lookup _orgName, which must exist or throw
299  BESDEBUG("ncml", "Looking up the existence of a variable with name=" << _orgName << "..." <<endl);
300  BaseType* pOrgVar = p.getVariableInCurrentVariableContainer(_orgName);
301  if (!pOrgVar) {
302  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
303  "Renaming variable failed for element='" + toString() + "' since no variable with orgName='" + _orgName
304  + "' exists at current parser scope='" + p.getScopeString()+"'");
305  }
306  BESDEBUG("ncml", "Found variable with name=" << _orgName << endl);
307 
308  // Lookup _name, which must NOT exist or throw
309  BESDEBUG("ncml", "Making sure new name=" << _name << " does not exist at this scope already..." << endl);
310  BaseType* pExisting = p.getVariableInCurrentVariableContainer(_name);
311  if (pExisting) {
312  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
313  "Renaming variable failed for element='" + toString() + "' since a variable with name='" + _name
314  + "' already exists at current parser scope='" + p.getScopeString()+"'");
315  }
316  BESDEBUG("ncml", "Success, new variable name is open at this scope." << endl);
317 
318  // Call set_name on the orgName variable.
319  BESDEBUG("ncml", "Renaming variable '" << _orgName << "' to '" << _name << "'" << endl);
320 
321  // If we are doing data, we need to handle some variables (Array)
322  // specially since they might refer to underlying data by the new name
323 
324  // After extensive work, we've come to this version of the code. It reads
325  // every renamed variable before renaming it and uses the same process to
326  // rename the variable for both data and metadata responses. Previous versions
327  // did something different (and wrong) for 'p.parsingDataRequest' and I
328  // discovered that the baselines were not correct for several tests. This
329  // version produces responses I think are correct.
330  //
331  // The old version of this code is preserved below in #if 0 .. #endif
332  //
333  // jhrg 8/13/18
334 
335  if (p.parsingDataRequest()) {
336  // Read the variable that's about to be renamed if this is a data request.
337  // jhrg 8/13/18
338 
339  bool send_p_was_set = false;
340  if (!pOrgVar->send_p()) {
341  // need set send flag to get read() to read in data
342  // See https://opendap.atlassian.net/browse/HYRAX-539 and
343  // https://opendap.atlassian.net/browse/HYRAX-802
344  pOrgVar->set_send_p(true);
345  // record the fact so we can back it out
346  send_p_was_set = true;
347  }
348 
349  pOrgVar->read();
350 
351  if (send_p_was_set) {
352  // Trick: Reset/clear send_p so that when a CE is applied the libdap CE
353  // parser does not think this variable has already had a CE applied. If
354  // it sees send_p as true, it will think the Array was already projected
355  // (and as the full size of the array). Any subsequent attempt to re-project
356  // it to a different size will fail. Similarly, is pOrgVar is a structure,
357  // the call to set_send_p(true) will cause all fields to be sent, regardless
358  // of any Structure field projection. jhrg 8/2/18
359  pOrgVar->set_send_p(false);
360  }
361  }
362 
363  auto_ptr<BaseType> pCopy = auto_ptr<BaseType>(pOrgVar->ptr_duplicate());
364  pCopy->set_name(_name);
365  if (pCopy->type() == libdap::dods_grid_c) dynamic_cast<libdap::Grid*>(pCopy.get())->array_var()->set_name(_name);
366 
367  // Nuke the old
368  p.deleteVariableAtCurrentScope(pOrgVar->name());
369 
370  // Add renamed
371  NetcdfElement* pCdf = dynamic_cast<NetcdfElement*>(p.getCurrentDataset());
372  if (pCdf->getChildAggregation()) {
373  AggregationElement* pAgg = pCdf->getChildAggregation();
374  pAgg->addAggregationVariable(_name);
375  }
376  // Add the new, which copies under the hood. auto_ptr will clean pCopy.
377  p.addCopyOfVariableAtCurrentScope(*pCopy);
378 
379 #if 0
380  auto_ptr<BaseType> pCopy = auto_ptr<BaseType>(pOrgVar->ptr_duplicate());
381  pCopy->set_name(_name);
382  if (pCopy->type() == libdap::dods_grid_c) dynamic_cast<libdap::Grid*>(pCopy.get())->array_var()->set_name(_name);
383 
384  // Nuke the old
385  p.deleteVariableAtCurrentScope(pOrgVar->name());
386 
387  // Add renamed
388  NetcdfElement* pCdf = dynamic_cast<NetcdfElement*>(p.getCurrentDataset());
389  if (pCdf->getChildAggregation()) {
390  AggregationElement* pAgg = pCdf->getChildAggregation();
391  pAgg->addAggregationVariable(_name);
392  }
393  // Add the new, which copies under the hood. auto_ptr will clean pCopy.
394  p.addCopyOfVariableAtCurrentScope(*pCopy);
395  }
396  else {
397  // The above branch will reorder the output for the DataDDS case,
398  // so we need to remove and read even if we don't convert to preserve order!
399 
400  // Need to copy unfortunately, since delete will kill storage...
401  auto_ptr<BaseType> pCopy = auto_ptr<BaseType>(pOrgVar->ptr_duplicate());
402 
403  pCopy->set_name(_name);
404  if (pCopy->type() == libdap::dods_grid_c)
405  dynamic_cast<libdap::Grid*>(pCopy.get())->array_var()->set_name(_name);
406 
407  // Nuke the old
408  p.deleteVariableAtCurrentScope(pOrgVar->name());
409 
410  // Add renamed
411  NetcdfElement* pCdf = dynamic_cast<NetcdfElement*>(p.getCurrentDataset());
412  if (pCdf->getChildAggregation()) {
413  AggregationElement* pAgg = pCdf->getChildAggregation();
414  pAgg->addAggregationVariable(_name);
415  }
416  // Add the new, which copies under the hood. auto_ptr will clean pCopy.
417  p.addCopyOfVariableAtCurrentScope(*pCopy);
418  }
419 #endif
420 
421 
422 #if 0
423  // This is an older version of the code that seemed to work, but which produced
424  // broken data responses. jhrg 8/13/18
425 
426  if (p.parsingDataRequest()) {
427  // If not an Array, force it to read or we won't find the new name in the file for HDF at least...
428  if (pOrgVar->type() != libdap::dods_array_c /* dynamic_cast<Array*>(pOrgVar)*/) {
429 
430  bool send_p_was_set = false;
431  if (!pOrgVar->send_p()) {
432  // need set send flag to get read() to read in data
433  // See https://opendap.atlassian.net/browse/HYRAX-539 and
434  // https://opendap.atlassian.net/browse/HYRAX-802
435  pOrgVar->set_send_p(true);
436  // record the fact so we can back it out
437  send_p_was_set = true;
438  }
439 
440  pOrgVar->read();
441 
442  if (send_p_was_set) {
443  // Trick: Reset/clear send_p so that when a CE is applied the libdap CE
444  // parser does not think this variable has already had a CE applied. If
445  // it sees send_p as true, it will think the Array was already projected
446  // (and as the full size of the array). Any subsequent attempt to re-project
447  // it to a different size will fail. Similarly, is pOrgVar is a structure,
448  // the call to set_send_p(true) will cause all fields to be sent, regardless
449  // of any Structure field projection. jhrg 8/2/18
450  pOrgVar->set_send_p(false);
451  }
452  }
453 
454  // If the variable is an Array, we need to wrap it in a RenamedArrayWrapper
455  // so that it finds it data correctly.
456  // This will remove the old one and replace our wrapper under the new _name if it's an Array subclass!
457  pOrgVar = replaceArrayIfNeeded(p, pOrgVar, _name);
458  // Rename variable
459 
460  pOrgVar->set_name(_name);
461  // This code was renaming in all cases, but I think we only should do this for
462  // Grids. Vector::set_name() was fixed a long time ago (see comment in the
463  // following block) and renaming parts of a structure or Sequence is probably
464  // wrong. Here we are basically hacking in a new behavior, but one that fixes
465  // problems with many NetCDF clients - that the Array of a Grid _should_ have
466  // the same name as the Grid itself. Not a DAP2/DAP4 requirement, but a CF req.
467  // jhrg 8/1/18
468  if (pOrgVar->type() == libdap::dods_grid_c) dynamic_cast<libdap::Grid*>(pOrgVar)->array_var()->set_name(_name);
469 
470  // If the variable is an Array, we need to wrap it in a RenamedArrayWrapper
471  // so that it finds it data correctly.
472  // This will remove the old one and replace our wrapper under the new _name if it's an Array subclass!
473  pOrgVar = replaceArrayIfNeeded(p, pOrgVar, _name);
474  }
475  else {
476  // The above branch will reorder the output for the DataDDS case,
477  // so we need to remove and read even if we don't convert to preserve order!
478 
479  // BaseType::set_name fails for Vector (Array etc) subtypes since it doesn't
480  // set the template's BaseType var's name as well. This function does that until
481  // a fix in libdap lets us call pOrgName->set_name(_name) directly.
482 
483  // pOrgVar->set_name(_name); // switch to this call when bug is fixed.
484  // NCMLUtil::setVariableNameProperly(pOrgVar, _name);
485 
486  // This (Vector::set_name()) was fixed long ago. jhrg 7/10/18
487 
488  // Need to copy unfortunately, since delete will kill storage...
489  auto_ptr<BaseType> pCopy = auto_ptr<BaseType>(pOrgVar->ptr_duplicate());
490 
491  NCMLUtil::setVariableNameProperly(pCopy.get(), _name);
492 
493  pCopy->set_name(_name);
494  if (pCopy->type() == libdap::dods_grid_c) dynamic_cast<libdap::Grid*>(pCopy.get())->array_var()->set_name(_name);
495 
496  // Nuke the old
497  p.deleteVariableAtCurrentScope(pOrgVar->name());
498 
499  // Add renamed
500  NetcdfElement* pCdf = dynamic_cast<NetcdfElement*>(p.getCurrentDataset());
501  if (pCdf->getChildAggregation()) {
502  AggregationElement* pAgg = pCdf->getChildAggregation();
503  pAgg->addAggregationVariable(_name);
504  }
505  // Add the new, which copies under the hood. auto_ptr will clean pCopy.
506  p.addCopyOfVariableAtCurrentScope(*pCopy);
507  }
508 #endif
509 
510 
511  // Make sure we find it under the new name
512  BaseType* pRenamedVar = p.getVariableInCurrentVariableContainer(_name);
513  NCML_ASSERT_MSG(pRenamedVar, "Renamed variable not found! Logic error!");
514 
515  // Finally change the scope to the variable.
516  enterScope(p, pRenamedVar);
517  BESDEBUG("ncml", "Entering scope of the renamed variable. Scope is now: '" << p.getTypedScopeString() << "'" << endl);
518 }
519 
520 void VariableElement::processNewVariable(NCMLParser& p)
521 {
522  BESDEBUG("ncml", "Entered VariableElement::processNewVariable..." << endl);
523 
524  // ASSERT: We know the variable with name doesn't exist, or we wouldn't call this function.
525 
526  // Type cannot be empty for a new variable!!
527  if (_type.empty()) {
528  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
529  "Must have non-empty variable@type when creating new variable='" + toString() + "'");
530  }
531 
532  // Convert the type to the canonical type...
533  string type = p.convertNcmlTypeToCanonicalType(_type);
534  if (_type.empty()) {
535  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(), "Unknown type for new variable='" + toString()+"'");
536  }
537 
538  // Tokenize the _shape string
539  NCMLUtil::tokenize(_shape, _shapeTokens, NCMLUtil::WHITESPACE);
540 
541  // Add new variables of the given type...
542  // On exit, each of these leave _parser->getCurrentVariable() with
543  // the new variable that exists within the current dataset's DDS
544  // at the current containing scope.
545  if (_type == p.STRUCTURE_TYPE) {
546  processNewStructure(p);
547  }
548  else if (_shape.empty()) // a scalar
549  {
550  processNewScalar(p, type);
551  }
552  else if (!_shape.empty()) {
553  processNewArray(p, type);
554  }
555  else {
556  THROW_NCML_INTERNAL_ERROR("UNIMPLEMENTED METHOD: Cannot create non-scalar Array types yet.");
557  }
558 
559  // Keep track that it's new so we can error if we get values for non-new.
560  // All the process new will have entered it into the dataset as scope, so
561  // getCurrentVariable() on the parser is the actual one in the dataset always.
562  _pNewlyCreatedVar = _parser->getCurrentVariable();
563 }
564 
565 void VariableElement::processNewStructure(NCMLParser& p)
566 {
567  // First, make sure we are at a parse scope that ALLOWS variables to be added!
568  if (!(p.isScopeCompositeVariable() || p.isScopeGlobal())) {
569  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(), "Cannot add a new Structure variable at current scope! "
570  "TypedScope='" + p.getTypedScopeString()+"'");
571  }
572 
573  auto_ptr<BaseType> pNewVar = MyBaseTypeFactory::makeVariable("Structure", _name);
574  NCML_ASSERT_MSG(pNewVar.get(),
575  "VariableElement::processNewStructure: factory failed to make a new Structure variable for name='" + _name +"'");
576 
577  // Add the copy, let auto_ptr clean up
578  p.addCopyOfVariableAtCurrentScope(*pNewVar);
579 
580  // Lookup the variable we just added since it is added as a copy!
581  BaseType* pActualVar = p.getVariableInCurrentVariableContainer(_name);
582  VALID_PTR(pActualVar);
583  // Make sure the copy mechanism did the right thing so we don't delete the var we just added.
584  NCML_ASSERT(pActualVar != pNewVar.get());
585 
586  // Make it be the scope for any incoming new attributes.
587  enterScope(p, pActualVar);
588 
589  // Structures technically don't have values, but we check later that we set them, so say we're good.
590  setGotValues();
591 }
592 
593 void VariableElement::processNewScalar(NCMLParser&p, const string& dapType)
594 {
595  addNewVariableAndEnterScope(p, dapType);
596 }
597 
598 void VariableElement::processNewArray(NCMLParser& p, const std::string& dapType)
599 {
600  // For now, we can reuse the processNewScalar to make the variable and enter scope and all that.
601  // Use the new template form for the Array so we get the NCMLArray<T> subclass that handles constraints.
602  addNewVariableAndEnterScope(p, "Array<" + dapType + ">");
603 
604  // Now look up the added variable so we can set it's template and dimensionality.
605  // it should be the current variable since we entered its scope above!
606  Array* pNewVar = dynamic_cast<Array*>(p.getCurrentVariable());
607  NCML_ASSERT_MSG(pNewVar,
608  "VariableElement::processNewArray: Expected non-null getCurrentVariable() in parser but got NULL!");
609 
610  // Now make the template variable of the array entry type with the same name and add it
611  auto_ptr<BaseType> pTemplateVar = MyBaseTypeFactory::makeVariable(dapType, _name);
612 #if 0
613  pNewVar->add_var(pTemplateVar.get());
614 #endif
615 
616  pNewVar->add_var_nocopy(pTemplateVar.release());
617 
618  // For each dimension in the shape, append it to make an N-D array...
619  for (unsigned int i = 0; i < _shapeTokens.size(); ++i) {
620  unsigned int dim = getSizeForDimension(p, _shapeTokens.at(i));
621  string dimName = ((isDimensionNumericConstant(_shapeTokens.at(i))) ? ("") : (_shapeTokens.at(i)));
622  BESDEBUG("ncml",
623  "Appending dimension name=\"" << dimName << "\" of size=" << dim << " to the Array name=" << pNewVar->name() << endl);
624  pNewVar->append_dim(dim, dimName);
625  }
626 
627  // Make sure the size of the flattened Array in memory (product of dimensions) is within the DAP2 limit...
628  if (getProductOfDimensionSizes(p) > static_cast<unsigned int>(DODS_MAX_ARRAY)) // actually the call itself will throw...
629  {
630  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
631  "Product of dimension sizes for Array must be < (2^31-1).");
632  }
633 }
634 
635 #if 0 // old school copy method
636 libdap::BaseType*
637 VariableElement::replaceArrayIfNeeded(NCMLParser& p, libdap::BaseType* pOrgVar, const string& name)
638 {
639  VALID_PTR(pOrgVar);
640  Array* pOrgArray = dynamic_cast<libdap::Array*>(pOrgVar);
641  if (!pOrgArray)
642  {
643  return pOrgVar;
644  }
645 
646  BESDEBUG("ncml", "VariableElement::replaceArray if needed. Renaming an Array means we need to convert it to NCMLArray." << endl);
647 
648  // Make a copy of it
649  auto_ptr<NCMLBaseArray> pNewArray = NCMLBaseArray::createFromArray(*pOrgArray);
650  VALID_PTR(pNewArray.get());
651 
652  // Remove the old one.
653  p.deleteVariableAtCurrentScope(pOrgArray->name());
654 
655  // Make sure the new name is set.
656  NCMLUtil::setVariableNameProperly(pNewArray.get(), name);
657 
658  // Add the new one. Unfortunately this copies it under the libdap hood. ARGH!
659  // So just use the get() and let the auto_ptr kill our copy.
660  p.addCopyOfVariableAtCurrentScope(*(pNewArray.get()));
661 
662  return p.getVariableInCurrentVariableContainer(name);
663 }
664 #endif
665 
666 libdap::BaseType*
667 VariableElement::replaceArrayIfNeeded(NCMLParser& p, libdap::BaseType* pOrgVar, const string& name)
668 {
669  VALID_PTR(pOrgVar);
670  Array* pOrgArray = dynamic_cast<libdap::Array*>(pOrgVar);
671  if (!pOrgArray) {
672  return pOrgVar;
673  }
674 
675  BESDEBUG("ncml",
676  "VariableElement::replaceArray if needed. Renaming an Array means we need to wrap it with RenamedArrayWrapper!" << endl);
677 
678  // Must make a clone() since deleteVariableAtCurrentScope from container below will destroy pOrgArray!
679  auto_ptr<RenamedArrayWrapper> pNewArray = auto_ptr<RenamedArrayWrapper>(
680  new RenamedArrayWrapper(static_cast<Array*>(pOrgArray->ptr_duplicate())));
681  p.deleteVariableAtCurrentScope(pOrgArray->name());
682 
683  // Make sure the new name is set.
684 #if 0
685  NCMLUtil::setVariableNameProperly(pNewArray.get(), name);
686 #endif
687  pNewArray->set_name(name);
688  if (pNewArray->type() == libdap::dods_grid_c)
689  dynamic_cast<libdap::Grid*>(pNewArray.get())->array_var()->set_name(name);
690 
691  // Add the new one. Unfortunately this copies it under the libdap hood. ARGH!
692  // So just use the get() and let the auto_ptr kill our copy.
693  p.addCopyOfVariableAtCurrentScope(*(pNewArray.get()));
694 
695  return p.getVariableInCurrentVariableContainer(name);
696 }
697 
698 void VariableElement::addNewVariableAndEnterScope(NCMLParser& p, const std::string& dapType)
699 {
700  // First, make sure we are at a parse scope that ALLOWS variables to be added!
701  if (!(p.isScopeCompositeVariable() || p.isScopeGlobal())) {
702  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
703  "Cannot add a new scalar variable at current scope! TypedScope='" + p.getTypedScopeString()+"'");
704  }
705 
706  // Destroy it no matter what sicne add_var copies it
707  auto_ptr<BaseType> pNewVar = MyBaseTypeFactory::makeVariable(dapType, _name);
708  NCML_ASSERT_MSG(pNewVar.get(),
709  "VariableElement::addNewVariable: factory failed to make a new variable of type: '" + dapType + "' for element: '"
710  + toString() +"'");
711 
712  // Now that we have it, we need to add it to the parser at current scope
713  // Internally, the add will copy the arg, not store it.
714  p.addCopyOfVariableAtCurrentScope(*pNewVar);
715 
716  // Lookup the variable we just added since it is added as a copy!
717  BaseType* pActualVar = p.getVariableInCurrentVariableContainer(_name);
718  VALID_PTR(pActualVar);
719  // Make sure the copy mechanism did the right thing so we don't delete the var we just added.
720  NCML_ASSERT(pActualVar != pNewVar.get());
721 
722  // Make it be the scope for any incoming new attributes.
723  enterScope(p, pActualVar);
724 
725 }
726 
727 void VariableElement::enterScope(NCMLParser& p, BaseType* pVar)
728 {
729  VALID_PTR(pVar);
730 
731  // Add the proper variable scope to the stack
732  if (pVar->is_constructor_type()) {
733  p.enterScope(_name, ScopeStack::VARIABLE_CONSTRUCTOR);
734  }
735  else {
736  p.enterScope(_name, ScopeStack::VARIABLE_ATOMIC);
737  }
738 
739  // this sets the _pCurrentTable to the variable's table.
740  p.setCurrentVariable(pVar);
741 }
742 
743 void VariableElement::exitScope(NCMLParser& p)
744 {
745  // Set the new variable container to the parent of the current.
746  // This could be NULL if we're a top level variable, making the DDS the variable container.
747  p.setCurrentVariable(p.getCurrentVariable()->get_parent());
748  p.exitScope();
749  p.printScope();
750 }
751 
752 bool VariableElement::isDimensionNumericConstant(const std::string& dimToken) const
753 {
754  // for now just test the first character is a number and assume it's a number
755  return isdigit(dimToken.at(0));
756 }
757 
758 unsigned int VariableElement::getSizeForDimension(NCMLParser& p, const std::string& dimToken) const
759 {
760  unsigned int dim = 0;
761  // First, if the first char is a number, then assume it's an explicit non-negative integer
762  if (isDimensionNumericConstant(dimToken)) {
763  stringstream token;
764  token.str(dimToken);
765  token >> dim;
766  if (token.fail()) {
767  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
768  "Trying to get the dimension size in shape=" + _shape + " for token " + dimToken
769  + " failed to parse the unsigned int!");
770  }
771  }
772  else {
773  const DimensionElement* pDim = p.getDimensionAtLexicalScope(dimToken);
774  if (pDim) {
775  return pDim->getLengthNumeric();
776  }
777  else {
778  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
779  "Failed to find a dimension with name=" + dimToken + " for variable=" + toString()
780  + " with dimension table= " + p.printAllDimensionsAtLexicalScope() + " at scope='"
781  + p.getScopeString()+"'");
782  }
783  }
784  return dim;
785 }
786 
787 unsigned int VariableElement::getProductOfDimensionSizes(NCMLParser& p) const
788 {
789  // If no shape, then it's size 0 (scalar)
790  if (_shape.empty()) {
791  return 0;
792  }
793 
794  // Otherwise compute it
795  unsigned int product = 1;
796  vector<string>::const_iterator endIt = _shapeTokens.end();
797  vector<string>::const_iterator it;
798  for (it = _shapeTokens.begin(); it != endIt; ++it) {
799  const string& dimName = *it;
800  unsigned int dimSize = getSizeForDimension(p, dimName); // might throw if not found...
801  // if multiplying this in will cause over DODS_MAX_ARRAY, then error
802  // Added test for product == 0. Coverity. jhrg 2/7/17
803  if (product == 0 || dimSize > (DODS_MAX_ARRAY / product)) {
804  THROW_NCML_PARSE_ERROR(_parser->getParseLineNumber(),
805  "Product of dimension sizes exceeds the maximum DAP2 size of 2147483647 (2^31-1)!");
806  }
807  // otherwise, multiply it in
808  product *= dimSize;
809  }
810  return product;
811 }
812 
813 vector<string> VariableElement::getValidAttributes()
814 {
815  vector<string> validAttrs;
816  validAttrs.reserve(4);
817  validAttrs.push_back("name");
818  validAttrs.push_back("type");
819  validAttrs.push_back("shape");
820  validAttrs.push_back("orgName");
821  return validAttrs;
822 }
823 }
ncml_module::VariableElement::toString
virtual string toString() const
Definition: VariableElement.cc:129
ncml_module::VariableElement::setAttributes
virtual void setAttributes(const XMLAttributeMap &attrs)
Definition: VariableElement.cc:98
ncml_module::VariableElement::handleBegin
virtual void handleBegin()
Definition: VariableElement.cc:108
ncml_module::VariableElement::setGotValues
void setGotValues()
Definition: VariableElement.cc:146
ncml_module::VariableElement::isNewVariable
bool isNewVariable() const
Definition: VariableElement.cc:136
ncml_module::NCMLParser
Definition: NCMLParser.h:158
ncml_module::VariableElement::getTypeName
virtual const string & getTypeName() const
Definition: VariableElement.cc:87
ncml_module::XMLAttributeMap
Definition: XMLHelpers.h:93
ncml_module::VariableElement::checkGotValues
bool checkGotValues() const
Definition: VariableElement.cc:141
libdap
Definition: BESDapFunctionResponseCache.h:35
ncml_module::VariableElement
Concrete class for NcML <variable> element.
Definition: VariableElement.h:58
ncml_module::NCMLUtil::isAllWhitespace
static bool isAllWhitespace(const std::string &str)
Definition: NCMLUtil.cc:103
ncml_module
NcML Parser for adding/modifying/removing metadata (attributes) to existing local datasets using NcML...
Definition: AggregationElement.cc:72
ncml_module::XMLAttributeMap::getValueForLocalNameOrDefault
const std::string getValueForLocalNameOrDefault(const std::string &localname, const std::string &defVal="") const
Definition: XMLHelpers.cc:181
ncml_module::NCMLParser::getParseLineNumber
int getParseLineNumber() const
Definition: NCMLParser.cc:200
ncml_module::NCMLUtil::setVariableNameProperly
static void setVariableNameProperly(libdap::BaseType *pVar, const std::string &name)
Definition: NCMLUtil.cc:460
ncml_module::NetcdfElement::addVariableToValidateOnClose
void addVariableToValidateOnClose(libdap::BaseType *pNewVar, VariableElement *pVE)
Definition: NetcdfElement.cc:476
ncml_module::NCMLElement::validateAttributes
virtual bool validateAttributes(const XMLAttributeMap &attrs, const std::vector< std::string > &validAttrs, std::vector< std::string > *pInvalidAttrs=0, bool printInvalid=true, bool throwOnError=true)
Definition: NCMLElement.cc:174
ncml_module::VariableElement::clone
virtual VariableElement * clone() const
Definition: VariableElement.cc:93
ncml_module::VariableElement::handleEnd
virtual void handleEnd()
Definition: VariableElement.cc:124