bes  Updated for version 3.20.6
AggregationUtil.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2009 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 #include "AggregationUtil.h"
30 
31 // agg_util includes
32 #include "AggMemberDataset.h"
33 #include "AggregationException.h"
34 #include "Dimension.h"
35 
36 // libdap includes
37 #include <Array.h> // libdap
38 #include <AttrTable.h>
39 #include <BaseType.h>
40 #include <DataDDS.h>
41 #include <DDS.h>
42 #include <Grid.h>
43 #include "BESDebug.h"
44 #include "BESStopWatch.h"
45 
46 // Outside includes (MINIMIZE THESE!)
47 #include "NCMLDebug.h" // This the ONLY dependency on NCML Module I want in this class since the macros there are general it's ok...
48 
49 using libdap::Array;
50 using libdap::AttrTable;
51 using libdap::BaseType;
52 using libdap::Constructor;
53 using libdap::DataDDS;
54 using libdap::DDS;
55 using libdap::Grid;
56 using libdap::Vector;
57 using std::string;
58 using std::vector;
59 
60 namespace agg_util {
61 // Static class member used to track the position of the last CVs insertion
62 // when building a JoinExisting aggregation.
63 int AggregationUtil::d_last_added_cv_position = 0;
64 
66 // ArrayGetterInterface impls
67 
68 /* virtual */
69 ArrayGetterInterface::~ArrayGetterInterface()
70 {
71 }
72 
74 // TopLevelArrayGetter impl
75 
76 TopLevelArrayGetter::TopLevelArrayGetter() :
77  ArrayGetterInterface()
78 {
79 }
80 
81 /* virtual */
82 TopLevelArrayGetter::~TopLevelArrayGetter()
83 {
84 }
85 
86 /* virtual */
87 TopLevelArrayGetter*
89 {
90  return new TopLevelArrayGetter(*this);
91 }
92 
93 /* virtual */
94 libdap::Array*
95 TopLevelArrayGetter::readAndGetArray(const std::string& name, const libdap::DDS& dds,
96  const libdap::Array* const pConstraintTemplate, const std::string& debugChannel) const
97 {
98 
99  BESStopWatch sw;
100  if (BESISDEBUG(TIMING_LOG)) sw.start("TopLevelArrayGetter::readAndGetArray", "");
101 
102  // First, look up the BaseType
103  BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, name);
104 
105  // Next, if it's not there, throw exception.
106  if (!pBT) {
107  throw AggregationException("TopLevelArrayGetter: "
108  "Did not find a variable named \"" + name + "\" at the top-level of the DDS!");
109  }
110 
111  // Next, make sure it's an Array before we cast it
112  // Prefer using the enum type for speed rather than RTTI
113  if (pBT->type() != libdap::dods_array_c) {
114  throw AggregationException("TopLevelArrayGetter: "
115  "The top-level DDS variable named \"" + name + "\" was not of the expected type!"
116  " Expected:Array Found:" + pBT->type_name());
117  }
118 
119  libdap::Array* pDatasetArray = static_cast<libdap::Array*>(pBT);
120 
121  // If given, copy the constraints over to the found Array
122  if (pConstraintTemplate) {
123  agg_util::AggregationUtil::transferArrayConstraints(pDatasetArray, // into this dataset array to be read
124  *pConstraintTemplate, // from this template
125  false, // same rank Array's in template and loaded, don't skip first dim
126  false, // or the to dimension. copy whole thing.
127  !(debugChannel.empty()), // printDebug
128  debugChannel);
129  }
130 
131  // Force a read() perhaps with constraints
132  pDatasetArray->set_send_p(true);
133  pDatasetArray->set_in_selection(true);
134  pDatasetArray->read();
135 
136  return pDatasetArray;
137 }
138 
140 // TopLevelGridDataArrayGetter impl
141 
142 TopLevelGridDataArrayGetter::TopLevelGridDataArrayGetter() :
144 {
145 }
146 
147 /* virtual */
148 TopLevelGridDataArrayGetter::~TopLevelGridDataArrayGetter()
149 {
150 }
151 
152 /* virtual */
153 TopLevelGridDataArrayGetter*
155 {
156  return new TopLevelGridDataArrayGetter(*this);
157 }
158 
159 /* virtual */
160 libdap::Array*
161 TopLevelGridDataArrayGetter::readAndGetArray(const std::string& name, const libdap::DDS& dds,
162  const libdap::Array* const pConstraintTemplate, const std::string& debugChannel) const
163 {
164  BESStopWatch sw;
165  if (BESISDEBUG(TIMING_LOG)) sw.start("TopLevelGridDataArrayGetter::readAndGetArray", "");
166 
167  // First, look up the BaseType
168  BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, name);
169 
170  // Next, if it's not there, throw exception.
171  if (!pBT) {
172  throw AggregationException("TopLevelGridArrayGetter: "
173  "Did not find a variable named \"" + name + "\" at the top-level of the DDS!");
174  }
175 
176  // Next, make sure it's a Grid before we cast it
177  // Prefer using the enum type for speed rather than RTTI
178  if (pBT->type() != libdap::dods_grid_c) {
179  throw AggregationException("TopLevelGridArrayGetter: "
180  "The top-level DDS variable named \"" + name + "\" was not of the expected type!"
181  " Expected:Grid Found:" + pBT->type_name());
182  }
183 
184  // Grab the array and return it.
185  Grid* pDataGrid = static_cast<Grid*>(pBT);
186  Array* pDataArray = static_cast<Array*>(pDataGrid->array_var());
187  if (!pDataArray) {
188  throw AggregationException("TopLevelGridArrayGetter: "
189  "The data Array var for variable name=\"" + name + "\" was unexpectedly null!");
190  }
191 
192  // If given, copy the constraints over to the found Array
193  if (pConstraintTemplate) {
194  agg_util::AggregationUtil::transferArrayConstraints(pDataArray, // into this data array to be read
195  *pConstraintTemplate, // from this template
196  false, // same rank Array's in template and loaded, don't skip first dim
197  false, // also don't skip in the to array
198  !(debugChannel.empty()), // printDebug
199  debugChannel);
200  }
201 
202  // Force the read() on the Grid level since some handlers
203  // cannot handle a read on a subobject unless read() is called
204  // on the parent object. We have given the constraints to the
205  // data Array already.
206  // TODO make an option on whether to load the Grid's map
207  // vectors or not! I think for these cases we do not want them ever!
208  pDataGrid->set_send_p(true);
209  pDataGrid->set_in_selection(true);
210  pDataGrid->read();
211 
212  // Also make sure the Array was read and if not call it as well.
213  if (!pDataArray->read_p()) {
214  pDataArray->set_send_p(true);
215  pDataArray->set_in_selection(true);
216  pDataArray->read();
217  }
218 
219  return pDataArray;
220 }
221 
223 // TopLevelGridMapArrayGetter impl
224 
225 TopLevelGridMapArrayGetter::TopLevelGridMapArrayGetter(const std::string& gridName) :
226  ArrayGetterInterface(), _gridName(gridName)
227 {
228 }
229 
230 /* virtual */
231 TopLevelGridMapArrayGetter::~TopLevelGridMapArrayGetter()
232 {
233 }
234 
235 /* virtual */
236 TopLevelGridMapArrayGetter*
238 {
239  return new TopLevelGridMapArrayGetter(*this);
240 }
241 
242 /* virtual */
243 libdap::Array*
244 TopLevelGridMapArrayGetter::readAndGetArray(const std::string& arrayName, const libdap::DDS& dds,
245  const libdap::Array* const pConstraintTemplate, const std::string& debugChannel) const
246 {
247 
248  BESStopWatch sw;
249  if (BESISDEBUG(TIMING_LOG)) sw.start("TopLevelGridMapArrayGetter::readAndGetArray", "");
250 
251  // First, look up the Grid the map is in
252  BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, _gridName);
253 
254  // Next, if it's not there, throw exception.
255  if (!pBT) {
256  throw AggregationException("Did not find a variable named \"" + _gridName + "\" at the top-level of the DDS!");
257  }
258 
259  // Next, make sure it's a Grid before we cast it
260  // Prefer using the enum type for speed rather than RTTI
261  if (pBT->type() != libdap::dods_grid_c) {
262  throw AggregationException(
263  "The top-level DDS variable named \"" + _gridName + "\" was not of the expected type!"
264  " Expected:Grid Found:" + pBT->type_name());
265  }
266 
267  // Find the correct map
268  Grid* pDataGrid = static_cast<Grid*>(pBT);
269  Array* pMap = const_cast<Array*>(AggregationUtil::findMapByName(*pDataGrid, arrayName));
270  NCML_ASSERT_MSG(pMap,
271  "Expected to find the map with name " + arrayName + " within the Grid " + _gridName + " but failed to find it!");
272 
273  // Prepare it to be read in so we can get the data
274  pMap->set_send_p(true);
275  pMap->set_in_selection(true);
276 
277  // If given, copy the constraints over to the found Array
278  if (pConstraintTemplate) {
279  agg_util::AggregationUtil::transferArrayConstraints(pMap, // into this data array to be read
280  *pConstraintTemplate, // from this template
281  false, // same rank Array's in template and loaded, don't skip first dim
282  false, // also don't skip in the to array
283  !(debugChannel.empty()), // printDebug
284  debugChannel);
285  }
286 
287  // Do the read
288  pMap->read();
289 
290  return pMap;
291 }
292 
293 /*********************************************************************************************************
294  * AggregationUtil Impl
295  */
296 void AggregationUtil::performUnionAggregation(DDS* pOutputUnion, const ConstDDSList& datasetsInOrder)
297 {
298  VALID_PTR(pOutputUnion);
299 
300  // Union any non-aggregated variables from the template dataset into the aggregated dataset
301  // Because we want the joinExistingaggregation to build up the Coordinate Variables (CVs)
302  // in the order they are declared in the NCML file, we need to track the current position
303  // where the last one was inserted. We can do that with a field in the AggregationUtil
304  // class. Here we reset that field so that it starts at position 0. 12.13.11 jhrg
306 
307  vector<const DDS*>::const_iterator endIt = datasetsInOrder.end();
308  vector<const DDS*>::const_iterator it;
309  for (it = datasetsInOrder.begin(); it != endIt; ++it) {
310  const DDS* pDDS = *it;
311  VALID_PTR(pDDS);
312 
313  // Merge in the global attribute tables
314  unionAttrsInto(&(pOutputUnion->get_attr_table()),
315  // TODO there really should be const version of this in libdap::DDS
316  const_cast<DDS*>(pDDS)->get_attr_table());
317 
318  // Merge in the variables, which carry their tables along with them since the AttrTable is
319  // within the variable's "namespace", or lexical scope.
320  unionAllVariablesInto(pOutputUnion, *pDDS);
321  }
322 }
323 
324 void AggregationUtil::unionAttrsInto(AttrTable* pOut, const AttrTable& fromTableIn)
325 {
326  // semantically const
327  AttrTable& fromTable = const_cast<AttrTable&>(fromTableIn);
328  AttrTable::Attr_iter endIt = fromTable.attr_end();
329  AttrTable::Attr_iter it;
330  for (it = fromTable.attr_begin(); it != endIt; ++it) {
331  const string& name = fromTable.get_name(it);
332  AttrTable::Attr_iter attrInOut;
333  bool foundIt = findAttribute(*pOut, name, attrInOut);
334  // If it's already in the output, then skip it
335  if (foundIt) {
336  BESDEBUG("ncml",
337  "Union of AttrTable: an attribute named " << name << " already exist in output, skipping it..." << endl);
338  continue;
339  }
340  else // put a copy of it into the output
341  {
342  // containers need deep copy
343  if (fromTable.is_container(it)) {
344  AttrTable* pOrigAttrContainer = fromTable.get_attr_table(it);
345  NCML_ASSERT_MSG(pOrigAttrContainer,
346  "AggregationUtil::mergeAttrTables(): expected non-null AttrTable for the attribute container: "
347  + name);
348  AttrTable* pClonedAttrContainer = new AttrTable(*pOrigAttrContainer);
349  VALID_PTR(pClonedAttrContainer);
350  pOut->append_container(pClonedAttrContainer, name);
351  BESDEBUG("ncml",
352  "Union of AttrTable: adding a deep copy of attribute=" << name << " to the merged output." << endl);
353  }
354  else // for a simple type
355  {
356  string type = fromTable.get_type(it);
357  vector<string>* pAttrTokens = fromTable.get_attr_vector(it);
358  VALID_PTR(pAttrTokens);
359  // append_attr makes a copy of the vector, so we don't have to do so here.
360  pOut->append_attr(name, type, pAttrTokens);
361  }
362  }
363  }
364 }
365 
366 bool AggregationUtil::findAttribute(const AttrTable& inTable, const string& name, AttrTable::Attr_iter& attr)
367 {
368  AttrTable& inTableSemanticConst = const_cast<AttrTable&>(inTable);
369  attr = inTableSemanticConst.simple_find(name);
370  return (attr != inTableSemanticConst.attr_end());
371 }
372 
373 void AggregationUtil::unionAllVariablesInto(libdap::DDS* pOutputUnion, const ConstDDSList& datasetsInOrder)
374 {
375  ConstDDSList::const_iterator endIt = datasetsInOrder.end();
376  ConstDDSList::const_iterator it;
377  for (it = datasetsInOrder.begin(); it != endIt; ++it) {
378  unionAllVariablesInto(pOutputUnion, *(*it));
379  }
380 }
381 
382 void AggregationUtil::unionAllVariablesInto(libdap::DDS* pOutputUnion, const libdap::DDS& fromDDS, bool add_at_top)
383 {
384  DDS& dds = const_cast<DDS&>(fromDDS); // semantically const
385  DDS::Vars_iter endIt = dds.var_end();
386  DDS::Vars_iter it;
387  for (it = dds.var_begin(); it != endIt; ++it) {
388  BaseType* var = *it;
389  if (var) {
390  bool addedVar = addCopyOfVariableIfNameIsAvailable(pOutputUnion, *var, add_at_top);
391  if (addedVar) {
392  BESDEBUG("ncml", "Variable name=" << var->name() << " wasn't in the union yet and was added." << endl);
393  }
394  else {
395  BESDEBUG("ncml",
396  "Variable name=" << var->name() << " was already in the union and was skipped." << endl);
397  }
398  }
399  }
400 }
401 
402 // This method is used to 'initialize' a new JoinExisting aggregation so that
403 // A set of Coordinate Variables (CVs) will be inserted _in the order they are
404 // listed_ in the .ncml file.
406 {
407  //cerr << "Called resetCVInsertionPosition" << endl;
408  d_last_added_cv_position = 0;
409 }
410 
411 bool AggregationUtil::addCopyOfVariableIfNameIsAvailable(libdap::DDS* pOutDDS, const libdap::BaseType& varProto,
412  bool add_at_top)
413 {
414  bool ret = false;
415  BaseType* existingVar = findVariableAtDDSTopLevel(*pOutDDS, varProto.name());
416  if (!existingVar) {
417  // Add the var. add_var does a clone, so we don't need to.
418  BESDEBUG("ncml2", "AggregationUtil::addCopyOfVariableIfNameIsAvailable: " << varProto.name() << endl);
419  if (add_at_top) {
420  // This provides a way to remember where the last CV was inserted and adds
421  // this one after it. That provides the behavior that all of the CVs are
422  // added at the beginning of the DDS but in the order they appear in the NCML.
423  // That will translate into a greater chance of success for users, I think ...
424  //
425  // See also similar code in AggregationElement::createAndAddCoordinateVariableForNewDimensio
426  // jhrg 10/17/11
427  //cerr << "d_last_added_cv_position: " << d_last_added_cv_position << endl;
428  DDS::Vars_iter pos = pOutDDS->var_begin() + d_last_added_cv_position;
429 
430  pOutDDS->insert_var(pos, const_cast<BaseType*>(&varProto));
431 
432  ++d_last_added_cv_position;
433  }
434  else {
435  pOutDDS->add_var(const_cast<BaseType*>(&varProto));
436  }
437 
438  ret = true;
439  }
440  return ret;
441 }
442 
443 void AggregationUtil::addOrReplaceVariableForName(libdap::DDS* pOutDDS, const libdap::BaseType& varProto)
444 {
445  BaseType* existingVar = findVariableAtDDSTopLevel(*pOutDDS, varProto.name());
446 
447  // If exists, nuke it.
448  if (existingVar) {
449  pOutDDS->del_var(varProto.name());
450  }
451 
452  // Add the var. add_var does a clone, so we don't need to clone it here.
453  pOutDDS->add_var(const_cast<BaseType*>(&varProto));
454 }
455 
456 libdap::BaseType*
457 AggregationUtil::findVariableAtDDSTopLevel(const libdap::DDS& dds_const, const string& name)
458 {
459  BaseType* ret = 0;
460  DDS& dds = const_cast<DDS&>(dds_const); // semantically const
461  DDS::Vars_iter endIt = dds.var_end();
462  DDS::Vars_iter it;
463  for (it = dds.var_begin(); it != endIt; ++it) {
464  BaseType* var = *it;
465  if (var && var->name() == name) {
466  ret = var;
467  break;
468  }
469  }
470  return ret;
471 }
472 
473 template<class LibdapType>
474 LibdapType*
475 AggregationUtil::findTypedVariableAtDDSTopLevel(const libdap::DDS& dds, const string& name)
476 {
477  BaseType* pBT = findVariableAtDDSTopLevel(dds, name);
478  if (pBT) {
479  return dynamic_cast<LibdapType*>(pBT);
480  }
481  else {
482  return 0;
483  }
484 }
485 
486 #if 0
487 void AggregationUtil::produceOuterDimensionJoinedArray(Array* pJoinedArray, const std::string& joinedArrayName,
488  const std::string& newOuterDimName, const std::vector<libdap::Array*>& fromVars, bool copyData)
489 {
490  string funcName = "AggregationUtil::createOuterDimensionJoinedArray:";
491 
492  NCML_ASSERT_MSG(fromVars.size() > 0, funcName + "Must be at least one Array in input!");
493 
494  // uses the first one as template for type and shape
495  if (!validateArrayTypesAndShapesMatch(fromVars, true)) {
496  throw AggregationException(
497  funcName + " The input arrays must all have the same data type and dimensions but do not!");
498  }
499 
500  // The first will be used to "set up" the pJoinedArray
501  Array* templateArray = fromVars[0];
502  VALID_PTR(templateArray);
503  BaseType* templateVar = templateArray->var();
504  NCML_ASSERT_MSG(templateVar, funcName + "Expected a non-NULL prototype BaseType in the first Array!");
505 
506  // Set the template var for the type.
507  pJoinedArray->add_var(templateVar);
508  // and force the name to be the desired one, not the prototype's
509  pJoinedArray->set_name(joinedArrayName);
510 
511  // Copy the attribute table from the template over... We're not merging or anything.
512  pJoinedArray->set_attr_table(templateArray->get_attr_table());
513 
514  // Create a new outer dimension based on the number of inputs we have.
515  // These append_dim calls go left to right, so we need to add the new dim FIRST.
516  pJoinedArray->append_dim(fromVars.size(), newOuterDimName);
517 
518  // Use the template to add inner dimensions to the new array
519  for (Array::Dim_iter it = templateArray->dim_begin(); it != templateArray->dim_end(); ++it) {
520  int dimSize = templateArray->dimension_size(it);
521  string dimName = templateArray->dimension_name(it);
522  pJoinedArray->append_dim(dimSize, dimName);
523  }
524 
525  if (copyData) {
526  // Make sure we have capacity for the full length of the up-ranked shape.
527  pJoinedArray->reserve_value_capacity(pJoinedArray->length());
528  // Glom the data together in
529  joinArrayData(pJoinedArray, fromVars, false, // we already reserved the space
530  true); // but please clear the Vector buffers after you use each Array in fromVars to help on memory.
531  }
532 }
533 #endif
534 
535 bool AggregationUtil::validateArrayTypesAndShapesMatch(const std::vector<libdap::Array*>& arrays,
536  bool enforceMatchingDimNames)
537 {
538  NCML_ASSERT(arrays.size() > 0);
539  bool valid = true;
540  Array* pTemplate = 0;
541  for (vector<Array*>::const_iterator it = arrays.begin(); it != arrays.end(); ++it) {
542  // Set the template from the first one.
543  if (!pTemplate) {
544  pTemplate = *it;
545  VALID_PTR(pTemplate);
546  continue;
547  }
548 
549  valid = (valid && doTypesMatch(*pTemplate, **it) && doShapesMatch(*pTemplate, **it, enforceMatchingDimNames));
550  // No use continuing
551  if (!valid) {
552  break;
553  }
554  }
555  return valid;
556 }
557 
558 bool AggregationUtil::doTypesMatch(const libdap::Array& lhsC, const libdap::Array& rhsC)
559 {
560  // semantically const
561  Array& lhs = const_cast<Array&>(lhsC);
562  Array& rhs = const_cast<Array&>(rhsC);
563  return (lhs.var() && rhs.var() && lhs.var()->type() == rhs.var()->type());
564 }
565 
566 bool AggregationUtil::doShapesMatch(const libdap::Array& lhsC, const libdap::Array& rhsC, bool checkDimNames)
567 {
568  // semantically const
569  Array& lhs = const_cast<Array&>(lhsC);
570  Array& rhs = const_cast<Array&>(rhsC);
571 
572  // Check the number of dims matches first.
573  bool valid = true;
574  if (lhs.dimensions() != rhs.dimensions()) {
575  valid = false;
576  }
577  else {
578  // Then iterate on both in sync and compare.
579  Array::Dim_iter rhsIt = rhs.dim_begin();
580  for (Array::Dim_iter lhsIt = lhs.dim_begin(); lhsIt != lhs.dim_end(); (++lhsIt, ++rhsIt)) {
581  valid = (valid && (lhs.dimension_size(lhsIt) == rhs.dimension_size(rhsIt)));
582 
583  if (checkDimNames) {
584  valid = (valid && (lhs.dimension_name(lhsIt) == rhs.dimension_name(rhsIt)));
585  }
586  }
587  }
588  return valid;
589 }
590 
591 unsigned int AggregationUtil::collectVariableArraysInOrder(std::vector<Array*>& varArrays,
592  const std::string& collectVarName, const ConstDDSList& datasetsInOrder)
593 {
594  unsigned int count = 0;
595  ConstDDSList::const_iterator endIt = datasetsInOrder.end();
596  ConstDDSList::const_iterator it;
597  for (it = datasetsInOrder.begin(); it != endIt; ++it) {
598  DDS* pDDS = const_cast<DDS*>(*it);
599  VALID_PTR(pDDS);
600  Array* pVar = dynamic_cast<Array*>(findVariableAtDDSTopLevel(*pDDS, collectVarName));
601  if (pVar) {
602  varArrays.push_back(pVar);
603  count++;
604  }
605  }
606  return count;
607 }
608 
610 {
611  Array* pArr = dynamic_cast<Array*>(pBT);
612  if (pArr && (pArr->dimensions() == 1)) {
613  // only one dimension, so grab the first and make sure we only got one.
614  Array::Dim_iter it = pArr->dim_begin();
615  bool matches = (pArr->dimension_name(it) == pArr->name());
616  NCML_ASSERT_MSG((++it == pArr->dim_end()),
617  "Logic error: NCMLUtil::isCoordinateVariable(): expected one dimension from Array, but got more!");
618  return matches;
619  }
620  else {
621  return false;
622  }
623 }
624 
625 #if 0
626 void AggregationUtil::joinArrayData(Array* pAggArray, const std::vector<Array*>& varArrays,
627  bool reserveStorage/*=true*/, bool clearDataAfterUse/*=false*/)
628 {
629  // Make sure we get a pAggArray with a type var we can deal with.
630  VALID_PTR(pAggArray->var());
631  NCML_ASSERT_MSG(pAggArray->var()->is_simple_type(),
632  "AggregationUtil::joinArrayData: the output Array is not of a simple type! Can't aggregate!");
633 
634  // If the caller wants us to do it, sum up length() and reserve that much.
635  if (reserveStorage) {
636  // Figure it how much we need...
637  unsigned int totalLength = 0;
638  {
639  vector<Array*>::const_iterator it;
640  vector<Array*>::const_iterator endIt = varArrays.end();
641  for (it = varArrays.begin(); it != endIt; ++it) {
642  Array* pArr = *it;
643  if (pArr) {
644  totalLength += pArr->length();
645  }
646  }
647  }
648  pAggArray->reserve_value_capacity(totalLength);
649  }
650 
651  // For each Array, make sure it's read in and copy its data into the output.
652  unsigned int nextElt = 0; // keeps track of where we are to write next in the output
653  vector<Array*>::const_iterator it;
654  vector<Array*>::const_iterator endIt = varArrays.end();
655  for (it = varArrays.begin(); it != endIt; ++it) {
656  Array* pArr = *it;
657  VALID_PTR(pArr);
658  NCML_ASSERT_MSG(pArr->var() && (pArr->var()->type() == pAggArray->var()->type()),
659  "AggregationUtil::joinArrayData: one of the arrays to join has different type than output! Can't aggregate!");
660 
661  // Make sure it was read in...
662  if (!pArr->read_p()) {
663  pArr->read();
664  }
665 
666  // Copy it in with the Vector call and update our location
667  nextElt += pAggArray->set_value_slice_from_row_major_vector(*pArr, nextElt);
668 
669  if (clearDataAfterUse) {
670  pArr->clear_local_data();
671  }
672  }
673 
674  // That's all folks!
675 }
676 #endif
677 
679 // struct dimension
680 // {
681 // int size; ///< The unconstrained dimension size.
682 // string name; ///< The name of this dimension.
683 // int start; ///< The constraint start index
684 // int stop; ///< The constraint end index
685 // int stride; ///< The constraint stride
686 // int c_size; ///< Size of dimension once constrained
687 // };
688 
690 void AggregationUtil::printDimensions(std::ostream& os, const libdap::Array& fromArray)
691 {
692  os << "Array dimensions: " << endl;
693  Array& theArray = const_cast<Array&>(fromArray);
694  Array::Dim_iter it;
695  Array::Dim_iter endIt = theArray.dim_end();
696  for (it = theArray.dim_begin(); it != endIt; ++it) {
697  Array::dimension d = *it;
698  os << "Dim = {" << endl;
699  os << "name=" << d.name << endl;
700  os << "size=" << d.size << endl;
701  os << " }" << endl;
702  }
703  os << "End Array dimensions." << endl;
704 }
705 
706 void AggregationUtil::printConstraints(std::ostream& os, const Array& rcArray)
707 {
708  os << "Array constraints: " << endl;
709  Array& theArray = const_cast<Array&>(rcArray);
710  Array::Dim_iter it;
711  Array::Dim_iter endIt = theArray.dim_end();
712  for (it = theArray.dim_begin(); it != endIt; ++it) {
713  Array::dimension d = *it;
714  os << "Dim = {" << endl;
715  os << "name=" << d.name << endl;
716  os << "start=" << d.start << endl;
717  os << "stride=" << d.stride << endl;
718  os << "stop=" << d.stop << endl;
719  os << " }" << endl;
720  }
721  os << "End Array constraints" << endl;
722 }
723 
724 void AggregationUtil::printConstraintsToDebugChannel(const std::string& debugChannel, const libdap::Array& fromArray)
725 {
726  ostringstream oss;
727  BESDEBUG(debugChannel, "Printing constraints for Array: " << fromArray.name() << ": " << oss.str() << endl);
728  AggregationUtil::printConstraints(oss, fromArray);
729  BESDEBUG(debugChannel, oss.str() << endl);
730 }
731 
732 void AggregationUtil::transferArrayConstraints(Array* pToArray, const Array& fromArrayConst, bool skipFirstFromDim,
733  bool skipFirstToDim, bool printDebug /* = false */, const std::string& debugChannel /* = "agg_util" */)
734 {
735  VALID_PTR(pToArray);
736  Array& fromArray = const_cast<Array&>(fromArrayConst);
737 
738  // Make sure there's no constraints on output. Shouldn't be, but...
739  pToArray->reset_constraint();
740 
741  // Ensure the dimensionalities will work out.
742  int skipDelta = ((skipFirstFromDim) ? (1) : (0));
743  // If skipping output as well, subtract out the delta.
744  // If we go negative, also an error.
745  if (skipFirstToDim) {
746  skipDelta -= 1;
747  }
748  if (skipDelta < 0 || (pToArray->dimensions() + skipDelta != const_cast<Array&>(fromArrayConst).dimensions())) {
749  throw AggregationException("AggregationUtil::transferArrayConstraints: "
750  "Mismatched dimensionalities!");
751  }
752 
753  if (printDebug) {
754  BESDEBUG(debugChannel,
755  "Printing constraints on fromArray name= " << fromArray.name() << " before transfer..." << endl);
756  printConstraintsToDebugChannel(debugChannel, fromArray);
757  }
758 
759  // Only real way to the constraints is with the iterator,
760  // so we'll iterator on the fromArray and move
761  // to toarray iterator in sync.
762  Array::Dim_iter fromArrIt = fromArray.dim_begin();
763  Array::Dim_iter fromArrEndIt = fromArray.dim_end();
764  Array::Dim_iter toArrIt = pToArray->dim_begin();
765  for (; fromArrIt != fromArrEndIt; ++fromArrIt) {
766  if (skipFirstFromDim && (fromArrIt == fromArray.dim_begin())) {
767  // If we skip first to array as well, increment
768  // before the next call.
769  if (skipFirstToDim) {
770  ++toArrIt;
771  }
772  continue;
773  }
774 // If aggregates with renaming dimensions do not match each other. SK 07/26/18
775 // NCML_ASSERT_MSG(fromArrIt->name == toArrIt->name, "GAggregationUtil::transferArrayConstraints: "
776 // "Expected the dimensions to have the same name but they did not.");
777  pToArray->add_constraint(toArrIt, fromArrIt->start, fromArrIt->stride, fromArrIt->stop);
778  ++toArrIt;
779  }
780 
781  if (printDebug) {
782  BESDEBUG(debugChannel, "Printing constrains on pToArray after transfer..." << endl);
783  printConstraintsToDebugChannel(debugChannel, *pToArray);
784  }
785 }
786 
787 BaseType*
788 AggregationUtil::getVariableNoRecurse(const libdap::DDS& ddsConst, const std::string& name)
789 {
790  BaseType* ret = 0;
791  DDS& dds = const_cast<DDS&>(ddsConst); // semantically const
792  DDS::Vars_iter endIt = dds.var_end();
793  DDS::Vars_iter it;
794  for (it = dds.var_begin(); it != endIt; ++it) {
795  BaseType* var = *it;
796  if (var && var->name() == name) {
797  ret = var;
798  break;
799  }
800  }
801  return ret;
802 }
803 
804 // Ugh cut and pasted from the other...
805 // TODO REFACTOR DDS and Constructor really need a common abstract interface,
806 // like IVariableContainer that declares the iterators and associated methods.
807 BaseType*
808 AggregationUtil::getVariableNoRecurse(const libdap::Constructor& varContainerConst, const std::string& name)
809 {
810  BaseType* ret = 0;
811  Constructor& varContainer = const_cast<Constructor&>(varContainerConst); // semantically const
812  Constructor::Vars_iter endIt = varContainer.var_end();
813  Constructor::Vars_iter it;
814  for (it = varContainer.var_begin(); it != endIt; ++it) {
815  BaseType* var = *it;
816  if (var && var->name() == name) {
817  ret = var;
818  break;
819  }
820  }
821  return ret;
822 }
823 
824 /*static*/
825 Array*
827 {
828  if (!pBT) {
829  return 0;
830  }
831 
832  // After switch():
833  // if Array, will be cast to Array.
834  // if Grid, will be cast data Array member of Grid.
835  // Other types, will be null.
836  libdap::Array* pArray(0);
837  switch (pBT->type()) {
838  case libdap::dods_array_c:
839  pArray = static_cast<libdap::Array*>(pBT);
840  break;
841 
842  case libdap::dods_grid_c:
843  pArray = static_cast<Grid*>(pBT)->get_array();
844  break;
845 
846  default:
847  pArray = 0;
848  break;
849  }
850  return pArray;
851 }
852 
853 const Array*
854 AggregationUtil::findMapByName(const libdap::Grid& inGrid, const string& findName)
855 {
856  Grid& grid = const_cast<Grid&>(inGrid);
857  Array* pRet = 0;
858  Grid::Map_iter it;
859  Grid::Map_iter endIt = grid.map_end();
860  for (it = grid.map_begin(); it != endIt; ++it) {
861  if ((*it)->name() == findName) {
862  pRet = static_cast<Array*>(*it);
863  break;
864  }
865  }
866  return pRet;
867 }
868 
869 Array* AggregationUtil::readDatasetArrayDataForAggregation(const Array& constrainedTemplateArray,
870  const std::string& varName, AggMemberDataset& dataset, const ArrayGetterInterface& arrayGetter,
871  const std::string& debugChannel)
872 {
873  BESStopWatch sw;
874  if (BESISDEBUG(TIMING_LOG)) sw.start("AggregationUtil::readDatasetArrayDataForAggregation", "");
875 
876  const libdap::DDS* pDDS = dataset.getDDS();
877  NCML_ASSERT_MSG(pDDS, "GridAggregateOnOuterDimension::read(): Got a null DataDDS "
878  "while loading dataset = " + dataset.getLocation());
879 
880  // Grab the Array from the DataDDS with the getter
881  Array* pDatasetArray = arrayGetter.readAndGetArray(varName, *pDDS, &constrainedTemplateArray, debugChannel);
882  NCML_ASSERT_MSG(pDatasetArray, "In aggregation member dataset, failed to get the array! "
883  "Dataset location = " + dataset.getLocation());
884 
885  // Make sure that the data was read in or I dunno what went on.
886  if (!pDatasetArray->read_p()) {
887  NCML_ASSERT_MSG(pDatasetArray->read_p(),
888  "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: pDatasetArray was not read_p()!");
889  }
890 
891  // Make sure it matches the prototype or somthing went wrong
892  if (!AggregationUtil::doTypesMatch(constrainedTemplateArray, *pDatasetArray)) {
893  throw AggregationException(
894  "Invalid aggregation! "
895  "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
896  "We found the aggregation variable name=" + varName
897  + " but it was not of the same type as the prototype variable!");
898  }
899 
900  // Make sure the subshapes match! (true means check dimension names too... debate this)
901  if (!AggregationUtil::doShapesMatch(constrainedTemplateArray, *pDatasetArray, true)) {
902  throw AggregationException(
903  "Invalid aggregation! "
904  "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
905  "We found the aggregation variable name=" + varName
906  + " but it was not of the same shape as the prototype!");
907  }
908 
909  // Make sure the length of the data array also matches the proto
910  if (constrainedTemplateArray.length() != pDatasetArray->length()) {
911  NCML_ASSERT_MSG(constrainedTemplateArray.length() == pDatasetArray->length(),
912  "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
913  "The prototype array and the loaded dataset array length()'s were not equal, even "
914  "though their shapes matched. Logic problem.");
915  }
916 
917  return pDatasetArray;
918 }
919 
920 void AggregationUtil::addDatasetArrayDataToAggregationOutputArray(libdap::Array& oOutputArray, unsigned int atIndex,
921  const Array& constrainedTemplateArray, const std::string& varName, AggMemberDataset& dataset,
922  const ArrayGetterInterface& arrayGetter, const std::string& debugChannel)
923 {
924  BESStopWatch sw;
925  if (BESISDEBUG(TIMING_LOG)) sw.start("AggregationUtil::addDatasetArrayDataToAggregationOutputArray", "");
926 
927  libdap::Array* pDatasetArray = readDatasetArrayDataForAggregation(constrainedTemplateArray, varName, dataset, arrayGetter,
928  debugChannel);
929 
930  // FINALLY, we get to stream the data!
931  oOutputArray.set_value_slice_from_row_major_vector(*pDatasetArray, atIndex);
932 
933  // Now that we have copied it - let the memory go! Free! Let the bytes be freed! - ndp 08/12/2015
934  pDatasetArray->clear_local_data();
935 }
936 
937 void AggregationUtil::gatherMetadataChangesFrom(BaseType* pIntoVar, const BaseType& fromVarC)
938 {
939  BaseType& fromVar = const_cast<BaseType&>(fromVarC); //semantic const
940  // The output will end up here.
941  AttrTable finalAT;
942 
943  // First, seed it with the changes in the fromVar.
944  unionAttrsInto(&finalAT, fromVar.get_attr_table());
945 
946  // Then union in the items from the to var
947  unionAttrsInto(&finalAT, pIntoVar->get_attr_table());
948 
949  // HACK BUG In the set_attr_table call through AttrTable operator=
950  // means we keep bad memory around. Workaround until fixed!
951  pIntoVar->get_attr_table().erase();
952 
953  // Finally, replace the output var's table with the constructed one!
954  pIntoVar->set_attr_table(finalAT);
955 }
956 
957 } // namespace agg_util
agg_util::AggregationUtil::resetCVInsertionPosition
static void resetCVInsertionPosition()
Definition: AggregationUtil.cc:405
BESStopWatch::start
virtual bool start(std::string name)
Definition: BESStopWatch.cc:58
agg_util::AggregationUtil::unionAllVariablesInto
static void unionAllVariablesInto(libdap::DDS *pOutputUnion, const ConstDDSList &datasetsInOrder)
Definition: AggregationUtil.cc:373
agg_util::AggregationUtil::doTypesMatch
static bool doTypesMatch(const libdap::Array &lhs, const libdap::Array &rhs)
Definition: AggregationUtil.cc:558
agg_util::AggregationUtil::doShapesMatch
static bool doShapesMatch(const libdap::Array &lhs, const libdap::Array &rhs, bool checkDimNames)
Definition: AggregationUtil.cc:566
agg_util::TopLevelGridDataArrayGetter::readAndGetArray
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const
Definition: AggregationUtil.cc:161
agg_util::AggMemberDataset::getDDS
virtual const libdap::DDS * getDDS()=0
agg_util::AggregationUtil::findAttribute
static bool findAttribute(const libdap::AttrTable &inTable, const string &name, libdap::AttrTable::Attr_iter &attr)
Definition: AggregationUtil.cc:366
agg_util::AggregationUtil::addDatasetArrayDataToAggregationOutputArray
static void addDatasetArrayDataToAggregationOutputArray(libdap::Array &oOutputArray, unsigned int atIndex, const libdap::Array &constrainedTemplateArray, const string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const string &debugChannel)
Definition: AggregationUtil.cc:920
agg_util::TopLevelGridMapArrayGetter::clone
virtual TopLevelGridMapArrayGetter * clone() const
Definition: AggregationUtil.cc:237
agg_util::TopLevelGridMapArrayGetter::readAndGetArray
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const
Definition: AggregationUtil.cc:244
agg_util::AggregationUtil::transferArrayConstraints
static void transferArrayConstraints(libdap::Array *pToArray, const libdap::Array &fromArray, bool skipFirstFromDim, bool skipFirstToDim, bool printDebug=false, const std::string &debugChannel="agg_util")
Definition: AggregationUtil.cc:732
agg_util::TopLevelGridDataArrayGetter::clone
virtual TopLevelGridDataArrayGetter * clone() const
Definition: AggregationUtil.cc:154
agg_util::AggregationUtil::getAsArrayIfPossible
static libdap::Array * getAsArrayIfPossible(libdap::BaseType *pBT)
Definition: AggregationUtil.cc:826
agg_util::ArrayGetterInterface::readAndGetArray
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const =0
agg_util::AggregationUtil::printDimensions
static void printDimensions(std::ostream &os, const libdap::Array &fromArray)
Definition: AggregationUtil.cc:690
agg_util
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
Definition: AggMemberDataset.cc:38
agg_util::AggregationUtil::addCopyOfVariableIfNameIsAvailable
static bool addCopyOfVariableIfNameIsAvailable(libdap::DDS *pOutDDS, const libdap::BaseType &varProto, bool add_at_top=false)
Definition: AggregationUtil.cc:411
agg_util::AggregationUtil::printConstraints
static void printConstraints(std::ostream &os, const libdap::Array &fromArray)
Definition: AggregationUtil.cc:706
agg_util::AggregationUtil::printConstraintsToDebugChannel
static void printConstraintsToDebugChannel(const std::string &debugChannel, const libdap::Array &fromArray)
Definition: AggregationUtil.cc:724
agg_util::TopLevelArrayGetter::clone
virtual TopLevelArrayGetter * clone() const
Definition: AggregationUtil.cc:88
agg_util::TopLevelGridDataArrayGetter
Definition: AggregationUtil.h:111
agg_util::ArrayGetterInterface
Definition: AggregationUtil.h:59
agg_util::AggregationUtil::gatherMetadataChangesFrom
static void gatherMetadataChangesFrom(libdap::BaseType *pIntoVar, const libdap::BaseType &fromVar)
Definition: AggregationUtil.cc:937
agg_util::TopLevelArrayGetter
Definition: AggregationUtil.h:94
agg_util::AggregationUtil::findTypedVariableAtDDSTopLevel
static LibdapType * findTypedVariableAtDDSTopLevel(const libdap::DDS &dds, const string &name)
agg_util::TopLevelGridMapArrayGetter
Definition: AggregationUtil.h:133
BESStopWatch
Definition: BESStopWatch.h:55
agg_util::AggregationUtil::findVariableAtDDSTopLevel
static libdap::BaseType * findVariableAtDDSTopLevel(const libdap::DDS &dds, const string &name)
Definition: AggregationUtil.cc:457
agg_util::AggregationException
Definition: AggregationException.h:41
agg_util::AggregationUtil::findMapByName
static const libdap::Array * findMapByName(const libdap::Grid &inGrid, const std::string &findName)
Definition: AggregationUtil.cc:854
agg_util::AggregationUtil::readDatasetArrayDataForAggregation
static libdap::Array * readDatasetArrayDataForAggregation(const libdap::Array &constrainedTemplateArray, const std::string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const std::string &debugChannel)
Definition: AggregationUtil.cc:869
agg_util::AggregationUtil::couldBeCoordinateVariable
static bool couldBeCoordinateVariable(libdap::BaseType *pBT)
Definition: AggregationUtil.cc:609
agg_util::AggMemberDataset
Definition: AggMemberDataset.h:63
agg_util::TopLevelArrayGetter::readAndGetArray
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const
Definition: AggregationUtil.cc:95
agg_util::AggregationUtil::validateArrayTypesAndShapesMatch
static bool validateArrayTypesAndShapesMatch(const std::vector< libdap::Array * > &arrays, bool enforceMatchingDimNames)
Definition: AggregationUtil.cc:535
agg_util::AggregationUtil::performUnionAggregation
static void performUnionAggregation(libdap::DDS *pOutputUnion, const ConstDDSList &datasetsInOrder)
Definition: AggregationUtil.cc:296
agg_util::AggregationUtil::unionAttrsInto
static void unionAttrsInto(libdap::AttrTable *pOut, const libdap::AttrTable &fromTable)
Definition: AggregationUtil.cc:324
agg_util::AggregationUtil::addOrReplaceVariableForName
static void addOrReplaceVariableForName(libdap::DDS *pOutDDS, const libdap::BaseType &varProto)
Definition: AggregationUtil.cc:443
agg_util::AggregationUtil::collectVariableArraysInOrder
static unsigned int collectVariableArraysInOrder(std::vector< libdap::Array * > &varArrays, const std::string &collectVarName, const ConstDDSList &datasetsInOrder)
Definition: AggregationUtil.cc:591
agg_util::AggMemberDataset::getLocation
const std::string & getLocation() const
Definition: AggMemberDataset.cc:62
agg_util::AggregationUtil::getVariableNoRecurse
static libdap::BaseType * getVariableNoRecurse(const libdap::DDS &dds, const std::string &name)
Definition: AggregationUtil.cc:788