29 #include "AggregationUtil.h"
32 #include "AggMemberDataset.h"
33 #include "AggregationException.h"
34 #include "Dimension.h"
38 #include <AttrTable.h>
44 #include "BESStopWatch.h"
47 #include "NCMLDebug.h"
50 using libdap::AttrTable;
51 using libdap::BaseType;
52 using libdap::Constructor;
53 using libdap::DataDDS;
63 int AggregationUtil::d_last_added_cv_position = 0;
69 ArrayGetterInterface::~ArrayGetterInterface()
76 TopLevelArrayGetter::TopLevelArrayGetter() :
77 ArrayGetterInterface()
82 TopLevelArrayGetter::~TopLevelArrayGetter()
96 const libdap::Array*
const pConstraintTemplate,
const std::string& debugChannel)
const
100 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"TopLevelArrayGetter::readAndGetArray",
"");
108 "Did not find a variable named \"" + name +
"\" at the top-level of the DDS!");
113 if (pBT->type() != libdap::dods_array_c) {
115 "The top-level DDS variable named \"" + name +
"\" was not of the expected type!"
116 " Expected:Array Found:" + pBT->type_name());
119 libdap::Array* pDatasetArray =
static_cast<libdap::Array*
>(pBT);
122 if (pConstraintTemplate) {
124 *pConstraintTemplate,
127 !(debugChannel.empty()),
132 pDatasetArray->set_send_p(
true);
133 pDatasetArray->set_in_selection(
true);
134 pDatasetArray->read();
136 return pDatasetArray;
142 TopLevelGridDataArrayGetter::TopLevelGridDataArrayGetter() :
148 TopLevelGridDataArrayGetter::~TopLevelGridDataArrayGetter()
153 TopLevelGridDataArrayGetter*
162 const libdap::Array*
const pConstraintTemplate,
const std::string& debugChannel)
const
165 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"TopLevelGridDataArrayGetter::readAndGetArray",
"");
173 "Did not find a variable named \"" + name +
"\" at the top-level of the DDS!");
178 if (pBT->type() != libdap::dods_grid_c) {
180 "The top-level DDS variable named \"" + name +
"\" was not of the expected type!"
181 " Expected:Grid Found:" + pBT->type_name());
185 Grid* pDataGrid =
static_cast<Grid*
>(pBT);
186 Array* pDataArray =
static_cast<Array*
>(pDataGrid->array_var());
189 "The data Array var for variable name=\"" + name +
"\" was unexpectedly null!");
193 if (pConstraintTemplate) {
195 *pConstraintTemplate,
198 !(debugChannel.empty()),
208 pDataGrid->set_send_p(
true);
209 pDataGrid->set_in_selection(
true);
213 if (!pDataArray->read_p()) {
214 pDataArray->set_send_p(
true);
215 pDataArray->set_in_selection(
true);
225 TopLevelGridMapArrayGetter::TopLevelGridMapArrayGetter(
const std::string& gridName) :
231 TopLevelGridMapArrayGetter::~TopLevelGridMapArrayGetter()
236 TopLevelGridMapArrayGetter*
245 const libdap::Array*
const pConstraintTemplate,
const std::string& debugChannel)
const
249 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"TopLevelGridMapArrayGetter::readAndGetArray",
"");
256 throw AggregationException(
"Did not find a variable named \"" + _gridName +
"\" at the top-level of the DDS!");
261 if (pBT->type() != libdap::dods_grid_c) {
263 "The top-level DDS variable named \"" + _gridName +
"\" was not of the expected type!"
264 " Expected:Grid Found:" + pBT->type_name());
268 Grid* pDataGrid =
static_cast<Grid*
>(pBT);
270 NCML_ASSERT_MSG(pMap,
271 "Expected to find the map with name " + arrayName +
" within the Grid " + _gridName +
" but failed to find it!");
274 pMap->set_send_p(
true);
275 pMap->set_in_selection(
true);
278 if (pConstraintTemplate) {
280 *pConstraintTemplate,
283 !(debugChannel.empty()),
298 VALID_PTR(pOutputUnion);
307 vector<const DDS*>::const_iterator endIt = datasetsInOrder.end();
308 vector<const DDS*>::const_iterator it;
309 for (it = datasetsInOrder.begin(); it != endIt; ++it) {
310 const DDS* pDDS = *it;
316 const_cast<DDS*
>(pDDS)->get_attr_table());
327 AttrTable& fromTable =
const_cast<AttrTable&
>(fromTableIn);
328 AttrTable::Attr_iter endIt = fromTable.attr_end();
329 AttrTable::Attr_iter it;
330 for (it = fromTable.attr_begin(); it != endIt; ++it) {
331 const string& name = fromTable.get_name(it);
332 AttrTable::Attr_iter attrInOut;
337 "Union of AttrTable: an attribute named " << name <<
" already exist in output, skipping it..." << endl);
343 if (fromTable.is_container(it)) {
344 AttrTable* pOrigAttrContainer = fromTable.get_attr_table(it);
345 NCML_ASSERT_MSG(pOrigAttrContainer,
346 "AggregationUtil::mergeAttrTables(): expected non-null AttrTable for the attribute container: "
348 AttrTable* pClonedAttrContainer =
new AttrTable(*pOrigAttrContainer);
349 VALID_PTR(pClonedAttrContainer);
350 pOut->append_container(pClonedAttrContainer, name);
352 "Union of AttrTable: adding a deep copy of attribute=" << name <<
" to the merged output." << endl);
356 string type = fromTable.get_type(it);
357 vector<string>* pAttrTokens = fromTable.get_attr_vector(it);
358 VALID_PTR(pAttrTokens);
360 pOut->append_attr(name, type, pAttrTokens);
368 AttrTable& inTableSemanticConst =
const_cast<AttrTable&
>(inTable);
369 attr = inTableSemanticConst.simple_find(name);
370 return (attr != inTableSemanticConst.attr_end());
375 ConstDDSList::const_iterator endIt = datasetsInOrder.end();
376 ConstDDSList::const_iterator it;
377 for (it = datasetsInOrder.begin(); it != endIt; ++it) {
384 DDS& dds =
const_cast<DDS&
>(fromDDS);
385 DDS::Vars_iter endIt = dds.var_end();
387 for (it = dds.var_begin(); it != endIt; ++it) {
392 BESDEBUG(
"ncml",
"Variable name=" << var->name() <<
" wasn't in the union yet and was added." << endl);
396 "Variable name=" << var->name() <<
" was already in the union and was skipped." << endl);
408 d_last_added_cv_position = 0;
418 BESDEBUG(
"ncml2",
"AggregationUtil::addCopyOfVariableIfNameIsAvailable: " << varProto.name() << endl);
428 DDS::Vars_iter pos = pOutDDS->var_begin() + d_last_added_cv_position;
430 pOutDDS->insert_var(pos,
const_cast<BaseType*
>(&varProto));
432 ++d_last_added_cv_position;
435 pOutDDS->add_var(
const_cast<BaseType*
>(&varProto));
449 pOutDDS->del_var(varProto.name());
453 pOutDDS->add_var(
const_cast<BaseType*
>(&varProto));
460 DDS& dds =
const_cast<DDS&
>(dds_const);
461 DDS::Vars_iter endIt = dds.var_end();
463 for (it = dds.var_begin(); it != endIt; ++it) {
465 if (var && var->name() == name) {
473 template<
class LibdapType>
479 return dynamic_cast<LibdapType*
>(pBT);
487 void AggregationUtil::produceOuterDimensionJoinedArray(Array* pJoinedArray,
const std::string& joinedArrayName,
488 const std::string& newOuterDimName,
const std::vector<libdap::Array*>& fromVars,
bool copyData)
490 string funcName =
"AggregationUtil::createOuterDimensionJoinedArray:";
492 NCML_ASSERT_MSG(fromVars.size() > 0, funcName +
"Must be at least one Array in input!");
496 throw AggregationException(
497 funcName +
" The input arrays must all have the same data type and dimensions but do not!");
501 Array* templateArray = fromVars[0];
502 VALID_PTR(templateArray);
503 BaseType* templateVar = templateArray->var();
504 NCML_ASSERT_MSG(templateVar, funcName +
"Expected a non-NULL prototype BaseType in the first Array!");
507 pJoinedArray->add_var(templateVar);
509 pJoinedArray->set_name(joinedArrayName);
512 pJoinedArray->set_attr_table(templateArray->get_attr_table());
516 pJoinedArray->append_dim(fromVars.size(), newOuterDimName);
519 for (Array::Dim_iter it = templateArray->dim_begin(); it != templateArray->dim_end(); ++it) {
520 int dimSize = templateArray->dimension_size(it);
521 string dimName = templateArray->dimension_name(it);
522 pJoinedArray->append_dim(dimSize, dimName);
527 pJoinedArray->reserve_value_capacity(pJoinedArray->length());
529 joinArrayData(pJoinedArray, fromVars,
false,
536 bool enforceMatchingDimNames)
538 NCML_ASSERT(arrays.size() > 0);
540 Array* pTemplate = 0;
541 for (vector<Array*>::const_iterator it = arrays.begin(); it != arrays.end(); ++it) {
545 VALID_PTR(pTemplate);
561 Array& lhs =
const_cast<Array&
>(lhsC);
562 Array& rhs =
const_cast<Array&
>(rhsC);
563 return (lhs.var() && rhs.var() && lhs.var()->type() == rhs.var()->type());
569 Array& lhs =
const_cast<Array&
>(lhsC);
570 Array& rhs =
const_cast<Array&
>(rhsC);
574 if (lhs.dimensions() != rhs.dimensions()) {
579 Array::Dim_iter rhsIt = rhs.dim_begin();
580 for (Array::Dim_iter lhsIt = lhs.dim_begin(); lhsIt != lhs.dim_end(); (++lhsIt, ++rhsIt)) {
581 valid = (valid && (lhs.dimension_size(lhsIt) == rhs.dimension_size(rhsIt)));
584 valid = (valid && (lhs.dimension_name(lhsIt) == rhs.dimension_name(rhsIt)));
592 const std::string& collectVarName,
const ConstDDSList& datasetsInOrder)
594 unsigned int count = 0;
595 ConstDDSList::const_iterator endIt = datasetsInOrder.end();
596 ConstDDSList::const_iterator it;
597 for (it = datasetsInOrder.begin(); it != endIt; ++it) {
598 DDS* pDDS =
const_cast<DDS*
>(*it);
602 varArrays.push_back(pVar);
611 Array* pArr =
dynamic_cast<Array*
>(pBT);
612 if (pArr && (pArr->dimensions() == 1)) {
614 Array::Dim_iter it = pArr->dim_begin();
615 bool matches = (pArr->dimension_name(it) == pArr->name());
616 NCML_ASSERT_MSG((++it == pArr->dim_end()),
617 "Logic error: NCMLUtil::isCoordinateVariable(): expected one dimension from Array, but got more!");
626 void AggregationUtil::joinArrayData(Array* pAggArray,
const std::vector<Array*>& varArrays,
627 bool reserveStorage,
bool clearDataAfterUse)
630 VALID_PTR(pAggArray->var());
631 NCML_ASSERT_MSG(pAggArray->var()->is_simple_type(),
632 "AggregationUtil::joinArrayData: the output Array is not of a simple type! Can't aggregate!");
635 if (reserveStorage) {
637 unsigned int totalLength = 0;
639 vector<Array*>::const_iterator it;
640 vector<Array*>::const_iterator endIt = varArrays.end();
641 for (it = varArrays.begin(); it != endIt; ++it) {
644 totalLength += pArr->length();
648 pAggArray->reserve_value_capacity(totalLength);
652 unsigned int nextElt = 0;
653 vector<Array*>::const_iterator it;
654 vector<Array*>::const_iterator endIt = varArrays.end();
655 for (it = varArrays.begin(); it != endIt; ++it) {
658 NCML_ASSERT_MSG(pArr->var() && (pArr->var()->type() == pAggArray->var()->type()),
659 "AggregationUtil::joinArrayData: one of the arrays to join has different type than output! Can't aggregate!");
662 if (!pArr->read_p()) {
667 nextElt += pAggArray->set_value_slice_from_row_major_vector(*pArr, nextElt);
669 if (clearDataAfterUse) {
670 pArr->clear_local_data();
692 os <<
"Array dimensions: " << endl;
693 Array& theArray =
const_cast<Array&
>(fromArray);
695 Array::Dim_iter endIt = theArray.dim_end();
696 for (it = theArray.dim_begin(); it != endIt; ++it) {
697 Array::dimension d = *it;
698 os <<
"Dim = {" << endl;
699 os <<
"name=" << d.name << endl;
700 os <<
"size=" << d.size << endl;
703 os <<
"End Array dimensions." << endl;
708 os <<
"Array constraints: " << endl;
709 Array& theArray =
const_cast<Array&
>(rcArray);
711 Array::Dim_iter endIt = theArray.dim_end();
712 for (it = theArray.dim_begin(); it != endIt; ++it) {
713 Array::dimension d = *it;
714 os <<
"Dim = {" << endl;
715 os <<
"name=" << d.name << endl;
716 os <<
"start=" << d.start << endl;
717 os <<
"stride=" << d.stride << endl;
718 os <<
"stop=" << d.stop << endl;
721 os <<
"End Array constraints" << endl;
727 BESDEBUG(debugChannel,
"Printing constraints for Array: " << fromArray.name() <<
": " << oss.str() << endl);
729 BESDEBUG(debugChannel, oss.str() << endl);
733 bool skipFirstToDim,
bool printDebug ,
const std::string& debugChannel )
736 Array& fromArray =
const_cast<Array&
>(fromArrayConst);
739 pToArray->reset_constraint();
742 int skipDelta = ((skipFirstFromDim) ? (1) : (0));
745 if (skipFirstToDim) {
748 if (skipDelta < 0 || (pToArray->dimensions() + skipDelta !=
const_cast<Array&
>(fromArrayConst).dimensions())) {
750 "Mismatched dimensionalities!");
754 BESDEBUG(debugChannel,
755 "Printing constraints on fromArray name= " << fromArray.name() <<
" before transfer..." << endl);
762 Array::Dim_iter fromArrIt = fromArray.dim_begin();
763 Array::Dim_iter fromArrEndIt = fromArray.dim_end();
764 Array::Dim_iter toArrIt = pToArray->dim_begin();
765 for (; fromArrIt != fromArrEndIt; ++fromArrIt) {
766 if (skipFirstFromDim && (fromArrIt == fromArray.dim_begin())) {
769 if (skipFirstToDim) {
777 pToArray->add_constraint(toArrIt, fromArrIt->start, fromArrIt->stride, fromArrIt->stop);
782 BESDEBUG(debugChannel,
"Printing constrains on pToArray after transfer..." << endl);
791 DDS& dds =
const_cast<DDS&
>(ddsConst);
792 DDS::Vars_iter endIt = dds.var_end();
794 for (it = dds.var_begin(); it != endIt; ++it) {
796 if (var && var->name() == name) {
811 Constructor& varContainer =
const_cast<Constructor&
>(varContainerConst);
812 Constructor::Vars_iter endIt = varContainer.var_end();
813 Constructor::Vars_iter it;
814 for (it = varContainer.var_begin(); it != endIt; ++it) {
816 if (var && var->name() == name) {
836 libdap::Array* pArray(0);
837 switch (pBT->type()) {
838 case libdap::dods_array_c:
839 pArray =
static_cast<libdap::Array*
>(pBT);
842 case libdap::dods_grid_c:
843 pArray =
static_cast<Grid*
>(pBT)->get_array();
856 Grid& grid =
const_cast<Grid&
>(inGrid);
859 Grid::Map_iter endIt = grid.map_end();
860 for (it = grid.map_begin(); it != endIt; ++it) {
861 if ((*it)->name() == findName) {
862 pRet =
static_cast<Array*
>(*it);
871 const std::string& debugChannel)
874 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationUtil::readDatasetArrayDataForAggregation",
"");
876 const libdap::DDS* pDDS = dataset.
getDDS();
877 NCML_ASSERT_MSG(pDDS,
"GridAggregateOnOuterDimension::read(): Got a null DataDDS "
878 "while loading dataset = " + dataset.
getLocation());
881 Array* pDatasetArray = arrayGetter.
readAndGetArray(varName, *pDDS, &constrainedTemplateArray, debugChannel);
882 NCML_ASSERT_MSG(pDatasetArray,
"In aggregation member dataset, failed to get the array! "
886 if (!pDatasetArray->read_p()) {
887 NCML_ASSERT_MSG(pDatasetArray->read_p(),
888 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: pDatasetArray was not read_p()!");
894 "Invalid aggregation! "
895 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
896 "We found the aggregation variable name=" + varName
897 +
" but it was not of the same type as the prototype variable!");
903 "Invalid aggregation! "
904 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
905 "We found the aggregation variable name=" + varName
906 +
" but it was not of the same shape as the prototype!");
910 if (constrainedTemplateArray.length() != pDatasetArray->length()) {
911 NCML_ASSERT_MSG(constrainedTemplateArray.length() == pDatasetArray->length(),
912 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
913 "The prototype array and the loaded dataset array length()'s were not equal, even "
914 "though their shapes matched. Logic problem.");
917 return pDatasetArray;
921 const Array& constrainedTemplateArray,
const std::string& varName,
AggMemberDataset& dataset,
925 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationUtil::addDatasetArrayDataToAggregationOutputArray",
"");
931 oOutputArray.set_value_slice_from_row_major_vector(*pDatasetArray, atIndex);
934 pDatasetArray->clear_local_data();
939 BaseType& fromVar =
const_cast<BaseType&
>(fromVarC);
951 pIntoVar->get_attr_table().erase();
954 pIntoVar->set_attr_table(finalAT);