33 #include "AggregationElement.h"
34 #include "AggMemberDatasetUsingLocationRef.h"
35 #include "AggMemberDatasetSharedDDSWrapper.h"
36 #include "AggregationUtil.h"
37 #include "ArrayAggregateOnOuterDimension.h"
38 #include "ArrayJoinExistingAggregation.h"
39 #include "GridAggregateOnOuterDimension.h"
40 #include "GridJoinExistingAggregation.h"
41 #include "AggMemberDatasetDimensionCache.h"
45 #include <AttrTable.h>
47 #include <AttrTable.h>
48 #include "DDSAccessInterface.h"
49 #include "Dimension.h"
50 #include "DimensionElement.h"
52 #include "MyBaseTypeFactory.h"
53 #include "NCMLBaseArray.h"
54 #include "NCMLDebug.h"
55 #include "NCMLParser.h"
56 #include "NetcdfElement.h"
57 #include "ScanElement.h"
59 #include "BESStopWatch.h"
65 using agg_util::AMDList;
73 const string AggregationElement::_sTypeName =
"aggregation";
75 const vector<string> AggregationElement::_sValidAttrs = getValidAttributes();
77 AggregationElement::AggregationElement() :
78 NCMLElement(0), _type(
""), _dimName(
""), _recheckEvery(
""), _parent(0), _datasets(), _scanners(), _aggVars(), _gotVariableAggElement(
79 false), _wasAggregatedMapAddedForJoinExistingGrid(
false), _coordinateAxisType(
"")
84 RCObjectInterface(),
NCMLElement(proto), _type(proto._type), _dimName(proto._dimName), _recheckEvery(
85 proto._recheckEvery), _parent(proto._parent)
88 , _aggVars(proto._aggVars), _gotVariableAggElement(
false), _wasAggregatedMapAddedForJoinExistingGrid(
false), _coordinateAxisType(
93 if (!proto._datasets.empty()) {
95 "WARNING: AggregationElement copy ctor is deep copying all contained datasets! This might be memory and time intensive!");
99 _datasets.reserve(proto._datasets.size());
100 for (vector<NetcdfElement*>::const_iterator it = proto._datasets.begin(); it != proto._datasets.end(); ++it) {
102 addChildDataset(elt->
clone());
104 NCML_ASSERT(_datasets.size() == proto._datasets.size());
106 _scanners.reserve(proto._scanners.size());
107 for (vector<ScanElement*>::const_iterator it = proto._scanners.begin(); it != proto._scanners.end(); ++it) {
109 addScanElement(elt->
clone());
111 NCML_ASSERT(_scanners.size() == proto._scanners.size());
114 AggregationElement::~AggregationElement()
116 BESDEBUG(
"ncml:memory",
"~AggregationElement called...");
121 _wasAggregatedMapAddedForJoinExistingGrid =
false;
124 while (!_datasets.empty()) {
126 _datasets.pop_back();
131 while (!_scanners.empty()) {
133 _scanners.pop_back();
164 if (BESISDEBUG( TIMING_LOG ))
165 sw.
start(
"AggregationElement::handleBegin",
"");
168 NCML_ASSERT(!getParentDataset());
171 if (!_parser->isScopeNetcdf()) {
173 "Got an <aggregation> = " +
toString()
174 +
" at incorrect parse location. They can only be direct children of <netcdf>. Scope="
175 + _parser->getScopeString());
179 NCML_ASSERT_MSG(dataset,
180 "We expected a non-noll current dataset while processing AggregationElement::handleBegin() for " +
toString());
184 "Got <aggregation> = " +
toString() +
" but the enclosing dataset = " + dataset->
toString()
185 +
" already had an aggregation set! There can be only one!");
192 void AggregationElement::handleContent(
const string& content)
197 "Got non-whitespace for content and didn't expect it. Element=" +
toString() +
" content=\"" + content
206 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationElement::handleEnd",
"");
209 BESDEBUG(
"ncml",
"AggregationElement::handleEnd() - Processing the aggregation!!" << endl);
211 if (isUnionAggregation()) {
212 BESDEBUG(
"ncml2",
"AggregationElement::handleEnd() - isUnionAggregation" << endl);
215 else if (isJoinNewAggregation()) {
216 BESDEBUG(
"ncml2",
"AggregationElement::handleEnd() - isJoinNewAggregation" << endl);
219 else if (isJoinExistingAggregation()) {
220 BESDEBUG(
"ncml2",
"AggregationElement::handleEnd() - isJoinExistingAggregation" << endl);
221 processJoinExisting();
223 else if (_type ==
"forecastModelRunCollection" || _type ==
"forecastModelSingleRunCollection") {
225 "Sorry, we do not implement the forecastModelRunCollection aggregations in this version of the NCML Module!");
229 "Unknown aggregation type=" + _type +
" at scope=" + _parser->getScopeString());
239 bool AggregationElement::isJoinNewAggregation()
const
241 return (_type ==
"joinNew");
244 bool AggregationElement::isUnionAggregation()
const
246 return (_type ==
"union");
249 bool AggregationElement::isJoinExistingAggregation()
const
251 return (_type ==
"joinExisting");
257 BESDEBUG(
"ncml",
"AggregationElement: adding child dataset: " << pDataset->
toString() << endl);
261 _datasets.push_back(pDataset);
271 "Tried to add an aggregation variable twice: name=" + name +
" at scope=" + _parser->getScopeString());
274 _aggVars.push_back(name);
275 BESDEBUG(
"ncml",
"Added aggregation variable name=" + name << endl);
282 AggVarIter endIt = endAggVarIter();
283 AggVarIter it = beginAggVarIter();
284 for (; it != endIt; ++it) {
293 string AggregationElement::printAggregationVariables()
const
296 AggVarIter endIt = endAggVarIter();
297 AggVarIter it = beginAggVarIter();
298 for (; it != endIt; ++it) {
306 AggregationElement::AggVarIter AggregationElement::beginAggVarIter()
const
308 return _aggVars.begin();
311 AggregationElement::AggVarIter AggregationElement::endAggVarIter()
const
313 return _aggVars.end();
318 return _gotVariableAggElement;
323 _gotVariableAggElement =
true;
329 _scanners.push_back(pScanner);
336 BESDEBUG(
"ncml",
"AggregationElement::processParentDatasetComplete() called..." << endl);
338 if (_type ==
"joinNew") {
339 processParentDatasetCompleteForJoinNew();
341 else if (_type ==
"joinExisting") {
342 processParentDatasetCompleteForJoinExisting();
357 void AggregationElement::processUnion()
359 BESDEBUG(
"ncml",
"Processing a union aggregation..." << endl);
368 vector<const DDS*> datasetsInOrder;
371 collectDatasetsInOrder(datasetsInOrder);
373 if (getParentDataset()) {
374 pUnion = getParentDataset()->
getDDS();
376 AggregationUtil::performUnionAggregation(pUnion, datasetsInOrder);
379 void AggregationElement::processJoinNew()
382 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationElement::processJoinNew",
"");
385 processAnyScanElements();
388 "AggregationElement::processJoinNew() - beginning joinNew on the following aggVars=" + printAggregationVariables() << endl);
391 BESDEBUG(
"ncml",
"Merging dimensions from children into aggregated dataset..." << endl);
395 unsigned int newDimSize = _datasets.size();
399 if (_datasets.empty()) {
400 THROW_NCML_PARSE_ERROR(
line(),
"In joinNew aggregation we cannot have zero datasets specified!");
404 DDS* pAggDDS = getParentDataset()->
getDDS();
406 DDS* pTemplateDDS = _datasets[0]->getDDS();
407 NCML_ASSERT_MSG(pTemplateDDS,
"AggregationElement::processJoinNew() - NULL template dataset!");
410 AggregationUtil::unionAttrsInto(&(pAggDDS->get_attr_table()), pTemplateDDS->get_attr_table());
415 vector<string>::const_iterator endIt = _aggVars.end();
416 for (vector<string>::const_iterator it = _aggVars.begin(); it != endIt; ++it) {
417 const string& varName = *it;
419 "AggregationElement::processJoinNew() - Aggregating with joinNew on variable=" << varName <<
"..." << endl);
420 processJoinNewOnAggVar(pAggDDS, varName, *pTemplateDDS);
428 AggregationUtil::resetCVInsertionPosition();
431 AggregationUtil::unionAllVariablesInto(pAggDDS, *pTemplateDDS,
true);
441 doAllScannersSpecifyNCoords(
const vector<ScanElement*>& scanners)
444 for (vector<ScanElement*>::const_iterator it = scanners.begin();
445 it != scanners.end();
449 if ((*it)->ncoords().empty())
459 void AggregationElement::processJoinExisting()
461 BESDEBUG(
"ncml:2",
"Called AggregationElement::processJoinExisting()...");
464 processAnyScanElements();
467 if (_datasets.empty()) {
468 THROW_NCML_PARSE_ERROR(
line(),
"In joinExisting aggregation we cannot have zero datasets specified!");
478 granuleList.reserve(_datasets.size());
479 fillDimensionCacheForJoinExistingDimension(granuleList, _dimName);
483 addNewDimensionForJoinExisting(granuleList);
488 BESDEBUG(
"ncml:2",
"Merging dimensions from children into aggregated dataset..." << endl);
489 mergeDimensions(
true, _dimName);
492 DDS* pAggDDS = getParentDataset()->
getDDS();
495 DDS* pTemplateDDS = _datasets[0]->getDDS();
496 NCML_ASSERT_MSG(pTemplateDDS,
"AggregationElement::processJoinExisting(): NULL template dataset!");
499 AggregationUtil::unionAttrsInto(&(pAggDDS->get_attr_table()), pTemplateDDS->get_attr_table());
502 decideWhichVariablesToJoinExist(*pTemplateDDS);
506 vector<string>::const_iterator endIt = _aggVars.end();
507 for (vector<string>::const_iterator it = _aggVars.begin(); it != endIt; ++it) {
508 const string& varName = *it;
509 BESDEBUG(
"ncml",
"Aggregating with joinExisting on variable=" << varName <<
"..." << endl);
510 processJoinExistingOnAggVar(pAggDDS, varName, *pTemplateDDS);
516 unionAddAllRequiredNonAggregatedVariablesFrom(*pTemplateDDS);
519 void AggregationElement::unionAddAllRequiredNonAggregatedVariablesFrom(
const DDS& templateDDS)
526 AggregationUtil::resetCVInsertionPosition();
529 if (isJoinExistingAggregation()) {
531 AggregationUtil::unionAllVariablesInto(getParentDataset()->getDDS(), templateDDS,
true);
538 else if (isJoinNewAggregation())
541 AggregationUtil::unionAllVariablesInto(getParentDataset()->getDDS(), templateDDS,
true);
545 void AggregationElement::decideWhichVariablesToJoinExist(
const DDS& templateDDS)
548 if (_aggVars.empty()) {
550 "Searching the the template DDS for variables with outer " "dimension matching the join dimension = " << _dimName <<
" in order to add them to the aggregation output list." << endl);
554 vector<string> matchingVars;
555 findVariablesWithOuterDimensionName(matchingVars, templateDDS, _dimName);
556 for (vector<string>::const_iterator it = matchingVars.begin(); it != matchingVars.end(); ++it) {
563 "joinExist aggregation had variableAgg specified... " "Validating these variables have outer dimension named " << _dimName << endl);
565 for (vector<string>::const_iterator it = _aggVars.begin(); it != _aggVars.end(); ++it) {
566 BaseType* pVar = AggregationUtil::findVariableAtDDSTopLevel(templateDDS, *it);
570 std::ostringstream msg;
571 msg <<
"Error validating the variableAgg list. The variable named " << *it
572 <<
" was not found in the top-level DDS!";
573 THROW_NCML_PARSE_ERROR(
line(), msg.str());
577 Array* pArray = AggregationUtil::getAsArrayIfPossible(pVar);
579 std::ostringstream msg;
580 msg <<
"The declared variableAgg aggregation variable named " << *it
581 <<
" was not of a type able to be aggregated!";
582 THROW_NCML_PARSE_ERROR(
line(), msg.str());
586 if (pArray->dimension_name(pArray->dim_begin()) != _dimName) {
587 std::ostringstream msg;
588 msg <<
"The declared variableAgg variable named " << *it <<
" did not match the outer dimension name "
589 << _dimName <<
" for this joinExisting aggregation!";
590 THROW_NCML_PARSE_ERROR(
line(), msg.str());
594 std::ostringstream msg;
595 msg <<
"The variable named " << *it <<
" is a valid joinExisting variable. Will be added to output.";
596 BESDEBUG(
"ncml", msg.str() << endl);
602 void AggregationElement::fillDimensionCacheForJoinExistingDimension(AMDList& granuleList,
608 vector<NetcdfElement*>::iterator endIt = _datasets.end();
609 for (vector<NetcdfElement*>::iterator it = _datasets.begin(); it != endIt; ++it) {
610 granuleList.push_back((*it)->getAggMemberDataset());
615 if (doesFirstGranuleSpecifyNcoords()) {
617 if (!doAllGranulesSpecifyNcoords()) {
618 THROW_NCML_PARSE_ERROR(-1,
"In a joinExisting aggregation we found that the first "
619 "granule specified an ncoords but not all of the others "
620 "did. Either all or none of them should have ncoords specified.");
624 seedDimensionCacheFromUserSpecs(granuleList);
630 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"LOAD_AGGREGATION_DIMENSIONS_CACHE",
"");
634 AMDList::iterator endIt = granuleList.end();
635 for (AMDList::iterator it = granuleList.begin(); it != endIt; ++it) {
638 BESDEBUG(
"ncml",
"AggregationElement::fillDimensionCacheForJoinExistingDimension() - Loading dimension cache for: " << (*it)->getLocation() <<
"..." << endl);
642 BESDEBUG(
"ncml",
"AggregationElement::fillDimensionCacheForJoinExistingDimension() - " <<
643 "WARNING NcML Dimension Caching is not configured or is not working! Loading dimensions from DDS for dataset: " <<
644 (*it)->getLocation() <<
"" << endl);
655 bool AggregationElement::doesFirstGranuleSpecifyNcoords()
const
657 if (_datasets.size() > 0) {
658 return _datasets.at(0)->hasNcoords();
665 bool AggregationElement::doAllGranulesSpecifyNcoords()
const
668 vector<NetcdfElement*>::const_iterator endIt = _datasets.end();
669 for (vector<NetcdfElement*>::const_iterator it = _datasets.begin(); it != endIt; ++it) {
670 success = success && (*it)->hasNcoords();
678 void AggregationElement::seedDimensionCacheFromUserSpecs(agg_util::AMDList& rGranuleList)
const
680 NCML_ASSERT(_datasets.size() == rGranuleList.size());
682 vector<NetcdfElement*>::const_iterator datasetIt;
683 AMDList::iterator amdIt;
684 for (datasetIt = _datasets.begin(), amdIt = rGranuleList.begin(); datasetIt != _datasets.end();
685 ++datasetIt, ++amdIt) {
687 const NetcdfElement* pDataset = *datasetIt;
688 if (!pDataset->hasNcoords()) {
690 THROW_NCML_INTERNAL_ERROR(
"Expected netcdf element member of a joinExisting "
691 "aggregation to have the ncoords attribute specified "
694 unsigned int ncoords = pDataset->getNcoordsAsUnsignedInt();
696 VALID_PTR(pAMD.get());
703 "Dimension cache bug");
706 NCML_ASSERT(amdIt == rGranuleList.end());
711 void AggregationElement::addNewDimensionForJoinExisting(
const agg_util::AMDList& rGranuleList)
714 unsigned int aggDimSize = 0;
715 for (AMDList::const_iterator it = rGranuleList.begin(); it != rGranuleList.end(); ++it) {
716 NCML_ASSERT((*it)->isDimensionCached(_dimName));
717 aggDimSize += (*it)->getCachedDimensionSize(_dimName);
721 NCML_ASSERT(getParentDataset());
722 NCML_ASSERT_MSG(!(getParentDataset()->getDimensionInLocalScope(_dimName)),
723 "AggregationElement::addNewDimensionForJoinExisting() found a dimension "
724 "named " + _dimName +
" already but did not expect it!");
731 oss <<
"Added joinExisting aggregation dimension "
732 " name=" << _dimName <<
" with aggregated size= " << aggDimSize;
733 BESDEBUG(
"ncml:2", oss.str());
736 void AggregationElement::findVariablesWithOuterDimensionName(vector<string>& oMatchingVars,
const DDS& templateDDS,
737 const string& outerDimName)
const
739 for (DDS::Vars_iter it =
const_cast<DDS&
>(templateDDS).var_begin(); it !=
const_cast<DDS&
>(templateDDS).var_end();
741 Array* pArray = AggregationUtil::getAsArrayIfPossible(*it);
743 if (pArray && outerDimName == pArray->dimension_name(pArray->dim_begin())) {
744 oMatchingVars.push_back(pArray->name());
749 void AggregationElement::getParamsForJoinAggOnVariable(JoinAggParams* pOutParams,
const DDS& ,
750 const std::string& varName,
const DDS& templateDDS)
752 VALID_PTR(pOutParams);
755 pOutParams->_pAggVarTemplate = AggregationUtil::getVariableNoRecurse(templateDDS, varName);
756 if (!(pOutParams->_pAggVarTemplate)) {
757 THROW_NCML_PARSE_ERROR(
line(),
758 " We could not find a template for the specified aggregation variable=" + varName
759 +
" so we cannot continue the aggregation.");
764 NCML_ASSERT_MSG(pDim,
"Didn't find a DimensionElement with the aggregation dimName=" + _dimName);
765 pOutParams->_pAggDim = &(pDim->getDimension());
775 BaseType* pExists = AggregationUtil::getVariableNoRecurse(aggOutputDDS, varName);
776 NCML_ASSERT_MSG(!pExists,
777 "Failed since the name of the new variable to add (name="
779 +
") already exists in the "
780 " output aggregation DDS! What happened?!");
785 collectAggMemberDatasets(pOutParams->_memberDatasets);
788 void AggregationElement::processJoinNewOnAggVar(DDS* pAggDDS,
const std::string& varName,
const DDS& templateDDS)
791 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationElement::processJoinNewOnAggVar",
"");
794 JoinAggParams joinAggParams;
795 getParamsForJoinAggOnVariable(&joinAggParams,
796 *pAggDDS, varName, templateDDS);
799 BaseType* pAggVarTemplate = joinAggParams._pAggVarTemplate;
800 if (pAggVarTemplate->type() == dods_array_c) {
801 processAggVarJoinNewForArray(*pAggDDS, *(
static_cast<Array*
>(pAggVarTemplate)), *(joinAggParams._pAggDim),
802 joinAggParams._memberDatasets);
804 else if (pAggVarTemplate->type() == dods_grid_c) {
805 processAggVarJoinNewForGrid(*pAggDDS, *(
static_cast<Grid*
>(pAggVarTemplate)), *(joinAggParams._pAggDim),
806 joinAggParams._memberDatasets);
809 THROW_NCML_PARSE_ERROR(
line(),
810 "Got an aggregation variable not of type Array or Grid, but of: " + pAggVarTemplate->type_name()
811 +
" which we cannot aggregate!");
816 void AggregationElement::processJoinExistingOnAggVar(DDS* pAggDDS,
const std::string& varName,
const DDS& templateDDS)
820 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationElement::processJoinExistingOnAggVar",
"");
823 JoinAggParams joinAggParams;
824 getParamsForJoinAggOnVariable(&joinAggParams,
825 *pAggDDS, varName, templateDDS);
828 BaseType* pAggVarTemplate = joinAggParams._pAggVarTemplate;
829 if (pAggVarTemplate->type() == dods_array_c) {
830 processAggVarJoinExistingForArray(*pAggDDS, *(
static_cast<Array*
>(pAggVarTemplate)), *(joinAggParams._pAggDim),
831 joinAggParams._memberDatasets);
833 else if (pAggVarTemplate->type() == dods_grid_c) {
834 processAggVarJoinExistingForGrid(*pAggDDS, *(
static_cast<Grid*
>(pAggVarTemplate)), *(joinAggParams._pAggDim),
835 joinAggParams._memberDatasets);
838 THROW_NCML_PARSE_ERROR(
line(),
839 "Got an aggregation variable not of type Array or Grid, but of: " + pAggVarTemplate->type_name()
840 +
" which we cannot aggregate!");
845 void AggregationElement::processAggVarJoinNewForArray(DDS& aggDDS,
const libdap::Array& arrayTemplate,
849 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationElement::processJoinExistingOnAggVar",
"");
854 auto_ptr<ArrayAggregateOnOuterDimension> pAggArray(
859 NCML_ASSERT_MSG(!(arrayGetter.get()),
"Expected auto_ptr owner xfer, failed!");
865 "Adding new ArrayAggregateOnOuterDimension with name=" << arrayTemplate.name() <<
" to aggregated dataset!" << endl);
873 aggDDS.add_var(pAggArray.get());
876 aggDDS.add_var_nocopy(pAggArray.release());
879 void AggregationElement::processAggVarJoinNewForGrid(DDS& aggDDS,
const Grid& gridTemplate,
883 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationElement::processAggVarJoinNewForGrid",
"");
885 auto_ptr<GridAggregateOnOuterDimension> pAggGrid(
891 "Adding new GridAggregateOnOuterDimension with name=" << gridTemplate.name() <<
" to aggregated dataset!" << endl);
894 aggDDS.add_var(pAggGrid.get());
897 aggDDS.add_var_nocopy(pAggGrid.release());
903 void AggregationElement::processAggVarJoinExistingForArray(DDS& aggDDS,
const libdap::Array& arrayTemplate,
908 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationElement::processAggVarJoinExistingForArray",
"");
913 auto_ptr<ArrayJoinExistingAggregation> pAggArray(
918 NCML_ASSERT_MSG(!(arrayGetter.get()),
"Expected auto_ptr owner xfer, failed!");
924 "Adding new ArrayJoinExistingAggregation with name=" << arrayTemplate.name() <<
" to aggregated dataset!" << endl);
927 aggDDS.add_var(pAggArray.get());
930 aggDDS.add_var_nocopy(pAggArray.release());
933 void AggregationElement::processAggVarJoinExistingForGrid(DDS& aggDDS,
const Grid& gridTemplate,
938 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationElement::processAggVarJoinExistingForGrid",
"");
940 auto_ptr<GridJoinExistingAggregation> pAggGrid(
944 "Adding new GridJoinExistingAggregation with name=" << gridTemplate.name() <<
" to aggregated dataset!" << endl);
947 aggDDS.add_var(pAggGrid.get());
950 aggDDS.add_var_nocopy(pAggGrid.release());
953 void AggregationElement::processParentDatasetCompleteForJoinNew()
956 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationElement::processParentDatasetCompleteForJoinNew",
"");
958 NetcdfElement* pParentDataset = getParentDataset();
959 VALID_PTR(pParentDataset);
960 DDS* pParentDDS = pParentDataset->getDDS();
961 VALID_PTR(pParentDDS);
964 NCML_ASSERT_MSG(pDim,
" AggregationElement::processParentDatasetCompleteForJoinNew(): "
965 " didn't find a DimensionElement with the joinNew dimName=" + _dimName);
969 BaseType* pBT = AggregationUtil::getVariableNoRecurse(*pParentDDS, dim.name);
974 pCV = createAndAddCoordinateVariableForNewDimension(*pParentDDS, dim);
975 NCML_ASSERT_MSG(pCV,
"processParentDatasetCompleteForJoinNew(): "
976 "failed to create a new coordinate variable for dim=" + dim.name);
982 VariableElement* pVarElt = pParentDataset->findVariableElementForLibdapVar(pBT);
987 pCV = ensureVariableIsProperNewCoordinateVariable(pBT, dim,
true);
992 pCV = processDeferredCoordinateVariable(pBT, dim);
1001 if (!_coordinateAxisType.empty()) {
1002 addCoordinateAxisType(*pCV, _coordinateAxisType);
1009 AggVarIter endIt = endAggVarIter();
1010 for (it = beginAggVarIter(); it != endIt; ++it) {
1011 const string& aggVar = *it;
1012 BaseType* pBT = AggregationUtil::getVariableNoRecurse(*pParentDDS, aggVar);
1016 pGrid->prepend_map(pCV,
true);
1021 void AggregationElement::processParentDatasetCompleteForJoinExisting()
1024 if (BESISDEBUG(TIMING_LOG)) sw.
start(
"AggregationElement::processParentDatasetCompleteForJoinExisting",
"");
1026 NetcdfElement* pParentDataset = getParentDataset();
1027 VALID_PTR(pParentDataset);
1028 DDS* pAggDDS = pParentDataset->getDDS();
1032 NCML_ASSERT_MSG(pDim,
" Didn't find a DimensionElement with the joinExisting dimName=" + _dimName);
1036 BaseType* pDimNameVar = AggregationUtil::getVariableNoRecurse(*pAggDDS, dim.name);
1038 bool placeholderExists =
false;
1044 VariableElement* pVarElt = pParentDataset->findVariableElementForLibdapVar(pDimNameVar);
1049 pCV = ensureVariableIsProperNewCoordinateVariable(pDimNameVar, dim,
true);
1051 placeholderExists =
false;
1056 placeholderExists =
true;
1064 auto_ptr<ArrayJoinExistingAggregation> pNewMap(0);
1071 AggVarIter endIt = endAggVarIter();
1072 for (it = beginAggVarIter(); it != endIt; ++it) {
1073 const string& aggVar = *it;
1074 BaseType* pAggVar = AggregationUtil::getVariableNoRecurse(*pAggDDS, aggVar);
1082 if (!pCV || placeholderExists) {
1084 VALID_PTR(pNewMap.get());
1089 if (placeholderExists) {
1090 processPlaceholderCoordinateVariableForJoinExisting(*pDimNameVar, pNewMap.get());
1094 AggregationUtil::addOrReplaceVariableForName(pAggDDS, *(pNewMap.get()));
1097 pCV = pNewMap.get();
1101 NCML_ASSERT_MSG(pCV,
"Expected a coordinate variable since a Grid exists... what happened?");
1104 pGrid->prepend_map(pCV,
true);
1109 void AggregationElement::processPlaceholderCoordinateVariableForJoinExisting(
const libdap::BaseType& placeholderVar,
1110 libdap::Array* pNewVar)
1115 BaseType* pNewEltProto = pNewVar->var();
1116 VALID_PTR(pNewEltProto);
1117 if (placeholderVar.type() != pNewEltProto->type()) {
1118 THROW_NCML_PARSE_ERROR(
line(),
1119 " We expected the type of the placeholder coordinate variable to be the same "
1120 " as that created by the aggregation. Expected type=" + pNewEltProto->type_name()
1121 + +
" but placeholder has type=" + placeholderVar.type_name()
1122 +
" Please make sure these match in the input file!");
1126 AggregationUtil::gatherMetadataChangesFrom(pNewVar, placeholderVar);
1135 _coordinateAxisType = cat;
1141 return _coordinateAxisType;
1145 AggregationElement::ensureVariableIsProperNewCoordinateVariable(libdap::BaseType* pBT,
const agg_util::Dimension& dim,
1146 bool throwOnInvalidCV)
const
1152 if (AggregationUtil::couldBeCoordinateVariable(pBT)) {
1154 Array* pArr =
static_cast<Array*
>(pBT);
1155 if (pArr->length() ==
static_cast<int>(dim.size)) {
1162 oss << string(
"In the aggregation for dimension=") << dim.name
1163 <<
": The coordinate variable we found does NOT have the same dimensionality as the"
1164 "aggregated dimension! We expected dimensionality=" << dim.size
1165 <<
" but the coordinate variable had dimensionality=" << pArr->length();
1166 BESDEBUG(
"ncml", oss.str() << endl);
1167 if (throwOnInvalidCV) {
1168 THROW_NCML_PARSE_ERROR(
line(), oss.str());
1175 std::ostringstream msg;
1176 msg <<
"Aggregation found a variable matching aggregated dimension name=" << dim.name
1177 <<
" but it was not a coordinate variable. "
1178 " It must be a 1D array whose dimension name is the same as its name. ";
1179 BESDEBUG(
"ncml",
"AggregationElement::ensureVariableIsProperNewCoordinateVariable: " + msg.str() << endl);
1180 if (throwOnInvalidCV) {
1181 THROW_NCML_PARSE_ERROR(
line(), msg.str())
1189 AggregationElement::findMatchingCoordinateVariable(
const DDS& dds,
const agg_util::Dimension& dim,
1190 bool throwOnInvalidCV)
const
1192 BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, dim.name);
1199 return ensureVariableIsProperNewCoordinateVariable(pBT, dim, throwOnInvalidCV);
1215 AggregationElement::processDeferredCoordinateVariable(libdap::BaseType* pBT,
const agg_util::Dimension& dim)
1220 "Processing the placeholder coordinate variable (no values) for the " "current aggregation to add placeholder metadata to the generated values..." << endl);
1225 auto_ptr<Array> pNewArrCV = createCoordinateVariableForNewDimension(dim);
1226 NCML_ASSERT_MSG(pNewArrCV.get(),
" createCoordinateVariableForNewDimension()"
1230 BaseType* pNewEltProto = pNewArrCV->var();
1231 VALID_PTR(pNewEltProto);
1232 if (pBT->type() != pNewEltProto->type()) {
1233 THROW_NCML_PARSE_ERROR(
line(),
1234 " We expected the type of the placeholder coordinate variable to be the same "
1235 " as that created by the aggregation. Expected type=" + pNewEltProto->type_name()
1236 + +
" but placeholder has type=" + pBT->type_name()
1237 +
" Please make sure these match in the input file!");
1245 pNewArrCV->get_attr_table() = pBT->get_attr_table();
1248 DDS* pDDS = getParentDataset()->
getDDS();
1250 pDDS->del_var(pBT->name());
1254 BESDEBUG(
"ncml",
"Adding CV: " << pNewArrCV->name() << endl);
1256 pDDS->add_var(pNewArrCV.get());
1258 pDDS->add_var_nocopy(pNewArrCV.release());
1261 Array* pArrCV =
static_cast<Array*
>(AggregationUtil::getVariableNoRecurse(*pDDS, dim.name));
1266 auto_ptr<libdap::Array> AggregationElement::createCoordinateVariableForNewDimension(
1270 NCML_ASSERT(_datasets.size() > 0);
1271 bool hasCoordValue = !(_datasets[0]->coordValue().empty());
1272 if (hasCoordValue) {
1273 return createCoordinateVariableForNewDimensionUsingCoordValue(dim);
1276 return createCoordinateVariableForNewDimensionUsingLocation(dim);
1281 AggregationElement::createAndAddCoordinateVariableForNewDimension(DDS& dds,
const agg_util::Dimension& dim)
1283 auto_ptr<libdap::Array> pNewCV = createCoordinateVariableForNewDimension(dim);
1286 NCML_ASSERT_MSG(pNewCV.get(),
1287 "AgregationElement::createCoordinateVariableForNewDimension() failed to create a coordinate variable!");
1295 BESDEBUG(
"ncml2",
"AggregationElement::createAndAddCoordinateVariableForNewDimension: " << pNewCV->name());
1297 dds.add_var(pNewCV.get());
1306 static int last_added = 0;
1307 DDS::Vars_iter pos = dds.var_begin();
1308 for (
int i = 0; i < last_added; ++i)
1311 dds.insert_var(pos, pNewCV.get());
1315 Array* pCV =
static_cast<Array*
>(AggregationUtil::getVariableNoRecurse(dds, dim.name));
1317 NCML_ASSERT_MSG(pCV,
"Logic Error: tried to add a new coordinate variable while processing joinNew"
1318 " but we couldn't locate it!");
1322 auto_ptr<libdap::Array> AggregationElement::createCoordinateVariableForNewDimensionUsingCoordValue(
1325 NCML_ASSERT(_datasets.size() > 0);
1326 NCML_ASSERT_MSG(_datasets.size() == dim.size,
"Logic error: Number of datasets doesn't match dimension!");
1328 double doubleVal = 0;
1329 if (_datasets[0]->getCoordValueAsDouble(doubleVal)) {
1330 return createCoordinateVariableForNewDimensionUsingCoordValueAsDouble(dim);
1333 return createCoordinateVariableForNewDimensionUsingCoordValueAsString(dim);
1337 auto_ptr<libdap::Array> AggregationElement::createCoordinateVariableForNewDimensionUsingCoordValueAsDouble(
1340 vector<dods_float64> coords;
1341 coords.reserve(dim.size);
1342 double doubleVal = 0;
1344 for (
unsigned int i = 0; i < _datasets.size(); ++i) {
1345 const NetcdfElement* pDataset = _datasets[i];
1346 if (!pDataset->getCoordValueAsDouble(doubleVal)) {
1347 THROW_NCML_PARSE_ERROR(
line(),
1348 "In creating joinNew coordinate variable from coordValue, expected a coordValue of type double"
1349 " but failed! coordValue=" + pDataset->coordValue() +
" which was in the dataset location="
1350 + pDataset->location() +
" with title=\"" + pDataset->title() +
"\"");
1354 coords.push_back(
static_cast<dods_float64
>(doubleVal));
1361 NCML_ASSERT_MSG(pNewCV.get(),
"createCoordinateVariableForNewDimensionUsingCoordValueAsDouble: failed to create"
1362 " the new Array<Float64> for variable: " + dim.name);
1363 pNewCV->append_dim(dim.size, dim.name);
1364 pNewCV->set_value(coords, coords.size());
1368 auto_ptr<libdap::Array> AggregationElement::createCoordinateVariableForNewDimensionUsingCoordValueAsString(
1372 vector<string> coords;
1373 coords.reserve(dim.size);
1374 for (
unsigned int i = 0; i < _datasets.size(); ++i) {
1375 const NetcdfElement* pDataset = _datasets[i];
1376 if (pDataset->coordValue().empty()) {
1377 int parseLine =
line();
1378 THROW_NCML_PARSE_ERROR(parseLine,
1379 "In creating joinNew coordinate variable from coordValue, expected a coordValue of type string"
1380 " but it was empty! dataset location=" + pDataset->location() +
" with title=\"" + pDataset->title()
1385 coords.push_back(pDataset->coordValue());
1391 NCML_ASSERT_MSG(pNewCV.get(),
"createCoordinateVariableForNewDimensionUsingCoordValueAsString: failed to create"
1392 " the new Array<String> for variable: " + dim.name);
1393 pNewCV->append_dim(dim.size, dim.name);
1394 pNewCV->set_value(coords, coords.size());
1398 auto_ptr<libdap::Array> AggregationElement::createCoordinateVariableForNewDimensionUsingLocation(
1402 vector<string> coords;
1403 coords.reserve(dim.size);
1404 for (
unsigned int i = 0; i < _datasets.size(); ++i) {
1405 const NetcdfElement* pDataset = _datasets[i];
1406 string location(
"");
1407 if (pDataset->location().empty()) {
1408 std::ostringstream oss;
1409 oss <<
"Virtual_Dataset_" << i;
1410 location = oss.str();
1414 location = pDataset->location();
1416 coords.push_back(location);
1421 NCML_ASSERT_MSG(pNewCV.get(),
1422 "createCoordinateVariableForNewDimensionUsingCoordValueUsingLocation: failed to create"
1423 " the new Array<String> for variable: " + dim.name);
1425 pNewCV->append_dim(dim.size, dim.name);
1426 pNewCV->set_value(coords, coords.size());
1430 void AggregationElement::collectDatasetsInOrder(vector<const DDS*>& ddsList)
const
1433 ddsList.reserve(_datasets.size());
1434 vector<NetcdfElement*>::const_iterator endIt = _datasets.end();
1435 vector<NetcdfElement*>::const_iterator it;
1436 for (it = _datasets.begin(); it != endIt; ++it) {
1437 const NetcdfElement* elt = *it;
1439 const DDS* pDDS = elt->getDDS();
1441 ddsList.push_back(pDDS);
1445 void AggregationElement::collectAggMemberDatasets(AMDList& rMemberDatasets)
const
1447 rMemberDatasets.resize(0);
1448 rMemberDatasets.reserve(_datasets.size());
1450 for (vector<NetcdfElement*>::const_iterator it = _datasets.begin(); it != _datasets.end(); ++it) {
1453 VALID_PTR(pAGM.get());
1456 if (!((*it)->ncoords().empty()) && !_dimName.empty()) {
1457 if (!(pAGM->isDimensionCached(_dimName))) {
1458 unsigned int ncoords = (*it)->getNcoordsAsUnsignedInt();
1465 rMemberDatasets.push_back(pAGM);
1469 void AggregationElement::processAnyScanElements()
1471 if (_scanners.size() > 0) {
1472 BESDEBUG(
"ncml",
"Started to process " << _scanners.size() <<
" scan elements..." << endl);
1475 vector<ScanElement*>::iterator it;
1476 vector<ScanElement*>::iterator endIt = _scanners.end();
1477 vector<NetcdfElement*> scannedDatasets;
1478 for (it = _scanners.begin(); it != endIt; ++it) {
1479 BESDEBUG(
"ncml",
"Processing scan element = " << (*it)->toString() <<
" ..." << endl);
1483 (*it)->getDatasetList(scannedDatasets);
1488 vector<NetcdfElement*>::iterator datasetIt;
1489 vector<NetcdfElement*>::iterator datasetEndIt = scannedDatasets.end();
1490 for (datasetIt = scannedDatasets.begin(); datasetIt != datasetEndIt; ++datasetIt) {
1492 _parser->addChildDatasetToCurrentDataset(*datasetIt);
1494 (*datasetIt)->unref();
1497 scannedDatasets.clear();
1501 void AggregationElement::mergeDimensions(
bool checkDimensionMismatch,
const std::string& dimToSkip)
1503 NetcdfElement* pParent = getParentDataset();
1505 vector<NetcdfElement*>::const_iterator datasetsEndIt = _datasets.end();
1506 vector<NetcdfElement*>::const_iterator datasetsIt;
1507 for (datasetsIt = _datasets.begin(); datasetsIt != datasetsEndIt; ++datasetsIt) {
1509 const NetcdfElement* dataset = *datasetsIt;
1511 const vector<DimensionElement*>& dimensions = dataset->getDimensionElements();
1512 vector<DimensionElement*>::const_iterator dimEndIt = dimensions.end();
1513 vector<DimensionElement*>::const_iterator dimIt;
1514 for (dimIt = dimensions.begin(); dimIt != dimEndIt; ++dimIt) {
1515 const DimensionElement* pDim = *dimIt;
1518 if (!dimToSkip.empty() && (pDim->name() == dimToSkip)) {
1522 const DimensionElement* pUnionDim = pParent->getDimensionInLocalScope(pDim->name());
1525 if (!pUnionDim->checkDimensionsMatch(*pDim)) {
1526 string msg = string(
"The union aggregation already had a dimension=") + pUnionDim->toString()
1527 +
" but we found another with different cardinality: " + pDim->toString()
1528 +
" This is likely an error and could cause a later exception.";
1529 BESDEBUG(
"ncml",
"WARNING: " + msg);
1530 if (checkDimensionMismatch) {
1532 msg +
" Scope=" + _parser->getScopeString());
1540 "Dimension name=" << pDim->name() <<
" was not found in the union yet, so adding it. The full elt is: " << pDim->toString() << endl);
1541 pParent->addDimension(
const_cast<DimensionElement*
>(pDim));
1547 static const string COORDINATE_AXIS_TYPE_ATTR(
"_CoordinateAxisType");
1548 void AggregationElement::addCoordinateAxisType(libdap::Array& rCV,
const std::string& cat)
1550 AttrTable& rAT = rCV.get_attr_table();
1551 AttrTable::Attr_iter foundIt = rAT.simple_find(COORDINATE_AXIS_TYPE_ATTR);
1553 if (foundIt != rAT.attr_end()) {
1554 rAT.del_attr(COORDINATE_AXIS_TYPE_ATTR);
1558 "Adding attribute to the aggregation variable " << rCV.name() <<
" Attr is " << COORDINATE_AXIS_TYPE_ATTR <<
" = " << cat << endl);
1561 rAT.append_attr(COORDINATE_AXIS_TYPE_ATTR,
"String", cat);
1564 vector<string> AggregationElement::getValidAttributes()
1566 vector<string> attrs;
1567 attrs.push_back(
"type");
1568 attrs.push_back(
"dimName");
1569 attrs.push_back(
"recheckEvery");