29 #include "AggMemberDatasetWithDimensionCacheBase.h"
42 #include "Constructor.h"
46 #include "AggregationException.h"
47 #include "AggMemberDatasetDimensionCache.h"
48 #include "NCMLDebug.h"
49 #include "TheBESKeys.h"
52 using libdap::BaseType;
53 using libdap::Constructor;
54 using libdap::DataDDS;
58 static const string BES_DATA_ROOT(
"BES.Data.RootDirectory");
59 static const string BES_CATALOG_ROOT(
"BES.Catalog.catalog.RootDirectory");
62 #define MAX_DIMENSION_COUNT_KEY "NCML.DimensionCache.maxDimensions"
63 #define DEFAULT_MAX_DIMENSIONS 100
65 static const string DEBUG_CHANNEL(
"agg_util");
70 static const unsigned int DIMENSION_CACHE_INITIAL_SIZE = 0;
72 AggMemberDatasetWithDimensionCacheBase::AggMemberDatasetWithDimensionCacheBase(
const std::string& location) :
73 AggMemberDataset(location), _dimensionCache(DIMENSION_CACHE_INITIAL_SIZE)
78 AggMemberDatasetWithDimensionCacheBase::~AggMemberDatasetWithDimensionCacheBase()
80 _dimensionCache.clear();
81 _dimensionCache.resize(0);
84 AggMemberDatasetWithDimensionCacheBase::AggMemberDatasetWithDimensionCacheBase(
85 const AggMemberDatasetWithDimensionCacheBase& proto) :
86 RCObjectInterface(), AggMemberDataset(proto), _dimensionCache(proto._dimensionCache)
90 AggMemberDatasetWithDimensionCacheBase&
91 AggMemberDatasetWithDimensionCacheBase::operator=(
const AggMemberDatasetWithDimensionCacheBase& rhs)
94 AggMemberDataset::operator=(rhs);
95 _dimensionCache.clear();
96 _dimensionCache = rhs._dimensionCache;
109 std::ostringstream oss;
110 oss << __PRETTY_FUNCTION__ <<
" Dimension " << dimName <<
" was not found in the cache!";
124 Dimension* pExistingDim = findDimension(dim.name);
134 std::ostringstream msg;
135 msg << __PRETTY_FUNCTION__ <<
" Dimension name=" << dim.name
136 <<
" already exists and we were asked to set uniquely!";
141 _dimensionCache.push_back(dim);
149 DDS* pDDS =
const_cast<DDS*
>(
getDDS());
153 for (DataDDS::Vars_iter it = pDDS->var_begin(); it != pDDS->var_end(); ++it) {
156 addDimensionsForVariableRecursive(*pBT);
163 _dimensionCache.clear();
169 saveDimensionCacheInternal(ostr);
175 loadDimensionCacheInternal(istr);
179 AggMemberDatasetWithDimensionCacheBase::findDimension(
const std::string& dimName)
182 for (vector<Dimension>::iterator it = _dimensionCache.begin(); it != _dimensionCache.end(); ++it) {
183 if (it->name == dimName) {
187 BESDEBUG(DEBUG_CHANNEL,
"AggMemberDatasetWithDimensionCacheBase::findDimension(dimName='"<<dimName<<
"') - " << (ret?
"Found " + ret->name:
"Dimension Not Found") << endl);
192 void AggMemberDatasetWithDimensionCacheBase::addDimensionsForVariableRecursive(libdap::BaseType& var)
194 BESDEBUG_FUNC(DEBUG_CHANNEL,
"Adding dimensions for variable name=" << var.name() << endl);
196 if (var.type() == libdap::dods_array_c) {
197 BESDEBUG(DEBUG_CHANNEL,
" Adding dimensions for array variable name = " << var.name() << endl);
199 libdap::Array& arrVar =
dynamic_cast<libdap::Array&
>(var);
200 libdap::Array::Dim_iter it;
201 for (it = arrVar.dim_begin(); it != arrVar.dim_end(); ++it) {
202 libdap::Array::dimension& dim = *it;
204 Dimension newDim(dim.name, dim.size);
207 BESDEBUG(DEBUG_CHANNEL,
208 " Adding dimension: " << newDim.toString() <<
" to the dataset granule cache..." << endl);
213 else if (var.is_constructor_type())
215 BESDEBUG(DEBUG_CHANNEL,
" Recursing on all variables for constructor variable name = " << var.name() << endl);
217 libdap::Constructor& containerVar =
dynamic_cast<libdap::Constructor&
>(var);
218 libdap::Constructor::Vars_iter it;
219 for (it = containerVar.var_begin(); it != containerVar.var_end(); ++it) {
220 BESDEBUG(DEBUG_CHANNEL,
" Recursing on variable name=" << (*it)->name() << endl);
222 addDimensionsForVariableRecursive(*(*it));
228 static bool sIsDimNameLessThan(
const Dimension& lhs,
const Dimension& rhs)
230 return (lhs.name < rhs.name);
233 void AggMemberDatasetWithDimensionCacheBase::saveDimensionCacheInternal(std::ostream& ostr)
235 BESDEBUG(
"ncml",
"Saving dimension cache for dataset location = " <<
getLocation() <<
" ..." << endl);
238 std::sort(_dimensionCache.begin(), _dimensionCache.end(), sIsDimNameLessThan);
245 unsigned int n = _dimensionCache.size();
247 for (
unsigned int i = 0; i < n; ++i) {
248 const Dimension& dim = _dimensionCache.at(i);
250 ostr << dim.name <<
'\n' << dim.size <<
'\n';
255 void AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal(std::istream& istr)
257 BESDEBUG(
"ncml",
"Loading dimension cache for dataset location = " <<
getLocation() << endl);
260 unsigned long maxDims;
264 maxDims = strtoul(maxDimsStr.c_str(), 0, 0);
266 throw BESError(
string(
"The value '") + maxDimsStr +
"' is not valid: " + strerror(errno),
267 BES_SYNTAX_USER_ERROR, __FILE__, __LINE__);
273 maxDims = DEFAULT_MAX_DIMENSIONS;
278 getline(istr, loc,
'\n');
284 ss <<
"Serialization error: the location loaded from the "
285 "dimensions cache was: \"" << loc <<
"\" but we expected it to be " <<
getLocation()
286 <<
"\". Unrecoverable!";
287 THROW_NCML_INTERNAL_ERROR(ss.str());
293 BESDEBUG(
"ncml",
"AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - n: " << n << endl);
294 for (
unsigned int i = 0; i < n; ++i) {
296 istr >> newDim.name >> ws;
297 BESDEBUG(
"ncml",
"AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - newDim.name: " << newDim.name << endl);
298 istr >> newDim.size >> ws;
299 BESDEBUG(
"ncml",
"AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - newDim.size: " << newDim.size << endl);
304 THROW_NCML_INTERNAL_ERROR(
"Parsing dimension cache failed to deserialize from stream.");
306 _dimensionCache.push_back(newDim);
310 unsigned long numDims = 0;
311 unsigned int dimCount = 0;
314 istr >> numDims >> ws;
316 THROW_NCML_INTERNAL_ERROR(
"Parsing dimension cache FAIL. Unable to read number of dimensions from cache file.");
320 msg <<
"Parsing dimension cache FAIL. Dimension count exceeds limits. Changing value of the ncml module configuration "
321 "key " << MAX_DIMENSION_COUNT_KEY <<
" may help. numDims: "<< numDims <<
" maxDims: "<< maxDims;
322 THROW_NCML_INTERNAL_ERROR(msg.str());
324 BESDEBUG(
"ncml",
"AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - numDims: " << numDims << endl);
326 while(istr.peek()!=EOF){
328 istr >> newDim.name >> ws;
330 THROW_NCML_INTERNAL_ERROR(
"Parsing dimension cache FAIL. Unable to read dimension name from cache.");
332 BESDEBUG(
"ncml",
"AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - newDim.name: " << newDim.name << endl);
335 if(istr.peek()==EOF){
336 THROW_NCML_INTERNAL_ERROR(
"Parsing dimension cache FAIL. Unexpected EOF. Expected to find dimension size value.");
339 istr >> newDim.size >> ws;
341 THROW_NCML_INTERNAL_ERROR(
"Parsing dimension cache FAIL. Unable to read dimension size from cache.");
343 BESDEBUG(
"ncml",
"AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - newDim.size: " << newDim.size << endl);
347 if(dimCount > numDims){
349 msg <<
"Parsing the dimension cache failed because the number of dimensions found in the cache did "
350 "not match the number indicated in the cache header. Expected " << numDims <<
" Found: " << dimCount;
351 BESDEBUG(
"ncml",
"AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - " << msg.str() << endl);
352 THROW_NCML_INTERNAL_ERROR(msg.str());
354 _dimensionCache.push_back(newDim);
357 if(dimCount != numDims){
359 msg <<
"Parsing the dimension cache failed because the number of dimensions found in the cache did "
360 "not match the number indicated in the cache header. Expected " << numDims <<
" Found: " << dimCount;
361 BESDEBUG(
"ncml",
"AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - " << msg.str() << endl);
362 THROW_NCML_INTERNAL_ERROR(msg.str());
366 BESDEBUG(
"ncml",
"Loaded dimension cache ("<< numDims <<
" dimensions) for dataset location = " <<
getLocation() << endl);