bes  Updated for version 3.20.6
AggMemberDatasetWithDimensionCacheBase.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2010 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 #include "AggMemberDatasetWithDimensionCacheBase.h"
30 
31 #include <string.h>
32 #include <errno.h>
33 
34 #include <sys/stat.h>
35 
36 #include <sstream>
37 #include <algorithm>
38 #include <fstream>
39 
40 #include "Array.h" // libdap
41 #include "BaseType.h" // libdap
42 #include "Constructor.h" // libdap
43 #include "DataDDS.h" // libdap
44 #include "DDS.h" // libdap
45 
46 #include "AggregationException.h" // agg_util
47 #include "AggMemberDatasetDimensionCache.h"
48 #include "NCMLDebug.h"
49 #include "TheBESKeys.h"
50 
51 using std::string;
52 using libdap::BaseType;
53 using libdap::Constructor;
54 using libdap::DataDDS;
55 using libdap::DDS;
56 
57 #if 0
58 static const string BES_DATA_ROOT("BES.Data.RootDirectory");
59 static const string BES_CATALOG_ROOT("BES.Catalog.catalog.RootDirectory");
60 #endif
61 
62 #define MAX_DIMENSION_COUNT_KEY "NCML.DimensionCache.maxDimensions"
63 #define DEFAULT_MAX_DIMENSIONS 100
64 
65 static const string DEBUG_CHANNEL("agg_util");
66 
67 namespace agg_util {
68 
69 // Used to init the DimensionCache below with an estimated number of dimensions
70 static const unsigned int DIMENSION_CACHE_INITIAL_SIZE = 0;
71 
72 AggMemberDatasetWithDimensionCacheBase::AggMemberDatasetWithDimensionCacheBase(const std::string& location) :
73  AggMemberDataset(location), _dimensionCache(DIMENSION_CACHE_INITIAL_SIZE)
74 {
75 }
76 
77 /* virtual */
78 AggMemberDatasetWithDimensionCacheBase::~AggMemberDatasetWithDimensionCacheBase()
79 {
80  _dimensionCache.clear();
81  _dimensionCache.resize(0);
82 }
83 
84 AggMemberDatasetWithDimensionCacheBase::AggMemberDatasetWithDimensionCacheBase(
85  const AggMemberDatasetWithDimensionCacheBase& proto) :
86  RCObjectInterface(), AggMemberDataset(proto), _dimensionCache(proto._dimensionCache)
87 {
88 }
89 
90 AggMemberDatasetWithDimensionCacheBase&
91 AggMemberDatasetWithDimensionCacheBase::operator=(const AggMemberDatasetWithDimensionCacheBase& rhs)
92 {
93  if (&rhs != this) {
94  AggMemberDataset::operator=(rhs);
95  _dimensionCache.clear();
96  _dimensionCache = rhs._dimensionCache;
97  }
98  return *this;
99 }
100 
101 /* virtual */
102 unsigned int AggMemberDatasetWithDimensionCacheBase::getCachedDimensionSize(const std::string& dimName) const
103 {
104  Dimension* pDim = const_cast<AggMemberDatasetWithDimensionCacheBase*>(this)->findDimension(dimName);
105  if (pDim) {
106  return pDim->size;
107  }
108  else {
109  std::ostringstream oss;
110  oss << __PRETTY_FUNCTION__ << " Dimension " << dimName << " was not found in the cache!";
111  throw DimensionNotFoundException(oss.str());
112  }
113 }
114 
115 /* virtual */
116 bool AggMemberDatasetWithDimensionCacheBase::isDimensionCached(const std::string& dimName) const
117 {
118  return bool(const_cast<AggMemberDatasetWithDimensionCacheBase*>(this)->findDimension(dimName));
119 }
120 
121 /* virtual */
123 {
124  Dimension* pExistingDim = findDimension(dim.name);
125  if (pExistingDim) {
126  if (!throwIfFound) {
127  // This discards the object that was in the vector with this name
128  // and replaces it with the information passed in via 'dim'. NB:
129  // the values of the object referenced by 'dim' are copied into
130  // the object pointed to by 'pExistingDim'.
131  *pExistingDim = dim;
132  }
133  else {
134  std::ostringstream msg;
135  msg << __PRETTY_FUNCTION__ << " Dimension name=" << dim.name
136  << " already exists and we were asked to set uniquely!";
137  throw AggregationException(msg.str());
138  }
139  }
140  else {
141  _dimensionCache.push_back(dim);
142  }
143 }
144 
145 /* virtual */
147 {
148  // Get the dds
149  DDS* pDDS = const_cast<DDS*>(getDDS());
150  VALID_PTR(pDDS);
151 
152  // Recursive add on all of them
153  for (DataDDS::Vars_iter it = pDDS->var_begin(); it != pDDS->var_end(); ++it) {
154  BaseType* pBT = *it;
155  VALID_PTR(pBT);
156  addDimensionsForVariableRecursive(*pBT);
157  }
158 }
159 
160 /* virtual */
162 {
163  _dimensionCache.clear();
164 }
165 
166 /* virtual */
168 {
169  saveDimensionCacheInternal(ostr);
170 }
171 
172 /* virtual */
174 {
175  loadDimensionCacheInternal(istr);
176 }
177 
178 Dimension*
179 AggMemberDatasetWithDimensionCacheBase::findDimension(const std::string& dimName)
180 {
181  Dimension* ret = 0;
182  for (vector<Dimension>::iterator it = _dimensionCache.begin(); it != _dimensionCache.end(); ++it) {
183  if (it->name == dimName) {
184  ret = &(*it);
185  }
186  }
187  BESDEBUG(DEBUG_CHANNEL,"AggMemberDatasetWithDimensionCacheBase::findDimension(dimName='"<<dimName<<"') - " << (ret?"Found " + ret->name:"Dimension Not Found") << endl);
188 
189  return ret;
190 }
191 
192 void AggMemberDatasetWithDimensionCacheBase::addDimensionsForVariableRecursive(libdap::BaseType& var)
193 {
194  BESDEBUG_FUNC(DEBUG_CHANNEL, "Adding dimensions for variable name=" << var.name() << endl);
195 
196  if (var.type() == libdap::dods_array_c) {
197  BESDEBUG(DEBUG_CHANNEL, " Adding dimensions for array variable name = " << var.name() << endl);
198 
199  libdap::Array& arrVar = dynamic_cast<libdap::Array&>(var);
200  libdap::Array::Dim_iter it;
201  for (it = arrVar.dim_begin(); it != arrVar.dim_end(); ++it) {
202  libdap::Array::dimension& dim = *it;
203  if (!isDimensionCached(dim.name)) {
204  Dimension newDim(dim.name, dim.size);
205  setDimensionCacheFor(newDim, false);
206 
207  BESDEBUG(DEBUG_CHANNEL,
208  " Adding dimension: " << newDim.toString() << " to the dataset granule cache..." << endl);
209  }
210  }
211  }
212 
213  else if (var.is_constructor_type()) // then recurse
214  {
215  BESDEBUG(DEBUG_CHANNEL, " Recursing on all variables for constructor variable name = " << var.name() << endl);
216 
217  libdap::Constructor& containerVar = dynamic_cast<libdap::Constructor&>(var);
218  libdap::Constructor::Vars_iter it;
219  for (it = containerVar.var_begin(); it != containerVar.var_end(); ++it) {
220  BESDEBUG(DEBUG_CHANNEL, " Recursing on variable name=" << (*it)->name() << endl);
221 
222  addDimensionsForVariableRecursive(*(*it));
223  }
224  }
225 }
226 
227 // Sort function
228 static bool sIsDimNameLessThan(const Dimension& lhs, const Dimension& rhs)
229 {
230  return (lhs.name < rhs.name);
231 }
232 
233 void AggMemberDatasetWithDimensionCacheBase::saveDimensionCacheInternal(std::ostream& ostr)
234 {
235  BESDEBUG("ncml", "Saving dimension cache for dataset location = " << getLocation() << " ..." << endl);
236 
237  // Not really necessary, but might help with trying to read output
238  std::sort(_dimensionCache.begin(), _dimensionCache.end(), sIsDimNameLessThan);
239 
240  // Save out the location first, ASSUMES \n is NOT in the location for read back
241  const std::string& loc = getLocation();
242  ostr << loc << '\n';
243 
244  // Now save each dimension
245  unsigned int n = _dimensionCache.size();
246  ostr << n << '\n';
247  for (unsigned int i = 0; i < n; ++i) {
248  const Dimension& dim = _dimensionCache.at(i);
249  // @TODO This assumes the dimension names don't contain spaces. We should fix this, and the loader, to work with any name.
250  ostr << dim.name << '\n' << dim.size << '\n';
251  }
252 }
253 
254 
255 void AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal(std::istream& istr)
256 {
257  BESDEBUG("ncml", "Loading dimension cache for dataset location = " << getLocation() << endl);
258 
259  string maxDimsStr;
260  unsigned long maxDims;
261  bool found;
262  TheBESKeys::TheKeys()->get_value(MAX_DIMENSION_COUNT_KEY,maxDimsStr, found);
263  if(found){
264  maxDims = strtoul(maxDimsStr.c_str(), 0, 0);
265  if (maxDims == 0)
266  throw BESError(string("The value '") + maxDimsStr + "' is not valid: " + strerror(errno),
267  BES_SYNTAX_USER_ERROR, __FILE__, __LINE__);
268  // Replace 2011 string function for Debian (and CentOS?).
269  // jhrg 10/27/15
270  // maxDims = stoul(maxDimsStr,0);
271  }
272  else {
273  maxDims = DEFAULT_MAX_DIMENSIONS;
274  }
275 
276  // Read in the location string
277  std::string loc;
278  getline(istr, loc, '\n');
279 
280  // Make sure the location we read is the same as the location
281  // for this AMD or there's an unrecoverable serialization bug
282  if (loc != getLocation()) {
283  stringstream ss;
284  ss << "Serialization error: the location loaded from the "
285  "dimensions cache was: \"" << loc << "\" but we expected it to be " << getLocation()
286  << "\". Unrecoverable!";
287  THROW_NCML_INTERNAL_ERROR(ss.str());
288  }
289 
290 #if 0
291  unsigned int n = 0;
292  istr >> n >> ws;
293  BESDEBUG("ncml", "AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - n: " << n << endl);
294  for (unsigned int i = 0; i < n; ++i) {
295  Dimension newDim;
296  istr >> newDim.name >> ws;
297  BESDEBUG("ncml", "AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - newDim.name: " << newDim.name << endl);
298  istr >> newDim.size >> ws;
299  BESDEBUG("ncml", "AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - newDim.size: " << newDim.size << endl);
300  if (istr.bad()) {
301  // Best we can do is throw an internal error for now.
302  // Perhaps later throw something else that causes a
303  // recreation of the cache
304  THROW_NCML_INTERNAL_ERROR("Parsing dimension cache failed to deserialize from stream.");
305  }
306  _dimensionCache.push_back(newDim);
307  }
308 #endif
309 
310  unsigned long numDims = 0;
311  unsigned int dimCount = 0;
312  string dimName;
313 
314  istr >> numDims >> ws;
315  if(istr.fail()){
316  THROW_NCML_INTERNAL_ERROR("Parsing dimension cache FAIL. Unable to read number of dimensions from cache file.");
317  }
318  if(numDims>maxDims){
319  stringstream msg;
320  msg << "Parsing dimension cache FAIL. Dimension count exceeds limits. Changing value of the ncml module configuration "
321  "key " << MAX_DIMENSION_COUNT_KEY << " may help. numDims: "<< numDims << " maxDims: "<< maxDims;
322  THROW_NCML_INTERNAL_ERROR(msg.str());
323  }
324  BESDEBUG("ncml", "AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - numDims: " << numDims << endl);
325 
326  while(istr.peek()!=EOF){
327  Dimension newDim;
328  istr >> newDim.name >> ws;
329  if(istr.fail()){
330  THROW_NCML_INTERNAL_ERROR("Parsing dimension cache FAIL. Unable to read dimension name from cache.");
331  }
332  BESDEBUG("ncml", "AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - newDim.name: " << newDim.name << endl);
333 
334 
335  if(istr.peek()==EOF){
336  THROW_NCML_INTERNAL_ERROR("Parsing dimension cache FAIL. Unexpected EOF. Expected to find dimension size value.");
337  }
338 
339  istr >> newDim.size >> ws;
340  if(istr.fail()){
341  THROW_NCML_INTERNAL_ERROR("Parsing dimension cache FAIL. Unable to read dimension size from cache.");
342  }
343  BESDEBUG("ncml", "AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - newDim.size: " << newDim.size << endl);
344 
345  dimCount++;
346 
347  if(dimCount > numDims){
348  stringstream msg;
349  msg << "Parsing the dimension cache failed because the number of dimensions found in the cache did "
350  "not match the number indicated in the cache header. Expected " << numDims << " Found: " << dimCount;
351  BESDEBUG("ncml", "AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - " << msg.str() << endl);
352  THROW_NCML_INTERNAL_ERROR(msg.str());
353  }
354  _dimensionCache.push_back(newDim);
355  }
356 
357  if(dimCount != numDims){
358  stringstream msg;
359  msg << "Parsing the dimension cache failed because the number of dimensions found in the cache did "
360  "not match the number indicated in the cache header. Expected " << numDims << " Found: " << dimCount;
361  BESDEBUG("ncml", "AggMemberDatasetWithDimensionCacheBase::loadDimensionCacheInternal() - " << msg.str() << endl);
362  THROW_NCML_INTERNAL_ERROR(msg.str());
363  }
364 
365 
366  BESDEBUG("ncml", "Loaded dimension cache ("<< numDims << " dimensions) for dataset location = " << getLocation() << endl);
367 
368 
369 }
370 
371 }
agg_util::AggMemberDatasetWithDimensionCacheBase
Definition: AggMemberDatasetWithDimensionCacheBase.h:41
agg_util::AggMemberDatasetWithDimensionCacheBase::fillDimensionCacheByUsingDDS
virtual void fillDimensionCacheByUsingDDS()
Definition: AggMemberDatasetWithDimensionCacheBase.cc:146
agg_util::AggMemberDataset::getDDS
virtual const libdap::DDS * getDDS()=0
agg_util::Dimension
Definition: Dimension.h:49
agg_util::AggMemberDatasetWithDimensionCacheBase::getCachedDimensionSize
virtual unsigned int getCachedDimensionSize(const std::string &dimName) const
Definition: AggMemberDatasetWithDimensionCacheBase.cc:102
agg_util::DimensionNotFoundException
Definition: AggregationException.h:51
agg_util
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
Definition: AggMemberDataset.cc:38
agg_util::AggMemberDatasetWithDimensionCacheBase::flushDimensionCache
virtual void flushDimensionCache()
Definition: AggMemberDatasetWithDimensionCacheBase.cc:161
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:62
agg_util::AggMemberDatasetWithDimensionCacheBase::isDimensionCached
virtual bool isDimensionCached(const std::string &dimName) const
Definition: AggMemberDatasetWithDimensionCacheBase.cc:116
TheBESKeys::get_value
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:272
agg_util::AggregationException
Definition: AggregationException.h:41
BESError
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
agg_util::AggMemberDatasetWithDimensionCacheBase::setDimensionCacheFor
virtual void setDimensionCacheFor(const Dimension &dim, bool throwIfFound)
Definition: AggMemberDatasetWithDimensionCacheBase.cc:122
agg_util::AggMemberDataset::getLocation
const std::string & getLocation() const
Definition: AggMemberDataset.cc:62
agg_util::AggMemberDatasetWithDimensionCacheBase::loadDimensionCache
virtual void loadDimensionCache(std::istream &istr)
Definition: AggMemberDatasetWithDimensionCacheBase.cc:173
agg_util::AggMemberDatasetWithDimensionCacheBase::saveDimensionCache
virtual void saveDimensionCache(std::ostream &ostr)
Definition: AggMemberDatasetWithDimensionCacheBase.cc:167