bes  Updated for version 3.20.6
GridAggregationBase.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2010 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 
30 
31 #include <Array.h> // libdap
32 #include <D4Group.h>
33 #include <Constructor.h>
34 #include <D4Maps.h>
35 #include <InternalErr.h>
36 
37 #include "BESStopWatch.h"
38 
39 #include "AggregationUtil.h" // agg_util
40 #include "GridAggregationBase.h" // agg_util
41 
42 #include "NCMLDebug.h"
43 
44 using libdap::Array;
45 using libdap::BaseType;
46 using libdap::Grid;
47 
48 using libdap::D4Group;
49 using libdap::Constructor;
50 using libdap::InternalErr;
51 using libdap::D4Maps;
52 using libdap::D4Map;
53 
54 // Local debug flags
55 static const string DEBUG_CHANNEL("agg_util");
56 static const bool PRINT_CONSTRAINTS(false);
57 
58 // Timeouts are now handled in/by the BES framework in BESInterface.
59 // jhrg 12/29/15
60 #undef USE_LOCAL_TIMEOUT_SCHEME
61 
62 namespace agg_util {
63 GridAggregationBase::GridAggregationBase(const libdap::Grid& proto, const AMDList& memberDatasets,
64  const DDSLoader& loaderProto) :
65  Grid(proto), _loader(loaderProto.getDHI()), _pSubGridProto(cloneSubGridProto(proto)), _memberDatasets(
66  memberDatasets)
67 {
68 }
69 
70 GridAggregationBase::GridAggregationBase(const string& name, const AMDList& memberDatasets,
71  const DDSLoader& loaderProto) :
72  Grid(name), _loader(loaderProto.getDHI()), _pSubGridProto(0), _memberDatasets(memberDatasets)
73 {
74 }
75 
76 GridAggregationBase::GridAggregationBase(const GridAggregationBase& proto) :
77  Grid(proto), _loader(proto._loader.getDHI()), _pSubGridProto(0) // init below
78  , _memberDatasets()
79 {
80  duplicate(proto);
81 }
82 
83 /* virtual */
84 GridAggregationBase::~GridAggregationBase()
85 {
86  cleanup();
87 }
88 
89 GridAggregationBase&
90 GridAggregationBase::operator=(const GridAggregationBase& rhs)
91 {
92  if (this != &rhs) {
93  cleanup();
94  Grid::operator=(rhs);
95  duplicate(rhs);
96  }
97  return *this;
98 }
99 
100 
101 void
102 GridAggregationBase::transform_to_dap4(D4Group *root, Constructor *container)
103 {
104  Grid::transform_to_dap4(root,container);
105 
106 #if 0 // I removed this method because I think the parent class implementation should work correctly.
107  BaseType *btp = array_var()->transform_to_dap4(root, container);
108  Array *coverage = static_cast<Array*>(btp);
109  if (!coverage) throw InternalErr(__FILE__, __LINE__, "Expected an Array while transforming a Grid (coverage)");
110 
111  coverage->set_parent(container);
112 
113  // Next find the maps; add them to the coverage and to the container,
114  // the latter only on the condition that they are not already there.
115 
116  for (Map_iter i = map_begin(), e = map_end(); i != e; ++i) {
117  btp = (*i)->transform_to_dap4(root, container);
118  Array *map = static_cast<Array*>(btp);
119  if (!map) throw InternalErr(__FILE__, __LINE__, "Expected an Array while transforming a Grid (map)");
120 
121  // map must be non-null (Grids cannot contain Grids in DAP2)
122  if (map) {
123  // Only add the map/array if it not already present; given the scoping rules
124  // for DAP2 and the assumption the DDS is valid, testing for the same name
125  // is good enough.
126  if (!root->var(map->name())) {
127  map->set_parent(container);
128  container->add_var_nocopy(map); // this adds the array to the container
129  }
130  D4Map *dap4_map = new D4Map(map->name(), map, coverage); // bind the 'map' to the coverage
131  coverage->maps()->add_map(dap4_map); // bind the coverage to the map
132  }
133  else {
134  throw InternalErr(__FILE__, __LINE__,
135  "transform_to_dap4() returned a null value where there can be no Grid.");
136  }
137  }
138 
139  container->add_var_nocopy(coverage);
140 #endif
141 }
142 
143 void GridAggregationBase::setShapeFrom(const libdap::Grid& constProtoSubGrid, bool addMaps)
144 {
145  // calls used are semantically const, but not syntactically.
146  Grid& protoSubGrid = const_cast<Grid&>(constProtoSubGrid);
147 
148  // Save a clone of the template for read() to use.
149  // We always use these maps...
150  _pSubGridProto = auto_ptr<Grid>(cloneSubGridProto(protoSubGrid));
151 
152  // Pass in the data array and maps from the proto by hand.
153  Array* pDataArrayTemplate = protoSubGrid.get_array();
154  VALID_PTR(pDataArrayTemplate);
155  set_array(static_cast<Array*>(pDataArrayTemplate->ptr_duplicate()));
156 
157  // Now the maps in order if asked
158  if (addMaps) {
159  Grid::Map_iter endIt = protoSubGrid.map_end();
160  for (Grid::Map_iter it = protoSubGrid.map_begin(); it != endIt; ++it) {
161  // have to case, the iter is for some reason BaseType*
162  Array* pMap = dynamic_cast<Array*>(*it);
163  VALID_PTR(pMap);
164  add_map(pMap, true); // add as a copy
165  }
166  }
167 }
168 
169 /* virtual */
170 const AMDList&
172 {
173  return _memberDatasets;
174 }
175 
176 /* virtual */
178 {
179  BESDEBUG_FUNC(DEBUG_CHANNEL, "Function entered..." << endl);
180 
181  if (read_p()) {
182  BESDEBUG_FUNC(DEBUG_CHANNEL, "read_p() set, early exit!");
183  return true;
184  }
185 
186  if (PRINT_CONSTRAINTS) {
187  printConstraints(*(get_array()));
188  }
189 
190  // Call the subclass hook methods to do this work properly
192 
193  // Now make the read call on the data array.
194  // The aggregation subclass will do the right thing.
195  Array* pAggArray = get_array();
196  VALID_PTR(pAggArray);
197 
198  // Only do this portion if the array part is supposed to serialize!
199  if (pAggArray->send_p() || pAggArray->is_in_selection()) {
200  pAggArray->read();
201  }
202 
203  // Set the cache bit.
204  set_read_p(true);
205  return true;
206 }
207 
208 #define PIPELINING 1
209 
230 bool
231 GridAggregationBase::serialize(libdap::ConstraintEvaluator &eval, libdap::DDS &dds, libdap::Marshaller &m,
232  bool ce_eval)
233 {
234  BESStopWatch sw;
235  if (BESISDEBUG(TIMING_LOG)) sw.start("GridAggregationBase::serialize", "");
236 
237  bool status = false;
238 
239  if (!read_p()) {
240  if (PRINT_CONSTRAINTS) {
241  printConstraints(*(get_array()));
242  }
243 #if USE_LOCAL_TIMEOUT_SCHEME
244  dds.timeout_on();
245 #endif
246  // Call the subclass hook methods to do this work properly
247  // *** Replace Map code readAndAggregateConstrainedMapsHook();
248 
249  // Transfers constraints to the proto grid and reads it
251 
252  // Make the call to serialize the data array.
253  // The aggregation subclass will do the right thing.
254  Array* pAggArray = get_array();
255  VALID_PTR(pAggArray);
256 
257  // Only do this portion if the array part is supposed to serialize!
258  if (pAggArray->send_p() || pAggArray->is_in_selection()) {
259 #if PIPELINING
260  pAggArray->serialize(eval, dds, m, ce_eval);
261 #else
262  pAggArray->read();
263 #endif
264  }
265 
266  // Get the read-in, constrained maps from the proto grid and serialize them.
267  // *** Replace copyProtoMapsIntoThisGrid(getAggregationDimension());
268 
269  Grid* pSubGridTemplate = getSubGridTemplate();
270  VALID_PTR(pSubGridTemplate);
271 
272  Map_iter mapIt;
273  Map_iter mapEndIt = map_end();
274  for (mapIt = map_begin(); mapIt != mapEndIt; ++mapIt) {
275  Array* pOutMap = static_cast<Array*>(*mapIt);
276  VALID_PTR(pOutMap);
277 
278  // If it isn't getting dumped, then don't bother with it
279  if (!(pOutMap->send_p() || pOutMap->is_in_selection())) {
280  continue;
281  }
282 
283  // We don't want to touch the aggregation dimension since it's
284  // handled specially.
285  if (pOutMap->name() == getAggregationDimension().name) {
286  if (PRINT_CONSTRAINTS) {
287  BESDEBUG_FUNC(DEBUG_CHANNEL,
288  "About to call read() on the map for the new outer dimension name=" << getAggregationDimension().name << " It's constraints are:" << endl);
289  printConstraints(*pOutMap);
290  }
291 
292  // Make sure it's read with these constraints.
293 #if PIPELINING
294  pOutMap->serialize(eval, dds, m, ce_eval);
295 #else
296  pOutMap->read();
297 #endif
298  continue;
299  }
300 
301  // Otherwise, find the map in the protogrid and copy it's data into this.
302  Array* pProtoGridMap = const_cast<Array*>(AggregationUtil::findMapByName(*pSubGridTemplate, pOutMap->name()));
303  NCML_ASSERT_MSG(pProtoGridMap, "Couldn't find map in prototype grid for map name=" + pOutMap->name());
304  BESDEBUG_FUNC(DEBUG_CHANNEL,
305  "About to call read() on prototype map vector name=" << pOutMap->name() << " and calling transfer constraints..." << endl);
306 
307  // Make sure the protogrid maps were properly read
308  NCML_ASSERT_MSG(pProtoGridMap->read_p(), "Expected the prototype map to have been read but it wasn't.");
309 
310  // Make sure the lengths match to be sure we're not gonna blow memory up
311  NCML_ASSERT_MSG(pOutMap->length() == pProtoGridMap->length(),
312  "Expected the prototype and output maps to have same length() after transfer of constraints, but they were not so we can't copy the data!");
313 
314  // The dimensions will have been set up correctly now so length() is correct...
315  // We assume the pProtoGridMap matches at this point as well.
316  // So we can use this call to copy from one vector to the other
317  // so we don't use temp storage in between
318 #if PIPELINING
319  pProtoGridMap->serialize(eval, dds, m, ce_eval);
320 #else
321  pOutMap->reserve_value_capacity(); // reserves mem for length
322  pOutMap->set_value_slice_from_row_major_vector(*pProtoGridMap, 0);
323 #endif
324  pOutMap->set_read_p(true);
325  }
326 
327  // *** End replaced Map code
328 #if USE_LOCAL_TIMEOUT_SCHEME
329  dds.timeout_off();
330 #endif
331  // Set the cache bit.
332  set_read_p(true);
333 
334 #if PIPELINING
335  status = true;
336 #else
337  status = libdap::Grid::serialize(eval, dds, m, ce_eval);
338 #endif
339  }
340  else {
341  status = libdap::Grid::serialize(eval, dds, m, ce_eval);
342  }
343 
344  return status;
345 }
346 
349 
350 Grid*
352 {
353  return _pSubGridProto.get();
354 }
355 
356 void GridAggregationBase::duplicate(const GridAggregationBase& rhs)
357 {
358  _loader = DDSLoader(rhs._loader.getDHI());
359 
360  std::auto_ptr<Grid> pGridTemplateClone(
361  ((rhs._pSubGridProto.get()) ? (static_cast<Grid*>(rhs._pSubGridProto->ptr_duplicate())) : (0)));
362  _pSubGridProto = pGridTemplateClone;
363 
364  _memberDatasets = rhs._memberDatasets;
365 }
366 
367 void GridAggregationBase::cleanup()
368 {
369  _loader.cleanup();
370 
371  _memberDatasets.clear();
372  _memberDatasets.resize(0);
373 }
374 
375 /* virtual */
377 {
378  // Transfers constraints to the proto grid and reads it
380 
381  // Copy the read-in, constrained maps from the proto grid
382  // into our output maps.
384 }
385 
386 /* static */
387 libdap::Grid*
388 GridAggregationBase::cloneSubGridProto(const libdap::Grid& proto)
389 {
390  return static_cast<Grid*>(const_cast<Grid&>(proto).ptr_duplicate());
391 }
392 
393 void GridAggregationBase::printConstraints(const Array& fromArray)
394 {
395  ostringstream oss;
396  AggregationUtil::printConstraints(oss, fromArray);
397  BESDEBUG("ncml:2", "Constraints for Grid: " << name() << ": " << oss.str() << endl);
398 }
399 
401 {
402  Grid* pSubGridTemplate = getSubGridTemplate();
403  VALID_PTR(pSubGridTemplate);
404 
405  // Call the specialized subclass constraint transfer method
406  transferConstraintsToSubGridHook(pSubGridTemplate);
407 
408  // Pass it the values for the aggregated grid...
409  pSubGridTemplate->set_send_p(send_p());
410  pSubGridTemplate->set_in_selection(is_in_selection());
411 
412  // Those settings will be used by read.
413  pSubGridTemplate->read();
414 
415  // For some reason, some handlers only set read_p for the parts, not the whole!!
416  pSubGridTemplate->set_read_p(true);
417 }
418 
420 {
421  Grid* pSubGridTemplate = getSubGridTemplate();
422  VALID_PTR(pSubGridTemplate);
423 
424  Map_iter mapIt;
425  Map_iter mapEndIt = map_end();
426  for (mapIt = map_begin(); mapIt != mapEndIt; ++mapIt) {
427  Array* pOutMap = static_cast<Array*>(*mapIt);
428  VALID_PTR(pOutMap);
429 
430  // If it isn't getting dumped, then don't bother with it
431  if (!(pOutMap->send_p() || pOutMap->is_in_selection())) {
432  continue;
433  }
434 
435  // We don't want to touch the aggregation dimension since it's
436  // handled specially.
437  if (pOutMap->name() == aggDim.name) {
438  if (PRINT_CONSTRAINTS) {
439  BESDEBUG_FUNC(DEBUG_CHANNEL,
440  "About to call read() on the map for the new outer dimension name=" << aggDim.name << " It's constraints are:" << endl);
441  printConstraints(*pOutMap);
442  }
443 
444  // Make sure it's read with these constraints.
445  pOutMap->read();
446  continue;
447  }
448 
449  // Otherwise, find the map in the protogrid and copy it's data into this.
450  Array* pProtoGridMap = const_cast<Array*>(AggregationUtil::findMapByName(*pSubGridTemplate, pOutMap->name()));
451  NCML_ASSERT_MSG(pProtoGridMap, "Couldn't find map in prototype grid for map name=" + pOutMap->name());
452  BESDEBUG_FUNC(DEBUG_CHANNEL,
453  "About to call read() on prototype map vector name=" << pOutMap->name() << " and calling transfer constraints..." << endl);
454 
455  // Make sure the protogrid maps were properly read
456  NCML_ASSERT_MSG(pProtoGridMap->read_p(), "Expected the prototype map to have been read but it wasn't.");
457 
458  // Make sure the lengths match to be sure we're not gonna blow memory up
459  NCML_ASSERT_MSG(pOutMap->length() == pProtoGridMap->length(),
460  "Expected the prototype and output maps to have same length() "
461  "after transfer of constraints, but they were not so we can't "
462  "copy the data!");
463 
464  // The dimensions will have been set up correctly now so length() is correct...
465  // We assume the pProtoGridMap matches at this point as well.
466  // So we can use this call to copy from one vector to the other
467  // so we don't use temp storage in between
468  pOutMap->reserve_value_capacity(); // reserves mem for length
469  pOutMap->set_value_slice_from_row_major_vector(*pProtoGridMap, 0);
470  pOutMap->set_read_p(true);
471  }
472 }
473 
474 /* virtual */
476 {
477  THROW_NCML_INTERNAL_ERROR("Impl me!");
478 }
479 
480 }
agg_util::GridAggregationBase
Definition: GridAggregationBase.h:46
agg_util::GridAggregationBase::setShapeFrom
void setShapeFrom(const libdap::Grid &protoSubGrid, bool addMaps)
Definition: GridAggregationBase.cc:143
agg_util::GridAggregationBase::copyProtoMapsIntoThisGrid
void copyProtoMapsIntoThisGrid(const Dimension &aggDim)
Definition: GridAggregationBase.cc:419
agg_util::DDSLoader
Definition: DDSLoader.h:62
BESStopWatch::start
virtual bool start(std::string name)
Definition: BESStopWatch.cc:58
agg_util::GridAggregationBase::getDatasetList
virtual const AMDList & getDatasetList() const
Definition: GridAggregationBase.cc:171
agg_util::GridAggregationBase::readProtoSubGrid
void readProtoSubGrid()
Definition: GridAggregationBase.cc:400
agg_util::Dimension
Definition: Dimension.h:49
agg_util
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
Definition: AggMemberDataset.cc:38
agg_util::AggregationUtil::printConstraints
static void printConstraints(std::ostream &os, const libdap::Array &fromArray)
Definition: AggregationUtil.cc:706
agg_util::DDSLoader::cleanup
void cleanup()
restore dhi to clean state
Definition: DDSLoader.cc:256
agg_util::GridAggregationBase::getAggregationDimension
virtual const Dimension & getAggregationDimension() const =0
agg_util::GridAggregationBase::serialize
virtual bool serialize(libdap::ConstraintEvaluator &eval, libdap::DDS &dds, libdap::Marshaller &m, bool ce_eval)
Definition: GridAggregationBase.cc:231
agg_util::GridAggregationBase::read
virtual bool read()
Definition: GridAggregationBase.cc:177
agg_util::GridAggregationBase::transferConstraintsToSubGridHook
virtual void transferConstraintsToSubGridHook(Grid *pSubGrid)
Definition: GridAggregationBase.cc:475
BESStopWatch
Definition: BESStopWatch.h:55
agg_util::GridAggregationBase::readAndAggregateConstrainedMapsHook
virtual void readAndAggregateConstrainedMapsHook()
Definition: GridAggregationBase.cc:376
agg_util::AggregationUtil::findMapByName
static const libdap::Array * findMapByName(const libdap::Grid &inGrid, const std::string &findName)
Definition: AggregationUtil.cc:854
agg_util::GridAggregationBase::getSubGridTemplate
Grid * getSubGridTemplate()
Definition: GridAggregationBase.cc:351
agg_util::DDSLoader::getDHI
BESDataHandlerInterface & getDHI() const
Definition: DDSLoader.h:120