bes  Updated for version 3.20.6
ArrayAggregateOnOuterDimension.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2010 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 
30 #include "ArrayAggregateOnOuterDimension.h"
31 #include "AggregationException.h"
32 
33 #include <DataDDS.h> // libdap::DataDDS
34 #include <Marshaller.h>
35 
36 // only NCML backlinks we want in this agg_util class.
37 #include "NCMLDebug.h" // BESDEBUG and throw macros
38 #include "NCMLUtil.h" // SAFE_DELETE, NCMLUtil::getVariableNoRecurse
39 #include "BESDebug.h"
40 #include "BESStopWatch.h"
41 
42 // BES debug channel we output to
43 static const string DEBUG_CHANNEL("agg_util");
44 
45 // Local flag for whether to print constraints, to help debugging
46 static const bool PRINT_CONSTRAINTS = false;
47 
48 extern BESStopWatch *bes_timing::elapsedTimeToReadStart;
49 extern BESStopWatch *bes_timing::elapsedTimeToTransmitStart;
50 
51 // Timeouts are now handled in/by the BES framework in BESInterface.
52 // jhrg 12/29/15
53 #undef USE_LOCAL_TIMEOUT_SCHEME
54 
55 namespace agg_util {
56 
58  const AMDList& memberDatasets, std::auto_ptr<ArrayGetterInterface>& arrayGetter, const Dimension& newDim) :
59  ArrayAggregationBase(proto, memberDatasets, arrayGetter) // no new dim yet in super chain
60  , _newDim(newDim)
61 {
62  BESDEBUG(DEBUG_CHANNEL, "ArrayAggregateOnOuterDimension: ctor called!" << endl);
63 
64  // Up the rank of the array using the new dimension as outer (prepend)
65  BESDEBUG(DEBUG_CHANNEL, "ArrayAggregateOnOuterDimension: adding new outer dimension: " << _newDim.name << endl);
66  prepend_dim(_newDim.size, _newDim.name);
67 }
68 
70  ArrayAggregationBase(proto), _newDim()
71 {
72  BESDEBUG(DEBUG_CHANNEL, "ArrayAggregateOnOuterDimension() copy ctor called!" << endl);
73  duplicate(proto);
74 }
75 
77 {
78  BESDEBUG(DEBUG_CHANNEL, "~ArrayAggregateOnOuterDimension() dtor called!" << endl);
79  cleanup();
80 }
81 
84 {
85  return new ArrayAggregateOnOuterDimension(*this);
86 }
87 
90 {
91  if (this != &rhs) {
92  cleanup();
93  ArrayAggregationBase::operator=(rhs);
94  duplicate(rhs);
95  }
96  return *this;
97 }
98 
99 // Set this to 0 to get the old behavior where the entire response
100 // (for this variable) is built in memory and then sent to the client.
101 #define PIPELINING 1
102 
123 bool ArrayAggregateOnOuterDimension::serialize(libdap::ConstraintEvaluator &eval, libdap::DDS &dds,
124  libdap::Marshaller &m, bool ce_eval)
125 {
126 
127  BESStopWatch sw;
128  if (BESISDEBUG(TIMING_LOG)) sw.start("ArrayAggregateOnOuterDimension::serialize", "");
129 
130  // Only continue if we are supposed to serialize this object at all.
131  if (!(send_p() || is_in_selection())) {
132  BESDEBUG_FUNC(DEBUG_CHANNEL, "Object not in output, skipping... name=" << name() << endl);
133  return true;
134  }
135 
136  bool status = false;
137 
138  delete bes_timing::elapsedTimeToReadStart;
139  bes_timing::elapsedTimeToReadStart = 0;
140 
141  if (!read_p()) {
142 
143  if (PRINT_CONSTRAINTS) {
144  BESDEBUG_FUNC(DEBUG_CHANNEL, "Constraints on this Array are:" << endl);
145  printConstraints(*this);
146  }
147 
148  // call subclass impl
150 
151  if (PRINT_CONSTRAINTS) {
152  BESDEBUG_FUNC(DEBUG_CHANNEL, "After transfer, constraints on the member template Array are: " << endl);
154  }
155 
156  // outer one is the first in iteration
157  const Array::dimension& outerDim = *(dim_begin());
158  BESDEBUG(DEBUG_CHANNEL,
159  "Aggregating datasets array with outer dimension constraints: " << " start=" << outerDim.start << " stride=" << outerDim.stride << " stop=" << outerDim.stop << endl);
160 
161  // Be extra sure we have enough datasets for the given request
162  if (static_cast<unsigned int>(outerDim.size) != getDatasetList().size()) {
163  // Not sure whose fault it was, but tell the author
164  THROW_NCML_PARSE_ERROR(-1, "The new outer dimension of the joinNew aggregation doesn't "
165  " have the same size as the number of datasets in the aggregation!");
166  }
167 
168 #if PIPELINING
169  // Prepare our output buffer for our constrained length
170  m.put_vector_start(length());
171 #else
172  reserve_value_capacity();
173 #endif
174  // this index pointing into the value buffer for where to write.
175  // The buffer has a stride equal to the _pSubArrayProto->length().
176 
177  // Keep this to do some error checking
178  int nextElementIndex = 0;
179 
180  // Traverse the dataset array respecting hyperslab
181  for (int i = outerDim.start; i <= outerDim.stop && i < outerDim.size; i += outerDim.stride) {
182  AggMemberDataset& dataset = *((getDatasetList())[i]);
183 
184  try {
185 #if USE_LOCAL_TIMEOUT_SCHEME
186  dds.timeout_on();
187 #endif
189  name(), dataset, getArrayGetterInterface(), DEBUG_CHANNEL);
190 #if USE_LOCAL_TIMEOUT_SCHEME
191  dds.timeout_off();
192 #endif
193 #if PIPELINING
194  delete bes_timing::elapsedTimeToTransmitStart;
195  bes_timing::elapsedTimeToTransmitStart = 0;
196  m.put_vector_part(pDatasetArray->get_buf(), getGranuleTemplateArray().length(), var()->width(),
197  var()->type());
198 #else
199  this->set_value_slice_from_row_major_vector(*pDatasetArray, nextElementIndex);
200 #endif
201 
202  pDatasetArray->clear_local_data();
203  }
204  catch (agg_util::AggregationException& ex) {
205  std::ostringstream oss;
206  oss << "Got AggregationException while streaming dataset index=" << i << " data for location=\""
207  << dataset.getLocation() << "\" The error msg was: " << std::string(ex.what());
208  THROW_NCML_PARSE_ERROR(-1, oss.str());
209  }
210 
211  // Jump forward by the amount we added.
212  nextElementIndex += getGranuleTemplateArray().length();
213  }
214 
215  // If we succeeded, we are at the end of the array!
216  NCML_ASSERT_MSG(nextElementIndex == length(), "Logic error:\n"
217  "ArrayAggregateOnOuterDimension::read(): "
218  "At end of aggregating, expected the nextElementIndex to be the length of the "
219  "aggregated array, but it wasn't!");
220 
221 #if PIPELINING
222  m.put_vector_end();
223  status = true;
224 #else
225  // Set the cache bit to avoid recomputing
226  set_read_p(true);
227 
228  delete bes_timing::elapsedTimeToTransmitStart;
229  bes_timing::elapsedTimeToTransmitStart = 0;
230  status = libdap::Array::serialize(eval, dds, m, ce_eval);
231 #endif
232  }
233  else {
234  status = libdap::Array::serialize(eval, dds, m, ce_eval);
235  }
236 
237  return status;
238 }
239 
241 // helpers
242 
243 void ArrayAggregateOnOuterDimension::duplicate(const ArrayAggregateOnOuterDimension& rhs)
244 {
245  _newDim = rhs._newDim;
246 }
247 
248 void ArrayAggregateOnOuterDimension::cleanup() throw ()
249 {
250 }
251 
252 /* virtual */
254 {
255  // transfer the constraints from this object into the subArray template
256  // skipping our first dim which is the new one and not in the subArray.
258  *this, // from this
259  true, // skip first dim in the copy since we handle it special
260  false, // also skip it in the toArray for the same reason.
261  true, // print debug
262  DEBUG_CHANNEL); // on this channel
263 }
264 
265 /* virtual */
266 // In this version of the code, I broke apart the call to
267 // agg_util::AggregationUtil::addDatasetArrayDataToAggregationOutputArray()
268 // into two calls: AggregationUtil::readDatasetArrayDataForAggregation()
269 // and this->set_value_slice_from_row_major_vector(). This
271 {
272  BESStopWatch sw;
273  if (BESISDEBUG(TIMING_LOG))
274  sw.start("ArrayAggregateOnOuterDimension::readConstrainedGranuleArraysAndAggregateDataHook", "");
275 
276  // outer one is the first in iteration
277  const Array::dimension& outerDim = *(dim_begin());
278  BESDEBUG(DEBUG_CHANNEL,
279  "Aggregating datasets array with outer dimension constraints: " << " start=" << outerDim.start << " stride=" << outerDim.stride << " stop=" << outerDim.stop << endl);
280 
281  // Be extra sure we have enough datasets for the given request
282  if (static_cast<unsigned int>(outerDim.size) != getDatasetList().size()) {
283  // Not sure whose fault it was, but tell the author
284  THROW_NCML_PARSE_ERROR(-1, "The new outer dimension of the joinNew aggregation doesn't "
285  " have the same size as the number of datasets in the aggregation!");
286  }
287 
288  // Prepare our output buffer for our constrained length
289  reserve_value_capacity();
290 
291  // this index pointing into the value buffer for where to write.
292  // The buffer has a stride equal to the _pSubArrayProto->length().
293  int nextElementIndex = 0;
294 
295  // Traverse the dataset array respecting hyperslab
296  for (int i = outerDim.start; i <= outerDim.stop && i < outerDim.size; i += outerDim.stride) {
297  AggMemberDataset& dataset = *((getDatasetList())[i]);
298 
299  try {
300  agg_util::AggregationUtil::addDatasetArrayDataToAggregationOutputArray(*this, // into the output buffer of this object
301  nextElementIndex, // into the next open slice
302  getGranuleTemplateArray(), // constraints template
303  name(), // aggvar name
304  dataset, // Dataset who's DDS should be searched
305  getArrayGetterInterface(), DEBUG_CHANNEL);
306 #if 0
307  // The code above is conceptually similar to this, but
308  // makes more efficient use of memory. jhrg8/18/15
310  name(), dataset, getArrayGetterInterface(), DEBUG_CHANNEL);
311 
312  this->set_value_slice_from_row_major_vector(*pDatasetArray, nextElementIndex);
313 #endif
314  }
315  catch (agg_util::AggregationException& ex) {
316  std::ostringstream oss;
317  oss << "Got AggregationException while streaming dataset index=" << i << " data for location=\""
318  << dataset.getLocation() << "\" The error msg was: " << std::string(ex.what());
319  THROW_NCML_PARSE_ERROR(-1, oss.str());
320  }
321 
322  // Jump forward by the amount we added.
323  nextElementIndex += getGranuleTemplateArray().length();
324  }
325 
326  // If we succeeded, we are at the end of the array!
327  NCML_ASSERT_MSG(nextElementIndex == length(), "Logic error:\n"
328  "ArrayAggregateOnOuterDimension::read(): "
329  "At end of aggregating, expected the nextElementIndex to be the length of the "
330  "aggregated array, but it wasn't!");
331 }
332 
333 }
agg_util::ArrayAggregateOnOuterDimension::~ArrayAggregateOnOuterDimension
virtual ~ArrayAggregateOnOuterDimension()
Definition: ArrayAggregateOnOuterDimension.cc:76
agg_util::ArrayAggregationBase::printConstraints
void printConstraints(const Array &fromArray)
Definition: ArrayAggregationBase.cc:143
agg_util::ArrayAggregateOnOuterDimension::operator=
ArrayAggregateOnOuterDimension & operator=(const ArrayAggregateOnOuterDimension &rhs)
Definition: ArrayAggregateOnOuterDimension.cc:89
BESStopWatch::start
virtual bool start(std::string name)
Definition: BESStopWatch.cc:58
agg_util::Dimension
Definition: Dimension.h:49
agg_util::ArrayAggregationBase::getGranuleTemplateArray
libdap::Array & getGranuleTemplateArray()
Definition: ArrayAggregationBase.cc:151
agg_util::AggregationUtil::addDatasetArrayDataToAggregationOutputArray
static void addDatasetArrayDataToAggregationOutputArray(libdap::Array &oOutputArray, unsigned int atIndex, const libdap::Array &constrainedTemplateArray, const string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const string &debugChannel)
Definition: AggregationUtil.cc:920
agg_util::AggregationUtil::transferArrayConstraints
static void transferArrayConstraints(libdap::Array *pToArray, const libdap::Array &fromArray, bool skipFirstFromDim, bool skipFirstToDim, bool printDebug=false, const std::string &debugChannel="agg_util")
Definition: AggregationUtil.cc:732
agg_util
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
Definition: AggMemberDataset.cc:38
agg_util::ArrayAggregateOnOuterDimension::transferOutputConstraintsIntoGranuleTemplateHook
virtual void transferOutputConstraintsIntoGranuleTemplateHook()
Definition: ArrayAggregateOnOuterDimension.cc:253
agg_util::ArrayAggregationBase
Definition: ArrayAggregationBase.h:49
agg_util::ArrayAggregateOnOuterDimension::ArrayAggregateOnOuterDimension
ArrayAggregateOnOuterDimension(const libdap::Array &proto, const AMDList &memberDatasets, std::auto_ptr< ArrayGetterInterface > &arrayGetter, const Dimension &newDim)
Definition: ArrayAggregateOnOuterDimension.cc:57
agg_util::ArrayAggregateOnOuterDimension
Definition: ArrayAggregateOnOuterDimension.h:77
agg_util::ArrayAggregateOnOuterDimension::readConstrainedGranuleArraysAndAggregateDataHook
virtual void readConstrainedGranuleArraysAndAggregateDataHook()
Definition: ArrayAggregateOnOuterDimension.cc:270
agg_util::ArrayAggregationBase::getArrayGetterInterface
const ArrayGetterInterface & getArrayGetterInterface() const
Definition: ArrayAggregationBase.cc:158
agg_util::ArrayAggregateOnOuterDimension::serialize
virtual bool serialize(libdap::ConstraintEvaluator &eval, libdap::DDS &dds, libdap::Marshaller &m, bool ce_eval)
Definition: ArrayAggregateOnOuterDimension.cc:123
BESStopWatch
Definition: BESStopWatch.h:55
agg_util::AggregationException
Definition: AggregationException.h:41
agg_util::AggregationUtil::readDatasetArrayDataForAggregation
static libdap::Array * readDatasetArrayDataForAggregation(const libdap::Array &constrainedTemplateArray, const std::string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const std::string &debugChannel)
Definition: AggregationUtil.cc:869
agg_util::ArrayAggregateOnOuterDimension::ptr_duplicate
virtual ArrayAggregateOnOuterDimension * ptr_duplicate()
Definition: ArrayAggregateOnOuterDimension.cc:83
agg_util::AggMemberDataset
Definition: AggMemberDataset.h:63
agg_util::ArrayAggregationBase::getDatasetList
const AMDList & getDatasetList() const
Definition: ArrayAggregationBase.cc:136
agg_util::AggMemberDataset::getLocation
const std::string & getLocation() const
Definition: AggMemberDataset.cc:62