bes  Updated for version 3.20.6
FONcArray.cc
1 // FONcArray.cc
2 
3 // This file is part of BES Netcdf File Out Module
4 
5 // Copyright (c) 2004,2005 University Corporation for Atmospheric Research
6 // Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact University Corporation for Atmospheric Research at
23 // 3080 Center Green Drive, Boulder, CO 80301
24 
25 // (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
26 // Please read the full copyright statement in the file COPYRIGHT_UCAR.
27 //
28 // Authors:
29 // pwest Patrick West <pwest@ucar.edu>
30 // jgarcia Jose Garcia <jgarcia@ucar.edu>
31 
32 #include <BESInternalError.h>
33 #include <BESDebug.h>
34 
35 #include "FONcRequestHandler.h" // For access to the handler's keys
36 #include "FONcArray.h"
37 #include "FONcDim.h"
38 #include "FONcGrid.h"
39 #include "FONcMap.h"
40 #include "FONcUtils.h"
41 #include "FONcAttributes.h"
42 
43 vector<FONcDim *> FONcArray::Dimensions;
44 
45 const int MAX_CHUNK_SIZE = 1024;
46 
55 FONcArray::FONcArray(BaseType *b) :
56  FONcBaseType(), d_a(0), d_array_type(NC_NAT), d_ndims(0), d_actual_ndims(0), d_nelements(1), d_dim_ids(0),
57  d_dim_sizes(0), d_str_data(0), d_dont_use_it(false), d_chunksizes(0), d_grid_maps(0)
58 {
59  d_a = dynamic_cast<Array *>(b);
60  if (!d_a) {
61  string s = "File out netcdf, FONcArray was passed a variable that is not a DAP Array";
62  throw BESInternalError(s, __FILE__, __LINE__);
63  }
64 }
65 
77 {
78  // Added jhrg 8/28/13
79  vector<FONcDim*>::iterator d = d_dims.begin();
80  while (d != d_dims.end()) {
81  (*d)->decref();
82  ++d;
83  }
84 
85  // Added jhrg 8/28/13
86  vector<FONcMap*>::iterator i = d_grid_maps.begin();
87  while (i != d_grid_maps.end()) {
88  (*i)->decref();
89  ++i;
90  }
91 }
92 
107 void FONcArray::convert(vector<string> embed)
108 {
109  FONcBaseType::convert(embed);
110  _varname = FONcUtils::gen_name(embed, _varname, _orig_varname);
111 
112  BESDEBUG("fonc", "FONcArray::convert() - converting array " << _varname << endl);
113 
114  d_array_type = FONcUtils::get_nc_type(d_a->var());
115  d_ndims = d_a->dimensions();
116  d_actual_ndims = d_ndims; //replace this with _a->dimensions(); below TODO
117  if (d_array_type == NC_CHAR) {
118  // if we have array of strings then we need to add the string length
119  // dimension, so add one more to ndims
120  d_ndims++;
121  }
122 
123  // See HYRAX-805. When assigning values using [], set the size using resize
124  // not reserve. THe reserve method works to optimize push_back(), but apparently
125  // does not work with []. jhrg 8/3/18
126  d_dim_ids.resize(d_ndims);
127  d_dim_sizes.resize(d_ndims);
128 
129  Array::Dim_iter di = d_a->dim_begin();
130  Array::Dim_iter de = d_a->dim_end();
131  int dimnum = 0;
132  for (; di != de; di++) {
133  int size = d_a->dimension_size(di, true);
134  d_dim_sizes[dimnum] = size;
135  d_nelements *= size;
136 
137  // Set COMPRESSION CHUNK SIZE for each dimension.
138  d_chunksizes.push_back(size <= MAX_CHUNK_SIZE ? size: MAX_CHUNK_SIZE);
139 
140  BESDEBUG("fonc", "FONcArray::convert() - dim num: " << dimnum << ", dim size: " << size << ", chunk size: " << d_chunksizes[dimnum] << endl);
141 
142  // See if this dimension has already been defined. If it has the
143  // same name and same size as another dimension, then it is a
144  // shared dimension. Create it only once and share the FONcDim
145  FONcDim *use_dim = find_dim(embed, d_a->dimension_name(di), size);
146  d_dims.push_back(use_dim);
147  dimnum++;
148  }
149 
150  // if this array is a string array, then add the length dimension
151  if (d_array_type == NC_CHAR) {
152  // get the data from the dap array
153  int array_length = d_a->length();
154 
155  d_str_data.reserve(array_length);
156  d_a->value(d_str_data);
157 
158  // determine the max length of the strings
159  size_t max_length = 0;
160  for (int i = 0; i < array_length; i++) {
161  if (d_str_data[i].length() > max_length) {
162  max_length = d_str_data[i].length();
163  }
164  }
165  max_length++;
166  vector<string> empty_embed;
167  string lendim_name = _varname + "_len";
168 
169  FONcDim *use_dim = find_dim(empty_embed, lendim_name, max_length, true);
170  // Added static_cast to suppress warning. 12.27.2011 jhrg
171  if (use_dim->size() < static_cast<int>(max_length)) {
172  use_dim->update_size(max_length);
173  }
174 
175  d_dim_sizes[d_ndims - 1] = use_dim->size();
176  d_dim_ids[d_ndims - 1] = use_dim->dimid();
177  d_dims.push_back(use_dim);
178 
179  // Adding this fixes the bug reported by GSFC where arrays of strings
180  // caused the handler to throw an error stating that 'Bad chunk sizes'
181  // were used. When the dimension of the string array was extended (because
182  // strings become char arrays in netcdf3/4), the numbers of dimensions
183  // in 'chunksizes' was not bumped up. The code below in convert() that
184  // set the chunk sizes then tried to access data that had never been set.
185  // jhrg 11/25/15
186  d_chunksizes.push_back(max_length <= MAX_CHUNK_SIZE ? max_length: MAX_CHUNK_SIZE);
187  }
188 
189  // If this array has a single dimension, and the name of the array
190  // and the name of that dimension are the same, then this array
191  // might be used as a map for a grid defined elsewhere.
192  if (!FONcGrid::InGrid && d_actual_ndims == 1 && d_a->name() == d_a->dimension_name(d_a->dim_begin())) {
193  // is it already in there?
194  FONcMap *map = FONcGrid::InMaps(d_a);
195  if (!map) {
196  // This memory is/was leaked. jhrg 8/28/13
197  FONcMap *new_map = new FONcMap(this);
198  d_grid_maps.push_back(new_map); // save it here so we can free it later. jhrg 8/28/13
199  FONcGrid::Maps.push_back(new_map);
200  }
201  else {
202  d_dont_use_it = true;
203  }
204  }
205 
206  BESDEBUG("fonc", "FONcArray::convert() - done converting array " << _varname << endl);
207 }
208 
222 FONcDim *
223 FONcArray::find_dim(vector<string> &embed, const string &name, int size, bool ignore_size)
224 {
225  string oname;
226  string ename = FONcUtils::gen_name(embed, name, oname);
227  FONcDim *ret_dim = 0;
228  vector<FONcDim *>::iterator i = FONcArray::Dimensions.begin();
229  vector<FONcDim *>::iterator e = FONcArray::Dimensions.end();
230  for (; i != e && !ret_dim; i++) {
231  if (!((*i)->name().empty()) && ((*i)->name() == name)) {
232  if (ignore_size) {
233  ret_dim = (*i);
234  }
235  else if ((*i)->size() == size) {
236  ret_dim = (*i);
237  }
238  else {
239  if (embed.size() > 0) {
240  vector<string> tmp;
241  return find_dim(tmp, ename, size);
242  }
243  string err = "fileout_netcdf: dimension found with the same name, but different size";
244  throw BESInternalError(err, __FILE__, __LINE__);
245  }
246  }
247  }
248  if (!ret_dim) {
249  ret_dim = new FONcDim(name, size);
250  FONcArray::Dimensions.push_back(ret_dim);
251  }
252  else {
253  ret_dim->incref();
254  }
255  return ret_dim;
256 }
257 
272 void FONcArray::define(int ncid)
273 {
274  BESDEBUG("fonc", "FONcArray::define() - defining array '" << _varname << "'" << endl);
275 
276  if (!_defined && !d_dont_use_it) {
277  vector<FONcDim *>::iterator i = d_dims.begin();
278  vector<FONcDim *>::iterator e = d_dims.end();
279  int dimnum = 0;
280  for (; i != e; i++) {
281  FONcDim *fd = *i;
282  fd->define(ncid);
283  //d_dim_ids.at(dimnum) = fd->dimid();
284  d_dim_ids[dimnum] = fd->dimid();
285  BESDEBUG("fonc", "FONcArray::define() - dim_id: " << fd->dimid() << " size:" << fd->size() << endl);
286  dimnum++;
287  }
288 
289  int stax = nc_def_var(ncid, _varname.c_str(), d_array_type, d_ndims, &d_dim_ids[0], &_varid);
290  if (stax != NC_NOERR) {
291  string err = (string) "fileout.netcdf - Failed to define variable " + _varname;
292  FONcUtils::handle_error(stax, err, __FILE__, __LINE__);
293  }
294 
295  if (isNetCDF4()) {
296  BESDEBUG("fonc", "FONcArray::define() Working netcdf-4 branch " << endl);
297  if (FONcRequestHandler::chunk_size == 0)
298  // I have no idea if chunksizes is needed in this case.
299  stax = nc_def_var_chunking(ncid, _varid, NC_CONTIGUOUS, &d_chunksizes[0]);
300  else
301  stax = nc_def_var_chunking(ncid, _varid, NC_CHUNKED, &d_chunksizes[0]);
302 
303  if (stax != NC_NOERR) {
304  string err = "fileout.netcdf - Failed to define chunking for variable " + _varname;
305  FONcUtils::handle_error(stax, err, __FILE__, __LINE__);
306  }
307 
308  // TODO Make this more adaptable to the Array's data type. Find out when it's
309  // best to use shuffle, et c. jhrg 7/22/18
310  if (FONcRequestHandler::use_compression) {
311  int shuffle = 0;
312  int deflate = 1;
313  int deflate_level = 4;
314  stax = nc_def_var_deflate(ncid, _varid, shuffle, deflate, deflate_level);
315 
316  if (stax != NC_NOERR) {
317  string err = (string) "fileout.netcdf - Failed to define compression (deflate) level for variable "
318  + _varname;
319  FONcUtils::handle_error(stax, err, __FILE__, __LINE__);
320  }
321  }
322  }
323 
324  // If the array type is NC_SHORT it may have been an unsigned byte type before,
325  // so make sure the _FillValue is also 16 bits (SMAP NetCDF-3 reformatting issue
326  // for landcover_class).
327  //
328  // Contributed by abdul.g.khan@nasa.gov
329  //
330  // Question: Are there other cases where an unsigned type is 'promoted' and thus
331  // the type of the fill value attribute should be too? jhrg 10/12/15
332  AttrTable &attrs = d_a->get_attr_table();
333  if (d_array_type == NC_SHORT && attrs.get_size()) {
334  for (AttrTable::Attr_iter iter = attrs.attr_begin(); iter != attrs.attr_end(); iter++)
335  if (attrs.get_name(iter) == "_FillValue" && attrs.get_attr_type(iter) == Attr_byte)
336  (*iter)->type = Attr_int16;
337  }
338 
339  BESDEBUG("fonc", "FONcArray::define() - Adding attributes " << endl);
340  FONcAttributes::add_variable_attributes(ncid, _varid, d_a);
341  FONcAttributes::add_original_name(ncid, _varid, _varname, _orig_varname);
342 
343  _defined = true;
344  }
345  else {
346  if (_defined) {
347  BESDEBUG("fonc", "FONcArray::define() - variable " << _varname << " is already defined" << endl);
348  }
349  if (d_dont_use_it) {
350  BESDEBUG("fonc", "FONcArray::define() - variable " << _varname << " is not being used" << endl);
351  }
352  }
353 
354  BESDEBUG("fonc", "FONcArray::define() - done defining array '" << _varname << "'" << endl);
355 }
356 
366 void FONcArray::write(int ncid)
367 {
368  BESDEBUG("fonc", "FONcArray::write() BEGIN var: " << _varname << "[" << d_nelements << "]" << endl);
369 
370  if (d_dont_use_it) {
371  BESDEBUG("fonc", "FONcTransform::write not using variable " << _varname << endl);
372  return;
373  }
374 
375  ncopts = NC_VERBOSE;
376  int stax = NC_NOERR;
377 
378  if (d_array_type != NC_CHAR) {
379  string var_type = d_a->var()->type_name();
380 
381  // create array to hold data hyperslab
382  switch (d_array_type) {
383  case NC_BYTE: {
384  unsigned char *data = new unsigned char[d_nelements];
385  d_a->buf2val((void**) &data);
386  stax = nc_put_var_uchar(ncid, _varid, data);
387  delete[] data;
388 
389  if (stax != NC_NOERR) {
390  string err = "fileout.netcdf - Failed to create array of bytes for " + _varname;
391  FONcUtils::handle_error(stax, err, __FILE__, __LINE__);
392  }
393  break;
394  }
395 
396  case NC_SHORT: {
397  short *data = new short[d_nelements];
398 
399  // Given Byte/UInt8 will always be unsigned they must map
400  // to a NetCDF type that will support unsigned bytes. This
401  // detects the original variable was of type Byte and typecasts
402  // each data value to a short.
403  if (var_type == "Byte") {
404 
405  unsigned char *orig_data = new unsigned char[d_nelements];
406  d_a->buf2val((void**) &orig_data);
407 
408  for (int d_i = 0; d_i < d_nelements; d_i++)
409  data[d_i] = orig_data[d_i];
410 
411  delete[] orig_data;
412  }
413  else {
414  d_a->buf2val((void**) &data);
415  }
416  int stax = nc_put_var_short(ncid, _varid, data);
417  delete[] data;
418 
419  if (stax != NC_NOERR) {
420  string err = (string) "fileout.netcdf - Failed to create array of shorts for " + _varname;
421  FONcUtils::handle_error(stax, err, __FILE__, __LINE__);
422  }
423  break;
424  }
425 
426  case NC_INT: {
427  int *data = new int[d_nelements];
428  // Since UInt16 also maps to NC_INT, we need to obtain the data correctly
429  // KY 2012-10-25
430  if (var_type == "UInt16") {
431  unsigned short *orig_data = new unsigned short[d_nelements];
432  d_a->buf2val((void**) &orig_data);
433 
434  for (int d_i = 0; d_i < d_nelements; d_i++)
435  data[d_i] = orig_data[d_i];
436 
437  delete[] orig_data;
438  }
439  else {
440  d_a->buf2val((void**) &data);
441  }
442 
443  int stax = nc_put_var_int(ncid, _varid, data);
444  delete[] data;
445 
446  if (stax != NC_NOERR) {
447  string err = (string) "fileout.netcdf - Failed to create array of ints for " + _varname;
448  FONcUtils::handle_error(stax, err, __FILE__, __LINE__);
449  }
450  break;
451  }
452 
453  case NC_FLOAT: {
454  float *data = new float[d_nelements];
455  d_a->buf2val((void**) &data);
456  int stax = nc_put_var_float(ncid, _varid, data);
457  delete[] data;
458 
459  if (stax != NC_NOERR) {
460  string err = (string) "fileout.netcdf - Failed to create array of floats for " + _varname;
461  FONcUtils::handle_error(stax, err, __FILE__, __LINE__);
462  }
463  break;
464  }
465 
466  case NC_DOUBLE: {
467  double *data = new double[d_nelements];
468  d_a->buf2val((void**) &data);
469  int stax = nc_put_var_double(ncid, _varid, data);
470  delete[] data;
471 
472  if (stax != NC_NOERR) {
473  string err = (string) "fileout.netcdf - Failed to create array of doubles for " + _varname;
474  FONcUtils::handle_error(stax, err, __FILE__, __LINE__);
475  }
476  break;
477  }
478 
479  default:
480  string err = (string) "Failed to transform array of unknown type in file out netcdf";
481  throw BESInternalError(err, __FILE__, __LINE__);
482  }
483  }
484  else {
485  // special case for string data. Could have put this in the
486  // switch, but it's pretty big
487  size_t var_count[d_ndims];
488  size_t var_start[d_ndims];
489  int dim = 0;
490  for (dim = 0; dim < d_ndims; dim++) {
491  // the count for each of the dimensions will always be 1 except
492  // for the string length dimension
493  var_count[dim] = 1;
494 
495  // the start for each of the dimensions will start at 0. We will
496  // bump this up in the while loop below
497  var_start[dim] = 0;
498  }
499 
500  for (int element = 0; element < d_nelements; element++) {
501  var_count[d_ndims - 1] = d_str_data[element].size() + 1;
502  var_start[d_ndims - 1] = 0;
503 
504  // write out the string
505  int stax = nc_put_vara_text(ncid, _varid, var_start, var_count, d_str_data[element].c_str());
506  if (stax != NC_NOERR) {
507  string err = (string) "fileout.netcdf - Failed to create array of strings for " + _varname;
508  FONcUtils::handle_error(stax, err, __FILE__, __LINE__);
509  }
510 
511  // bump up the start.
512  if (element + 1 < d_nelements) {
513  bool done = false;
514  dim = d_ndims - 2;
515  while (!done) {
516  var_start[dim] = var_start[dim] + 1;
517  if (var_start[dim] == d_dim_sizes[dim]) {
518  var_start[dim] = 0;
519  dim--;
520  }
521  else {
522  done = true;
523  }
524  }
525  }
526  }
527  }
528 
529  BESDEBUG("fonc", "FONcArray::write() END var: " << _varname << "[" << d_nelements << "]" << endl);
530 }
531 
537 {
538  return d_a->name();
539 }
540 
549 void FONcArray::dump(ostream &strm) const
550 {
551  strm << BESIndent::LMarg << "FONcArray::dump - (" << (void *) this << ")" << endl;
552  BESIndent::Indent();
553  strm << BESIndent::LMarg << "name = " << _varname << endl;
554  strm << BESIndent::LMarg << "ndims = " << d_ndims << endl;
555  strm << BESIndent::LMarg << "actual ndims = " << d_actual_ndims << endl;
556  strm << BESIndent::LMarg << "nelements = " << d_nelements << endl;
557  if (d_dims.size()) {
558  strm << BESIndent::LMarg << "dimensions:" << endl;
559  BESIndent::Indent();
560  vector<FONcDim *>::const_iterator i = d_dims.begin();
561  vector<FONcDim *>::const_iterator e = d_dims.end();
562  for (; i != e; i++) {
563  (*i)->dump(strm);
564  }
565  BESIndent::UnIndent();
566  }
567  else {
568  strm << BESIndent::LMarg << "dimensions: none" << endl;
569  }
570  BESIndent::UnIndent();
571 }
FONcArray::FONcArray
FONcArray(libdap::BaseType *b)
Constructor for FONcArray that takes a DAP Array.
Definition: FONcArray.cc:55
FONcUtils::gen_name
static string gen_name(const vector< string > &embed, const string &name, string &original)
generate a new name for the embedded variable
Definition: FONcUtils.cc:148
FONcUtils::handle_error
static void handle_error(int stax, const string &err, const string &file, int line)
handle any netcdf errors
Definition: FONcUtils.cc:245
FONcMap
A map of a DAP Grid with file out netcdf information included.
Definition: FONcMap.h:52
FONcArray::write
virtual void write(int ncid)
Write the array out to the netcdf file.
Definition: FONcArray.cc:366
FONcUtils::get_nc_type
static nc_type get_nc_type(BaseType *element)
translate the OPeNDAP data type to a netcdf data type
Definition: FONcUtils.cc:102
FONcDim::define
virtual void define(int ncid)
define the DAP dimension in the netcdf file
Definition: FONcDim.cc:79
FONcGrid::Maps
static vector< FONcMap * > Maps
global list of maps that could be shared amongst the different grids
Definition: FONcGrid.h:74
FONcAttributes::add_variable_attributes
static void add_variable_attributes(int ncid, int varid, BaseType *b)
Add the attributes for an OPeNDAP variable to the netcdf file.
Definition: FONcAttributes.cc:77
FONcArray::~FONcArray
virtual ~FONcArray()
Destructor that cleans up the array.
Definition: FONcArray.cc:76
FONcArray::name
virtual std::string name()
returns the name of the DAP Array
Definition: FONcArray.cc:536
FONcDim
A class that represents the dimension of an array.
Definition: FONcDim.h:44
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
FONcAttributes::add_original_name
static void add_original_name(int ncid, int varid, const string &var_name, const string &orig)
Adds an attribute for the variable if the variable name had to be modified in any way.
Definition: FONcAttributes.cc:415
FONcBaseType
A DAP BaseType with file out netcdf information included.
Definition: FONcBaseType.h:58
FONcArray::convert
virtual void convert(std::vector< std::string > embed)
Converts the DAP Array to a FONcArray.
Definition: FONcArray.cc:107
FONcArray::define
virtual void define(int ncid)
define the DAP Array in the netcdf file
Definition: FONcArray.cc:272
FONcGrid::InGrid
static bool InGrid
tells whether we are converting or defining a grid.
Definition: FONcGrid.h:76
FONcBaseType::isNetCDF4
virtual bool isNetCDF4()
Returns true if NetCDF4 features will be required.
Definition: FONcBaseType.cc:97
FONcArray::dump
virtual void dump(std::ostream &strm) const
dumps information about this object for debugging purposes
Definition: FONcArray.cc:549