bes  Updated for version 3.20.6
HDF5CF.cc
Go to the documentation of this file.
1 // This file is part of the hdf5_handler implementing for the CF-compliant
2 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3 //
4 // This is free software; you can redistribute it and/or modify it under the
5 // terms of the GNU Lesser General Public License as published by the Free
6 // Software Foundation; either version 2.1 of the License, or (at your
7 // option) any later version.
8 //
9 // This software is distributed in the hope that it will be useful, but
10 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12 // License for more details.
13 //
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 //
18 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
20 // Suite 203, Champaign, IL 61820
21 
36 
37 #include <sstream>
38 #include <algorithm>
39 #include <functional>
40 #include <climits>
41 #include "HDF5CF.h"
42 #include "h5cfdaputil.h"
43 #include "HDF5RequestHandler.h"
44 #include "BESDebug.h"
45 
46 using namespace HDF5CF;
47 
48 Var::Var(Var *var)
49 {
50 
51  newname = var->newname;
52  name = var->name;
53  fullpath = var->fullpath;
54  rank = var->rank;
55  total_elems = var->total_elems;
56  dtype = var->dtype;
57  comp_ratio = var->comp_ratio;
58  unsupported_attr_dtype = var->unsupported_attr_dtype;
59  unsupported_attr_dspace = var->unsupported_attr_dspace;
60  unsupported_dspace = var->unsupported_dspace;
61  unsupported_attr_dspace = var->unsupported_attr_dspace;
62  dimnameflag = var->dimnameflag;
63 
64  for (vector<Attribute*>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
65  Attribute* attr = new Attribute();
66  attr->name = (*ira)->name;
67  attr->newname = (*ira)->newname;
68  attr->dtype = (*ira)->dtype;
69  attr->count = (*ira)->count;
70  attr->strsize = (*ira)->strsize;
71  attr->fstrsize = (*ira)->fstrsize;
72  attr->value = (*ira)->value;
73  attrs.push_back(attr);
74  }
75 
76  for (vector<Dimension*>::iterator ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
77  Dimension *dim = new Dimension((*ird)->size);
78  dim->name = (*ird)->name;
79  dim->newname = (*ird)->newname;
80  dim->unlimited_dim = (*ird)->unlimited_dim;
81  dims.push_back(dim);
82  }
83 
84 }
85 
86 bool CVar::isLatLon() const
87 {
88 
89  bool ret_value = false;
90  if (CV_EXIST == this->cvartype || CV_MODIFY == this->cvartype || CV_SPECIAL == this->cvartype) {
91  string attr_name = "units";
92  string lat_unit_value = "degrees_north";
93  string lon_unit_value = "degrees_east";
94 
95  for (vector<Attribute *>::const_iterator ira = this->attrs.begin(); ira != this->attrs.end(); ira++) {
96 
97  if ((H5FSTRING == (*ira)->getType()) || (H5VSTRING == (*ira)->getType())) {
98  if (attr_name == (*ira)->newname) {
99  string attr_value1((*ira)->getValue().begin(), (*ira)->getValue().end());
100 
101  if ((*ira)->getCount() == 1) {
102  string attr_value((*ira)->getValue().begin(), (*ira)->getValue().end());
103  if (attr_value.compare(0, lat_unit_value.size(), lat_unit_value) == 0) {
104  if (attr_value.size() == lat_unit_value.size()) {
105  ret_value = true;
106  break;
107  }
108  else if (attr_value.size() == (lat_unit_value.size() + 1)) {
109  if (attr_value[attr_value.size() - 1] == '\0'
110  || attr_value[attr_value.size() - 1] == ' ') {
111  ret_value = true;
112  break;
113  }
114  }
115  }
116  else if (attr_value.compare(0, lon_unit_value.size(), lon_unit_value) == 0) {
117  if (attr_value.size() == lon_unit_value.size()) {
118  ret_value = true;
119  break;
120  }
121  else if (attr_value.size() == (lon_unit_value.size() + 1)) {
122  if (attr_value[attr_value.size() - 1] == '\0'
123  || attr_value[attr_value.size() - 1] == ' ') {
124  ret_value = true;
125  break;
126  }
127  }
128 
129  }
130 
131  }
132  }
133  }
134  }
135  }
136  else if (this->cvartype == CV_LAT_MISS || this->cvartype == CV_LON_MISS) ret_value = true;
137  return ret_value;
138 
139 }
140 File::~File()
141 {
142 
143  if (this->fileid >= 0) {
144  if (this->rootid >= 0) {
145  for_each(this->groups.begin(), this->groups.end(), delete_elem());
146  for_each(this->vars.begin(), this->vars.end(), delete_elem());
147  for_each(this->root_attrs.begin(), this->root_attrs.end(), delete_elem());
148  H5Gclose(rootid);
149  }
150  }
151 }
152 
153 Group::~Group()
154 {
155  for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
156 }
157 
158 Var::~Var()
159 {
160  for_each(this->dims.begin(), this->dims.end(), delete_elem());
161  for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
162 }
163 
164 Attribute::~Attribute()
165 {
166 }
167 
168 void File::Retrieve_H5_Info(const char * /*path*/, hid_t file_id, bool include_attr)
169 {
170 
171  BESDEBUG("h5", "coming to Retrieve_H5_Info" <<endl);
172 
173  if (true == include_attr) {
174 
175  // Obtain the BES key to check the ignored objects
176  // We will only use DAS to output these information.
177  this->check_ignored = HDF5RequestHandler::get_check_ignore_obj();
178  if (true == this->check_ignored) this->add_ignored_info_page_header();
179 
180  }
181 
182  hid_t root_id;
183  if ((root_id = H5Gopen(file_id, "/", H5P_DEFAULT)) < 0) {
184  throw1("Cannot open the HDF5 root group ");
185  }
186  this->rootid = root_id;
187  try {
188  this->Retrieve_H5_Obj(root_id, "/", include_attr);
189  }
190  catch (...) {
191  throw;
192  }
193 
194  // Obtain attributes only necessary
195  if (true == include_attr) {
196 
197  // Find the file(root group) attribute
198 
199  // Obtain the object type, such as group or dataset.
200  H5O_info_t oinfo;
201  int num_attrs = 0;
202 
203  if (H5Oget_info(root_id, &oinfo) < 0)
204  throw1("Error obtaining the info for the root group");
205 
206  num_attrs = oinfo.num_attrs;
207  bool temp_unsup_attr_atype = false;
208  bool temp_unsup_attr_dspace = false;
209 
210  for (int j = 0; j < num_attrs; j++) {
211  Attribute * attr = new Attribute();
212  try {
213  this->Retrieve_H5_Attr_Info(attr, root_id, j, temp_unsup_attr_atype, temp_unsup_attr_dspace);
214  }
215  catch (...) {
216  delete attr;
217  throw;
218 
219  }
220  this->root_attrs.push_back(attr);
221  }
222 
223  this->unsupported_attr_dtype = temp_unsup_attr_atype;
224  this->unsupported_attr_dspace = temp_unsup_attr_dspace;
225  }
226 }
227 
228 void File::Retrieve_H5_Obj(hid_t grp_id, const char*gname, bool include_attr)
229 {
230 
231  // Iterate through the file to see the members of the group from the root.
232  H5G_info_t g_info;
233  hsize_t nelems = 0;
234 
235  if (H5Gget_info(grp_id, &g_info) < 0)
236  throw2("Counting hdf5 group elements error for ", gname);
237  nelems = g_info.nlinks;
238 
239  ssize_t oname_size = 0;
240  for (hsize_t i = 0; i < nelems; i++) {
241 
242  hid_t cgroup = -1;
243  hid_t cdset = -1;
244  Group *group = NULL;
245  Var *var = NULL;
246  Attribute *attr = NULL;
247 
248  try {
249 
250  size_t dummy_name_len = 1;
251 
252  // Query the length of object name.
253  oname_size = H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, NULL, dummy_name_len,
254  H5P_DEFAULT);
255  if (oname_size <= 0)
256  throw2("Error getting the size of the hdf5 object from the group: ", gname);
257 
258  // Obtain the name of the object
259  vector<char> oname;
260  oname.resize((size_t) oname_size + 1);
261 
262  if (H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, &oname[0], (size_t) (oname_size + 1),
263  H5P_DEFAULT) < 0)
264  throw2("Error getting the hdf5 object name from the group: ", gname);
265 
266  // Check if it is a hard link or a soft link
267  H5L_info_t linfo;
268  if (H5Lget_info(grp_id, &oname[0], &linfo, H5P_DEFAULT) < 0)
269  throw2("HDF5 link name error from ", gname);
270 
271  // We ignore soft links and external links for the CF options
272  if (H5L_TYPE_SOFT == linfo.type || H5L_TYPE_EXTERNAL == linfo.type) {
273  if (true == include_attr && true == check_ignored) {
274  this->add_ignored_info_links_header();
275  string full_path_name;
276  string temp_oname(oname.begin(), oname.end());
277  full_path_name = (
278  (string(gname) != "/") ?
279  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
280  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
281  this->add_ignored_info_links(full_path_name);
282 
283  }
284  continue;
285  }
286 
287  // Obtain the object type, such as group or dataset.
288  H5O_info_t oinfo;
289 
290  if (H5Oget_info_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, &oinfo, H5P_DEFAULT) < 0)
291  throw2("Error obtaining the info for the object ", string(oname.begin(), oname.end()));
292 
293  H5O_type_t obj_type = oinfo.type;
294 
295  switch (obj_type) {
296 
297  case H5O_TYPE_GROUP: {
298 
299  // Obtain the full path name
300  string full_path_name;
301  string temp_oname(oname.begin(), oname.end());
302 
303  full_path_name = (
304  (string(gname) != "/") ?
305  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
306  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
307 
308  cgroup = H5Gopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
309  if (cgroup < 0)
310  throw2("Error opening the group ", full_path_name);
311 
312  group = new Group();
313  group->path = full_path_name;
314  group->newname = full_path_name;
315 
316  // Retrieve group attribute if the attribute flag is true
317  if (true == include_attr) {
318 
319  int num_attrs = oinfo.num_attrs;
320  bool temp_unsup_attr_dtype = false;
321  bool temp_unsup_attr_dspace = false;
322 
323  for (int j = 0; j < num_attrs; j++) {
324 
325  attr = new Attribute();
326  Retrieve_H5_Attr_Info(attr, cgroup, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
327  group->attrs.push_back(attr);
328  attr = NULL;
329  }
330 
331  group->unsupported_attr_dtype = temp_unsup_attr_dtype;
332  group->unsupported_attr_dspace = temp_unsup_attr_dspace;
333  }
334  this->groups.push_back(group);
335  Retrieve_H5_Obj(cgroup, full_path_name.c_str(), include_attr);
336  if (H5Gclose(cgroup) < 0)
337  throw2("Error closing the group ", full_path_name);
338  }
339  break;
340  case H5O_TYPE_DATASET: {
341 
342  // Obtain the absolute path of the HDF5 dataset
343  string temp_oname(oname.begin(), oname.end());
344  string full_path_name = (
345  (string(gname) != "/") ?
346  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
347  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
348 
349  var = new Var();
350  var->name = temp_oname.substr(0, temp_oname.size() - 1);
351  var->fullpath = full_path_name;
352 
353  // newname is for the final CF name
354  var->newname = full_path_name;
355 
356  cdset = H5Dopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
357  if (cdset < 0)
358  throw2("Error opening the HDF5 dataset ", full_path_name);
359 
360  // Retrieve the HDF5 dataset datatype, return the flag for unsupported types.
361  bool temp_unsup_var_dtype = false;
362  Retrieve_H5_VarType(var, cdset, full_path_name, temp_unsup_var_dtype);
363 
364  // Update the unsupported datatype flag
365  if (!this->unsupported_var_dtype && temp_unsup_var_dtype) this->unsupported_var_dtype = true;
366 
367  // Retrieve the HDF5 dataset data space, return the flag for unsupported dataspaces.
368  bool temp_unsup_var_dspace = false;
369  Retrieve_H5_VarDim(var, cdset, full_path_name, temp_unsup_var_dspace);
370 
371  // Update the unsupported data space flag
372  if (!this->unsupported_var_dspace && temp_unsup_var_dspace) this->unsupported_var_dspace = true;
373 
374  var->comp_ratio = Retrieve_H5_VarCompRatio(var, cdset);
375 
376  // Retrieve the attribute info. if asked
377  if (true == include_attr) {
378 
379  int num_attrs = oinfo.num_attrs;
380  bool temp_unsup_attr_dtype = false;
381  bool temp_unsup_attr_dspace = false;
382 
383  for (int j = 0; j < num_attrs; j++) {
384 
385  attr = new Attribute();
386 
387  Retrieve_H5_Attr_Info(attr, cdset, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
388  var->attrs.push_back(attr);
389  attr = NULL;
390  }
391 
392  var->unsupported_attr_dtype = temp_unsup_attr_dtype;
393  var->unsupported_attr_dspace = temp_unsup_attr_dspace;
394 
395  if (!this->unsupported_var_attr_dspace && temp_unsup_attr_dspace)
396  this->unsupported_var_attr_dspace = true;
397  }
398 
399  this->vars.push_back(var);
400  if (H5Dclose(cdset) < 0)
401  throw2("Error closing the HDF5 dataset ", full_path_name);
402  }
403  break;
404 
405  case H5O_TYPE_NAMED_DATATYPE: {
406  // ignore the named datatype
407  if (true == include_attr && true == check_ignored) {
408  this->add_ignored_info_namedtypes(string(gname), string(oname.begin(), oname.end()));
409  }
410  }
411  break;
412  default:
413  break;
414  } // "switch (obj_type)"
415  } // try
416  catch (...) {
417 
418  if (attr != NULL) {
419  delete attr;
420  attr = NULL;
421  }
422 
423  if (var != NULL) {
424  delete var;
425  var = NULL;
426  }
427 
428  if (group != NULL) {
429  delete group;
430  group = NULL;
431  }
432 
433  if (cgroup != -1) H5Gclose(cgroup);
434 
435  if (cdset != -1) H5Dclose(cdset);
436  throw;
437 
438  } // catch
439  } // "for (hsize_t i = 0; i < nelems; i++)"
440 
441 }
442 
443 // Retrieve HDF5 dataset datatype
444 float File::Retrieve_H5_VarCompRatio(Var *var, hid_t dset_id)
445 {
446 
447  float comp_ratio = 1.0;
448  // Obtain the data type of the variable.
449  hid_t dset_create_plist = H5Dget_create_plist(dset_id);
450  if (dset_create_plist < 0)
451  throw1("unable to obtain hdf5 dataset creation property list ");
452  H5D_layout_t dset_layout = H5Pget_layout(dset_create_plist);
453  if (dset_layout < 0) {
454  H5Pclose(dset_create_plist);
455  throw1("unable to obtain hdf5 dataset creation property list storage layout");
456  }
457 
458  if (dset_layout == H5D_CHUNKED) {
459 
460  hsize_t dstorage_size = H5Dget_storage_size(dset_id);
461  if (dstorage_size > 0 && var->total_elems > 0) {
462  hid_t ty_id = -1;
463 
464  // Obtain the data type of the variable.
465  if ((ty_id = H5Dget_type(dset_id)) < 0)
466  throw1("unable to obtain hdf5 datatype for the dataset ");
467  size_t type_size = H5Tget_size(ty_id);
468  comp_ratio = ((float) (var->total_elems) * type_size) / dstorage_size;
469  H5Tclose(ty_id);
470  }
471 
472  }
473  H5Pclose(dset_create_plist);
474  return comp_ratio;
475 
476 }
477 // Retrieve HDF5 dataset datatype
478 void File::Retrieve_H5_VarType(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dtype)
479 {
480 
481  hid_t ty_id = -1;
482 
483  // Obtain the data type of the variable.
484  if ((ty_id = H5Dget_type(dset_id)) < 0)
485  throw2("unable to obtain hdf5 datatype for the dataset ", varname);
486 
487  // The following datatype class and datatype will not be supported for the CF option.
488  // H5T_TIME, H5T_BITFIELD
489  // H5T_OPAQUE, H5T_ENUM
490  // H5T_REFERENCE, H5T_COMPOUND
491  // H5T_VLEN,H5T_ARRAY
492  // 64-bit integer
493 
494  // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
495  // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
496  // 1-D variable length of string can also be mapped for the CF option..
497  // The variable length string class is H5T_STRING rather than H5T_VLEN,
498  // We also ignore the mapping of integer 64 bit since DAP2 doesn't
499  // support 64-bit integer. In theory, DAP2 doesn't support long double
500  // (128-bit or 92-bit floating point type).
501  //
502 
503  var->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
504  if (false == HDF5CFUtil::cf_strict_support_type(var->dtype)) unsup_var_dtype = true;
505 
506  if (H5Tclose(ty_id) < 0)
507  throw1("Unable to close the HDF5 datatype ");;
508 }
509 
510 // Retrieve the HDF5 dataset dimension information
511 void File::Retrieve_H5_VarDim(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dspace)
512 {
513 
514  vector<hsize_t> dsize;
515  vector<hsize_t> maxsize;
516 
517  hid_t dspace_id = -1;
518  hid_t ty_id = -1;
519 
520  try {
521  if ((dspace_id = H5Dget_space(dset_id)) < 0)
522  throw2("Cannot get hdf5 dataspace id for the variable ", varname);
523 
524  H5S_class_t space_class = H5S_NO_CLASS;
525  if ((space_class = H5Sget_simple_extent_type(dspace_id)) < 0)
526  throw2("Cannot obtain the HDF5 dataspace class for the variable ", varname);
527 
528  if (H5S_NULL == space_class)
529  unsup_var_dspace = true;
530  else {
531  if (false == unsup_var_dspace) {
532 
533  hssize_t h5_total_elms = H5Sget_simple_extent_npoints(dspace_id);
534  if (h5_total_elms < 0)
535  throw2("Cannot get the total number of elements of HDF5 dataset ", varname);
536  else
537  var->total_elems = (size_t) h5_total_elms;
538  int ndims = H5Sget_simple_extent_ndims(dspace_id);
539  if (ndims < 0)
540  throw2("Cannot get the hdf5 dataspace number of dimension for the variable ", varname);
541 
542  var->rank = ndims;
543  if (ndims != 0) {
544  dsize.resize(ndims);
545  maxsize.resize(ndims);
546  }
547 
548  // The netcdf DAP client supports the representation of the unlimited dimension.
549  // So we need to check.
550  if (H5Sget_simple_extent_dims(dspace_id, &dsize[0], &maxsize[0]) < 0)
551  throw2("Cannot obtain the dim. info for the variable ", varname);
552 
553  for (int i = 0; i < ndims; i++) {
554  Dimension * dim = new Dimension(dsize[i]);
555  if (maxsize[i] == H5S_UNLIMITED) {
556  dim->unlimited_dim = true;
557  if (false == have_udim) have_udim = true;
558  }
559  var->dims.push_back(dim);
560  }
561  }
562  }
563 
564  var->unsupported_dspace = unsup_var_dspace;
565 
566  if (H5Sclose(dspace_id) < 0)
567  throw1("Cannot close the HDF5 dataspace .");
568 
569  }
570 
571  catch (...) {
572 
573  if (dspace_id != -1) H5Sclose(dspace_id);
574 
575  if (ty_id != -1) H5Tclose(ty_id);
576  throw;
577  }
578 
579 }
580 
581 // Retrieve the HDF5 attribute information.
582 void File::Retrieve_H5_Attr_Info(Attribute * attr, hid_t obj_id, const int j, bool &unsup_attr_dtype,
583  bool &unsup_attr_dspace)
584 
585 {
586 
587  hid_t attrid = -1;
588  hid_t ty_id = -1;
589  hid_t aspace_id = -1;
590  hid_t memtype = -1;
591 
592  try {
593 
594  // Obtain the attribute ID.
595  if ((attrid = H5Aopen_by_idx(obj_id, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t) j, H5P_DEFAULT,
596  H5P_DEFAULT)) < 0)
597  throw1("Unable to open attribute by index ");
598 
599  // Obtain the size of attribute name.
600  ssize_t name_size = H5Aget_name(attrid, 0, NULL);
601  if (name_size < 0)
602  throw1("Unable to obtain the size of the hdf5 attribute name ");
603 
604  string attr_name;
605  attr_name.resize(name_size + 1);
606 
607  // Obtain the attribute name.
608  if ((H5Aget_name(attrid, name_size + 1, &attr_name[0])) < 0)
609  throw1("unable to obtain the hdf5 attribute name ");
610 
611  // Obtain the type of the attribute.
612  if ((ty_id = H5Aget_type(attrid)) < 0)
613  throw2("unable to obtain hdf5 datatype for the attribute ", attr_name);
614 
615  // The following datatype class and datatype will not be supported for the CF option.
616  // H5T_TIME, H5T_BITFIELD
617  // H5T_OPAQUE, H5T_ENUM
618  // H5T_REFERENCE, H5T_COMPOUND
619  // H5T_VLEN,H5T_ARRAY
620  // 64-bit integer
621 
622  // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
623  // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
624  // 1-D variable length of string can also be mapped for the CF option..
625  // The variable length string class is H5T_STRING rather than H5T_VLEN,
626  // We also ignore the mapping of integer 64 bit since DAP2 doesn't
627  // support 64-bit integer. In theory, DAP2 doesn't support long double
628  // (128-bit or 92-bit floating point type).
629  //
630  attr->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
631  if (false == HDF5CFUtil::cf_strict_support_type(attr->dtype)) unsup_attr_dtype = true;
632 
633  if(H5VSTRING == attr->dtype || H5FSTRING == attr->dtype) {
634  H5T_cset_t c_set_type = H5Tget_cset(ty_id);
635  if(c_set_type <0)
636  throw2("Cannot get hdf5 character set type for the attribute ", attr_name);
637  // This is a UTF-8 string
638  if(c_set_type == 1)
639  attr->is_cset_ascii = false;
640  }
641 
642  if ((aspace_id = H5Aget_space(attrid)) < 0)
643  throw2("Cannot get hdf5 dataspace id for the attribute ", attr_name);
644 
645  int ndims = H5Sget_simple_extent_ndims(aspace_id);
646  if (ndims < 0)
647  throw2("Cannot get the hdf5 dataspace number of dimension for attribute ", attr_name);
648 
649  hsize_t nelmts = 1;
650 
651  // if it is a scalar attribute, just define number of elements to be 1.
652  if (ndims != 0) {
653 
654  vector<hsize_t> asize;
655  vector<hsize_t> maxsize;
656  asize.resize(ndims);
657  maxsize.resize(ndims);
658 
659  // Obtain the attribute data space information.
660  if (H5Sget_simple_extent_dims(aspace_id, &asize[0], &maxsize[0]) < 0)
661  throw2("Cannot obtain the dim. info for the attribute ", attr_name);
662 
663  // Here we need to take care of 0-length attribute. This is legal in HDF5.
664  for (int dim_count = 0;dim_count < ndims; dim_count ++) {
665  // STOP adding unsupported_attr_dspace!
666  if (asize[dim_count] == 0) {
667  unsup_attr_dspace = true;
668  break;
669  }
670  }
671 
672  if (false == unsup_attr_dspace) {
673  // Return ndims and size[ndims].
674  for (int dim_count = 0; dim_count< ndims; dim_count++)
675  nelmts *= asize[dim_count];
676  }
677  else
678  nelmts = 0;
679  } // "if(ndims != 0)"
680 
681  size_t ty_size = H5Tget_size(ty_id);
682  if (0 == ty_size)
683  throw2("Cannot obtain the dtype size for the attribute ", attr_name);
684 
685  memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
686  if (memtype < 0)
687  throw2("Cannot obtain the memory datatype for the attribute ", attr_name);
688 
689  // Store the name and the count
690  string temp_aname(attr_name.begin(), attr_name.end());
691  attr->name = temp_aname.substr(0, temp_aname.size() - 1);
692  attr->newname = attr->name;
693  attr->count = nelmts;
694 
695  // Release HDF5 resources.
696  if (H5Tclose(ty_id) < 0)
697  throw1("Cannot successfully close the attribute datatype.");
698  if (H5Tclose(memtype) < 0)
699  throw1("Cannot successfully close the attribute memory datatype.");
700  if (H5Sclose(aspace_id) < 0)
701  throw1("Cannot successfully close the HDF5 dataspace.");
702  if (H5Aclose(attrid) < 0)
703  throw1("Cannot successfully close the HDF5 attribute.");
704 
705  } // try
706  catch (...) {
707 
708  if (ty_id != -1) H5Tclose(ty_id);
709 
710  if (memtype != -1) H5Tclose(memtype);
711 
712  if (aspace_id != -1) H5Sclose(aspace_id);
713 
714  if (attrid != -1) H5Aclose(attrid);
715 
716  throw;
717  }
718 
719 }
720 
721 // Retrieve all HDF5 supported attribute values.
723 {
724 
725  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira)
726  Retrieve_H5_Attr_Value(*ira, "/");
727 
728  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
729  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
730  Retrieve_H5_Attr_Value(*ira, (*irg)->path);
731  }
732  }
733 
734  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
735  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
736  Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
737  }
738  }
739 }
740 
742 {
743  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
744  Retrieve_H5_Attr_Value(*ira, var->fullpath);
745  }
746 }
747 
748 // Retrieve the values of a specific HDF5 attribute.
749 void File::Retrieve_H5_Attr_Value(Attribute *attr, string obj_name)
750 {
751 
752  // Define HDF5 object Ids.
753  hid_t obj_id = -1;
754  hid_t attr_id = -1;
755  hid_t ty_id = -1;
756  hid_t memtype_id = -1;
757  hid_t aspace_id = -1;
758 
759  try {
760 
761  // Open the object that hold this attribute
762  obj_id = H5Oopen(this->fileid, obj_name.c_str(), H5P_DEFAULT);
763  if (obj_id < 0)
764  throw2("Cannot open the object ", obj_name);
765 
766  attr_id = H5Aopen(obj_id, (attr->name).c_str(), H5P_DEFAULT);
767  if (attr_id < 0)
768  throw4("Cannot open the attribute ", attr->name, " of object ", obj_name);
769 
770  ty_id = H5Aget_type(attr_id);
771  if (ty_id < 0)
772  throw4("Cannot obtain the datatype of the attribute ", attr->name, " of object ", obj_name);
773 
774  memtype_id = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
775  if (memtype_id < 0)
776  throw2("Cannot obtain the memory datatype for the attribute ", attr->name);
777 
778  size_t ty_size = H5Tget_size(memtype_id);
779  if (0 == ty_size)
780  throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
781 
782  size_t total_bytes = attr->count * ty_size;
783 
784  // We have to handle variable length string differently.
785  if (H5VSTRING == attr->dtype) {
786 
787  // Variable length string attribute values only store pointers of the actual string value.
788  vector<char> temp_buf;
789  temp_buf.resize(total_bytes);
790 
791  if (H5Aread(attr_id, memtype_id, &temp_buf[0]) < 0)
792  throw4("Cannot obtain the value of the attribute ", attr->name, " of object ", obj_name);
793 
794  char *temp_bp = NULL;
795  char *ptr_1stvlen_ptr = &temp_buf[0];
796  temp_bp = &temp_buf[0];
797  char* onestring = NULL;
798  string total_vstring = "";
799 
800  attr->strsize.resize(attr->count);
801 
802  for (unsigned int temp_i = 0; temp_i < attr->count; temp_i++) {
803 
804  // This line will assure that we get the real variable length string value.
805  onestring = *(char **) temp_bp;
806  if (onestring != NULL) {
807  total_vstring += string(onestring);
808  attr->strsize[temp_i] = (string(onestring)).size();
809  }
810  else
811  attr->strsize[temp_i] = 0;
812 
813  // going to the next value.
814  temp_bp += ty_size;
815  }
816 
817  if (ptr_1stvlen_ptr != NULL) {
818  aspace_id = H5Aget_space(attr_id);
819  if (aspace_id < 0)
820  throw4("Cannot obtain space id for ", attr->name, " of object ", obj_name);
821 
822  // Reclaim any VL memory if necessary.
823  if (H5Dvlen_reclaim(memtype_id, aspace_id, H5P_DEFAULT, &temp_buf[0]) < 0)
824  throw4("Cannot reclaim VL memory for ", attr->name, " of object ", obj_name);
825 
826  H5Sclose(aspace_id);
827  }
828 
829  if (HDF5CFUtil::H5type_to_H5DAPtype(ty_id) != H5VSTRING)
830  throw4("Error to obtain the VL string type for attribute ", attr->name, " of object ", obj_name);
831 
832  attr->value.resize(total_vstring.size());
833 
834  copy(total_vstring.begin(), total_vstring.end(), attr->value.begin());
835 
836  }
837  else {
838 
839  if (attr->dtype == H5FSTRING) {
840  attr->fstrsize = ty_size;
841  }
842 
843  attr->value.resize(total_bytes);
844 
845  // Read HDF5 attribute data.
846  if (H5Aread(attr_id, memtype_id, (void *) &attr->value[0]) < 0)
847  throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
848 
849  if (attr->dtype == H5FSTRING) {
850 
851  size_t sect_size = ty_size;
852  int num_sect = 1;
853  if (sect_size > 0)
854  num_sect =
855  (total_bytes % sect_size == 0) ? (total_bytes / sect_size) : (total_bytes / sect_size + 1);
856  else
857  throw4("The attribute datatype size is not a positive integer ", attr->name, " of object ",
858  obj_name);
859 
860  vector<size_t> sect_newsize;
861  sect_newsize.resize(num_sect);
862 
863  string total_fstring = string(attr->value.begin(), attr->value.end());
864 
865  string new_total_fstring = HDF5CFUtil::trim_string(memtype_id, total_fstring, num_sect, sect_size,
866  sect_newsize);
867  attr->value.resize(new_total_fstring.size());
868  copy(new_total_fstring.begin(), new_total_fstring.end(), attr->value.begin());
869  attr->strsize.resize(num_sect);
870  for (int temp_i = 0; temp_i < num_sect; temp_i++)
871  attr->strsize[temp_i] = sect_newsize[temp_i];
872 
873 #if 0
874  // "h5","new string value " <<string(attr->value.begin(), attr->value.end()) <<endl;
875  for (int temp_i = 0; temp_i <num_sect; temp_i ++)
876  "h5","string new section size = " << attr->strsize[temp_i] <<endl;
877 #endif
878  }
879  }
880 
881  if (H5Tclose(memtype_id) < 0)
882  throw1("Fail to close the HDF5 memory datatype ID.");
883  if (H5Tclose(ty_id) < 0)
884  throw1("Fail to close the HDF5 datatype ID.");
885  if (H5Aclose(attr_id) < 0)
886  throw1("Fail to close the HDF5 attribute ID.");
887  if (H5Oclose(obj_id) < 0)
888  throw1("Fail to close the HDF5 object ID.");
889 
890  }
891 
892  catch (...) {
893 
894  if (memtype_id != -1) H5Tclose(memtype_id);
895 
896  if (ty_id != -1) H5Tclose(ty_id);
897 
898  if (aspace_id != -1) H5Sclose(aspace_id);
899 
900  if (attr_id != -1) H5Aclose(attr_id);
901 
902  if (obj_id != -1) H5Oclose(obj_id);
903 
904  throw;
905  }
906 
907 }
908 
909 // Handle the unsupported datatype
910 void File::Handle_Unsupported_Dtype(bool include_attr)
911 {
912 
913  if (true == include_attr) {
914  Handle_Group_Unsupported_Dtype();
915  Handle_VarAttr_Unsupported_Dtype();
916  }
917 
918  Handle_Var_Unsupported_Dtype();
919 }
920 
921 //Leave this code here.
922 #if 0
923 // First the root attributes
924 if (true == include_attr) {
925  if (true == this->unsupported_attr_dtype) {
926  for (vector<Attribute *>::iterator ira = this->root_attrs.begin();
927  ira != this->root_attrs.end(); ) {
928  H5DataType temp_dtype = (*ira)->getType();
929  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
930  delete (*ira);
931  ira = this->root_attrs.erase(ira);
932 
933  }
934  else {
935  ++ira;
936  }
937  }
938  }
939 }
940 
941 // Then the group attributes
942 if (false == this->groups.empty()) {
943  for (vector<Group *>::iterator irg = this->groups.begin();
944  irg != this->groups.end(); ++irg) {
945  if (false == (*irg)->attrs.empty()) {
946  if (true == (*irg)->unsupported_attr_dtype) {
947  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin();
948  ira != (*irg)->attrs.end(); ) {
949  H5DataType temp_dtype = (*ira)->getType();
950  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
951  delete (*ira);
952  ira = (*irg)->attrs.erase(ira);
953  }
954  else {
955  ++ira;
956  }
957  }
958  }
959  }
960  }
961 }
962 }
963 
964  // Then the variable(HDF5 dataset) and the correponding attributes.
965 if (false == this->vars.empty()) {
966 if (true == include_attr) {
967  for (vector<Var *>::iterator irv = this->vars.begin();
968  irv != this->vars.end();++irv ) {
969  if (false == (*irv)->attrs.empty()) {
970  if (true == (*irv)->unsupported_attr_dtype) {
971  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
972  ira != (*irv)->attrs.end(); ) {
973  H5DataType temp_dtype = (*ira)->getType();
974  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
975  delete (*ira);
976  ira = (*irv)->attrs.erase(ira);
977  //ira--;
978  }
979  else {
980  ++ira;
981  }
982  }
983  }
984  }
985  }
986 }
987 if (true == this->unsupported_var_dtype) {
988  // "h5","having unsupported variable datatype" <<endl;
989  for (vector<Var *>::iterator irv = this->vars.begin();
990  irv != this->vars.end(); ) {
991  H5DataType temp_dtype = (*irv)->getType();
992  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
993  delete (*irv);
994  irv = this->vars.erase(irv);
995  //irv--;
996  }
997  else
998  ++irv;
999  }
1000 }
1001 }
1002 #endif
1003 
1004 void File::Handle_Group_Unsupported_Dtype()
1005 {
1006 
1007  // First root
1008  if (false == this->root_attrs.empty()) {
1009  if (true == this->unsupported_attr_dtype) {
1010  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1011  H5DataType temp_dtype = (*ira)->getType();
1012  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
1013  delete (*ira);
1014  ira = this->root_attrs.erase(ira);
1015  }
1016  else {
1017  ++ira;
1018  }
1019  }
1020  }
1021  }
1022 
1023  // Then the group attributes
1024  if (false == this->groups.empty()) {
1025  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1026  if (false == (*irg)->attrs.empty()) {
1027  if (true == (*irg)->unsupported_attr_dtype) {
1028  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1029  H5DataType temp_dtype = (*ira)->getType();
1030  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
1031  delete (*ira);
1032  ira = (*irg)->attrs.erase(ira);
1033  }
1034  else {
1035  ++ira;
1036  }
1037  }
1038  }
1039  }
1040  }
1041  }
1042 }
1043 
1044 // Generate group unsupported datatype Information, this is for the BES ignored object key
1045 void File::Gen_Group_Unsupported_Dtype_Info()
1046 {
1047 
1048  // First root
1049  if (false == this->root_attrs.empty()) {
1050  //if (true == this->unsupported_attr_dtype) {
1051  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1052  H5DataType temp_dtype = (*ira)->getType();
1053  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
1054  this->add_ignored_info_attrs(true, "/", (*ira)->name);
1055  }
1056  }
1057  //}
1058  }
1059 
1060  // Then the group attributes
1061  if (false == this->groups.empty()) {
1062  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1063  if (false == (*irg)->attrs.empty()) {
1064  //if (true == (*irg)->unsupported_attr_dtype) {
1065  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1066  H5DataType temp_dtype = (*ira)->getType();
1067  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype==H5UINT64 ) {
1068  this->add_ignored_info_attrs(true, (*irg)->path, (*ira)->name);
1069  }
1070  }
1071  //}
1072  }
1073  }
1074  }
1075 }
1076 
1077 // Handler unsupported variable datatype
1078 void File::Handle_Var_Unsupported_Dtype()
1079 {
1080  if (false == this->vars.empty()) {
1081  if (true == this->unsupported_var_dtype) {
1082  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end();) {
1083  H5DataType temp_dtype = (*irv)->getType();
1084  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
1085  delete (*irv);
1086  irv = this->vars.erase(irv);
1087  }
1088  else {
1089  ++irv;
1090 
1091  }
1092  }
1093  }
1094  }
1095 }
1096 
1097 // Generate unsupported variable type info. This is for the ignored objects.
1098 void File::Gen_Var_Unsupported_Dtype_Info()
1099 {
1100 
1101  if (false == this->vars.empty()) {
1102  //if (true == this->unsupported_var_dtype) {
1103  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1104  H5DataType temp_dtype = (*irv)->getType();
1105  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)||(H5INT64 == temp_dtype) ||(H5UINT64 == temp_dtype)) {
1106  this->add_ignored_info_objs(false, (*irv)->fullpath);
1107  }
1108  }
1109  //}
1110  }
1111 
1112 }
1113 
1114 // Handling unsupported datatypes for variable(HDF5 dataset) and the correponding attributes.
1115 void File::Handle_VarAttr_Unsupported_Dtype()
1116 {
1117  if (false == this->vars.empty()) {
1118  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1119  if (false == (*irv)->attrs.empty()) {
1120  if (true == (*irv)->unsupported_attr_dtype) {
1121  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1122  H5DataType temp_dtype = (*ira)->getType();
1123  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
1124  delete (*ira);
1125  ira = (*irv)->attrs.erase(ira);
1126  }
1127  else {
1128  ++ira;
1129  }
1130  }
1131  }
1132  }
1133  }
1134  }
1135 }
1136 
1137 // Generated unsupported var/attribute unsupported datatype Info when the BES ignored object key is on.
1138 void File::Gen_VarAttr_Unsupported_Dtype_Info()
1139 {
1140 
1141  if (false == this->vars.empty()) {
1142  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1143  if (false == (*irv)->attrs.empty()) {
1144  //if (true == (*irv)->unsupported_attr_dtype) {
1145  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1146  H5DataType temp_dtype = (*ira)->getType();
1147  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || (temp_dtype==H5INT64) || (temp_dtype == H5UINT64)) {
1148  this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1149  }
1150  }
1151  //}
1152  }
1153  }
1154  }
1155 }
1156 
1157 // Generated unsupported datatype information for HDF5 dimension scales.
1158 // The datatypes of HDF5 dimension scales("DIMENSION_LIST" and "REFERENCE_LIST")
1159 // are not supported. However, the information
1160 // are retrieved by the handlers so we don't want to report them as ignored objects.
1161 void File::Gen_DimScale_VarAttr_Unsupported_Dtype_Info()
1162 {
1163 
1164  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1165 
1166  // If the attribute REFERENCE_LIST comes with the attribut CLASS, the
1167  // attribute REFERENCE_LIST is okay to ignore. No need to report.
1168  bool is_ignored = ignored_dimscale_ref_list((*irv));
1169  if (false == (*irv)->attrs.empty()) {
1170  //if (true == (*irv)->unsupported_attr_dtype) {
1171  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1172  H5DataType temp_dtype = (*ira)->getType();
1173  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || (temp_dtype == H5INT64) || (temp_dtype == H5UINT64)) {
1174  // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
1175  // is okay to ignore if the variable has another attribute
1176  // CLASS="DIMENSION_SCALE"
1177  if (("DIMENSION_LIST" != (*ira)->name)
1178  && ("REFERENCE_LIST" != (*ira)->name || true == is_ignored))
1179  this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1180  }
1181  }
1182  //}
1183  }
1184  }
1185 }
1186 
1187 // Handle unsupported dataspace for group attributes.
1188 void File::Handle_GroupAttr_Unsupported_Dspace()
1189 {
1190 
1191  // First root
1192  if (false == this->root_attrs.empty()) {
1193  if (true == this->unsupported_attr_dspace) {
1194  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1195  // Remove 0-size attribute
1196  if ((*ira)->count == 0) {
1197  delete (*ira);
1198  ira = this->root_attrs.erase(ira);
1199  }
1200  else {
1201  ++ira;
1202  }
1203  }
1204  }
1205  }
1206 
1207  // Then the group attributes
1208  if (false == this->groups.empty()) {
1209  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1210  if (false == (*irg)->attrs.empty()) {
1211  if (true == (*irg)->unsupported_attr_dspace) {
1212  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1213  if ((*ira)->count == 0) {
1214  delete (*ira);
1215  ira = (*irg)->attrs.erase(ira);
1216  }
1217  else {
1218  ++ira;
1219  }
1220  }
1221  }
1222  }
1223  }
1224  }
1225 }
1226 
1227 // Handle unsupported data space information for variable and attribute
1228 void File::Handle_VarAttr_Unsupported_Dspace()
1229 {
1230 
1231  if (false == this->vars.empty()) {
1232  if (true == this->unsupported_var_attr_dspace) {
1233  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1234  if (false == (*irv)->attrs.empty()) {
1235  if (true == (*irv)->unsupported_attr_dspace) {
1236  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1237  if (0 == (*ira)->count) {
1238  delete (*ira);
1239  ira = (*irv)->attrs.erase(ira);
1240  }
1241  else {
1242  ++ira;
1243  }
1244  }
1245  }
1246  }
1247  }
1248  }
1249  }
1250 }
1251 
1252 // Handle unsupported data space.
1253 void File::Handle_Unsupported_Dspace(bool include_attr)
1254 {
1255 
1256  // The unsupported data space
1257  if (false == this->vars.empty()) {
1258  if (true == this->unsupported_var_dspace) {
1259  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end();) {
1260  if (true == (*irv)->unsupported_dspace) {
1261  delete (*irv);
1262  irv = this->vars.erase(irv);
1263  }
1264  else {
1265  ++irv;
1266 
1267  }
1268  }
1269  }
1270  }
1271 
1272  if (true == include_attr) {
1273  Handle_GroupAttr_Unsupported_Dspace();
1274  Handle_VarAttr_Unsupported_Dspace();
1275  }
1276 }
1277 
1278 // Generated unsupported dataspace Info when the BES ignored object key is on.
1279 void File::Gen_Unsupported_Dspace_Info()
1280 {
1281 
1282  // Notice in this function, we deliberately don't put the case when an attribute dimension has 0 length.
1283  // Since doing this requires non-trivial change of the source code and the 0-size attribute case is really, really rare,
1284  // so we just "ignore" this case in the "ignored" information.
1285  // In fact, the zero size variable is allowed in both HDF5 and DAP2. So we don't ignore 0-size HDF5 dataset. So
1286  // the only case this function checks is the H5S_NULL case.
1287  if (false == this->vars.empty()) {
1288  if (true == this->unsupported_var_dspace) {
1289  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1290  if (true == (*irv)->unsupported_dspace) {
1291  this->add_ignored_info_objs(true, (*irv)->fullpath);
1292  }
1293  }
1294  }
1295  }
1296 
1297 }
1298 
1299 // Handle other unsupported information.
1300 void File::Handle_Unsupported_Others(bool include_attr)
1301 {
1302 
1303  if (true == this->check_ignored && true == include_attr) {
1304 
1305  if (true == HDF5RequestHandler::get_drop_long_string()) {
1306 
1307  // netCDF java doesn't have limitation for attributes
1308 #if 0
1309  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1310  if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1311  if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1312  this->add_ignored_droplongstr_hdr();
1313  this->add_ignored_grp_longstr_info("/", (*ira)->name);
1314  }
1315  }
1316  }
1317 
1318  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1319  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1320  if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1321  if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1322  this->add_ignored_droplongstr_hdr();
1323  this->add_ignored_grp_longstr_info((*irg)->path, (*ira)->name);
1324  }
1325  }
1326  }
1327  }
1328 #endif
1329  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1330  if (true == Check_DropLongStr((*irv), NULL)) {
1331  this->add_ignored_droplongstr_hdr();
1332  this->add_ignored_var_longstr_info((*irv), NULL);
1333  }
1334  // netCDF java doesn't have limitation for attributes
1335 #if 0
1336  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1337  if (true == Check_DropLongStr((*irv), (*ira))) {
1338  this->add_ignored_droplongstr_hdr();
1339  this->add_ignored_var_longstr_info((*irv), (*ira));
1340  }
1341  }
1342 #endif
1343  }
1344  }
1345  }
1346 
1347 }
1348 
1349 // Flatten the object name, mainly call get_CF_string.
1350 void File::Flatten_Obj_Name(bool include_attr)
1351 {
1352 
1353  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1354  (*irv)->newname = get_CF_string((*irv)->newname);
1355 
1356  for (vector<Dimension *>::iterator ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1357  (*ird)->newname = get_CF_string((*ird)->newname);
1358  }
1359  }
1360 
1361  if (true == include_attr) {
1362 
1363  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1364  (*ira)->newname = get_CF_string((*ira)->newname);
1365  }
1366 
1367  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1368  (*irg)->newname = get_CF_string((*irg)->newname);
1369  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1370  (*ira)->newname = get_CF_string((*ira)->newname);
1371  }
1372  }
1373 
1374  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1375  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1376  (*ira)->newname = get_CF_string((*ira)->newname);
1377  }
1378  }
1379  } // "if (true == include_attr)"
1380 }
1381 
1382 // Variable name clashing
1383 void File::Handle_Var_NameClashing(set<string>&objnameset)
1384 {
1385 
1386  Handle_General_NameClashing(objnameset, this->vars);
1387 }
1388 
1389 // Group name clashing
1390 void File::Handle_Group_NameClashing(set<string> &objnameset)
1391 {
1392 
1393  pair<set<string>::iterator, bool> setret;
1394 
1395  // Now for DAS, we need to handle name clashings for
1396  // DAS tables. Namely we need to make sure the global attribute
1397  // table(HDF5_GLOBAL) and the attribute tables mapped from
1398  // HDF5 groups will not have name clashings with the variable name
1399  // lists. If having the name clashings, the global attribute table and the
1400  // the attribute tables generated from the groups will be changed.
1401  // The file attribute name clashing
1402 
1403  setret = objnameset.insert(FILE_ATTR_TABLE_NAME);
1404  if (false == setret.second) {
1405 
1406  int clash_index = 1;
1407  string fa_clash_name = FILE_ATTR_TABLE_NAME;
1408  HDF5CFUtil::gen_unique_name(fa_clash_name, objnameset, clash_index);
1409  FILE_ATTR_TABLE_NAME = fa_clash_name;
1410  }
1411 
1412  // The group attribute name clashing
1413  Handle_General_NameClashing(objnameset, this->groups);
1414 
1415 }
1416 
1417 //Object attribute name clashing
1418 void File::Handle_Obj_AttrNameClashing()
1419 {
1420 
1421  // Now handling the possible name clashings for attributes
1422  // For attribute clashings, we only need to resolve the name clashings
1423  // for attributes within each variable, file attributes and attributes
1424  // within each group. The name clashings for attributes should be very rare.
1425  // Potentially the checking and the correcting may be costly.
1426  // This is another reason for special products, we may not even need to check
1427  // the name clashings. KY 2011-12-24
1428 
1429  set<string> objnameset;
1430 
1431  // For root attributes
1432  Handle_General_NameClashing(objnameset, this->root_attrs);
1433 
1434  // For group attributes
1435  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1436  objnameset.clear();
1437  Handle_General_NameClashing(objnameset, (*irg)->attrs);
1438  }
1439 
1440  // For variable attributes
1441  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1442  objnameset.clear();
1443  Handle_General_NameClashing(objnameset, (*irv)->attrs);
1444  }
1445 }
1446 
1447 // Handle General name clashing
1448 //class T must have member string newname. In our case, T is either groups, attributes or vars.
1449 template<class T> void File::Handle_General_NameClashing(set<string>&objnameset, vector<T*>& objvec)
1450 {
1451 
1452 // set<string> objnameset;
1453  pair<set<string>::iterator, bool> setret;
1454  set<string>::iterator iss;
1455 
1456  vector<string> clashnamelist;
1457  vector<string>::iterator ivs;
1458 
1459  map<int, int> cl_to_ol;
1460  int ol_index = 0;
1461  int cl_index = 0;
1462 
1463  /*class*/
1464  typename vector<T*>::iterator irv;
1465 
1466  for (irv = objvec.begin(); irv != objvec.end(); ++irv) {
1467  setret = objnameset.insert((*irv)->newname);
1468  if (false == setret.second) {
1469  clashnamelist.insert(clashnamelist.end(), (*irv)->newname);
1470  cl_to_ol[cl_index] = ol_index;
1471  cl_index++;
1472  }
1473  ol_index++;
1474  }
1475 
1476  // Now change the clashed elements to unique elements,
1477  // Generate the set which has the same size as the original vector.
1478  for (ivs = clashnamelist.begin(); ivs != clashnamelist.end(); ivs++) {
1479  int clash_index = 1;
1480  string temp_clashname = *ivs + '_';
1481  HDF5CFUtil::gen_unique_name(temp_clashname, objnameset, clash_index);
1482  *ivs = temp_clashname;
1483  }
1484 
1485  // Now go back to the original vector, make it unique.
1486  for (unsigned int i = 0; i < clashnamelist.size(); i++)
1487  objvec[cl_to_ol[i]]->newname = clashnamelist[i];
1488 
1489 }
1490 
1491 // Handle General object name clashing
1492 void File::Handle_GeneralObj_NameClashing(bool include_attr, set<string>& objnameset)
1493 {
1494 
1495  Handle_Var_NameClashing(objnameset);
1496  if (true == include_attr) {
1497  Handle_Group_NameClashing(objnameset);
1498  Handle_Obj_AttrNameClashing();
1499  }
1500 }
1501 
1502 // Get CF name, flatten the path, change the non-alphanumeric letters to underscore.
1503 string File::get_CF_string(string s)
1504 {
1505 
1506  if ("" == s) return s;
1507  string insertString(1, '_');
1508 
1509  // Always start with _ if the first character is not a letter
1510  if (true == isdigit(s[0])) s.insert(0, insertString);
1511 
1512  for (unsigned int i = 0; i < s.length(); i++)
1513  if ((false == isalnum(s[i])) && (s[i] != '_')) s[i] = '_';
1514 
1515  return s;
1516 
1517 }
1518 
1519 // For the connection of variable dimensions. Build dimname to dimsize(unlimited) maps
1520 void File::Insert_One_NameSizeMap_Element(string name, hsize_t size, bool unlimited)
1521 {
1522  pair<map<string, hsize_t>::iterator, bool> mapret;
1523  mapret = dimname_to_dimsize.insert(pair<string, hsize_t>(name, size));
1524  if (false == mapret.second)
1525  throw4("The dimension name ", name, " should map to ", size);
1526 
1527  pair<map<string, bool>::iterator, bool> mapret2;
1528  mapret2 = dimname_to_unlimited.insert(pair<string, bool>(name, unlimited));
1529  if (false == mapret2.second)
1530  throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1531 
1532 }
1533 
1534 // Similar to Inset_One_NameSizeMap_Element but the maps are provided as parameters.
1535 void File::Insert_One_NameSizeMap_Element2(map<string, hsize_t>& name_to_size, map<string, bool>& name_to_unlimited,
1536  string name, hsize_t size, bool unlimited)
1537 {
1538  pair<map<string, hsize_t>::iterator, bool> mapret;
1539  mapret = name_to_size.insert(pair<string, hsize_t>(name, size));
1540  if (false == mapret.second)
1541  throw4("The dimension name ", name, " should map to ", size);
1542 
1543  pair<map<string, bool>::iterator, bool> mapret2;
1544  mapret2 = name_to_unlimited.insert(pair<string, bool>(name, unlimited));
1545  if (false == mapret2.second)
1546  throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1547 
1548 }
1549 
1550 // For dimension names added by the handlers, by default,
1551 // Each dimension will have a unique dimension name. For example,
1552 // Int foo[100][200] will be Int foo[Fakedim1][Fakedim2]
1553 // If you have many variables, the dimension names may be too many.
1554 // To reduce numbers, we ASSUME that the dimension having the same
1555 // size shares the same dimension. In this way, the number of dimension names
1556 // will be reduced.
1557 // For example, Int foo2[100][300] will be Int foo2[Fakedim1][Fakedim3]
1558 // instead of foo2[Fakedim3][Fakedim4]. However, that may impose
1559 // another problem. Suppose Int Foosame[100][100] becomes
1560 // Int Foosame[FakeDim1][FakeDim1]. This doesn't make sense for some
1561 // applications. The fuction Adjust_Duplicate_FakeDim_Name will make sure
1562 // this case will not happen.
1563 void File::Add_One_FakeDim_Name(Dimension *dim)
1564 {
1565 
1566  stringstream sfakedimindex;
1567  string fakedimstr = "FakeDim";
1568  pair<set<string>::iterator, bool> setret;
1569  map<hsize_t, string>::iterator im;
1570  pair<map<hsize_t, string>::iterator, bool> mapret;
1571 
1572  sfakedimindex << addeddimindex;
1573  string added_dimname = fakedimstr + sfakedimindex.str();
1574 
1575  // Build up the size to fakedim map.
1576  mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, added_dimname));
1577  if (false == mapret.second) { //The dim size exists, use the corresponding name.
1578  dim->name = dimsize_to_fakedimname[dim->size];
1579  dim->newname = dim->name;
1580  }
1581  else { // Insert this (dimsize,dimname) pair to dimsize_to_fakedimname map successfully.
1582  //First make sure this new dim name doesn't have name clashing
1583  // with previous dim names, after the checking, inserting to the
1584  // dimname list set.
1585  // dimnamelist is a private memeber of File.
1586  setret = dimnamelist.insert(added_dimname);
1587  if (false == setret.second) {
1588  int clash_index = 1;
1589  string temp_clashname = added_dimname + '_';
1590  HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1591  dim->name = temp_clashname;
1592  dim->newname = dim->name;
1593  setret = dimnamelist.insert(dim->name);
1594  if (false == setret.second)
1595  throw2("Fail to insert the unique dimsizede name ", dim->name);
1596 
1597  // We have to adjust the dim. name of the dimsize_to_fakedimname map, since the
1598  // dimname has been updated for this size.
1599  dimsize_to_fakedimname.erase(dim->size);
1600  mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, dim->name));
1601  if (false == mapret.second)
1602  throw4("The dimension size ", dim->size, " should map to ", dim->name);
1603  } // "if(false == setret.second)"
1604 
1605  // New dim name is inserted successfully, update the dimname_to_dimsize map.
1606  dim->name = added_dimname;
1607  dim->newname = dim->name;
1608  Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1609 
1610  // Increase the dimindex since the new dimname has been inserted.
1611  addeddimindex++;
1612  } // else
1613 }
1614 
1615 // See the function comments of Add_One_FakeDim_Name
1616 void File::Adjust_Duplicate_FakeDim_Name(Dimension * dim)
1617 {
1618 
1619  // No need to adjust the dimsize_to_fakedimname map, only create a new Fakedim
1620  // The simplest way is to increase the dim index and resolve any name clashings with other dim names.
1621  // Note: No need to update the dimsize_to_dimname map since the original "FakeDim??" of this size
1622  // can be used as a dimension name of other variables. But we need to update the dimname_to_dimsize map
1623  // since this is a new dim name.
1624  stringstream sfakedimindex;
1625  pair<set<string>::iterator, bool> setret;
1626 
1627  addeddimindex++;
1628  sfakedimindex << addeddimindex;
1629  string added_dimname = "FakeDim" + sfakedimindex.str();
1630  setret = dimnamelist.insert(added_dimname);
1631  if (false == setret.second) {
1632  int clash_index = 1;
1633  string temp_clashname = added_dimname + '_';
1634  HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1635  dim->name = temp_clashname;
1636  dim->newname = dim->name;
1637  setret = dimnamelist.insert(dim->name);
1638  if (false == setret.second)
1639  throw2("Fail to insert the unique dimsizede name ", dim->name);
1640  }
1641  dim->name = added_dimname;
1642  dim->newname = dim->name;
1643  Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1644 
1645  // Need to prepare for the next unique FakeDim.
1646  addeddimindex++;
1647 }
1648 
1649 // Replace all dimension names, this function is currently not used. So comment out. May delete it in the future.
1650 #if 0
1651 void File::Replace_Dim_Name_All(const string orig_dim_name, const string new_dim_name) {
1652 
1653  // The newname of the original dimension should also be replaced by new_dim_name
1654  for (vector<Var *>::iterator irv = this->vars.begin();
1655  irv != this->vars.end(); ++irv) {
1656  for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1657  ird != (*irv)->dims.end(); ++ird) {
1658  if((*ird)->name == orig_dim_name) {
1659  (*ird)->name = new_dim_name;
1660  (*ird)->newname = new_dim_name;
1661  }
1662 
1663  }
1664  }
1665 }
1666 #endif
1667 
1668 #if 0
1669 void File::Use_Dim_Name_With_Size_All(const string dim_name, const size_t dim_size) {
1670 
1671  // The newname of the original dimension should also be replaced by new_dim_name
1672  for (vector<Var *>::iterator irv = this->vars.begin();
1673  irv != this->vars.end(); ++irv) {
1674  for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1675  ird != (*irv)->dims.end(); ++ird) {
1676  if((*ird)->size == orig_dim_name) {
1677  (*ird)->name = new_dim_name;
1678  (*ird)->newname = new_dim_name;
1679  }
1680 
1681  }
1682  }
1683 }
1684 #endif
1685 
1686 // Often times we need to add a CF attribute with string datatype because some products don't provide them
1687 // Examples are units, comment etc.
1688 void File::Add_Str_Attr(Attribute* attr, const string &attrname, const string& strvalue)
1689 {
1690 
1691  attr->name = attrname;
1692  attr->newname = attr->name;
1693  attr->dtype = H5FSTRING;
1694  attr->count = 1;
1695  attr->fstrsize = strvalue.size();
1696  attr->strsize.resize(1);
1697  attr->strsize[0] = attr->fstrsize;
1698  attr->value.resize(strvalue.size());
1699  copy(strvalue.begin(), strvalue.end(), attr->value.begin());
1700 }
1701 
1702 #if 0
1703 bool
1704 File:: Var_Has_Attr(Var*var,const string &attrname) {
1705 
1706  for (vector<Attribute *>:: iterator ira =var->attrs.begin(); ira !=var->attrs.end(); ++ira) {
1707 
1708  // We only check the original attribute name
1709  // Remove the original "coordinates" attribute.
1710  if((*ira)->name == attrname || (*ira)->newname == attrname) {
1711  return true;
1712  }
1713  }
1714  return false;
1715 }
1716 #endif
1717 
1718 // Rretrieve the variable attribute in string.var_path is the variable path.
1719 string File::Retrieve_Str_Attr_Value(Attribute *attr, const string var_path)
1720 {
1721 
1722  if (attr != NULL && var_path != "") {
1723  Retrieve_H5_Attr_Value(attr, var_path);
1724  string orig_attr_value(attr->value.begin(), attr->value.end());
1725  return orig_attr_value;
1726  }
1727  return "";
1728 
1729 }
1730 
1731 //Check if the attribute value of this variable is the input value.
1732 bool File::Is_Str_Attr(Attribute* attr, string varfullpath, const string &attrname, const string& strvalue)
1733 {
1734  bool ret_value = false;
1735  if (attrname == get_CF_string(attr->newname)) {
1736  Retrieve_H5_Attr_Value(attr, varfullpath);
1737  string attr_value(attr->value.begin(), attr->value.end());
1738  if (attr_value == strvalue) ret_value = true;
1739  }
1740  return ret_value;
1741 }
1742 
1743 // If this latitude or longitude units follows the CF
1744 bool File::has_latlon_cf_units(Attribute *attr, const string &varfullpath, bool is_lat)
1745 {
1746  string attr_name = "units";
1747  if (true == is_lat) {
1748  string lat_unit_value = "degrees_north";
1749  return Is_Str_Attr(attr, varfullpath, attr_name, lat_unit_value);
1750  }
1751  else {
1752  string lon_unit_value = "degrees_east";
1753  return Is_Str_Attr(attr, varfullpath, attr_name, lon_unit_value);
1754  }
1755 }
1756 
1757 // This function is mainly to add _FillValue.
1758 void File::Add_One_Float_Attr(Attribute* attr, const string &attrname, float float_value)
1759 {
1760  attr->name = attrname;
1761  attr->newname = attr->name;
1762  attr->dtype = H5FLOAT32;
1763  attr->count = 1;
1764  attr->value.resize(sizeof(float));
1765  memcpy(&(attr->value[0]), (void*) (&float_value), sizeof(float));
1766 }
1767 
1768 // Products like GPM use string type for MissingValue, we need to change them to the corresponding variable datatype and
1769 // get the value corrected.
1770 void File::Change_Attr_One_Str_to_Others(Attribute* attr, Var*var)
1771 {
1772 
1773  char *pEnd;
1774  // string to long int number.
1775  long int num_sli = 0;
1776  if (attr->dtype != H5FSTRING)
1777  throw2("Currently we only convert fixed-size string to other datatypes. ", attr->name);
1778  if (attr->count != 1)
1779  throw4("The fixed-size string count must be 1 and the current count is ", attr->count, " for the attribute ",
1780  attr->name);
1781 
1782  Retrieve_H5_Attr_Value(attr, var->fullpath);
1783  string attr_value;
1784  attr_value.resize(attr->value.size());
1785  copy(attr->value.begin(), attr->value.end(), attr_value.begin());
1786 
1787  switch (var->dtype) {
1788 
1789  case H5UCHAR: {
1790  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1791  if (num_sli < 0 || num_sli > UCHAR_MAX)
1792  throw5("Attribute type is unsigned char, the current attribute ", attr->name, " has the value ", num_sli,
1793  ". It is overflowed. ");
1794  else {
1795  unsigned char num_suc = (unsigned char) num_sli;
1796  attr->dtype = H5UCHAR;
1797  attr->value.resize(sizeof(unsigned char));
1798  memcpy(&(attr->value[0]), (void*) (&num_suc), sizeof(unsigned char));
1799  }
1800 
1801  }
1802  break;
1803  case H5CHAR: {
1804  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1805  if (num_sli < SCHAR_MIN || num_sli > SCHAR_MAX)
1806  throw5("Attribute type is signed char, the current attribute ", attr->name, " has the value ", num_sli,
1807  ". It is overflowed. ");
1808  else {
1809  char num_sc = (char) num_sli;
1810  attr->dtype = H5CHAR;
1811  attr->value.resize(sizeof(char));
1812  memcpy(&(attr->value[0]), (void*) (&num_sc), sizeof(char));
1813  }
1814 
1815  }
1816  break;
1817  case H5INT16: {
1818  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1819  if (num_sli < SHRT_MIN || num_sli > SHRT_MAX)
1820  throw5("Attribute type is 16-bit integer, the current attribute ", attr->name, " has the value ", num_sli,
1821  ". It is overflowed. ");
1822  else {
1823  short num_ss = (short) num_sli;
1824  attr->dtype = H5INT16;
1825  attr->value.resize(sizeof(short));
1826  memcpy(&(attr->value[0]), (void*) (&num_ss), sizeof(short));
1827  }
1828 
1829  }
1830  break;
1831  case H5UINT16: {
1832  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1833  if (num_sli < 0 || num_sli > USHRT_MAX)
1834  throw5("Attribute type is unsigned 16-bit integer, the current attribute ", attr->name, " has the value ",
1835  num_sli, ". It is overflowed. ");
1836  else {
1837  unsigned short num_uss = (unsigned short) num_sli;
1838  attr->dtype = H5UINT16;
1839  attr->value.resize(sizeof(unsigned short));
1840  memcpy(&(attr->value[0]), (void*) (&num_uss), sizeof(unsigned short));
1841  }
1842  }
1843  break;
1844  case H5INT32: {
1845  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1846  //No need to check overflow, the number will always be in the range.
1847 #if 0
1848  //if(num_sli <LONG_MIN || num_sli >LONG_MAX)
1849  // throw5("Attribute type is 32-bit integer, the current attribute ",attr->name, " has the value ",num_sli, ". It is overflowed. ");
1850 #endif
1851  attr->dtype = H5INT32;
1852  attr->value.resize(sizeof(long int));
1853  memcpy(&(attr->value[0]), (void*) (&num_sli), sizeof(long int));
1854 
1855  }
1856  break;
1857  case H5UINT32: {
1858  unsigned long int num_suli = strtoul(&(attr->value[0]), &pEnd, 10);
1859  // No need to check since num_suli will not be bigger than ULONG_MAX.
1860  attr->dtype = H5UINT32;
1861  attr->value.resize(sizeof(unsigned long int));
1862  memcpy(&(attr->value[0]), (void*) (&num_suli), sizeof(unsigned long int));
1863  }
1864  break;
1865  case H5FLOAT32: {
1866  float num_sf = strtof(&(attr->value[0]), NULL);
1867  // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1868  attr->dtype = H5FLOAT32;
1869  attr->value.resize(sizeof(float));
1870  memcpy(&(attr->value[0]), (void*) (&num_sf), sizeof(float));
1871  }
1872  break;
1873  case H5FLOAT64: {
1874  double num_sd = strtod(&(attr->value[0]), NULL);
1875  // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1876  attr->dtype = H5FLOAT64;
1877  attr->value.resize(sizeof(double));
1878  memcpy(&(attr->value[0]), (void*) (&num_sd), sizeof(double));
1879  }
1880  break;
1881 
1882  default:
1883  throw4("Unsupported HDF5 datatype that the string is converted to for the attribute ", attr->name,
1884  " of the variable ", var->fullpath);
1885  } // "switch(var->dtype)"
1886 
1887 }
1888 
1889 // Change a string type attribute to the input value
1890 void File::Replace_Var_Str_Attr(Var* var, const string &attr_name, const string& strvalue)
1891 {
1892 
1893  bool rep_attr = true;
1894  bool rem_attr = false;
1895  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1896  if ((*ira)->name == attr_name) {
1897  if (true == Is_Str_Attr(*ira, var->fullpath, attr_name, strvalue))
1898  rep_attr = false;
1899  else
1900  rem_attr = true;
1901  break;
1902  }
1903  }
1904 
1905  // Remove the attribute if the attribute value is not strvalue
1906  if (true == rem_attr) {
1907  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1908  if ((*ira)->name == attr_name) {
1909  delete (*ira);
1910  var->attrs.erase(ira);
1911  break;
1912  }
1913  }
1914  }
1915 
1916  // Add the attribute with strvalue
1917  if (true == rep_attr) {
1918  Attribute * attr = new Attribute();
1919  Add_Str_Attr(attr, attr_name, strvalue);
1920  var->attrs.push_back(attr);
1921  }
1922 }
1923 
1924 // Check if this variable if latitude,longitude.We check the three name pairs(lat,lon),(latitude,longitude),(Latitude,Longitude)
1925 bool File::Is_geolatlon(const string & var_name, bool is_lat)
1926 {
1927 
1928  bool ret_value = false;
1929  if (true == is_lat) {
1930  string lat1 = "lat";
1931  string lat2 = "latitude";
1932  string lat3 = "Latitude";
1933 
1934  if (var_name.compare(lat1) == 0 || var_name.compare(lat2) == 0 || var_name.compare(lat3) == 0) ret_value = true;
1935  }
1936 
1937  else {
1938  string lon1 = "lon";
1939  string lon2 = "longitude";
1940  string lon3 = "Longitude";
1941  if (var_name.compare(lon1) == 0 || var_name.compare(lon2) == 0 || var_name.compare(lon3) == 0) ret_value = true;
1942 
1943  }
1944  return ret_value;
1945 }
1946 
1947 // Add supplementary attributes.
1948 void File::Add_Supplement_Attrs(bool add_path)
1949 {
1950 
1951  if (false == add_path) return;
1952 
1953  // Adding variable original name(origname) and full path(fullpath)
1954  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1955  Attribute * attr = new Attribute();
1956  const string varname = (*irv)->name;
1957  const string attrname = "origname";
1958  Add_Str_Attr(attr, attrname, varname);
1959  (*irv)->attrs.push_back(attr);
1960  }
1961 
1962  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1963  Attribute * attr = new Attribute();
1964  const string varname = (*irv)->fullpath;
1965  const string attrname = "fullnamepath";
1966  Add_Str_Attr(attr, attrname, varname);
1967  (*irv)->attrs.push_back(attr);
1968  }
1969 
1970  // Adding group path
1971  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1972  // Only when this group has attributes, the original path of the group has some values. So add it.
1973  if (false == (*irg)->attrs.empty()) {
1974 
1975  Attribute * attr = new Attribute();
1976  const string varname = (*irg)->path;
1977  const string attrname = "fullnamepath";
1978  Add_Str_Attr(attr, attrname, varname);
1979  (*irg)->attrs.push_back(attr);
1980  }
1981  }
1982 
1983 }
1984 
1985 // Variable target will not be deleted, but rather its contents are replaced.
1986 // We may make this as an operator = in the future.
1987 // Note: the attributes can not be replaced.
1988 void File::Replace_Var_Info(Var *src, Var *target)
1989 {
1990 
1991 #if 0
1992  for_each (target->dims.begin (), target->dims.end (),
1993  delete_elem ());
1994  for_each (target->attrs.begin (), target->attrs.end (),
1995  delete_elem ());
1996 #endif
1997 
1998  target->newname = src->newname;
1999  target->name = src->name;
2000  target->fullpath = src->fullpath;
2001  target->rank = src->rank;
2002  target->dtype = src->dtype;
2003  target->unsupported_attr_dtype = src->unsupported_attr_dtype;
2004  target->unsupported_dspace = src->unsupported_dspace;
2005 #if 0
2006  for (vector<Attribute*>::iterator ira = target->attrs.begin();
2007  ira!=target->attrs.end(); ++ira) {
2008  delete (*ira);
2009  target->attrs.erase(ira);
2010  ira--;
2011  }
2012 #endif
2013  for (vector<Dimension*>::iterator ird = target->dims.begin(); ird != target->dims.end();) {
2014  delete (*ird);
2015  ird = target->dims.erase(ird);
2016  }
2017 
2018  // Somehow attributes cannot be replaced.
2019 #if 0
2020  for (vector<Attribute*>::iterator ira = src->attrs.begin();
2021  ira!=src->attrs.end(); ++ira) {
2022  Attribute* attr= new Attribute();
2023  attr->name = (*ira)->name;
2024  attr->newname = (*ira)->newname;
2025  attr->dtype =(*ira)->dtype;
2026  attr->count =(*ira)->count;
2027  attr->strsize = (*ira)->strsize;
2028  attr->fstrsize = (*ira)->fstrsize;
2029  attr->value =(*ira)->value;
2030  target->attrs.push_back(attr);
2031  }
2032 #endif
2033 
2034  for (vector<Dimension*>::iterator ird = src->dims.begin(); ird != src->dims.end(); ++ird) {
2035  Dimension *dim = new Dimension((*ird)->size);
2036  dim->name = (*ird)->name;
2037  dim->newname = (*ird)->newname;
2038  target->dims.push_back(dim);
2039  }
2040 
2041 }
2042 
2043 // Replace the attributes of target with src.
2044 void File::Replace_Var_Attrs(Var *src, Var *target)
2045 {
2046 
2047 #if 0
2048  for_each (target->dims.begin (), target->dims.end (),
2049  delete_elem ());
2050  for_each (target->attrs.begin (), target->attrs.end (),
2051  delete_elem ());
2052 #endif
2053 
2054  for (vector<Attribute*>::iterator ira = target->attrs.begin(); ira != target->attrs.end();) {
2055  delete (*ira);
2056  ira = target->attrs.erase(ira);
2057  }
2058  for (vector<Attribute*>::iterator ira = src->attrs.begin(); ira != src->attrs.end(); ++ira) {
2059  Attribute* attr = new Attribute();
2060  attr->name = (*ira)->name;
2061  attr->newname = (*ira)->newname;
2062  attr->dtype = (*ira)->dtype;
2063  attr->count = (*ira)->count;
2064  attr->strsize = (*ira)->strsize;
2065  attr->fstrsize = (*ira)->fstrsize;
2066  attr->value = (*ira)->value;
2067  target->attrs.push_back(attr);
2068  }
2069 
2070 }
2071 
2072 // Check if a variable with a var name is under a specific group with groupname
2073 // note: the variable's size at each dimension is also returned. The user must allocate the
2074 // memory for the dimension sizes(an array(vector is perferred).
2075 bool File::is_var_under_group(const string &varname, const string &grpname, const int var_rank,
2076  vector<size_t> & var_size)
2077 {
2078 
2079  bool ret_value = false;
2080  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2081 
2082  if ((*irv)->rank == var_rank) {
2083  if ((*irv)->name == varname) {
2084 
2085  // Obtain the variable path
2086  string var_path = HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
2087 
2088  // Check if we find the variable under this group
2089  if (grpname == var_path) {
2090  ret_value = true;
2091  for (int i = 0; i < var_rank; i++)
2092  var_size[i] = (*irv)->getDimensions()[i]->size;
2093  break;
2094  }
2095  }
2096  } // "if((*irv)->rank == var_rank)"
2097  } // "for (vector<Var *>::iterator irv = this->vars.begin()"
2098 
2099 
2100  return ret_value;
2101 
2102 }
2104 
2105  bool ret_value = false;
2106  for (vector<Var *>::iterator irv = this->vars.begin();
2107  irv != this->vars.end(); ++irv) {
2108  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2109  ira != (*irv)->attrs.end(); ++ira) {
2110  if((*ira)->name =="grid_mapping") {
2111  ret_value = true;
2112  break;
2113  }
2114  }
2115  if(true == ret_value)
2116  break;
2117  }
2118 
2119  return ret_value;
2120 
2121 }
2122 
2124 
2125  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2126  string attr_value;
2127  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
2128  if((*ira)->name =="grid_mapping") {
2129  Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
2130  attr_value.resize((*ira)->value.size());
2131  copy((*ira)->value.begin(), (*ira)->value.end(), attr_value.begin());
2132  break;
2133  }
2134 
2135  }
2136  if(attr_value.find('/') ==string::npos){
2137  string new_name = Check_Grid_Mapping_VarName(attr_value,(*irv)->fullpath);
2138  if(new_name != "")
2139  Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2140 
2141  }
2142  else {
2143  string new_name = Check_Grid_Mapping_FullPath(attr_value);
2144  //Using new_name as the attribute value
2145  if(new_name != "")
2146  Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2147  }
2148  }
2149 
2150 }
2151 
2152 string File::Check_Grid_Mapping_VarName(const string & a_value,const string & var_fpath) {
2153 
2154  string var_path = HDF5CFUtil::obtain_string_before_lastslash(var_fpath);
2155  string gmap_new_name;
2156  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2157  if((*irv)->name == a_value){
2158  if(var_path == HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) {
2159  gmap_new_name = (*irv)->newname;
2160  break;
2161  }
2162  }
2163  }
2164  return gmap_new_name;
2165 }
2166 
2167 
2168 string File::Check_Grid_Mapping_FullPath(const string & a_value) {
2169 
2170  string gmap_new_name;
2171  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2172  if((*irv)->fullpath == a_value){
2173  gmap_new_name = (*irv)->newname;
2174  break;
2175  }
2176  }
2177 
2178  return gmap_new_name;
2179 }
2180 
2181 void File::remove_netCDF_internal_attributes(bool include_attr) {
2182 
2183  if(true == include_attr) {
2184  for (vector<Var *>::iterator irv = this->vars.begin();
2185  irv != this->vars.end(); ++irv) {
2186  bool var_has_dimscale = false;
2187 
2188  for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2189  ira != (*irv)->attrs.end();) {
2190  if((*ira)->name == "CLASS") {
2191  string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2192 
2193  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2194  // "DIMENSION_SCALE", which is 15.
2195  if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
2196  delete(*ira);
2197  ira = (*irv)->attrs.erase(ira);
2198  var_has_dimscale = true;
2199 
2200  }
2201 #if 0
2202  else if(1) {// Add a BES key,also delete
2203 
2204  }
2205 #endif
2206  else {
2207  ++ira;
2208  }
2209  }
2210 #if 0
2211  else if((*ira)->name == "NAME") {// Add a BES Key
2212  string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2213  if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
2214  delete(*ira);
2215  ira =(*irv)->attrs.erase(ira);
2216  }
2217  else {
2218  string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
2219  if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
2220  delete(*ira);
2221  ira =(*irv)->attrs.erase(ira);
2222  }
2223  else {
2224  ++ira;
2225  }
2226  }
2227 
2228  }
2229 #endif
2230  else if((*ira)->name == "_Netcdf4Dimid") {
2231  delete(*ira);
2232  ira =(*irv)->attrs.erase(ira);
2233  }
2234 #if 0
2235  else if((*ira)->name == "_nc3_strict") {
2236  delete((*ira));
2237  ira =(*irv)->attrs.erase(ira);
2238  }
2239 #endif
2240  else {
2241  ++ira;
2242  }
2243  }
2244 
2245  if(true == var_has_dimscale) {
2246  for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2247  ira != (*irv)->attrs.end();++ira) {
2248  if((*ira)->name == "NAME") {// Add a BES Key
2249  delete(*ira);
2250  ira =(*irv)->attrs.erase(ira);
2251  break;
2252  }
2253  }
2254  }
2255  }
2256  }
2257 
2258 }
2259 // Add ignored page header info. Mainly a helper message.
2260 void File::add_ignored_info_page_header()
2261 {
2262  ignored_msg =
2263  " \n This page is for HDF5 CF hyrax data providers or distributors to check if any HDF5 object or attribute information are ignored during the mapping. \n\n";
2264 }
2265 
2266 // Add ignored object header info. Mainly a helper message.
2267 void File::add_ignored_info_obj_header()
2268 {
2269 
2270  ignored_msg += " Some HDF5 objects or the object information are ignored when mapping to DAP2 by the HDF5 OPeNDAP";
2271  ignored_msg += " handler due to the restrictions of DAP2, CF conventions or CF tools.";
2272  ignored_msg += " Please use HDF5 tools(h5dump or HDFView) to check carefully and make sure that these objects";
2273  ignored_msg +=
2274  " are OK to ignore for your service. For questions or requests to find a way to handle the ignored objects, please";
2275  ignored_msg += " contact the HDF5 OPeNDAP handler developer or send an email to help@hdfgroup.org.\n";
2276 
2277  ignored_msg += " \n In general, ignored HDF5 objects include HDF5 soft links, external links and named datatype.\n";
2278  ignored_msg +=
2279  " \n The HDF5 datasets(variables in the CF term) and attributes that have the following datatypes are ignored: \n";
2280  ignored_msg +=
2281  " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2282  ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2283 
2284  ignored_msg +=
2285  " \n The HDF5 datasets(variables in the CF term) and attributes associated with the following dimensions are ignored: \n";
2286  ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2287  ignored_msg += " 2) attributes that have any zero size dimensions(not reported due to extreme rarity and non-trivial coding)\n\n";
2288 
2289 }
2290 
2291 // Add the ignored links information.Mainly a helper message.
2292 void File::add_ignored_info_links_header()
2293 {
2294 
2295  if (false == this->have_ignored) {
2296  add_ignored_info_obj_header();
2297  have_ignored = true;
2298  }
2299  // Add ignored datatype header.
2300  string lh_msg = "******WARNING******\n";
2301  lh_msg += "IGNORED soft links or external links are: ";
2302  if (ignored_msg.rfind(lh_msg) == string::npos) ignored_msg += lh_msg + "\n";
2303 
2304 }
2305 
2306 // Leave the code for the time being.
2307 #if 0
2308 void
2309 File:: add_ignored_info_obj_dtype_header() {
2310 
2311  // Add ignored datatype header.
2312  ignored_msg += " \n Variables and attributes ignored due to the unsupported datatypes. \n";
2313  ignored_msg += " In general, the unsupported datatypes include: \n";
2314  ignored_msg += " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2315  ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2316 
2317 }
2318 
2319 void
2320 File:: add_ignored_info_obj_dspace_header() {
2321 
2322  // Add ignored dataspace header.
2323  ignored_msg += " \n Variables and attributes ignored due to the unsupported dimensions. \n";
2324  ignored_msg += " In general, the unsupported dimensions include: \n";
2325  ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2326  ignored_msg += " 2) variables that have any zero size dimensions\n";
2327 
2328 }
2329 #endif
2330 
2331 // Add the ignored link info.
2332 void File::add_ignored_info_links(const string & link_path)
2333 {
2334  if (ignored_msg.find("Link paths: ") == string::npos)
2335  ignored_msg += " Link paths: " + link_path;
2336  else
2337  ignored_msg += " " + link_path;
2338 }
2339 
2340 // Add the ignored name datatype info.
2341 void File::add_ignored_info_namedtypes(const string& grp_name, const string& named_dtype_name)
2342 {
2343 
2344  if (false == this->have_ignored) {
2345  add_ignored_info_obj_header();
2346  have_ignored = true;
2347  }
2348 
2349  string ignored_HDF5_named_dtype_hdr = "\n******WARNING******";
2350  ignored_HDF5_named_dtype_hdr += "\n IGNORED HDF5 named datatype objects:\n";
2351  string ignored_HDF5_named_dtype_msg = " Group name: " + grp_name + " HDF5 named datatype name: " + named_dtype_name.substr(0,named_dtype_name.size()-1)
2352  + "\n";
2353  if (ignored_msg.find(ignored_HDF5_named_dtype_hdr) == string::npos)
2354  ignored_msg += ignored_HDF5_named_dtype_hdr + ignored_HDF5_named_dtype_msg;
2355  else
2356  ignored_msg += ignored_HDF5_named_dtype_msg;
2357 
2358 }
2359 
2360 // Add the ignored attribute information. When is_grp is true, the ignored group attribute names are added.
2361 // Otherwise, the ignored dataset attribute names are added.
2362 void File::add_ignored_info_attrs(bool is_grp, const string & obj_path, const string & attr_name)
2363 {
2364 
2365  if (false == this->have_ignored) {
2366  add_ignored_info_obj_header();
2367  have_ignored = true;
2368  }
2369 
2370 
2371  string ignored_warning_str = "\n******WARNING******";
2372  string ignored_HDF5_grp_hdr = ignored_warning_str + "\n Ignored attributes under root and groups:\n";
2373  string ignored_HDF5_grp_msg = " Group path: " + obj_path + " Attribute names: " + attr_name + "\n";
2374  string ignored_HDF5_var_hdr = ignored_warning_str + "\n Ignored attributes for variables:\n";
2375  string ignored_HDF5_var_msg = " Variable path: " + obj_path + " Attribute names: " + attr_name + "\n";
2376 
2377 
2378  if (true == is_grp) {
2379  if (ignored_msg.find(ignored_HDF5_grp_hdr) == string::npos)
2380  ignored_msg += ignored_HDF5_grp_hdr + ignored_HDF5_grp_msg;
2381  else
2382  ignored_msg += ignored_HDF5_grp_msg;
2383  }
2384  else {
2385  if (ignored_msg.find(ignored_HDF5_var_hdr) == string::npos)
2386  ignored_msg += ignored_HDF5_var_hdr + ignored_HDF5_var_msg;
2387  else
2388  ignored_msg += ignored_HDF5_var_msg;
2389  }
2390 
2391 }
2392 
2393 //Ignored object information. When is_dim_related is true, ignored data space info. is present.
2394 //When is_dim_related is false, ignored data type info. is present.
2395 void File::add_ignored_info_objs(bool is_dim_related, const string & obj_path)
2396 {
2397 
2398  if (false == this->have_ignored) {
2399  add_ignored_info_obj_header();
2400  have_ignored = true;
2401  }
2402 
2403  string ignored_warning_str = "\n******WARNING******";
2404  string ignored_HDF5_dtype_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported datatypes:\n";
2405  string ignored_HDF5_dspace_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported dimensions:\n";
2406  string ignored_HDF5_var_msg = " Variable path: " + obj_path + "\n";
2407 
2408  if (true == is_dim_related) {
2409  if (ignored_msg.find(ignored_HDF5_dspace_var_hdr) == string::npos)
2410  ignored_msg += ignored_HDF5_dspace_var_hdr + ignored_HDF5_var_msg;
2411  else
2412  ignored_msg += ignored_HDF5_var_msg;
2413 
2414  }
2415  else {
2416  if (ignored_msg.find(ignored_HDF5_dtype_var_hdr) == string::npos)
2417  ignored_msg += ignored_HDF5_dtype_var_hdr + ignored_HDF5_var_msg;
2418  else
2419  ignored_msg += ignored_HDF5_var_msg;
2420  }
2421 
2422 }
2423 
2424 // No ignored info.
2425 void File::add_no_ignored_info()
2426 {
2427 
2428  ignored_msg += "There are no ignored HDF5 objects or attributes.";
2429 
2430 }
2431 
2432 // This function should only be used when the HDF5 file is following the netCDF data model.
2433 // Check if we should not report the Dimension scale related attributes as ignored.
2434 bool File::ignored_dimscale_ref_list(Var *var)
2435 {
2436 
2437  bool ignored_dimscale = true;
2438  // Only when "General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)"
2439 
2440  bool has_dimscale = false;
2441  bool has_reference_list = false;
2442  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
2443  if ((*ira)->name == "REFERENCE_LIST" && false == HDF5CFUtil::cf_strict_support_type((*ira)->getType()))
2444  has_reference_list = true;
2445  if ((*ira)->name == "CLASS") {
2446  Retrieve_H5_Attr_Value(*ira, var->fullpath);
2447  string class_value;
2448  class_value.resize((*ira)->value.size());
2449  copy((*ira)->value.begin(), (*ira)->value.end(), class_value.begin());
2450 
2451  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2452  // "DIMENSION_SCALE", which is 15.
2453  if (0 == class_value.compare(0, 15, "DIMENSION_SCALE")) {
2454  has_dimscale = true;
2455  }
2456  }
2457 
2458  if (true == has_dimscale && true == has_reference_list) {
2459  ignored_dimscale = false;
2460  break;
2461  }
2462 
2463  }
2464  //}
2465  return ignored_dimscale;
2466 }
2467 
2468 // Check if the long string can should be dropped from a dataset or an attribute. Users can set up a BES key to turn it off or on.
2469 bool File::Check_DropLongStr(Var *var, Attribute * attr)
2470 {
2471 
2472  bool drop_longstr = false;
2473  if (NULL == attr) {
2474  if (H5FSTRING == var->dtype || H5VSTRING == var->dtype) {
2475  try {
2476  drop_longstr = Check_VarDropLongStr(var->fullpath, var->dims, var->dtype);
2477  }
2478  catch (...) {
2479  throw1("Check_VarDropLongStr fails ");
2480  }
2481  }
2482  }
2483  // No limitation for the attributes. KY 2018-02-26
2484 #if 0
2485  else {
2486  if (H5FSTRING == attr->dtype || H5VSTRING == attr->dtype) {
2487  if (attr->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
2488  drop_longstr = true;
2489  }
2490  }
2491 
2492  }
2493 #endif
2494  return drop_longstr;
2495 }
2496 
2497 // Check if a long string dataset should be dropped. Users can turn on a BES key not to drop the long string.
2498 // However, the Java clients may not access.
2499 //
2500 bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype)
2501 
2502 {
2503 
2504  bool drop_longstr = false;
2505 
2506  hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2507  if (dset_id < 0)
2508  throw2("Cannot open the dataset ", varpath);
2509 
2510  hid_t dtype_id = -1;
2511  if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2512  H5Dclose(dset_id);
2513  throw2("Cannot obtain the datatype of the dataset ", varpath);
2514  }
2515 
2516  size_t ty_size = H5Tget_size(dtype_id);
2517  if (ty_size == 0) {
2518  H5Tclose(dtype_id);
2519  H5Dclose(dset_id);
2520  throw2("Cannot obtain the datatype size of the dataset ", varpath);
2521  }
2522 
2523  if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2524  if (ty_size > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2525  }
2526  else if (H5VSTRING == dtype) {
2527 
2528  unsigned long long total_elms = 1;
2529  if (dims.size() != 0) {
2530  for (unsigned int i = 0; i < dims.size(); i++)
2531  total_elms = total_elms * ((dims[i])->size);
2532  }
2533  vector<char> strval;
2534  strval.resize(total_elms * ty_size);
2535  hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) &strval[0]);
2536  if (read_ret < 0) {
2537  H5Tclose(dtype_id);
2538  H5Dclose(dset_id);
2539  throw2("Cannot read the data of the dataset ", varpath);
2540  }
2541 
2542  vector<string> finstrval;
2543  finstrval.resize(total_elms);
2544  char*temp_bp = &strval[0];
2545  char*onestring = NULL;
2546  for (unsigned long long i = 0; i < total_elms; i++) {
2547  onestring = *(char**) temp_bp;
2548  if (onestring != NULL) {
2549  finstrval[i] = string(onestring);
2550  if(finstrval[i].size()>NC_JAVA_STR_SIZE_LIMIT) {
2551  drop_longstr = true;
2552  break;
2553  }
2554  }
2555  temp_bp += ty_size;
2556  }
2557 
2558  if (false == strval.empty()) {
2559  herr_t ret_vlen_claim;
2560  hid_t dspace_id = H5Dget_space(dset_id);
2561  if (dspace_id < 0) {
2562  H5Tclose(dtype_id);
2563  H5Dclose(dset_id);
2564  throw2("Cannot obtain the dataspace id.", varpath);
2565  }
2566  ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) &strval[0]);
2567  if (ret_vlen_claim < 0) {
2568  H5Tclose(dtype_id);
2569  H5Sclose(dspace_id);
2570  H5Dclose(dset_id);
2571  throw2("Cannot reclaim the vlen space ", varpath);
2572  }
2573  if (H5Sclose(dspace_id) < 0) {
2574  H5Tclose(dtype_id);
2575  H5Dclose(dset_id);
2576  throw2("Cannot close the HDF5 data space.", varpath);
2577  }
2578  }
2579  }
2580  if (H5Tclose(dtype_id) < 0) {
2581  H5Dclose(dset_id);
2582  throw2("Cannot close the HDF5 data type.", varpath);
2583  }
2584  if (H5Dclose(dset_id) < 0)
2585  throw2("Cannot close the HDF5 data type.", varpath);
2586 
2587  return drop_longstr;
2588 }
2589 #if 0
2590 bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype)
2591 
2592 {
2593 
2594  bool drop_longstr = false;
2595 
2596  unsigned long long total_elms = 1;
2597  if (dims.size() != 0) {
2598  for (unsigned int i = 0; i < dims.size(); i++)
2599  total_elms = total_elms * ((dims[i])->size);
2600  }
2601 
2602  if (total_elms > NC_JAVA_STR_SIZE_LIMIT)
2603  drop_longstr = true;
2604 
2605  else { // We need to check both fixed-size and variable-length strings.
2606 
2607  hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2608  if (dset_id < 0)
2609  throw2("Cannot open the dataset ", varpath);
2610 
2611  hid_t dtype_id = -1;
2612  if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2613  H5Dclose(dset_id);
2614  throw2("Cannot obtain the datatype of the dataset ", varpath);
2615  }
2616 
2617  size_t ty_size = H5Tget_size(dtype_id);
2618  if (ty_size == 0) {
2619  H5Tclose(dtype_id);
2620  H5Dclose(dset_id);
2621  throw2("Cannot obtain the datatype size of the dataset ", varpath);
2622  }
2623 
2624  if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2625  if ((ty_size * total_elms) > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2626  }
2627  else if (H5VSTRING == dtype) {
2628 
2629  vector<char> strval;
2630  strval.resize(total_elms * ty_size);
2631  hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) &strval[0]);
2632  if (read_ret < 0) {
2633  H5Tclose(dtype_id);
2634  H5Dclose(dset_id);
2635  throw2("Cannot read the data of the dataset ", varpath);
2636  }
2637 
2638  vector<string> finstrval;
2639  finstrval.resize(total_elms);
2640  char*temp_bp = &strval[0];
2641  char*onestring = NULL;
2642  for (unsigned long long i = 0; i < total_elms; i++) {
2643  onestring = *(char**) temp_bp;
2644  if (onestring != NULL)
2645  finstrval[i] = string(onestring);
2646  else
2647  // We will add a NULL if onestring is NULL.
2648  finstrval[i] = "";
2649  temp_bp += ty_size;
2650  }
2651 
2652  if (false == strval.empty()) {
2653  herr_t ret_vlen_claim;
2654  hid_t dspace_id = H5Dget_space(dset_id);
2655  if (dspace_id < 0) {
2656  H5Tclose(dtype_id);
2657  H5Dclose(dset_id);
2658  throw2("Cannot obtain the dataspace id.", varpath);
2659  }
2660  ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) &strval[0]);
2661  if (ret_vlen_claim < 0) {
2662  H5Tclose(dtype_id);
2663  H5Sclose(dspace_id);
2664  H5Dclose(dset_id);
2665  throw2("Cannot reclaim the vlen space ", varpath);
2666  }
2667  if (H5Sclose(dspace_id) < 0) {
2668  H5Tclose(dtype_id);
2669  H5Dclose(dset_id);
2670  throw2("Cannot close the HDF5 data space.", varpath);
2671  }
2672  }
2673  unsigned long long total_str_size = 0;
2674  for (unsigned long long i = 0; i < total_elms; i++) {
2675  total_str_size += finstrval[i].size();
2676  if (total_str_size > NC_JAVA_STR_SIZE_LIMIT) {
2677  drop_longstr = true;
2678  break;
2679  }
2680  }
2681  }
2682  if (H5Tclose(dtype_id) < 0) {
2683  H5Dclose(dset_id);
2684  throw2("Cannot close the HDF5 data type.", varpath);
2685  }
2686  if (H5Dclose(dset_id) < 0)
2687  throw2("Cannot close the HDF5 data type.", varpath);
2688  }
2689  return drop_longstr;
2690 }
2691 #endif
2692 
2693 
2694 // Provide if the long string is dropped.
2695 void File::add_ignored_grp_longstr_info(const string& grp_path, const string & attr_name)
2696 {
2697 
2698  ignored_msg += "The HDF5 group: " + grp_path + " has an empty-set string attribute: " + attr_name + "\n";
2699 
2700  return;
2701 }
2702 
2703 // Provide if the long variable string is dropped.
2704 void File::add_ignored_var_longstr_info(Var *var, Attribute *attr)
2705 {
2706 
2707  if (NULL == attr)
2708  ignored_msg += "String variable: " + var->fullpath + " value is set to empty.\n";
2709  else {
2710  ignored_msg += "The variable: " + var->fullpath + " has an empty-set string attribute: " + attr->name + "\n";
2711 
2712  }
2713  return;
2714 }
2715 
2716 // The warnings of the drop of the long string header
2717 void File::add_ignored_droplongstr_hdr()
2718 {
2719 
2720  if (false == this->have_ignored) this->have_ignored = true;
2721  string hdr = "\n\n The values of the following string variables ";
2722  hdr += " are set to empty because at least one string size in this variable exceeds netCDF Java string limit(32767 bytes).\n";
2723  hdr += "To obtain the values, change the BES key H5.EnableDropLongString=true at the handler BES";
2724  hdr += " configuration file(h5.conf)\nto H5.EnableDropLongString=false.\n\n";
2725 
2726  if (ignored_msg.rfind(hdr) == string::npos) ignored_msg += hdr;
2727 
2728 }
2729 
2730 // Sometimes, we need to release the temporary added resources.
2731 void File::release_standalone_var_vector(vector<Var*>&temp_vars)
2732 {
2733 
2734  for (vector<Var *>::iterator i = temp_vars.begin(); i != temp_vars.end();) {
2735  delete (*i);
2736  i = temp_vars.erase(i);
2737  }
2738 
2739 }
HDF5CF::File::Handle_Grid_Mapping_Vars
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5CF.cc:2123
HDF5CFUtil::H5type_to_H5DAPtype
static H5DataType H5type_to_H5DAPtype(hid_t h5_type_id)
Map HDF5 Datatype to the intermediate H5DAPtype for the future use.
Definition: HDF5CFUtil.cc:54
HDF5CF::File::root_attrs
std::vector< Attribute * > root_attrs
Root attribute vectors.
Definition: HDF5CF.h:790
HDF5CF::File::Have_Grid_Mapping_Attrs
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5CF.cc:2103
HDF5CF::Var
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:259
throw1
#define throw1(a1)
The followings are convenient functions to throw exceptions with different.
Definition: HDF5CF.h:128
HDF5CF::Dimension
This class repersents one dimension of an HDF5 dataset(variable).
Definition: HDF5CF.h:145
HDF5RequestHandler.h
include the entry functions to execute the handlers
HDF5CF::File::Retrieve_H5_Var_Attr_Values
virtual void Retrieve_H5_Var_Attr_Values(Var *var)
Retrieve attribute values for a variable.
Definition: HDF5CF.cc:741
HDF5CF::File::Add_Supplement_Attrs
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name.
Definition: HDF5CF.cc:1948
HDF5CF::File::groups
std::vector< Group * > groups
Non-root group vectors.
Definition: HDF5CF.h:793
HDF5CF::File::Retrieve_H5_Info
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool)
Definition: HDF5CF.cc:168
HDF5CF::File::Handle_Unsupported_Dtype
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes.
Definition: HDF5CF.cc:910
HDF5CF::delete_elem
Definition: HDF5CF.h:134
HDF5CF::File::vars
std::vector< Var * > vars
Var vectors.
Definition: HDF5CF.h:787
h5cfdaputil.h
Helper functions for generating DAS attributes and a function to check BES Key.
HDF5CF::File::Handle_Unsupported_Dspace
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for datasets.
Definition: HDF5CF.cc:1253
HDF5CF.h
This class specifies the core engineering of mapping HDF5 to DAP by following CF.
HDF5CF::File::Flatten_Obj_Name
virtual void Flatten_Obj_Name(bool)
Flatten the object name.
Definition: HDF5CF.cc:1350
HDF5CF::File::dimsize_to_fakedimname
std::map< hsize_t, std::string > dimsize_to_fakedimname
Handle added dimension names.
Definition: HDF5CF.h:810
HDF5CF::Attribute
This class represents one attribute.
Definition: HDF5CF.h:189
HDF5CFUtil::trim_string
static std::string trim_string(hid_t dtypeid, const std::string s, int num_sect, size_t section_size, std::vector< size_t > &sect_newsize)
Definition: HDF5CFUtil.cc:223
HDF5CF::File::Handle_Unsupported_Others
virtual void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes.
Definition: HDF5CF.cc:1300
HDF5CF::File::Retrieve_H5_Supported_Attr_Values
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes.
Definition: HDF5CF.cc:722
HDF5CF::Group
This class represents an HDF5 group. The group will be flattened according to the CF conventions.
Definition: HDF5CF.h:540