bes  Updated for version 3.20.6
h5dmr.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler a HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Copyright (c) 2007-2015 The HDF Group, Inc. and OPeNDAP, Inc.
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
22 // Suite 203, Champaign, IL 61820
23 
36 // the correct DAP4 DMR layout(group's variables first and then the group).
39 
40 #include "config_hdf5.h"
41 
42 #include <InternalErr.h>
43 #include <BESDebug.h>
44 
45 #include <mime_util.h>
46 
47 #include "hdf5_handler.h"
48 #include "HDF5Int32.h"
49 #include "HDF5UInt32.h"
50 #include "HDF5UInt16.h"
51 #include "HDF5Int16.h"
52 #include "HDF5Byte.h"
53 #include "HDF5Array.h"
54 #include "HDF5Str.h"
55 #include "HDF5Float32.h"
56 #include "HDF5Float64.h"
57 #include "HDF5Url.h"
58 #include "HDF5Structure.h"
59 
60 // The HDF5CFUtil.h includes the utility function obtain_string_after_lastslash.
61 #include "HDF5CFUtil.h"
62 #include "h5dmr.h"
63 
64 using namespace std;
65 using namespace libdap;
68 
69 
71 static DS_t dt_inst;
72 
74 void map_h5_attrs_to_dap4(hid_t oid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag);
75 
76 #if 0
77 // \param par_grp DAP4 parent group
92 
93 //bool depth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname)
94 bool depth_first(hid_t pid, char *gname, D4Group* par_grp, const char *fname)
95 {
96  BESDEBUG("h5",
97  ">depth_first() for dmr "
98  << " pid: " << pid
99  << " gname: " << gname
100  << " fname: " << fname
101  << endl);
102 
104  int slinkindex = 0;
105 
106  H5G_info_t g_info;
107  hsize_t nelems = 0;
108 
110  if(H5Gget_info(pid,&g_info) <0) {
111  string msg =
112  "h5_dmr handler: counting hdf5 group elements error for ";
113  msg += gname;
114  throw InternalErr(__FILE__, __LINE__, msg);
115  }
116 
117  nelems = g_info.nlinks;
118 
119  ssize_t oname_size = 0;
120 
121  // Iterate through the file to see the members of the group from the root.
122  for (hsize_t i = 0; i < nelems; i++) {
123 
124  vector <char>oname;
125 
126  // Query the length of object name.
127  oname_size =
128  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
129  (size_t)DODS_NAMELEN, H5P_DEFAULT);
130  if (oname_size <= 0) {
131  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
132  msg += gname;
133  throw InternalErr(__FILE__, __LINE__, msg);
134  }
135 
136  // Obtain the name of the object
137  oname.resize((size_t) oname_size + 1);
138 
139  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
140  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
141  string msg =
142  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
143  msg += gname;
144  throw InternalErr(__FILE__, __LINE__, msg);
145  }
146 
147  // Check if it is the hard link or the soft link
148  H5L_info_t linfo;
149  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
150  string msg = "hdf5 link name error from: ";
151  msg += gname;
152  throw InternalErr(__FILE__, __LINE__, msg);
153  }
154 
155  // Information of soft links are stored as attributes
156  if(linfo.type == H5L_TYPE_SOFT) {
157  slinkindex++;
158  size_t val_size = linfo.u.val_size;
159  get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
160  //get_softlink(par_grp,pid,gname,&oname[0],slinkindex,val_size);
161  continue;
162  }
163 
164  // Ignore external links
165  if(linfo.type == H5L_TYPE_EXTERNAL)
166  continue;
167 
168  // Obtain the object type, such as group or dataset.
169  H5O_info_t oinfo;
170 
171  if (H5Oget_info_by_idx(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
172  i, &oinfo, H5P_DEFAULT)<0) {
173  string msg = "h5_dmr handler: Error obtaining the info for the object";
174  msg += string(oname.begin(),oname.end());
175  throw InternalErr(__FILE__, __LINE__, msg);
176  }
177 
178  H5O_type_t obj_type = oinfo.type;
179 
180  switch (obj_type) {
181 
182  case H5O_TYPE_GROUP:
183  {
184 
185  // Obtain the full path name
186  string full_path_name =
187  string(gname) + string(oname.begin(),oname.end()-1) + "/";
188 
189  BESDEBUG("h5", "=depth_first dmr ():H5G_GROUP " << full_path_name
190  << endl);
191 
192  vector <char>t_fpn;
193  t_fpn.resize(full_path_name.length()+1);
194  copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
195  t_fpn[full_path_name.length()] = '\0';
196 
197  hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
198  if (cgroup < 0){
199  throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
200  }
201 
202  string grp_name = string(oname.begin(),oname.end()-1);
203 
204  // Check the hard link loop and break the loop if it exists.
205  string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
206  if (oid == "") {
207  try {
208  D4Group* tem_d4_cgroup = new D4Group(grp_name);
209  // Map the HDF5 cgroup attributes to DAP4 group attributes.
210  // Note the last flag of map_h5_attrs_to_dap4 must be 0 for the group attribute mapping.
211  map_h5_attrs_to_dap4(cgroup,tem_d4_cgroup,NULL,NULL,0);
212 
213  // Add this new DAP4 group
214  par_grp->add_group_nocopy(tem_d4_cgroup);
215 
216  // Continue searching the objects under this group
217  //depth_first(cgroup, &t_fpn[0], dmr, tem_d4_cgroup,fname);
218  depth_first(cgroup, &t_fpn[0], tem_d4_cgroup,fname);
219  }
220  catch(...) {
221  H5Gclose(cgroup);
222  throw;
223  }
224  }
225  else {
226  // This group has been visited.
227  // Add the attribute table with the attribute name as HDF5_HARDLINK.
228  // The attribute value is the name of the group when it is first visited.
229  D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
230 
231  // Note attr_str_c is the DAP4 attribute string datatype
232  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
233 
234  d4_hlinfo->add_value(obj_paths.get_name(oid));
235  tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
236  par_grp->add_group_nocopy(tem_d4_cgroup);
237 
238  }
239 
240  if (H5Gclose(cgroup) < 0){
241  throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
242  }
243  break;
244  }
245 
246  case H5O_TYPE_DATASET:
247  {
248 
249  // Obtain the absolute path of the HDF5 dataset
250  string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
251 
252  // TOOOODOOOO
253  // Obtain the hdf5 dataset handle stored in the structure dt_inst.
254  // All the metadata information in the handler is stored in dt_inst.
255  // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
256  // Note: depth_first is for building DMR of an HDF5 file that doesn't use dim. scale.
257  // so passing the last parameter as false.
258  get_dataset(pid, full_path_name, &dt_inst,false);
259 
260  // Here we open the HDF5 dataset again to use the dataset id for dataset attributes.
261  // This is not necessary for DAP2 since DAS and DDS are separated.
262  hid_t dset_id = -1;
263  if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
264  string msg = "cannot open the HDF5 dataset ";
265  msg += full_path_name;
266  throw InternalErr(__FILE__, __LINE__, msg);
267  }
268 
269  try {
270  read_objects(par_grp, full_path_name, fname,dset_id);
271  }
272  catch(...) {
273  H5Dclose(dset_id);
274  throw;
275  }
276  if(H5Dclose(dset_id)<0) {
277  string msg = "cannot close the HDF5 dataset ";
278  msg += full_path_name;
279  throw InternalErr(__FILE__, __LINE__, msg);
280  }
281  }
282  break;
283 
284  case H5O_TYPE_NAMED_DATATYPE:
285  // ignore the named datatype
286  break;
287  default:
288  break;
289  }// switch(obj_type)
290  } // for i is 0 ... nelems
291 
292  BESDEBUG("h5", "<depth_first() for dmr" << endl);
293  return true;
294 }
295 #endif
296 // \param par_grp DAP4 parent group
313 
314 
315 // The reason to use breadth_first is that the DMR representation needs to show the dimension names and the variables under the group first and then the group names.
316 // So we use this search. In the future, we may just use the breadth_first search for all cases.??
317 //bool breadth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname,bool use_dimscale)
318 bool breadth_first(hid_t pid, char *gname, D4Group* par_grp, const char *fname,bool use_dimscale)
319 {
320  BESDEBUG("h5",
321  ">breadth_first() for dmr "
322  << " pid: " << pid
323  << " gname: " << gname
324  << " fname: " << fname
325  << endl);
326 
328  int slinkindex = 0;
329 
330  // Obtain the number of objects in this group
331  H5G_info_t g_info;
332  hsize_t nelems = 0;
333  if(H5Gget_info(pid,&g_info) <0) {
334  string msg =
335  "h5_dmr handler: counting hdf5 group elements error for ";
336  msg += gname;
337  throw InternalErr(__FILE__, __LINE__, msg);
338  }
339 
340  nelems = g_info.nlinks;
341 
342  ssize_t oname_size;
343 
344  // First iterate through the HDF5 datasets under the group.
345  for (hsize_t i = 0; i < nelems; i++) {
346 
347  vector <char>oname;
348 
349  // Query the length of object name.
350  oname_size =
351  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
352  (size_t)DODS_NAMELEN, H5P_DEFAULT);
353  if (oname_size <= 0) {
354  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
355  msg += gname;
356  throw InternalErr(__FILE__, __LINE__, msg);
357  }
358 
359  // Obtain the name of the object
360  oname.resize((size_t) oname_size + 1);
361 
362  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
363  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
364  string msg =
365  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
366  msg += gname;
367  throw InternalErr(__FILE__, __LINE__, msg);
368  }
369 
370  // Check if it is the hard link or the soft link
371  H5L_info_t linfo;
372  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
373  string msg = "hdf5 link name error from: ";
374  msg += gname;
375  throw InternalErr(__FILE__, __LINE__, msg);
376  }
377 
378  // Information of soft links are stored as attributes
379  if(linfo.type == H5L_TYPE_SOFT) {
380  slinkindex++;
381 
382  // Size of a soft link value
383  size_t val_size = linfo.u.val_size;
384  get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
385  continue;
386  }
387 
388  // Ignore external links
389  if(linfo.type == H5L_TYPE_EXTERNAL)
390  continue;
391 
392  // Obtain the object type, such as group or dataset.
393  H5O_info_t oinfo;
394 
395  if (H5Oget_info_by_idx(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
396  i, &oinfo, H5P_DEFAULT)<0) {
397  string msg = "h5_dmr handler: Error obtaining the info for the object";
398  msg += string(oname.begin(),oname.end());
399  throw InternalErr(__FILE__, __LINE__, msg);
400  }
401 
402  H5O_type_t obj_type = oinfo.type;
403 
404  if(H5O_TYPE_DATASET == obj_type) {
405 
406  // Obtain the absolute path of the HDF5 dataset
407  string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
408 
409  // TOOOODOOOO
410  // Obtain the hdf5 dataset handle stored in the structure dt_inst.
411  // All the metadata information in the handler is stored in dt_inst.
412  // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
413  get_dataset(pid, full_path_name, &dt_inst,use_dimscale);
414 
415  hid_t dset_id = -1;
416  if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
417  string msg = "cannot open the HDF5 dataset ";
418  msg += full_path_name;
419  throw InternalErr(__FILE__, __LINE__, msg);
420  }
421 
422  try {
423  read_objects(par_grp, full_path_name, fname,dset_id);
424  }
425  catch(...) {
426  H5Dclose(dset_id);
427  throw;
428  }
429  if(H5Dclose(dset_id)<0) {
430  string msg = "cannot close the HDF5 dataset ";
431  msg += full_path_name;
432  throw InternalErr(__FILE__, __LINE__, msg);
433  }
434 
435  }
436  }
437 
438  // The attributes of this group. Doing this order to follow ncdump's way (variable,attribute then groups)
439  map_h5_attrs_to_dap4(pid,par_grp,NULL,NULL,0);
440 
441  // Then HDF5 child groups
442  for (hsize_t i = 0; i < nelems; i++) {
443 
444  vector <char>oname;
445 
446  // Query the length of object name.
447  oname_size =
448  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
449  (size_t)DODS_NAMELEN, H5P_DEFAULT);
450  if (oname_size <= 0) {
451  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
452  msg += gname;
453  throw InternalErr(__FILE__, __LINE__, msg);
454  }
455 
456  // Obtain the name of the object
457  oname.resize((size_t) oname_size + 1);
458 
459  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
460  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
461  string msg =
462  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
463  msg += gname;
464  throw InternalErr(__FILE__, __LINE__, msg);
465  }
466 
467  // Check if it is the hard link or the soft link
468  H5L_info_t linfo;
469  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
470  string msg = "hdf5 link name error from: ";
471  msg += gname;
472  throw InternalErr(__FILE__, __LINE__, msg);
473  }
474 
475  // Information of soft links are handled already, the softlinks need to be ignored, otherwise
476  // the group it links will be mapped again in the block of if obj_type is H5O_TYPE_GROUP
477  if(linfo.type == H5L_TYPE_SOFT) {
478  continue;
479  }
480 
481  // Ignore external links
482  if(linfo.type == H5L_TYPE_EXTERNAL)
483  continue;
484 
485  // Obtain the object type, such as group or dataset.
486  H5O_info_t oinfo;
487 
488  if (H5Oget_info_by_idx(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
489  i, &oinfo, H5P_DEFAULT)<0) {
490  string msg = "h5_dmr handler: Error obtaining the info for the object in the breadth_first.";
491  throw InternalErr(__FILE__, __LINE__, msg);
492  }
493 
494  H5O_type_t obj_type = oinfo.type;
495 
496 
497  if(obj_type == H5O_TYPE_GROUP) {
498 
499  // Obtain the full path name
500  string full_path_name =
501  string(gname) + string(oname.begin(),oname.end()-1) + "/";
502 
503  BESDEBUG("h5", "=breadth_first dmr ():H5G_GROUP " << full_path_name
504  << endl);
505 
506  vector <char>t_fpn;
507  t_fpn.resize(full_path_name.length()+1);
508  copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
509  t_fpn[full_path_name.length()] = '\0';
510 
511  hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
512  if (cgroup < 0){
513  throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
514  }
515 
516  string grp_name = string(oname.begin(),oname.end()-1);
517 
518  // Check the hard link loop and break the loop if it exists.
519  string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
520  if (oid == "") {
521  try {
522  D4Group* tem_d4_cgroup = new D4Group(grp_name);
523 
524  // Add this new DAP4 group
525  par_grp->add_group_nocopy(tem_d4_cgroup);
526 
527  // Continue searching the objects under this group
528  breadth_first(cgroup, &t_fpn[0], tem_d4_cgroup,fname,use_dimscale);
529  }
530  catch(...) {
531  H5Gclose(cgroup);
532  throw;
533  }
534  }
535  else {
536  // This group has been visited.
537  // Add the attribute table with the attribute name as HDF5_HARDLINK.
538  // The attribute value is the name of the group when it is first visited.
539  D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
540 
541  // Note attr_str_c is the DAP4 attribute string datatype
542  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
543 
544  d4_hlinfo->add_value(obj_paths.get_name(oid));
545  tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
546  par_grp->add_group_nocopy(tem_d4_cgroup);
547  }
548 
549  if (H5Gclose(cgroup) < 0){
550  throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
551  }
552  }// end if
553  } // for i is 0 ... nelems
554 
555  BESDEBUG("h5", "<breadth_first() " << endl);
556  return true;
557 }
558 
574 //
575 void
576 read_objects( D4Group * d4_grp, const string &varname, const string &filename, const hid_t dset_id)
577 {
578 
579  switch (H5Tget_class(dt_inst.type)) {
580 
581  // HDF5 compound maps to DAP structure.
582  case H5T_COMPOUND:
583  read_objects_structure(d4_grp, varname, filename,dset_id);
584  break;
585 
586  case H5T_ARRAY:
587  H5Tclose(dt_inst.type);
588  throw InternalErr(__FILE__, __LINE__, "Currently don't support accessing data of Array datatype when array datatype is not inside the compound.");
589 
590  default:
591  read_objects_base_type(d4_grp,varname, filename,dset_id);
592  break;
593  }
594  // We must close the datatype obtained in the get_dataset routine since this is the end of reading DDS.
595  if(H5Tclose(dt_inst.type)<0) {
596  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
597  }
598 }
599 
614 //
615 
616 //void
617 //read_objects_base_type(DMR & dmr, D4Group * d4_grp,const string & varname,
618 void
619 read_objects_base_type(D4Group * d4_grp,const string & varname,
620  const string & filename,hid_t dset_id)
621 {
622 
623  // Obtain the relative path of the variable name under the leaf group
624  string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
625 
626  // Get a base type. It should be an HDF5 atomic datatype
627  // datatype.
628  BaseType *bt = Get_bt(newvarname, varname,filename, dt_inst.type,true);
629  if (!bt) {
630  throw
631  InternalErr(__FILE__, __LINE__,
632  "Unable to convert hdf5 datatype to dods basetype");
633  }
634 
635  // First deal with scalar data.
636  if (dt_inst.ndims == 0) {
637  // transform the DAP2 to DAP4 for this DAP base type and add it to d4_grp
638  bt->transform_to_dap4(d4_grp,d4_grp);
639  // Get it back - this may return null because the underlying type
640  // may have no DAP2 manifestation.
641  BaseType* new_var = d4_grp->var(bt->name());
642  if(new_var){
643  // Map the HDF5 dataset attributes to DAP4
644  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
645  // If this variable is a hardlink, stores the HARDLINK info. as an attribute.
646  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
647  }
648  delete bt;
649  bt = 0;
650  }
651  else {
652  // Next, deal with Array data. This 'else clause' runs to
653  // the end of the method.
654  HDF5Array *ar = new HDF5Array(newvarname, filename, bt);
655  delete bt; bt = 0;
656 
657  // set number of elements and variable name values.
658  // This essentially stores in the struct.
659  ar->set_memneed(dt_inst.need);
660  ar->set_numdim(dt_inst.ndims);
661  ar->set_numelm((int) (dt_inst.nelmts));
662  ar->set_varpath(varname);
663 
664 
665  // If we have dimension names(dimension scale is used.),we will see if we can add the names.
666  int dimnames_size = 0;
667  if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
668  {
669  delete ar;
670  throw
671  InternalErr(__FILE__, __LINE__,
672  "number of dimensions: overflow");
673  }
674  dimnames_size = (int)(dt_inst.dimnames.size());
675 
676  if(dimnames_size ==dt_inst.ndims) {
677  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
678  if(dt_inst.dimnames[dim_index] !="")
679  ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
680  else
681  ar->append_dim(dt_inst.size[dim_index]);
682  }
683  dt_inst.dimnames.clear();
684  }
685  else {
686  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
687  ar->append_dim(dt_inst.size[dim_index]);
688  }
689 
690  // We need to transform dimension info. to DAP4 group
691  BaseType* new_var = ar->h5dims_transform_to_dap4(d4_grp);
692 
693  // Map HDF5 dataset attributes to DAP4
694  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
695 
696  // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
697  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
698 #if 0
699  // Test the attribute
700  D4Attribute *test_attr = new D4Attribute("DAP4_test",attr_str_c);
701  test_attr->add_value("test_grp_attr");
702  new_var->attributes()->add_attribute_nocopy(test_attr);
703 #endif
704  // Add this var to DAP4 group.
705  d4_grp->add_var_nocopy(new_var);
706  delete ar; ar = 0;
707  }
708 
709  BESDEBUG("h5", "<read_objects_base_type(dmr)" << endl);
710 }
711 
725 void
726 read_objects_structure(D4Group *d4_grp, const string & varname,
727  const string & filename,hid_t dset_id)
728 {
729  // Obtain the relative path of the variable name under the leaf group
730  string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
731 
732  // Map HDF5 compound datatype to Structure
733  Structure *structure = Get_structure(newvarname, varname,filename, dt_inst.type,true);
734 
735  try {
736  BESDEBUG("h5", "=read_objects_structure(): Dimension is "
737  << dt_inst.ndims << endl);
738 
739  if (dt_inst.ndims != 0) { // Array of Structure
740  BESDEBUG("h5", "=read_objects_structure(): array of size " <<
741  dt_inst.nelmts << endl);
742  BESDEBUG("h5", "=read_objects_structure(): memory needed = " <<
743  dt_inst.need << endl);
744 
745  // Create the Array of structure.
746  HDF5Array *ar = new HDF5Array(newvarname, filename, structure);
747  delete structure; structure = 0;
748 
749 
750  // These parameters are used in the data read function.
751  ar->set_memneed(dt_inst.need);
752  ar->set_numdim(dt_inst.ndims);
753  ar->set_numelm((int) (dt_inst.nelmts));
754  ar->set_length((int) (dt_inst.nelmts));
755  ar->set_varpath(varname);
756 
757  // If having dimension names, add the dimension names to DAP.
758  int dimnames_size = 0;
759  if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
760  {
761  delete ar;
762  throw
763  InternalErr(__FILE__, __LINE__,
764  "number of dimensions: overflow");
765  }
766  dimnames_size = (int)(dt_inst.dimnames.size());
767 
768 
769  if(dimnames_size ==dt_inst.ndims) {
770  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
771  if(dt_inst.dimnames[dim_index] !="")
772  ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
773  else
774  ar->append_dim(dt_inst.size[dim_index]);
775  }
776  dt_inst.dimnames.clear();
777  }
778  else {
779  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
780  ar->append_dim(dt_inst.size[dim_index]);
781 
782  }
783 
784  // We need to transform dimension info. to DAP4 group
785  BaseType* new_var = ar->h5dims_transform_to_dap4(d4_grp);
786 
787  // Map HDF5 dataset attributes to DAP4
788  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
789 
790  // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
791  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
792 
793  // Add this var to DAP4 group
794  if(new_var)
795  d4_grp->add_var_nocopy(new_var);
796  delete ar; ar = 0;
797  }// end if
798  else {// A scalar structure
799 
800  structure->set_is_dap4(true);
801  map_h5_attrs_to_dap4(dset_id,NULL,NULL,structure,2);
802  map_h5_dset_hardlink_to_d4(dset_id,varname,NULL,structure,2);
803  if(structure)
804  d4_grp->add_var_nocopy(structure);
805  }
806  } // try Structure
807  catch (...) {
808  delete structure;
809  throw;
810  }
811 }
812 
813 
827 //
828 
829 void map_h5_attrs_to_dap4(hid_t h5_objid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag) {
830 
831  // Get the object info
832  H5O_info_t obj_info;
833  if (H5Oget_info(h5_objid, &obj_info) <0) {
834  string msg = "Fail to obtain the HDF5 object info. .";
835  throw InternalErr(__FILE__, __LINE__, msg);
836  }
837 
838  // Obtain the number of attributes
839  int num_attr = obj_info.num_attrs;
840  if (num_attr < 0 ) {
841  string msg = "Fail to get the number of attributes for the HDF5 object. ";
842  throw InternalErr(__FILE__, __LINE__,msg);
843  }
844 
845  string print_rep;
846  vector<char>temp_buf;
847 
848  bool ignore_attr = false;
849  hid_t attr_id = -1;
850  for (int j = 0; j < num_attr; j++) {
851 
852  // Obtain attribute information.
853  DSattr_t attr_inst;
854 
855  // Ignore the attributes of which the HDF5 datatype
856  // cannot be mapped to DAP4. The ignored attribute datatypes can be found
857  // at function get_attr_info in h5get.cc.
858  attr_id = get_attr_info(h5_objid, j, true,&attr_inst, &ignore_attr);
859  if (true == ignore_attr) {
860  H5Aclose(attr_id);
861  continue;
862  }
863 
864  // Get the corresponding DAP data type of the HDF5 datatype.
865  // The following line doesn't work in HDF5 1.10.
866 #if 0
867  //hid_t ty_id = attr_inst.type;
868 #endif
869  hid_t ty_id = H5Aget_type(attr_id);
870  string dap_type = get_dap_type(ty_id,true);
871 
872  // Need to have DAP4 representation of the attribute type
873  D4AttributeType dap4_attr_type = daptype_strrep_to_dap4_attrtype(dap_type);
874 
875  // We encounter an unsupported DAP4 attribute type.
876  if(attr_null_c == dap4_attr_type) {
877  H5Tclose(ty_id);
878  H5Aclose(attr_id);
879  throw InternalErr(__FILE__, __LINE__, "unsupported DAP4 attribute type");
880  }
881 
882  string attr_name = attr_inst.name;
883 
884  // Create the DAP4 attribute mapped from HDF5
885  D4Attribute *d4_attr = new D4Attribute(attr_name,dap4_attr_type);
886 
887  // We have to handle variable length string differently.
888  if (H5Tis_variable_str(ty_id)) {
889  write_vlen_str_attrs(attr_id,ty_id,&attr_inst,d4_attr,NULL,true);
890 #if 0
891  BESDEBUG("h5","attribute name " << attr_name <<endl);
892  BESDEBUG("h5","attribute size " <<attr_inst.need <<endl);
893  //BESDEBUG("h5","attribute type size " <<(int)(H5Tget_size(attr_inst.type))<<endl);
894  BESDEBUG("h5","attribute type size " <<(int)(H5Tget_size(ty_id))<<endl);
895 
896  hid_t temp_space_id = H5Aget_space(attr_id);
897  //BESDEBUG("h5","attribute calculated size "<<(int)(H5Tget_size(attr_inst.type)) *(int)(H5Sget_simple_extent_npoints(temp_space_id)) <<endl);
898  BESDEBUG("h5","attribute calculated size "<<(int)(H5Tget_size(ty_id)) *(int)(H5Sget_simple_extent_npoints(temp_space_id)) <<endl);
899  if(temp_space_id <0) {
900  H5Tclose(ty_id);
901  H5Aclose(attr_id);
902  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
903  }
904 
905  // Variable length string attribute values only store pointers of the actual string value.
906  temp_buf.resize((size_t)attr_inst.need);
907 
908  if (H5Aread(attr_id, ty_id, &temp_buf[0]) < 0) {
909  H5Sclose(temp_space_id);
910  H5Tclose(ty_id);
911  H5Aclose(attr_id);
912  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
913  }
914 
915  char *temp_bp;
916  temp_bp = &temp_buf[0];
917  char* onestring;
918  for (unsigned int temp_i = 0; temp_i <attr_inst.nelmts; temp_i++) {
919 
920  // This line will assure that we get the real variable length string value.
921  onestring =*(char **)temp_bp;
922 
923  // Change the C-style string to C++ STD string just for easy appending the attributes in DAP.
924  if (onestring !=NULL) {
925  string tempstring(onestring);
926  d4_attr->add_value(tempstring);
927  }
928 
929  // going to the next value.
930  //temp_bp +=H5Tget_size(attr_inst.type);
931  temp_bp +=H5Tget_size(ty_id);
932  }
933  if (temp_buf.empty() != true) {
934 
935  // Reclaim any VL memory if necessary.
936  herr_t ret_vlen_claim;
937  //ret_vlen_claim = H5Dvlen_reclaim(attr_inst.type,temp_space_id,H5P_DEFAULT,&temp_buf[0]);
938  ret_vlen_claim = H5Dvlen_reclaim(ty_id,temp_space_id,H5P_DEFAULT,&temp_buf[0]);
939  if(ret_vlen_claim < 0){
940  H5Sclose(temp_space_id);
941  throw InternalErr(__FILE__, __LINE__, "Cannot reclaim the memory buffer of the HDF5 variable length string.");
942  }
943 
944  temp_buf.clear();
945  }
946  H5Sclose(temp_space_id);
947 #endif
948  }// if (H5Tis_variable_str(ty_id)
949  else {
950 
951  vector<char> value;
952  value.resize(attr_inst.need + sizeof(char));
953  BESDEBUG("h5", "arttr_inst.need=" << attr_inst.need << endl);
954 
955  // Read HDF5 attribute data.
956  if (H5Aread(attr_id, ty_id, (void *) (&value[0])) < 0) {
957  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
958  }
959 
960  // For scalar data, just read data once.
961  if (attr_inst.ndims == 0) {
962  for (int loc = 0; loc < (int) attr_inst.nelmts; loc++) {
963  print_rep = print_attr(ty_id, loc, &value[0]);
964  if (print_rep.c_str() != NULL) {
965  d4_attr->add_value(print_rep);
966  }
967  }
968 
969  }
970  else {// The number of dimensions is > 0
971 
972  // Get the attribute datatype size
973  int elesize = (int) H5Tget_size(ty_id);
974  if (elesize == 0) {
975  H5Tclose(ty_id);
976  H5Aclose(attr_id);
977  throw InternalErr(__FILE__, __LINE__, "unable to get attibute size");
978  }
979 
980  // Due to the implementation of print_attr, the attribute value will be
981  // written one by one.
982  char *tempvalue = &value[0];
983 
984  // Write this value. the "loc" can always be set to 0 since
985  // tempvalue will be moved to the next value.
986  for( hsize_t temp_index = 0; temp_index < attr_inst.nelmts; temp_index ++) {
987  print_rep = print_attr(ty_id, 0, tempvalue);
988  if (print_rep.c_str() != NULL) {
989 
990  d4_attr->add_value(print_rep);
991  tempvalue = tempvalue + elesize;
992  BESDEBUG("h5",
993  "tempvalue=" << tempvalue
994  << "elesize=" << elesize
995  << endl);
996 
997  }
998  else {
999  H5Tclose(ty_id);
1000  H5Aclose(attr_id);
1001  throw InternalErr(__FILE__, __LINE__, "unable to convert attibute value to DAP");
1002  }
1003  }//for(hsize_t temp_index=0; .....
1004  } // if attr_inst.ndims != 0
1005  }
1006  if(H5Tclose(ty_id) < 0) {
1007  H5Aclose(attr_id);
1008  throw InternalErr(__FILE__, __LINE__, "unable to close HDF5 type id");
1009  }
1010  if (H5Aclose(attr_id) < 0) {
1011  throw InternalErr(__FILE__, __LINE__, "unable to close attibute id");
1012  }
1013 
1014  if(0 == flag) // D4group
1015  d4g->attributes()->add_attribute_nocopy(d4_attr);
1016  else if (1 == flag) // HDF5 dataset with atomic datatypes
1017  d4b->attributes()->add_attribute_nocopy(d4_attr);
1018  else if ( 2 == flag) // HDF5 dataset with compound datatype
1019  d4s->attributes()->add_attribute_nocopy(d4_attr);
1020  } // for (int j = 0; j < num_attr; j++)
1021 
1022  return;
1023 }
1024 
1038 
1039 
1040 void map_h5_dset_hardlink_to_d4(hid_t h5_dsetid,const string & full_path, BaseType* d4b,Structure * d4s,int flag) {
1041 
1042  // Obtain the unique object number info. If no hardlinks, empty string will return.
1043  string oid = get_hardlink_dmr(h5_dsetid, full_path);
1044 
1045  // Find that this is a hardlink,add the hardlink info to a DAP4 attribute.
1046  if(false == oid.empty()) {
1047 
1048  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
1049  d4_hlinfo->add_value(obj_paths.get_name(oid));
1050 
1051  if (1 == flag)
1052  d4b->attributes()->add_attribute_nocopy(d4_hlinfo);
1053  else if ( 2 == flag)
1054  d4s->attributes()->add_attribute_nocopy(d4_hlinfo);
1055  else
1056  delete d4_hlinfo;
1057  }
1058 
1059 }
1060 
1073 void get_softlink(D4Group* par_grp, hid_t h5obj_id, const string & oname, int index, size_t val_size)
1074 {
1075  BESDEBUG("h5", "dap4 >get_softlink():" << oname << endl);
1076 
1077  ostringstream oss;
1078  oss << string("HDF5_SOFTLINK");
1079  oss << "_";
1080  oss << index;
1081  string temp_varname = oss.str();
1082 
1083 
1084  BESDEBUG("h5", "dap4->get_softlink():" << temp_varname << endl);
1085  D4Attribute *d4_slinfo = new D4Attribute;
1086  d4_slinfo->set_name(temp_varname);
1087 
1088  // Make the type as a container
1089  d4_slinfo->set_type(attr_container_c);
1090 
1091  string softlink_name = "linkname";
1092 
1093  D4Attribute *softlink_src = new D4Attribute(softlink_name,attr_str_c);
1094  softlink_src->add_value(oname);
1095 
1096  d4_slinfo->attributes()->add_attribute_nocopy(softlink_src);
1097  string softlink_value_name ="LINKTARGET";
1098 
1099  // Get the link target information. We always return the link value in a string format.
1100  D4Attribute *softlink_tgt = 0;
1101 
1102  try {
1103  vector<char> buf;
1104  buf.resize(val_size + 1);
1105 
1106  // get link target name
1107  if (H5Lget_val(h5obj_id, oname.c_str(), (void*) &buf[0], val_size + 1, H5P_DEFAULT) < 0) {
1108  throw InternalErr(__FILE__, __LINE__, "unable to get link value");
1109  }
1110  softlink_tgt = new D4Attribute(softlink_value_name, attr_str_c);
1111  string link_target_name = string(buf.begin(), buf.end());
1112  softlink_tgt->add_value(link_target_name);
1113 
1114  d4_slinfo->attributes()->add_attribute_nocopy(softlink_tgt);
1115  }
1116  catch (...) {
1117  delete softlink_tgt;
1118  throw;
1119  }
1120 
1121  par_grp->attributes()->add_attribute_nocopy(d4_slinfo);
1122 }
1123 
1124 
1137 string get_hardlink_dmr( hid_t h5obj_id, const string & oname) {
1138 
1139  BESDEBUG("h5", "dap4->get_hardlink_dmr():" << oname << endl);
1140 
1141  // Get the object info
1142  H5O_info_t obj_info;
1143  if (H5Oget_info(h5obj_id, &obj_info) <0) {
1144  throw InternalErr(__FILE__, __LINE__, "H5Oget_info() failed.");
1145  }
1146 
1147  // If the reference count is greater than 1,that means
1148  // hard links are found. return the original object name this
1149  // hard link points to.
1150 
1151  if (obj_info.rc >1) {
1152 
1153  ostringstream oss;
1154  oss << hex << obj_info.addr;
1155  string objno = oss.str();
1156 
1157  BESDEBUG("h5", "dap4->get_hardlink_dmr() objno=" << objno << endl);
1158 
1159  // Add this hard link to the map.
1160  // obj_paths is a global variable defined at the beginning of this file.
1161  // it is essentially a id to obj name map. See HDF5PathFinder.h.
1162  if (!obj_paths.add(objno, oname)) {
1163  return objno;
1164  }
1165  else {
1166  return "";
1167  }
1168  }
1169  else {
1170  return "";
1171  }
1172 
1173 }
DSattr
A structure for DAS generation.
Definition: hdf5_handler.h:92
read_objects_structure
void read_objects_structure(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:310
breadth_first
bool breadth_first(hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale)
Definition: h5dmr.cc:318
DODS_NAMELEN
const int DODS_NAMELEN
Maximum length of variable or attribute name(default option only).
Definition: hdf5_handler.h:64
HDF5Float32.h
A class for mapping HDF5 32-bit float to DAP for the default option.
DSattr::ndims
int ndims
Number of dimensions.
Definition: hdf5_handler.h:98
HDF5Byte.h
This class provides a way to map HDF5 byte to DAP Byte for the default option.
DS
A structure for DDS generation.
Definition: hdf5_handler.h:70
HDF5CFUtil.h
This file includes several helper functions for translating HDF5 to CF-compliant.
HDF5UInt16.h
This class provides a way to map unsigned HDF5 16 bit integer to DAP UInt16 for the default option.
read_objects_base_type
void read_objects_base_type(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:256
HDF5Float64.h
A class for mapping HDF5 64-bit float to DAP for the default option.
HDF5UInt32.h
This class provides a way to map unsigned HDF5 32 bit integer to DAP UInt32.
HDF5Array::set_numdim
void set_numdim(int ndims)
remembers number of dimensions of this array.
Definition: HDF5Array.cc:1450
DSattr::name
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:94
HDF5Array.h
A class for handling all types of array in HDF5 for the default option.
HDF5Int16.h
A class for HDF5 signed 16 bit integer type.
DS::need
hsize_t need
Space needed.
Definition: hdf5_handler.h:89
DSattr::nelmts
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:102
read_objects
void read_objects(DAS &das, const string &varname, hid_t oid, int num_attr)
Definition: h5das.cc:295
libdap
Definition: BESDapFunctionResponseCache.h:35
HDF5Array::set_memneed
void set_memneed(size_t need)
remembers memory size needed.
Definition: HDF5Array.cc:1446
HDF5Int32.h
This class provides a way to map HDF5 32 bit integer to DAP Int32 for the default option.
DS::ndims
int ndims
HDF5 data space id.
Definition: hdf5_handler.h:82
HDF5Array::set_numelm
void set_numelm(int nelms)
remembers number of elements in this array.
Definition: HDF5Array.cc:1454
HDF5PathFinder
Definition: HDF5PathFinder.h:42
DS::size
int size[DODS_MAX_RANK]
Size of each dimension.
Definition: hdf5_handler.h:84
DSattr::need
hsize_t need
Memory space needed to hold nelmts type.
Definition: hdf5_handler.h:104
obj_paths
HDF5PathFinder obj_paths
A variable for remembering visited paths to break cyclic HDF5 groups.
Definition: h5dmr.cc:67
DS::type
hid_t type
HDF5 data set id.
Definition: hdf5_handler.h:78
HDF5Structure.h
This class converts HDF5 compound type into DAP structure for the default option.
HDF5PathFinder::add
bool add(std::string id, const std::string name)
Definition: HDF5PathFinder.cc:50
HDF5Str.h
This class that translates HDF5 string into DAP string for the default option.
h5dmr.h
Data structure and retrieval processing header for the default option.
get_dap_type
string get_dap_type(hid_t type, bool is_dap4)
Definition: h5get.cc:272
HDF5PathFinder::get_name
std::string get_name(std::string id)
Definition: HDF5PathFinder.cc:74
DS::nelmts
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:87
HDF5Array
Definition: HDF5Array.h:48
depth_first
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
get_softlink
void get_softlink(D4Group *par_grp, hid_t h5obj_id, const string &oname, int index, size_t val_size)
Definition: h5dmr.cc:1073
print_attr
string print_attr(hid_t type, int loc, void *sm_buf)
Definition: h5get.cc:642
HDF5Url.h
This class generates DAP URL type for the default option.
hdf5_handler.h
The main header of the HDF5 OPeNDAP handler.
get_attr_info
hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t *attr_inst_ptr, bool *ignore_attr_ptr)
Definition: h5get.cc:83
map_h5_attrs_to_dap4
void map_h5_attrs_to_dap4(hid_t oid, D4Group *d4g, BaseType *d4b, Structure *d4s, int flag)
A function that map HDF5 attributes to DAP4.
Definition: h5dmr.cc:829