40 #include "config_hdf5.h"
42 #include <InternalErr.h>
45 #include <mime_util.h>
94 bool depth_first(hid_t pid,
char *gname, D4Group* par_grp,
const char *fname)
97 ">depth_first() for dmr "
99 <<
" gname: " << gname
100 <<
" fname: " << fname
110 if(H5Gget_info(pid,&g_info) <0) {
112 "h5_dmr handler: counting hdf5 group elements error for ";
114 throw InternalErr(__FILE__, __LINE__, msg);
117 nelems = g_info.nlinks;
119 ssize_t oname_size = 0;
122 for (hsize_t i = 0; i < nelems; i++) {
128 H5Lget_name_by_idx(pid,
".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
130 if (oname_size <= 0) {
131 string msg =
"h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
133 throw InternalErr(__FILE__, __LINE__, msg);
137 oname.resize((
size_t) oname_size + 1);
139 if (H5Lget_name_by_idx(pid,
".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
140 (
size_t)(oname_size+1), H5P_DEFAULT) < 0){
142 "h5_dmr handler: Error getting the hdf5 object name from the group: ";
144 throw InternalErr(__FILE__, __LINE__, msg);
149 if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
150 string msg =
"hdf5 link name error from: ";
152 throw InternalErr(__FILE__, __LINE__, msg);
156 if(linfo.type == H5L_TYPE_SOFT) {
158 size_t val_size = linfo.u.val_size;
159 get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
165 if(linfo.type == H5L_TYPE_EXTERNAL)
171 if (H5Oget_info_by_idx(pid,
".", H5_INDEX_NAME, H5_ITER_NATIVE,
172 i, &oinfo, H5P_DEFAULT)<0) {
173 string msg =
"h5_dmr handler: Error obtaining the info for the object";
174 msg += string(oname.begin(),oname.end());
175 throw InternalErr(__FILE__, __LINE__, msg);
178 H5O_type_t obj_type = oinfo.type;
186 string full_path_name =
187 string(gname) + string(oname.begin(),oname.end()-1) +
"/";
189 BESDEBUG(
"h5",
"=depth_first dmr ():H5G_GROUP " << full_path_name
193 t_fpn.resize(full_path_name.length()+1);
194 copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
195 t_fpn[full_path_name.length()] =
'\0';
197 hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
199 throw InternalErr(__FILE__, __LINE__,
"h5_dmr handler: H5Gopen() failed.");
202 string grp_name = string(oname.begin(),oname.end()-1);
205 string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
208 D4Group* tem_d4_cgroup =
new D4Group(grp_name);
214 par_grp->add_group_nocopy(tem_d4_cgroup);
218 depth_first(cgroup, &t_fpn[0], tem_d4_cgroup,fname);
229 D4Group* tem_d4_cgroup =
new D4Group(
string(grp_name));
232 D4Attribute *d4_hlinfo =
new D4Attribute(
"HDF5_HARDLINK",attr_str_c);
235 tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
236 par_grp->add_group_nocopy(tem_d4_cgroup);
240 if (H5Gclose(cgroup) < 0){
241 throw InternalErr(__FILE__, __LINE__,
"Could not close the group.");
246 case H5O_TYPE_DATASET:
250 string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
258 get_dataset(pid, full_path_name, &dt_inst,
false);
263 if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
264 string msg =
"cannot open the HDF5 dataset ";
265 msg += full_path_name;
266 throw InternalErr(__FILE__, __LINE__, msg);
276 if(H5Dclose(dset_id)<0) {
277 string msg =
"cannot close the HDF5 dataset ";
278 msg += full_path_name;
279 throw InternalErr(__FILE__, __LINE__, msg);
284 case H5O_TYPE_NAMED_DATATYPE:
292 BESDEBUG(
"h5",
"<depth_first() for dmr" << endl);
318 bool breadth_first(hid_t pid,
char *gname, D4Group* par_grp,
const char *fname,
bool use_dimscale)
321 ">breadth_first() for dmr "
323 <<
" gname: " << gname
324 <<
" fname: " << fname
333 if(H5Gget_info(pid,&g_info) <0) {
335 "h5_dmr handler: counting hdf5 group elements error for ";
337 throw InternalErr(__FILE__, __LINE__, msg);
340 nelems = g_info.nlinks;
345 for (hsize_t i = 0; i < nelems; i++) {
351 H5Lget_name_by_idx(pid,
".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
353 if (oname_size <= 0) {
354 string msg =
"h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
356 throw InternalErr(__FILE__, __LINE__, msg);
360 oname.resize((
size_t) oname_size + 1);
362 if (H5Lget_name_by_idx(pid,
".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
363 (
size_t)(oname_size+1), H5P_DEFAULT) < 0){
365 "h5_dmr handler: Error getting the hdf5 object name from the group: ";
367 throw InternalErr(__FILE__, __LINE__, msg);
372 if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
373 string msg =
"hdf5 link name error from: ";
375 throw InternalErr(__FILE__, __LINE__, msg);
379 if(linfo.type == H5L_TYPE_SOFT) {
383 size_t val_size = linfo.u.val_size;
384 get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
389 if(linfo.type == H5L_TYPE_EXTERNAL)
395 if (H5Oget_info_by_idx(pid,
".", H5_INDEX_NAME, H5_ITER_NATIVE,
396 i, &oinfo, H5P_DEFAULT)<0) {
397 string msg =
"h5_dmr handler: Error obtaining the info for the object";
398 msg += string(oname.begin(),oname.end());
399 throw InternalErr(__FILE__, __LINE__, msg);
402 H5O_type_t obj_type = oinfo.type;
404 if(H5O_TYPE_DATASET == obj_type) {
407 string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
413 get_dataset(pid, full_path_name, &dt_inst,use_dimscale);
416 if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
417 string msg =
"cannot open the HDF5 dataset ";
418 msg += full_path_name;
419 throw InternalErr(__FILE__, __LINE__, msg);
429 if(H5Dclose(dset_id)<0) {
430 string msg =
"cannot close the HDF5 dataset ";
431 msg += full_path_name;
432 throw InternalErr(__FILE__, __LINE__, msg);
442 for (hsize_t i = 0; i < nelems; i++) {
448 H5Lget_name_by_idx(pid,
".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
450 if (oname_size <= 0) {
451 string msg =
"h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
453 throw InternalErr(__FILE__, __LINE__, msg);
457 oname.resize((
size_t) oname_size + 1);
459 if (H5Lget_name_by_idx(pid,
".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
460 (
size_t)(oname_size+1), H5P_DEFAULT) < 0){
462 "h5_dmr handler: Error getting the hdf5 object name from the group: ";
464 throw InternalErr(__FILE__, __LINE__, msg);
469 if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
470 string msg =
"hdf5 link name error from: ";
472 throw InternalErr(__FILE__, __LINE__, msg);
477 if(linfo.type == H5L_TYPE_SOFT) {
482 if(linfo.type == H5L_TYPE_EXTERNAL)
488 if (H5Oget_info_by_idx(pid,
".", H5_INDEX_NAME, H5_ITER_NATIVE,
489 i, &oinfo, H5P_DEFAULT)<0) {
490 string msg =
"h5_dmr handler: Error obtaining the info for the object in the breadth_first.";
491 throw InternalErr(__FILE__, __LINE__, msg);
494 H5O_type_t obj_type = oinfo.type;
497 if(obj_type == H5O_TYPE_GROUP) {
500 string full_path_name =
501 string(gname) + string(oname.begin(),oname.end()-1) +
"/";
503 BESDEBUG(
"h5",
"=breadth_first dmr ():H5G_GROUP " << full_path_name
507 t_fpn.resize(full_path_name.length()+1);
508 copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
509 t_fpn[full_path_name.length()] =
'\0';
511 hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
513 throw InternalErr(__FILE__, __LINE__,
"h5_dmr handler: H5Gopen() failed.");
516 string grp_name = string(oname.begin(),oname.end()-1);
519 string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
522 D4Group* tem_d4_cgroup =
new D4Group(grp_name);
525 par_grp->add_group_nocopy(tem_d4_cgroup);
528 breadth_first(cgroup, &t_fpn[0], tem_d4_cgroup,fname,use_dimscale);
539 D4Group* tem_d4_cgroup =
new D4Group(
string(grp_name));
542 D4Attribute *d4_hlinfo =
new D4Attribute(
"HDF5_HARDLINK",attr_str_c);
545 tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
546 par_grp->add_group_nocopy(tem_d4_cgroup);
549 if (H5Gclose(cgroup) < 0){
550 throw InternalErr(__FILE__, __LINE__,
"Could not close the group.");
555 BESDEBUG(
"h5",
"<breadth_first() " << endl);
576 read_objects( D4Group * d4_grp,
const string &varname,
const string &filename,
const hid_t dset_id)
579 switch (H5Tget_class(dt_inst.
type)) {
587 H5Tclose(dt_inst.
type);
588 throw InternalErr(__FILE__, __LINE__,
"Currently don't support accessing data of Array datatype when array datatype is not inside the compound.");
595 if(H5Tclose(dt_inst.
type)<0) {
596 throw InternalErr(__FILE__, __LINE__,
"Cannot close the HDF5 datatype.");
620 const string & filename,hid_t dset_id)
624 string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
628 BaseType *bt = Get_bt(newvarname, varname,filename, dt_inst.
type,
true);
631 InternalErr(__FILE__, __LINE__,
632 "Unable to convert hdf5 datatype to dods basetype");
636 if (dt_inst.
ndims == 0) {
638 bt->transform_to_dap4(d4_grp,d4_grp);
641 BaseType* new_var = d4_grp->var(bt->name());
646 map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
662 ar->set_varpath(varname);
666 int dimnames_size = 0;
667 if((
unsigned int)((
int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
671 InternalErr(__FILE__, __LINE__,
672 "number of dimensions: overflow");
674 dimnames_size = (int)(dt_inst.dimnames.size());
676 if(dimnames_size ==dt_inst.
ndims) {
677 for (
int dim_index = 0; dim_index < dt_inst.
ndims; dim_index++) {
678 if(dt_inst.dimnames[dim_index] !=
"")
679 ar->append_dim(dt_inst.
size[dim_index],dt_inst.dimnames[dim_index]);
681 ar->append_dim(dt_inst.
size[dim_index]);
683 dt_inst.dimnames.clear();
686 for (
int dim_index = 0; dim_index < dt_inst.
ndims; dim_index++)
687 ar->append_dim(dt_inst.
size[dim_index]);
691 BaseType* new_var = ar->h5dims_transform_to_dap4(d4_grp);
697 map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
700 D4Attribute *test_attr =
new D4Attribute(
"DAP4_test",attr_str_c);
701 test_attr->add_value(
"test_grp_attr");
702 new_var->attributes()->add_attribute_nocopy(test_attr);
705 d4_grp->add_var_nocopy(new_var);
709 BESDEBUG(
"h5",
"<read_objects_base_type(dmr)" << endl);
727 const string & filename,hid_t dset_id)
730 string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
733 Structure *structure = Get_structure(newvarname, varname,filename, dt_inst.
type,
true);
736 BESDEBUG(
"h5",
"=read_objects_structure(): Dimension is "
737 << dt_inst.
ndims << endl);
739 if (dt_inst.
ndims != 0) {
740 BESDEBUG(
"h5",
"=read_objects_structure(): array of size " <<
742 BESDEBUG(
"h5",
"=read_objects_structure(): memory needed = " <<
743 dt_inst.
need << endl);
747 delete structure; structure = 0;
754 ar->set_length((
int) (dt_inst.
nelmts));
755 ar->set_varpath(varname);
758 int dimnames_size = 0;
759 if((
unsigned int)((
int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
763 InternalErr(__FILE__, __LINE__,
764 "number of dimensions: overflow");
766 dimnames_size = (int)(dt_inst.dimnames.size());
769 if(dimnames_size ==dt_inst.
ndims) {
770 for (
int dim_index = 0; dim_index < dt_inst.
ndims; dim_index++) {
771 if(dt_inst.dimnames[dim_index] !=
"")
772 ar->append_dim(dt_inst.
size[dim_index],dt_inst.dimnames[dim_index]);
774 ar->append_dim(dt_inst.
size[dim_index]);
776 dt_inst.dimnames.clear();
779 for (
int dim_index = 0; dim_index < dt_inst.
ndims; dim_index++)
780 ar->append_dim(dt_inst.
size[dim_index]);
785 BaseType* new_var = ar->h5dims_transform_to_dap4(d4_grp);
791 map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
795 d4_grp->add_var_nocopy(new_var);
800 structure->set_is_dap4(
true);
802 map_h5_dset_hardlink_to_d4(dset_id,varname,NULL,structure,2);
804 d4_grp->add_var_nocopy(structure);
833 if (H5Oget_info(h5_objid, &obj_info) <0) {
834 string msg =
"Fail to obtain the HDF5 object info. .";
835 throw InternalErr(__FILE__, __LINE__, msg);
839 int num_attr = obj_info.num_attrs;
841 string msg =
"Fail to get the number of attributes for the HDF5 object. ";
842 throw InternalErr(__FILE__, __LINE__,msg);
846 vector<char>temp_buf;
848 bool ignore_attr =
false;
850 for (
int j = 0; j < num_attr; j++) {
858 attr_id =
get_attr_info(h5_objid, j,
true,&attr_inst, &ignore_attr);
859 if (
true == ignore_attr) {
869 hid_t ty_id = H5Aget_type(attr_id);
873 D4AttributeType dap4_attr_type = daptype_strrep_to_dap4_attrtype(dap_type);
876 if(attr_null_c == dap4_attr_type) {
879 throw InternalErr(__FILE__, __LINE__,
"unsupported DAP4 attribute type");
882 string attr_name = attr_inst.
name;
885 D4Attribute *d4_attr =
new D4Attribute(attr_name,dap4_attr_type);
888 if (H5Tis_variable_str(ty_id)) {
889 write_vlen_str_attrs(attr_id,ty_id,&attr_inst,d4_attr,NULL,
true);
891 BESDEBUG(
"h5",
"attribute name " << attr_name <<endl);
892 BESDEBUG(
"h5",
"attribute size " <<attr_inst.
need <<endl);
894 BESDEBUG(
"h5",
"attribute type size " <<(
int)(H5Tget_size(ty_id))<<endl);
896 hid_t temp_space_id = H5Aget_space(attr_id);
898 BESDEBUG(
"h5",
"attribute calculated size "<<(
int)(H5Tget_size(ty_id)) *(
int)(H5Sget_simple_extent_npoints(temp_space_id)) <<endl);
899 if(temp_space_id <0) {
902 throw InternalErr(__FILE__, __LINE__,
"unable to read HDF5 attribute data");
906 temp_buf.resize((
size_t)attr_inst.
need);
908 if (H5Aread(attr_id, ty_id, &temp_buf[0]) < 0) {
909 H5Sclose(temp_space_id);
912 throw InternalErr(__FILE__, __LINE__,
"unable to read HDF5 attribute data");
916 temp_bp = &temp_buf[0];
918 for (
unsigned int temp_i = 0; temp_i <attr_inst.
nelmts; temp_i++) {
921 onestring =*(
char **)temp_bp;
924 if (onestring !=NULL) {
925 string tempstring(onestring);
926 d4_attr->add_value(tempstring);
931 temp_bp +=H5Tget_size(ty_id);
933 if (temp_buf.empty() !=
true) {
936 herr_t ret_vlen_claim;
938 ret_vlen_claim = H5Dvlen_reclaim(ty_id,temp_space_id,H5P_DEFAULT,&temp_buf[0]);
939 if(ret_vlen_claim < 0){
940 H5Sclose(temp_space_id);
941 throw InternalErr(__FILE__, __LINE__,
"Cannot reclaim the memory buffer of the HDF5 variable length string.");
946 H5Sclose(temp_space_id);
952 value.resize(attr_inst.
need +
sizeof(
char));
953 BESDEBUG(
"h5",
"arttr_inst.need=" << attr_inst.
need << endl);
956 if (H5Aread(attr_id, ty_id, (
void *) (&value[0])) < 0) {
957 throw InternalErr(__FILE__, __LINE__,
"unable to read HDF5 attribute data");
961 if (attr_inst.
ndims == 0) {
962 for (
int loc = 0; loc < (int) attr_inst.
nelmts; loc++) {
963 print_rep =
print_attr(ty_id, loc, &value[0]);
964 if (print_rep.c_str() != NULL) {
965 d4_attr->add_value(print_rep);
973 int elesize = (int) H5Tget_size(ty_id);
977 throw InternalErr(__FILE__, __LINE__,
"unable to get attibute size");
982 char *tempvalue = &value[0];
986 for( hsize_t temp_index = 0; temp_index < attr_inst.
nelmts; temp_index ++) {
988 if (print_rep.c_str() != NULL) {
990 d4_attr->add_value(print_rep);
991 tempvalue = tempvalue + elesize;
993 "tempvalue=" << tempvalue
994 <<
"elesize=" << elesize
1001 throw InternalErr(__FILE__, __LINE__,
"unable to convert attibute value to DAP");
1006 if(H5Tclose(ty_id) < 0) {
1008 throw InternalErr(__FILE__, __LINE__,
"unable to close HDF5 type id");
1010 if (H5Aclose(attr_id) < 0) {
1011 throw InternalErr(__FILE__, __LINE__,
"unable to close attibute id");
1015 d4g->attributes()->add_attribute_nocopy(d4_attr);
1017 d4b->attributes()->add_attribute_nocopy(d4_attr);
1018 else if ( 2 == flag)
1019 d4s->attributes()->add_attribute_nocopy(d4_attr);
1040 void map_h5_dset_hardlink_to_d4(hid_t h5_dsetid,
const string & full_path, BaseType* d4b,Structure * d4s,
int flag) {
1043 string oid = get_hardlink_dmr(h5_dsetid, full_path);
1046 if(
false == oid.empty()) {
1048 D4Attribute *d4_hlinfo =
new D4Attribute(
"HDF5_HARDLINK",attr_str_c);
1052 d4b->attributes()->add_attribute_nocopy(d4_hlinfo);
1053 else if ( 2 == flag)
1054 d4s->attributes()->add_attribute_nocopy(d4_hlinfo);
1073 void get_softlink(D4Group* par_grp, hid_t h5obj_id,
const string & oname,
int index,
size_t val_size)
1075 BESDEBUG(
"h5",
"dap4 >get_softlink():" << oname << endl);
1078 oss << string(
"HDF5_SOFTLINK");
1081 string temp_varname = oss.str();
1084 BESDEBUG(
"h5",
"dap4->get_softlink():" << temp_varname << endl);
1085 D4Attribute *d4_slinfo =
new D4Attribute;
1086 d4_slinfo->set_name(temp_varname);
1089 d4_slinfo->set_type(attr_container_c);
1091 string softlink_name =
"linkname";
1093 D4Attribute *softlink_src =
new D4Attribute(softlink_name,attr_str_c);
1094 softlink_src->add_value(oname);
1096 d4_slinfo->attributes()->add_attribute_nocopy(softlink_src);
1097 string softlink_value_name =
"LINKTARGET";
1100 D4Attribute *softlink_tgt = 0;
1104 buf.resize(val_size + 1);
1107 if (H5Lget_val(h5obj_id, oname.c_str(), (
void*) &buf[0], val_size + 1, H5P_DEFAULT) < 0) {
1108 throw InternalErr(__FILE__, __LINE__,
"unable to get link value");
1110 softlink_tgt =
new D4Attribute(softlink_value_name, attr_str_c);
1111 string link_target_name = string(buf.begin(), buf.end());
1112 softlink_tgt->add_value(link_target_name);
1114 d4_slinfo->attributes()->add_attribute_nocopy(softlink_tgt);
1117 delete softlink_tgt;
1121 par_grp->attributes()->add_attribute_nocopy(d4_slinfo);
1137 string get_hardlink_dmr( hid_t h5obj_id,
const string & oname) {
1139 BESDEBUG(
"h5",
"dap4->get_hardlink_dmr():" << oname << endl);
1142 H5O_info_t obj_info;
1143 if (H5Oget_info(h5obj_id, &obj_info) <0) {
1144 throw InternalErr(__FILE__, __LINE__,
"H5Oget_info() failed.");
1151 if (obj_info.rc >1) {
1154 oss << hex << obj_info.addr;
1155 string objno = oss.str();
1157 BESDEBUG(
"h5",
"dap4->get_hardlink_dmr() objno=" << objno << endl);