35 #define H5S_MAX_RANK 32
36 #define H5O_LAYOUT_NDIMS (H5S_MAX_RANK+1)
37 #include <H5Ppublic.h>
38 #include <H5Dpublic.h>
39 #include <H5Epublic.h>
40 #include <H5Zpublic.h>
41 #include <H5Spublic.h>
54 typedef struct H5D_chunk_rec_t {
55 hsize_t scaled[H5O_LAYOUT_NDIMS];
62 #include <D4Attributes.h>
64 #include <D4ParserSax2.h>
67 #include <TheBESKeys.h>
72 #include <BESNotFoundError.h>
73 #include <BESInternalError.h>
74 #include <BESDataHandlerInterface.h>
76 #include "DmrppTypeFactory.h"
77 #include "DmrppD4Group.h"
78 #include "DmrppMetadataStore.h"
79 #include "BESDapNames.h"
83 using namespace dmrpp;
85 static bool verbose =
false;
86 #define VERBOSE(x) do { if (verbose) x; } while(false)
88 #define DEBUG_KEY "metadata_store,dmrpp_store,dmrpp"
89 #define ROOT_DIRECTORY "BES.Catalog.catalog.RootDirectory"
101 static void print_dataset_type_info(hid_t dataset, uint8_t layout_type)
103 hid_t dtype_id = H5Dget_type(dataset);
105 throw BESInternalError(
"Cannot obtain the correct HDF5 datatype.", __FILE__, __LINE__);
108 if (H5Tget_class(dtype_id) == H5T_INTEGER || H5Tget_class(dtype_id) == H5T_FLOAT) {
109 hid_t dcpl_id = H5Dget_create_plist(dataset);
111 throw BESInternalError(
"Cannot obtain the HDF5 dataset creation property list.", __FILE__, __LINE__);
117 H5D_fill_value_t fvalue_status;
118 if (H5Pfill_value_defined(dcpl_id, &fvalue_status) < 0) {
120 throw BESInternalError(
"Cannot obtain the fill value status.", __FILE__, __LINE__);
122 if (fvalue_status == H5D_FILL_VALUE_UNDEFINED) {
124 if (layout_type == 1)
125 cerr <<
" The storage size is 0 and the storage type is contiguous." << endl;
126 else if (layout_type == 2)
127 cerr <<
" The storage size is 0 and the storage type is chunking." << endl;
128 else if (layout_type == 3) cerr <<
" The storage size is 0 and the storage type is compact." << endl;
130 cerr <<
" The Fillvalue is undefined ." << endl;
133 if (layout_type == 1)
134 cerr <<
" The storage size is 0 and the storage type is contiguous." << endl;
135 else if (layout_type == 2)
136 cerr <<
" The storage size is 0 and the storage type is chunking." << endl;
137 else if (layout_type == 3) cerr <<
" The storage size is 0 and the storage type is compact." << endl;
140 size_t fv_size = H5Tget_size(dtype_id);
142 fvalue = (
char*) (malloc(1));
143 else if (fv_size == 2)
144 fvalue = (
char*) (malloc(2));
145 else if (fv_size == 4)
146 fvalue = (
char*) (malloc(4));
147 else if (fv_size == 8) fvalue = (
char*) (malloc(8));
150 if (H5Pget_fill_value(dcpl_id, dtype_id, (
void*) (fvalue)) < 0) {
152 throw BESInternalError(
"Cannot obtain the fill value status.", __FILE__, __LINE__);
154 if (H5Tget_class(dtype_id) == H5T_INTEGER) {
155 H5T_sign_t fv_sign = H5Tget_sign(dtype_id);
157 if (fv_sign == H5T_SGN_NONE) {
158 cerr <<
"This dataset's datatype is unsigned char " << endl;
159 cerr <<
"and the fillvalue is " << *fvalue << endl;
162 cerr <<
"This dataset's datatype is char and the fillvalue is " << *fvalue << endl;
165 else if (fv_size == 2) {
166 if (fv_sign == H5T_SGN_NONE) {
167 cerr <<
"This dataset's datatype is unsigned short and the fillvalue is " << *fvalue << endl;
170 cerr <<
"This dataset's datatype is short and the fillvalue is " << *fvalue << endl;
173 else if (fv_size == 4) {
174 if (fv_sign == H5T_SGN_NONE) {
175 cerr <<
"This dataset's datatype is unsigned int and the fillvalue is " << *fvalue << endl;
178 cerr <<
"This dataset's datatype is int and the fillvalue is " << *fvalue << endl;
181 else if (fv_size == 8) {
182 if (fv_sign == H5T_SGN_NONE) {
183 cerr <<
"This dataset's datatype is unsigned long long and the fillvalue is " << *fvalue << endl;
186 cerr <<
"This dataset's datatype is long long and the fillvalue is " << *fvalue << endl;
190 if (H5Tget_class(dtype_id) == H5T_FLOAT) {
192 cerr <<
"This dataset's datatype is float and the fillvalue is " << *fvalue << endl;
194 else if (fv_size == 8) {
195 cerr <<
"This dataset's datatype is double and the fillvalue is " << *fvalue << endl;
199 if (fvalue != NULL) free(fvalue);
203 <<
"The size of the datatype is greater than 8 bytes, Use HDF5 API H5Pget_fill_value() to retrieve the fill value of this dataset."
214 if (layout_type == 1)
215 cerr <<
" The storage size is 0 and the storage type is contiguous." << endl;
216 else if (layout_type == 2)
217 cerr <<
" The storage size is 0 and the storage type is chunking." << endl;
218 else if (layout_type == 3) cerr <<
" The storage size is 0 and the storage type is compact." << endl;
220 cerr <<
"The datatype is neither float nor integer,use HDF5 API H5Pget_fill_value() to retrieve the fill value of this dataset." << endl;
241 static void set_filter_information(hid_t dataset_id,
DmrppCommon *dc)
243 hid_t plist_id = H5Dget_create_plist(dataset_id);
246 int numfilt = H5Pget_nfilters(plist_id);
247 VERBOSE(cerr <<
"Number of filters associated with dataset: " << numfilt << endl);
249 for (
int filter = 0; filter < numfilt; filter++) {
251 unsigned int flags, filter_info;
252 H5Z_filter_t filter_type = H5Pget_filter2(plist_id, filter, &flags, &nelmts, NULL, 0, NULL, &filter_info);
253 VERBOSE(cerr <<
"Filter Type: ");
255 switch (filter_type) {
256 case H5Z_FILTER_DEFLATE:
257 VERBOSE(cerr <<
"H5Z_FILTER_DEFLATE" << endl);
260 case H5Z_FILTER_SHUFFLE:
261 VERBOSE(cerr <<
"H5Z_FILTER_SHUFFLE" << endl);
265 ostringstream oss(
"Unsupported HDF5 filter: ", std::ios::ate);
290 static void get_variable_chunk_info(hid_t dataset,
DmrppCommon *dc)
293 hid_t dcpl = H5Dget_create_plist(dataset);
294 uint8_t layout_type = H5Pget_layout(dcpl);
296 hid_t fspace_id = H5Dget_space(dataset);
298 unsigned int dataset_rank = H5Sget_simple_extent_ndims(fspace_id);
301 switch (layout_type) {
303 case H5D_CONTIGUOUS: {
304 haddr_t cont_addr = 0;
305 hsize_t cont_size = 0;
306 VERBOSE(cerr <<
"Storage: contiguous" << endl);
308 cont_addr = H5Dget_offset(dataset);
312 cont_size = H5Dget_storage_size(dataset);
314 throw BESInternalError(
"Cannot obtain the storage size.", __FILE__, __LINE__);
316 VERBOSE(cerr <<
" Addr: " << cont_addr << endl);
317 VERBOSE(cerr <<
" Size: " << cont_size << endl);
319 if (dc) dc->
add_chunk(
"", cont_size, cont_addr,
"" );
325 hsize_t num_chunks = 0;
326 herr_t status = H5Dget_num_chunks(dataset, fspace_id, &num_chunks);
332 VERBOSE(cerr <<
"storage: chunked." << endl);
333 VERBOSE(cerr <<
"Number of chunks is " << num_chunks << endl);
336 set_filter_information(dataset, dc);
339 vector<size_t> chunk_dims(dataset_rank);
340 unsigned int chunk_rank = H5Pget_chunk(dcpl, dataset_rank, (hsize_t*) &chunk_dims[0]);
341 if (chunk_rank != dataset_rank)
342 throw BESNotFoundError(
"Found a chunk with rank different than the dataset's (aka variables's) rank", __FILE__, __LINE__);
346 for (
unsigned int i = 0; i < num_chunks; ++i) {
348 vector<hsize_t> temp_coords(dataset_rank);
349 vector<unsigned int> chunk_coords(dataset_rank);
355 status = H5Dget_chunk_info(dataset, fspace_id, i, &temp_coords[0], NULL, &addr, &size);
357 VERBOSE(cerr <<
"ERROR" << endl);
358 throw BESInternalError(
"Cannot get HDF5 dataset storage info.", __FILE__, __LINE__);
361 VERBOSE(cerr <<
"chk_idk: " << i <<
", addr: " << addr <<
", size: " << size << endl);
365 for (
unsigned int j = 0; j < chunk_coords.size(); ++j) {
366 chunk_coords[j] = temp_coords[j];
372 if (dc) dc->
add_chunk(
"", size, addr, chunk_coords);
380 VERBOSE(cerr <<
"Storage: compact" << endl);
381 size_t comp_size = 0;
385 comp_size = H5Dget_storage_size(dataset);
387 throw BESInternalError(
"Cannot obtain the compact storage size.", __FILE__, __LINE__);
389 VERBOSE(cerr <<
" Size: " << comp_size << endl);
395 ostringstream oss(
"Unsupported HDF5 dataset layout type: ", std::ios::ate);
396 oss << layout_type <<
".";
416 static void get_chunks_for_all_variables(hid_t file, D4Group *group)
419 for (Constructor::Vars_iter v = group->var_begin(), ve = group->var_end(); v != ve; ++v) {
422 D4Attributes *d4_attrs = (*v)->attributes();
424 throw BESInternalError(
"Expected to find an attribute table for " + (*v)->name() +
" but did not.", __FILE__, __LINE__);
429 D4Attribute *attr = d4_attrs->get(
"fullnamepath");
440 if(attr->num_values() == 1)
441 FQN = attr->value(0);
444 BESDEBUG(
"dmrpp",
"Working on: " <<FQN<<endl);
445 dataset = H5Dopen2(file, FQN.c_str(), H5P_DEFAULT);
447 throw BESInternalError(
"HDF5 dataset '" + FQN +
"' cannot be opened.", __FILE__, __LINE__);
453 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
455 BESDEBUG(
"dmrpp",
"Working on: " <<FQN<<endl);
456 dataset = H5Dopen2(file, FQN.c_str(), H5P_DEFAULT);
463 if (attr && attr->num_values() == 1)
464 FQN = attr->value(0);
468 VERBOSE(cerr <<
"Working on: " << FQN << endl);
469 hid_t dataset = H5Dopen2(file, FQN.c_str(), H5P_DEFAULT);
473 if (dataset < 0 && attr == 0) {
474 cerr<<
"Unable to open dataset name "<<FQN <<endl;
477 else if (dataset < 0)
478 throw BESInternalError(
"HDF5 dataset '" + FQN +
"' cannot be opened.", __FILE__, __LINE__);
480 get_variable_chunk_info(dataset,
dynamic_cast<DmrppCommon*
>(*v));
484 D4Group::groupsIter g = group->grp_begin();
485 D4Group::groupsIter ge = group->grp_end();
487 get_chunks_for_all_variables(file, *g++);
490 int main(
int argc,
char*argv[])
492 string h5_file_name =
"";
493 string h5_dset_path =
"";
494 string dmr_name =
"";
495 string url_name =
"";
498 GetOpt getopt(argc, argv,
"c:f:r:u:dhv");
500 while ((option_char = getopt()) != -1) {
501 switch (option_char) {
511 h5_file_name = getopt.optarg;
514 dmr_name = getopt.optarg;
517 url_name = getopt.optarg;
523 cerr <<
"build_dmrpp [-v] -c <bes.conf> -f <data file> [-u <href url>] | build_dmrpp -f <data file> -r <dmr file> | build_dmrpp -h" << endl;
530 if (h5_file_name.empty()) {
531 cerr <<
"HDF5 file name must be given (-f <input>)." << endl;
543 if (!dmr_name.empty()) {
545 unique_ptr<DMRpp> dmrpp(
new DMRpp);
547 dmrpp->set_factory(&dtf);
549 ifstream in(dmr_name.c_str());
551 parser.intern(in, dmrpp.get(),
false);
554 file = H5Fopen(h5_file_name.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
556 cerr <<
"Error: HDF5 file '" + h5_file_name +
"' cannot be opened." << endl;
561 get_chunks_for_all_variables(file, dmrpp->root());
564 dmrpp->print_dmrpp(writer, url_name);
566 cout << writer.get_doc();
570 string bes_data_root;
574 cerr <<
"Error: Could not find the BES root directory key." << endl;
586 cerr <<
"The Metadata Store (MDS) must be configured for this command to work." << endl;
602 cerr <<
"Expected a DMR++ object from the DmrppMetadataStore." << endl;
607 file = H5Fopen(h5_file_path.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
609 cerr <<
"Error: HDF5 file '" + h5_file_path +
"' cannot be opened." << endl;
613 get_chunks_for_all_variables(file, dmrpp->root());
615 dmrpp->set_href(url_name);
617 mds->add_dmrpp_response(dmrpp.get(), h5_file_name );
620 dmrpp->set_print_chunks(
true);
621 dmrpp->print_dap4(writer);
623 cout << writer.get_doc();
626 cerr <<
"Error: Could not get a lock on the DMR for '" + h5_file_path +
"'." << endl;
635 catch (std::exception &e) {
636 cerr <<
"std::exception: " << e.what() << endl;
640 cerr <<
"Unknown error." << endl;