39 #include <sys/types.h>
45 #include <D4BaseTypeFactory.h>
46 #include <BESDMRResponse.h>
47 #include <ObjMemCache.h>
50 #include <mime_util.h>
55 #include <BESDASResponse.h>
56 #include <BESDDSResponse.h>
57 #include <BESDataDDSResponse.h>
58 #include <Ancillary.h>
60 #include <BESDapNames.h>
61 #include <BESResponseNames.h>
62 #include <BESContainer.h>
63 #include <BESResponseHandler.h>
64 #include <BESVersionInfo.h>
65 #include <BESServiceRegistry.h>
67 #include <BESDapError.h>
68 #include <BESInternalFatalError.h>
69 #include <TheBESKeys.h>
72 #include "config_hdf5.h"
74 #define HDF5_NAME "h5"
81 void get_attr_contents(AttrTable* temp_table);
85 void write_das_to_file(DAS*das_ptr,FILE* das_file);
88 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
91 void write_container_name_to_file(
const string&,FILE* das_file);
94 void write_das_attr_info(AttrTable* dtp,
const string&,
const string&,FILE * das_file);
97 char* copy_str(
char *temp_ptr,
const string & str);
101 char* obtain_str(
char*temp_ptr,
string & str);
104 char* get_attr_info_from_dc(
char*temp_pointer,DAS *das,AttrTable *at);
107 bool check_beskeys(
const string);
110 static unsigned int get_uint_key(
const string &key,
unsigned int def_val);
111 static unsigned long get_ulong_key(
const string &key,
unsigned long def_val);
114 static float get_float_key(
const string &key,
float def_val);
117 static string get_beskeys(
const string&);
120 extern void read_cfdas(DAS &das,
const string & filename,hid_t fileid);
121 extern void read_cfdds(DDS &dds,
const string & filename,hid_t fileid);
125 unsigned int HDF5RequestHandler::_mdcache_entries = 500;
126 unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
127 unsigned int HDF5RequestHandler::_srdcache_entries = 0;
128 float HDF5RequestHandler::_cache_purge_level = 0.2;
133 ObjMemCache *HDF5RequestHandler::datadds_cache = 0;
136 ObjMemCache *HDF5RequestHandler::lrdata_mem_cache = 0;
137 ObjMemCache *HDF5RequestHandler::srdata_mem_cache = 0;
140 bool HDF5RequestHandler::_usecf =
false;
141 bool HDF5RequestHandler::_pass_fileid =
false;
142 bool HDF5RequestHandler::_disable_structmeta =
false;
143 bool HDF5RequestHandler::_disable_ecsmeta =
false;
144 bool HDF5RequestHandler::_keep_var_leading_underscore =
false;
145 bool HDF5RequestHandler::_check_name_clashing =
false;
146 bool HDF5RequestHandler::_add_path_attrs =
false;
147 bool HDF5RequestHandler::_drop_long_string =
false;
148 bool HDF5RequestHandler::_fillvalue_check =
false;
149 bool HDF5RequestHandler::_check_ignore_obj =
false;
150 bool HDF5RequestHandler::_flatten_coor_attr =
false;
151 bool HDF5RequestHandler::_default_handle_dimension =
false;
152 bool HDF5RequestHandler::_eos5_rm_convention_attr_path =
true;
154 bool HDF5RequestHandler::_common_cache_dirs =
false;
156 bool HDF5RequestHandler::_use_disk_cache =
false;
157 bool HDF5RequestHandler::_use_disk_dds_cache =
false;
158 string HDF5RequestHandler::_disk_cache_dir =
"";
159 string HDF5RequestHandler::_disk_cachefile_prefix =
"";
160 unsigned long long HDF5RequestHandler::_disk_cache_size =0;
163 bool HDF5RequestHandler::_disk_cache_comp_data =
false;
164 bool HDF5RequestHandler::_disk_cache_float_only_comp_data =
false;
165 float HDF5RequestHandler::_disk_cache_comp_threshold =1.0;
166 unsigned long HDF5RequestHandler::_disk_cache_var_size =0;
168 bool HDF5RequestHandler::_use_disk_meta_cache =
false;
169 string HDF5RequestHandler::_disk_meta_cache_path =
"";
171 bool HDF5RequestHandler::_use_latlon_disk_cache =
false;
172 long HDF5RequestHandler::_latlon_disk_cache_size =0;
173 string HDF5RequestHandler::_latlon_disk_cache_dir =
"";
174 string HDF5RequestHandler::_latlon_disk_cachefile_prefix=
"";
176 DMR* HDF5RequestHandler::dmr_int64 = 0;
183 string HDF5RequestHandler::_stp_east_filename;
184 string HDF5RequestHandler::_stp_north_filename;
185 vector<string> HDF5RequestHandler::lrd_cache_dir_list;
186 vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
187 vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
194 HDF5RequestHandler::HDF5RequestHandler(
const string & name)
198 BESDEBUG(HDF5_NAME,
"In HDF5RequestHandler::HDF5RequestHandler" << endl);
200 add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
201 add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
202 add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
203 add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
204 add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
206 add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
207 add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
210 HDF5RequestHandler::_mdcache_entries = get_uint_key(
"H5.MetaDataMemCacheEntries", 0);
211 HDF5RequestHandler::_lrdcache_entries = get_uint_key(
"H5.LargeDataMemCacheEntries", 0);
212 HDF5RequestHandler::_srdcache_entries = get_uint_key(
"H5.SmallDataMemCacheEntries", 0);
213 HDF5RequestHandler::_cache_purge_level = get_float_key(
"H5.CachePurgeLevel", 0.2);
215 if (get_mdcache_entries()) {
216 das_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
217 dds_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
218 datadds_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
219 dmr_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
223 _usecf = check_beskeys(
"H5.EnableCF");
226 _pass_fileid = check_beskeys(
"H5.EnablePassFileID");
227 _disable_structmeta = check_beskeys(
"H5.DisableStructMetaAttr");
228 _disable_ecsmeta = check_beskeys(
"H5.DisableECSMetaAttr");
229 _keep_var_leading_underscore = check_beskeys(
"H5.KeepVarLeadingUnderscore");
230 _check_name_clashing = check_beskeys(
"H5.EnableCheckNameClashing");
231 _add_path_attrs = check_beskeys(
"H5.EnableAddPathAttrs");
232 _drop_long_string = check_beskeys(
"H5.EnableDropLongString");
233 _fillvalue_check = check_beskeys(
"H5.EnableFillValueCheck");
234 _check_ignore_obj = check_beskeys(
"H5.CheckIgnoreObj");
235 _flatten_coor_attr = check_beskeys(
"H5.ForceFlattenNDCoorAttr");
236 _default_handle_dimension = check_beskeys(
"H5.DefaultHandleDimension");
237 _eos5_rm_convention_attr_path= check_beskeys(
"H5.RmConventionAttrPath");
239 _use_disk_cache = check_beskeys(
"H5.EnableDiskDataCache");
240 _disk_cache_dir = get_beskeys(
"H5.DiskCacheDataPath");
241 _disk_cachefile_prefix = get_beskeys(
"H5.DiskCacheFilePrefix");
242 _disk_cache_size = get_ulong_key(
"H5.DiskCacheSize",0);
244 _disk_cache_comp_data = check_beskeys(
"H5.DiskCacheComp");
245 _disk_cache_float_only_comp_data = check_beskeys(
"H5.DiskCacheFloatOnlyComp");
246 _disk_cache_comp_threshold = get_float_key(
"H5.DiskCacheCompThreshold",1.0);
247 _disk_cache_var_size = 1024*get_uint_key(
"H5.DiskCacheCompVarSize",0);
249 _use_disk_meta_cache = check_beskeys(
"H5.EnableDiskMetaDataCache");
250 _use_disk_dds_cache = check_beskeys(
"H5.EnableDiskDDSCache");
251 _disk_meta_cache_path = get_beskeys(
"H5.DiskMetaDataCachePath");
253 _use_latlon_disk_cache = check_beskeys(
"H5.EnableEOSGeoCacheFile");
254 _latlon_disk_cache_size = get_uint_key(
"H5.Cache.latlon.size",0);
255 _latlon_disk_cache_dir = get_beskeys(
"H5.Cache.latlon.path");
256 _latlon_disk_cachefile_prefix= get_beskeys(
"H5.Cache.latlon.prefix");
259 if(get_lrdcache_entries()) {
260 lrdata_mem_cache =
new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
261 if(
true == check_beskeys(
"H5.LargeDataMemCacheConfig")) {
262 _common_cache_dirs =obtain_lrd_common_cache_dirs();
264 if(
false == _common_cache_dirs)
265 cerr<<
"No specific cache info"<<endl;
270 if(get_srdcache_entries()) {
272 BESDEBUG(HDF5_NAME,
"Generate memory cache for smaller coordinate variables" << endl);
273 srdata_mem_cache =
new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
277 if(_disk_cache_comp_data ==
true && _use_disk_cache ==
true) {
278 if(_disk_cache_comp_threshold < 1.0) {
280 ss<< _disk_cache_comp_threshold;
281 string _comp_threshold_str(ss.str());
282 string invalid_comp_threshold =
"The Compression Threshold is the total size of the variable array";
283 invalid_comp_threshold+=
" divided by the storage size of compressed array. It should always be >1";
284 invalid_comp_threshold+=
" The current threhold set at h5.conf is ";
285 invalid_comp_threshold+=_comp_threshold_str;
286 invalid_comp_threshold+=
" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
290 _stp_east_filename = get_beskeys(
"H5.STPEastFileName");
291 _stp_north_filename = get_beskeys(
"H5.STPNorthFileName");
295 BESDEBUG(HDF5_NAME,
"Exiting HDF5RequestHandler::HDF5RequestHandler" << endl);
298 HDF5RequestHandler::~HDF5RequestHandler()
303 delete datadds_cache;
305 delete lrdata_mem_cache;
306 delete srdata_mem_cache;
315 hid_t cf_fileid = -1;
330 DAS *das = bdas->get_das();
333 DAS *cached_das_ptr = 0;
334 bool use_das_cache =
false;
336 cached_das_ptr =
static_cast<DAS*
>(das_cache->
get(filename));
338 use_das_cache =
true;
340 if (
true == use_das_cache) {
343 BESDEBUG(HDF5_NAME,
"DAS Cached hit for : " << filename << endl);
344 *das = *cached_das_ptr;
348 bool das_from_dc =
false;
349 string das_cache_fname;
352 if(_use_disk_meta_cache ==
true) {
354 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
355 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
357 if(access(das_cache_fname.c_str(),F_OK) !=-1)
363 if(
true == das_from_dc) {
364 read_das_from_disk_cache(das_cache_fname,das);
369 BESDEBUG(HDF5_NAME,
"HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
370 das_cache->
add(
new DAS(*das), filename);
375 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
376 if (
true == _usecf) {
378 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
380 string invalid_file_msg=
"Could not open this HDF5 file ";
381 invalid_file_msg +=filename;
382 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
383 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
384 invalid_file_msg +=
" distributor.";
388 if(HDF5RequestHandler::get_dmr_64bit_int()!=NULL)
389 HDF5RequestHandler::set_dmr_64bit_int(NULL);
390 read_cfdas( *das,filename,cf_fileid);
396 string invalid_file_msg=
"Could not open this HDF5 file ";
397 invalid_file_msg +=filename;
398 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
399 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
400 invalid_file_msg +=
" distributor.";
410 Ancillary::read_ancillary_das( *das, filename ) ;
414 AttrTable* top_table = das->get_top_level_attributes();
415 get_attr_contents(top_table);
418 AttrTable::Attr_iter start_aiter=das->var_begin();
419 AttrTable::Attr_iter it = start_aiter;
420 AttrTable::Attr_iter end_aiter = das->var_end();
421 while(it != end_aiter) {
422 AttrTable* temp_table = das->get_table(it);
424 cerr<<
"var_begin"<<endl;
425 temp_table->print(cerr);
433 BESDEBUG(HDF5_NAME,
"DAS added to the cache for : " << filename << endl);
434 das_cache->
add(
new DAS(*das), filename);
439 if(das_cache_fname!=
"") {
440 BESDEBUG(HDF5_NAME,
"HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
441 write_das_to_disk_cache(das_cache_fname,das);
448 catch(InternalErr & e) {
452 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
459 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
466 string s =
"unknown exception caught building HDF5 DAS";
476 void HDF5RequestHandler::get_dds_with_attributes(
BESDDSResponse*bdds,
BESDataDDSResponse*data_bdds,
const string &container_name,
const string& filename,
const string &dds_cache_fname,
const string &das_cache_fname,
bool dds_from_dc,
bool das_from_dc,
bool build_data)
479 if(
true == build_data)
480 dds = data_bdds->get_dds();
485 hid_t cf_fileid = -1;
490 DDS* cached_dds_ptr = 0;
491 bool use_dds_cache =
false;
493 cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(filename));
495 use_dds_cache =
true;
496 if (
true == use_dds_cache) {
499 BESDEBUG(HDF5_NAME,
"DDS Metadata Cached hit for : " << filename << endl);
500 *dds = *cached_dds_ptr;
502 else if (
true ==dds_from_dc) {
503 read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,das_cache_fname,-1,das_from_dc);
506 BESDEBUG(HDF5_NAME,
"Build DDS from the HDF5 file. " << filename << endl);
507 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
508 dds->filename(filename);
513 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
515 string invalid_file_msg=
"Could not open this HDF5 file ";
516 invalid_file_msg +=filename;
517 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
518 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
519 invalid_file_msg +=
" distributor.";
524 if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
525 HDF5RequestHandler::set_dmr_64bit_int(NULL);
526 read_cfdds(*dds,filename,cf_fileid);
532 string invalid_file_msg=
"Could not open this HDF5 file ";
533 invalid_file_msg +=filename;
534 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
535 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
536 invalid_file_msg +=
" distributor.";
540 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
544 if (!dds->check_semantics()) {
546 throw InternalErr(__FILE__, __LINE__,
547 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
550 Ancillary::read_ancillary_dds( *dds, filename ) ;
553 if(dds_cache_fname!=
"" && dds_from_dc ==
false)
554 write_dds_to_disk_cache(dds_cache_fname,dds);
563 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
569 BESDEBUG(HDF5_NAME,
"DDS added to the cache for : " << filename << endl);
570 dds_cache->
add(
new DDS(*dds), filename);
581 catch(InternalErr & e) {
589 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
599 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
609 string s =
"unknown exception caught building HDF5 DDS";
615 void HDF5RequestHandler::get_dds_without_attributes_datadds(
BESDataDDSResponse*data_bdds,
const string &container_name,
const string& filename)
617 DDS *dds = data_bdds->get_dds();
621 hid_t cf_fileid = -1;
626 DDS* cached_dds_ptr = 0;
627 bool use_datadds_cache =
false;
629 cached_dds_ptr =
static_cast<DDS*
>(datadds_cache->
get(filename));
631 use_datadds_cache =
true;
632 if (
true == use_datadds_cache) {
635 BESDEBUG(HDF5_NAME,
"DataDDS Metadata Cached hit for : " << filename << endl);
636 *dds = *cached_dds_ptr;
639 BESDEBUG(HDF5_NAME,
"Build DDS from the HDF5 file. " << filename << endl);
640 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
641 dds->filename(filename);
646 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
648 string invalid_file_msg=
"Could not open this HDF5 file ";
649 invalid_file_msg +=filename;
650 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
651 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
652 invalid_file_msg +=
" distributor.";
657 if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
658 HDF5RequestHandler::set_dmr_64bit_int(NULL);
659 read_cfdds(*dds,filename,cf_fileid);
665 string invalid_file_msg=
"Could not open this HDF5 file ";
666 invalid_file_msg +=filename;
667 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
668 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
669 invalid_file_msg +=
" distributor.";
673 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
677 if (!dds->check_semantics()) {
679 throw InternalErr(__FILE__, __LINE__,
680 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
683 Ancillary::read_ancillary_dds( *dds, filename ) ;
693 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
700 BESDEBUG(HDF5_NAME,
"DataDDS added to the cache for : " << filename << endl);
701 datadds_cache->
add(
new DDS(*dds), filename);
710 BESDEBUG(HDF5_NAME,
"Data ACCESS build_data(): set the including attribute flag to false: "<<filename << endl);
711 data_bdds->set_ia_flag(
false);
714 catch(InternalErr & e) {
722 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
732 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
742 string s =
"unknown exception caught building HDF5 DDS";
752 void HDF5RequestHandler::get_dds_with_attributes(
const string &filename,
const string &container_name, DDS*dds) {
756 hid_t cf_fileid = -1;
761 DDS* cached_dds_ptr = 0;
762 if (dds_cache && (cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(filename)))) {
765 BESDEBUG(HDF5_NAME,
"DDS Cached hit for : " << filename << endl);
766 *dds = *cached_dds_ptr;
771 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
772 if (!container_name.empty())
773 dds->container_name(container_name);
774 dds->filename(filename);
780 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
781 string dds_filename =
"/tmp/"+base_filename+
"_dds";
782 FILE *dds_file = fopen(dds_filename.c_str(),
"r");
783 cerr<<
"before parsing "<<endl;
785 DDS tdds(&tf,name_path(filename),
"3.2");
786 tdds.filename(filename);
788 tdds.parse(dds_file);
790 cache_dds =
new DDS(tdds);
796 cerr<<
"after parsing "<<endl;
805 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
807 string invalid_file_msg=
"Could not open this HDF5 file ";
808 invalid_file_msg +=filename;
809 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
810 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
811 invalid_file_msg +=
" distributor.";
815 read_cfdds(*dds,filename,cf_fileid);
818 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
819 string dds_filename =
"/tmp/"+base_filename+
"_dds";
820 FILE *dds_file = fopen(dds_filename.c_str(),
"w");
821 dds->print(dds_file);
829 string invalid_file_msg=
"Could not open this HDF5 file ";
830 invalid_file_msg +=filename;
831 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
832 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
833 invalid_file_msg +=
" distributor.";
837 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
843 if (!dds->check_semantics()) {
845 throw InternalErr(__FILE__, __LINE__,
846 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
849 Ancillary::read_ancillary_dds( *dds, filename ) ;
855 if (das_cache && (das =
static_cast<DAS*
>(das_cache->
get(filename)))) {
856 BESDEBUG(HDF5_NAME,
"DAS Cached hit for : " << filename << endl);
857 dds->transfer_attributes(das);
864 if (!container_name.empty())
865 das->container_name(container_name);
867 if (
true == _usecf) {
870 read_cfdas( *das,filename,cf_fileid);
883 Ancillary::read_ancillary_das( *das, filename ) ;
885 dds->transfer_attributes(das);
891 BESDEBUG(HDF5_NAME,
"DAS added to the cache for : " << filename << endl);
893 das_cache->
add(das, filename);
902 BESDEBUG(HDF5_NAME,
"DDS added to the cache for : " << filename << endl);
903 dds_cache->
add(
new DDS(*dds), filename);
911 catch(InternalErr & e) {
919 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
929 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
939 string s =
"unknown exception caught building HDF5 DDS";
962 bool dds_from_dc =
false;
963 bool das_from_dc =
false;
964 bool build_data =
false;
965 string dds_cache_fname;
966 string das_cache_fname;
968 if(_use_disk_meta_cache ==
true) {
970 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
973 if(_use_disk_dds_cache ==
true) {
974 dds_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_dds";
975 if(access(dds_cache_fname.c_str(),F_OK) !=-1)
979 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
981 if(access(das_cache_fname.c_str(),F_OK) !=-1)
986 get_dds_with_attributes(bdds, NULL,container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
990 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
991 string dds_filename =
"/tmp/"+base_filename+
"_dds";
994 DDS tdds(&tf,name_path(filename),
"3.2");
995 tdds.filename(filename);
998 FILE *dds_file = fopen(dds_filename.c_str(),
"r");
999 tdds.parse(dds_file);
1001 DDS* cache_dds =
new DDS(tdds);
1012 catch(InternalErr & e) {
1014 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1015 __FILE__, __LINE__);
1019 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1020 __FILE__, __LINE__);
1024 string s =
"unknown exception caught building HDF5 DDS";
1036 if(
true == _pass_fileid)
1037 return hdf5_build_data_with_IDs(dhi);
1051 bool dds_from_dc =
false;
1052 bool das_from_dc =
false;
1053 bool build_data =
true;
1054 string dds_cache_fname;
1055 string das_cache_fname;
1059 if(_use_disk_meta_cache ==
true) {
1061 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1062 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
1064 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1070 get_dds_without_attributes_datadds(bdds,container_name,filename);
1076 catch(InternalErr & e) {
1078 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1079 __FILE__, __LINE__);
1083 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1084 __FILE__, __LINE__);
1088 string s =
"unknown exception caught building HDF5 DDS";
1099 BESDEBUG(
"h5",
"Building DataDDS by passing file IDs. "<<endl);
1100 hid_t cf_fileid = -1;
1104 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1105 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1107 string invalid_file_msg=
"Could not open this HDF5 file ";
1108 invalid_file_msg +=filename;
1109 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1110 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1111 invalid_file_msg +=
" distributor.";
1125 delete bdds->get_dds();
1129 hdds->setHDF5Dataset(cf_fileid);
1131 read_cfdds( *hdds,filename,cf_fileid);
1133 if (!hdds->check_semantics()) {
1135 throw InternalErr(__FILE__, __LINE__,
1136 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1139 Ancillary::read_ancillary_dds( *hdds, filename ) ;
1141 DAS *das =
new DAS ;
1144 read_cfdas( *das,filename,cf_fileid);
1145 Ancillary::read_ancillary_das( *das, filename ) ;
1147 hdds->transfer_attributes(das);
1153 catch(InternalErr & e) {
1155 H5Fclose(cf_fileid);
1156 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1157 __FILE__, __LINE__);
1161 H5Fclose(cf_fileid);
1162 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1163 __FILE__, __LINE__);
1167 H5Fclose(cf_fileid);
1168 string s =
"unknown exception caught building HDF5 DataDDS";
1186 DMR *dmr = bes_dmr.get_dmr();
1190 hid_t cf_fileid = -1;
1194 DMR* cached_dmr_ptr = 0;
1195 bool use_dmr_cache =
false;
1197 cached_dmr_ptr =
static_cast<DMR*
>(dmr_cache->
get(filename));
1199 use_dmr_cache =
true;
1200 if (
true == use_dmr_cache) {
1202 BESDEBUG(HDF5_NAME,
"DMR Cached hit for : " << filename << endl);
1203 *dmr = *cached_dmr_ptr;
1207 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1208 D4BaseTypeFactory MyD4TypeFactory;
1209 dmr->set_factory(&MyD4TypeFactory);
1213 if(
true == _pass_fileid)
1214 return hdf5_build_dmr_with_IDs(dhi);
1216 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1218 string invalid_file_msg=
"Could not open this HDF5 file ";
1219 invalid_file_msg +=filename;
1220 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1221 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1222 invalid_file_msg +=
" distributor.";
1226 BaseTypeFactory factory;
1227 DDS dds(&factory, name_path(filename),
"3.2");
1228 dds.filename(filename);
1234 HDF5RequestHandler::set_dmr_64bit_int(dmr);
1235 read_cfdds( dds,filename,cf_fileid);
1236 if (!dds.check_semantics()) {
1238 throw InternalErr(__FILE__, __LINE__,
1239 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1242 read_cfdas(das,filename,cf_fileid);
1243 Ancillary::read_ancillary_das( das, filename ) ;
1245 dds.transfer_attributes(&das);
1249 H5Fclose(cf_fileid);
1251 dmr->build_using_dds(dds);
1259 string invalid_file_msg=
"Could not open this HDF5 file ";
1260 invalid_file_msg +=filename;
1261 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1262 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1263 invalid_file_msg +=
" distributor.";
1267 bool use_dimscale =
false;
1268 if(
true == _default_handle_dimension)
1269 use_dimscale = check_dimscale(fileid);
1270 dmr->set_name(name_path(filename));
1271 dmr->set_filename(name_path(filename));
1273 D4Group* root_grp = dmr->root();
1274 breadth_first(fileid,(
char*)
"/",root_grp,filename.c_str(),use_dimscale);
1277 if(
true == use_dimscale)
1279 breadth_first(fileid,(
char*)
"/",root_grp,filename.c_str(),
true);
1281 depth_first(fileid,(
char*)
"/",root_grp,filename.c_str());
1292 BESDEBUG(HDF5_NAME,
"DMR added to the cache for : " << filename << endl);
1293 dmr_cache->
add(
new DMR(*dmr), filename);
1298 catch(InternalErr & e) {
1301 H5Fclose(cf_fileid);
1305 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1306 __FILE__, __LINE__);
1311 H5Fclose(cf_fileid);
1314 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1315 __FILE__, __LINE__);
1320 H5Fclose(cf_fileid);
1323 string s =
"unknown exception caught building HDF5 DMR";
1336 dmr->set_factory(0);
1345 BESDEBUG(
"h5",
"Building DMR with passing file IDs. "<<endl);
1347 hid_t cf_fileid = -1;
1349 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1350 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1352 string invalid_file_msg=
"Could not open this HDF5 file ";
1353 invalid_file_msg +=filename;
1354 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1355 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1356 invalid_file_msg +=
" distributor.";
1360 BaseTypeFactory factory;
1361 DDS dds(&factory, name_path(filename),
"3.2");
1362 dds.filename(filename);
1370 read_cfdds( dds,filename,cf_fileid);
1372 if (!dds.check_semantics()) {
1374 throw InternalErr(__FILE__, __LINE__,
1375 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1378 Ancillary::read_ancillary_dds( dds, filename ) ;
1381 read_cfdas(das,filename,cf_fileid);
1383 Ancillary::read_ancillary_das( das, filename ) ;
1385 dds.transfer_attributes(&das);
1392 catch(InternalErr & e) {
1395 H5Fclose(cf_fileid);
1397 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1398 __FILE__, __LINE__);
1403 H5Fclose(cf_fileid);
1405 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1406 __FILE__, __LINE__);
1411 H5Fclose(cf_fileid);
1413 string s =
"unknown exception caught building HDF5 DataDDS";
1427 DMR *dmr = bes_dmr.get_dmr();
1428 D4BaseTypeFactory MyD4TypeFactory;
1429 dmr->set_factory(&MyD4TypeFactory);
1430 dmr->build_using_dds(dds);
1433 hdf5_dmr->setHDF5Dataset(cf_fileid);
1435 bes_dmr.set_dmr(hdf5_dmr);
1444 hdf5_dmr->set_factory(0);
1456 string add_info=
"Just for Test";
1458 map<string,string> attrs ;
1459 attrs[
"name"] = MODULE_NAME ;
1460 attrs[
"version"] = MODULE_VERSION ;
1461 list<string> services ;
1462 BESServiceRegistry::TheRegistry()->
services_handled( HDF5_NAME, services );
1463 if( services.size() > 0 )
1466 attrs[
"handles"] = handles ;
1468 info->begin_tag(
"module", &attrs ) ;
1469 info->end_tag(
"module" ) ;
1482 info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1488 bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1490 string lrd_config_fpath;
1491 string lrd_config_fname;
1494 lrd_config_fpath = get_beskeys(
"H5.DataCachePath");
1497 lrd_config_fname = get_beskeys(
"H5.LargeDataMemCacheFileName");
1500 if(lrd_config_fpath==
"" || lrd_config_fname==
"")
1507 string mcache_config_fname = lrd_config_fpath+
"/"+lrd_config_fname;
1511 ifstream mcache_config_file(mcache_config_fname.c_str());
1514 if(mcache_config_file.is_open()==
false){
1515 BESDEBUG(HDF5_NAME,
"The large data memory cache configure file "<<mcache_config_fname );
1516 BESDEBUG(HDF5_NAME,
" cannot be opened."<<endl);
1521 while(getline(mcache_config_file,temp_line)) {
1524 if(temp_line.size()>1 && temp_line.at(1)==
' ') {
1526 string subline = temp_line.substr(2);
1527 vector<string> temp_name_list;
1530 if(temp_line.at(0)==
'1') {
1531 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1533 lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1536 else if(temp_line.at(0)==
'0'){
1537 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1539 lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1542 else if(temp_line.at(0)==
'2') {
1548 for(
unsigned int i = 0; i<subline.size();i++){
1549 if(subline[i]==
'"') {
1550 dq_pos.push_back(i);
1552 else if(subline[i]==
'\'')
1553 sq_pos.push_back(i);
1555 if(dq_pos.size()==0 && sq_pos.size()==0)
1556 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1557 else if((dq_pos.size()!=0) &&(dq_pos.size()%2==0)&& sq_pos.size()==0) {
1558 unsigned int dq_index= 0;
1559 while(dq_index < dq_pos.size()){
1560 if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1561 temp_name_list.push_back
1562 (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1564 dq_index = dq_index + 2;
1567 else if((sq_pos.size()!=0) &&(sq_pos.size()%2==0)&& dq_pos.size()==0) {
1568 unsigned int sq_index= 0;
1569 while(sq_index < sq_pos.size()){
1570 if(sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1571 temp_name_list.push_back
1572 (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1574 sq_index = sq_index+2;
1578 lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1586 for(
int i =0; i<lrd_cache_dir_list.size();i++)
1587 cerr<<
"lrd cache list is "<<lrd_cache_dir_list[i] <<endl;
1588 for(
int i =0; i<lrd_non_cache_dir_list.size();i++)
1589 cerr<<
"lrd non cache list is "<<lrd_non_cache_dir_list[i] <<endl;
1590 for(
int i =0; i<lrd_var_cache_file_list.size();i++)
1591 cerr<<
"lrd var cache file list is "<<lrd_var_cache_file_list[i] <<endl;
1595 mcache_config_file.close();
1596 if(lrd_cache_dir_list.size()==0 && lrd_non_cache_dir_list.size()==0 && lrd_var_cache_file_list.size()==0)
1603 bool HDF5RequestHandler::read_das_from_disk_cache(
const string & cache_filename,DAS *das_ptr) {
1605 BESDEBUG(HDF5_NAME,
"Coming to read_das_from_disk_cache() " << cache_filename << endl);
1606 bool ret_value =
true;
1607 FILE *md_file = NULL;
1608 md_file = fopen(cache_filename.c_str(),
"rb");
1610 if(NULL == md_file) {
1611 string bes_error =
"An error occurred trying to open a metadata cache file " + cache_filename;
1616 int fd_md = fileno(md_file);
1618 l_md = lock(F_RDLCK);
1621 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1624 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1631 if(stat(cache_filename.c_str(),&sb) != 0) {
1632 string bes_error =
"An error occurred trying to stat a metadata cache file size " + cache_filename;
1638 size_t bytes_expected_read=(size_t)sb.st_size;
1639 BESDEBUG(HDF5_NAME,
"DAS Disk cache file size is " << bytes_expected_read << endl);
1642 buf.resize(bytes_expected_read);
1643 size_t bytes_to_read =fread((
void*)&buf[0],1,bytes_expected_read,md_file);
1644 if(bytes_to_read != bytes_expected_read)
1645 throw InternalErr(__FILE__,__LINE__,
"Fail to read the data from the das cache file.");
1647 char* temp_pointer =&buf[0];
1649 AttrTable*at = NULL;
1652 temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1658 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1660 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1664 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a das cache file.");
1668 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1670 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1679 bool HDF5RequestHandler::write_dds_to_disk_cache(
const string& dds_cache_fname,DDS *dds_ptr) {
1681 BESDEBUG(HDF5_NAME,
"Write DDS to disk cache " << dds_cache_fname << endl);
1682 FILE *dds_file = fopen(dds_cache_fname.c_str(),
"w");
1684 if(NULL == dds_file) {
1685 string bes_error =
"An error occurred trying to open a metadata cache file " + dds_cache_fname;
1690 int fd_md = fileno(dds_file);
1692 l_md = lock(F_WRLCK);
1695 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1698 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1703 dds_ptr->print(dds_file);
1706 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1708 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1712 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a dds cache file.");
1715 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1717 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1727 bool HDF5RequestHandler::write_das_to_disk_cache(
const string & das_cache_fname, DAS *das_ptr) {
1729 BESDEBUG(HDF5_NAME,
"Write DAS to disk cache " << das_cache_fname << endl);
1730 FILE *das_file = fopen(das_cache_fname.c_str(),
"wb");
1731 if(NULL == das_file) {
1732 string bes_error =
"An error occurred trying to open a metadata cache file " + das_cache_fname;
1736 int fd_md = fileno(das_file);
1738 l_md = lock(F_WRLCK);
1741 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1744 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1749 write_das_to_file(das_ptr,das_file);
1752 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1754 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1758 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a dds cache file.");
1761 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1763 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1775 void write_das_to_file(DAS*das_ptr,FILE* das_file) {
1778 uint8_t category_flag = 2;
1779 AttrTable* top_table = das_ptr->get_top_level_attributes();
1780 write_das_table_to_file(top_table,das_file);
1783 fwrite((
const void*)&category_flag,1,1,das_file);
1789 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
1791 if(temp_table !=NULL) {
1794 uint8_t category_flag = 2;
1797 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
1798 AttrTable::Attr_iter top_endit = temp_table->attr_end();
1799 AttrTable::Attr_iter top_it = top_startit;
1800 while(top_it !=top_endit) {
1801 AttrType atype = temp_table->get_attr_type(top_it);
1802 if(atype == Attr_unknown)
1803 throw InternalErr(__FILE__,__LINE__,
"Unsupported DAS Attribute type");
1804 else if(atype!=Attr_container) {
1805 BESDEBUG(HDF5_NAME,
"DAS to the disk cache, attr name is: "
1806 << temp_table->get_name(top_it) << endl);
1807 BESDEBUG(HDF5_NAME,
"DAS to the disk cache, attr type is: "
1808 << temp_table->get_type(top_it) << endl);
1817 write_das_attr_info(temp_table,temp_table->get_name(top_it),temp_table->get_type(top_it),das_file);
1820 BESDEBUG(HDF5_NAME,
"DAS to the disk cache, attr container name is: "
1821 << (*top_it)->name << endl);
1823 AttrTable* sub_table = temp_table->get_attr_table(top_it);
1824 write_container_name_to_file(sub_table->get_name(),das_file);
1825 write_das_table_to_file(sub_table,das_file);
1828 fwrite((
const void*)&category_flag,1,1,das_file);
1838 void write_container_name_to_file(
const string& cont_name,FILE *das_file) {
1841 uint8_t category_flag = 1;
1843 size_t bytes_to_write = cont_name.size()+
sizeof(size_t)+1;
1844 buf.resize(bytes_to_write);
1845 char*temp_pointer =&buf[0];
1846 memcpy((
void*)temp_pointer,(
void*)&category_flag,1);
1848 temp_pointer=copy_str(temp_pointer,cont_name);
1850 size_t bytes_to_be_written = fwrite((
const void*)&buf[0],1,bytes_to_write,das_file);
1851 if(bytes_to_be_written != bytes_to_write)
1852 throw InternalErr(__FILE__, __LINE__,
"Failed to write a DAS container name to a cache");
1858 void write_das_attr_info(AttrTable* dtp,
const string& attr_name,
const string & attr_type,FILE * das_file) {
1861 uint8_t category_flag = 0;
1863 unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
1864 vector<string> attr_values;
1865 size_t total_attr_values_size = 0;
1866 for (
unsigned int i = 0; i <num_attr_elems;i++){
1867 attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
1868 total_attr_values_size += attr_values[i].size();
1872 size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2*
sizeof(size_t);
1878 bytes_to_write_attr +=
sizeof(
unsigned int) + num_attr_elems*
sizeof(
size_t)+total_attr_values_size;
1880 vector<char>attr_buf;
1881 attr_buf.resize(bytes_to_write_attr);
1882 char* temp_attrp =&attr_buf[0];
1885 memcpy((
void*)temp_attrp,(
void*)&category_flag,1);
1889 temp_attrp=copy_str(temp_attrp,attr_name);
1890 temp_attrp=copy_str(temp_attrp,attr_type);
1893 memcpy((
void*)temp_attrp,(
void*)&num_attr_elems,
sizeof(
unsigned int));
1894 temp_attrp+=
sizeof(
unsigned int);
1897 for (
unsigned int i = 0; i <num_attr_elems;i++)
1898 temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
1900 size_t bytes_to_be_written = fwrite((
const void*)&attr_buf[0],1,bytes_to_write_attr,das_file);
1901 if(bytes_to_be_written != bytes_to_write_attr)
1902 throw InternalErr(__FILE__, __LINE__,
"Failed to write a DAS attribute to a cache");
1910 bool build_data,
const string & container_name,
const string & h5_fname,
1911 const string & dds_cache_fname,
const string &das_cache_fname, hid_t h5_fd,
1915 BESDEBUG(HDF5_NAME,
"Read DDS from disk cache " << dds_cache_fname << endl);
1918 if(
true == build_data)
1919 dds = data_bdds->get_dds();
1925 DDS tdds(&tf,name_path(h5_fname),
"3.2");
1926 tdds.filename(h5_fname);
1928 FILE *dds_file = fopen(dds_cache_fname.c_str(),
"r");
1929 tdds.parse(dds_file);
1930 DDS* cache_dds =
new DDS(tdds);
1932 cerr<<
"before dds "<<endl;
1934 cerr<<
"after dds "<<endl;
1935 cerr<<
"before tdds "<<endl;
1936 cache_dds->dump(cerr);
1937 cerr<<
"after tdds "<<endl;
1942 Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
1944 add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
1945 if(
true == build_data)
1946 data_bdds->
set_dds(cache_dds);
1953 BESDEBUG(HDF5_NAME,
"Reading DDS from Disk Cache routine, For memory cache, DDS added to the cache for : " << h5_fname << endl);
1954 dds_cache->
add(
new DDS(*cache_dds), h5_fname);
1960 void HDF5RequestHandler::add_das_to_dds(DDS *dds,
const string &,
const string &filename,
1961 const string &das_cache_fname, hid_t h5_fd,
bool das_from_dc) {
1963 BESDEBUG(HDF5_NAME,
"Coming to add_das_to_dds() " << endl);
1967 bool use_das_cache =
false;
1969 das =
static_cast<DAS*
>(das_cache->
get(filename));
1971 use_das_cache =
true;
1973 if (
true == use_das_cache) {
1974 BESDEBUG(HDF5_NAME,
"DAS Cached hit for : " << filename << endl);
1975 dds->transfer_attributes(das);
1982 if (!container_name.empty())
1983 das->container_name(container_name);
1985 if(das_from_dc ==
true)
1986 read_das_from_disk_cache(das_cache_fname,das);
1989 bool h5_file_open =
true;
1991 h5_file_open =
false;
1992 if (
true == _usecf) {
1994 if(h5_file_open ==
false)
1995 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1997 read_cfdas( *das,filename,h5_fd);
1998 if(h5_file_open ==
false)
2002 if(h5_file_open ==
false)
2006 if(h5_file_open ==
false)
2010 Ancillary::read_ancillary_das( *das, filename ) ;
2012 if(das_cache_fname!=
"" && das_from_dc ==
false)
2013 write_das_to_disk_cache(das_cache_fname,das);
2016 dds->transfer_attributes(das);
2020 BESDEBUG(HDF5_NAME,
"Reading DDS from Disk Cache routine, For memory cache, DAS added to the cache for : " << filename << endl);
2021 das_cache->
add(
new DAS(*das), filename);
2029 bool check_beskeys(
const string key) {
2033 const string dosettrue =
"true";
2034 const string dosetyes =
"yes";
2037 if(
true == found ) {
2039 if( dosettrue == doset || dosetyes == doset )
2048 static unsigned int get_uint_key(
const string &key,
unsigned int def_val)
2054 if (
true == found) {
2056 return atoi(doset.c_str());
2063 static unsigned long get_ulong_key(
const string &key,
unsigned long def_val)
2069 if (
true == found) {
2071 return atol(doset.c_str());
2077 static float get_float_key(
const string &key,
float def_val)
2083 if (
true == found) {
2084 return atof(doset.c_str());
2091 static string get_beskeys(
const string &key) {
2094 string ret_value =
"";
2102 char* copy_str(
char*temp_ptr,
const string & str) {
2104 size_t str_size=str.size();
2105 memcpy((
void*)temp_ptr,(
void*)&str_size,
sizeof(
size_t));
2106 temp_ptr+=
sizeof(size_t);
2107 vector<char>temp_vc2(str.begin(),str.end());
2108 memcpy((
void*)temp_ptr,(
void*)&temp_vc2[0],str.size());
2109 temp_ptr+=str.size();
2118 char* obtain_str(
char*temp_ptr,
string & str) {
2120 size_t oname_size = *((
size_t *)temp_ptr);
2121 temp_ptr = temp_ptr +
sizeof(size_t);
2123 for(
unsigned int i =0; i<oname_size; i++){
2124 oname.push_back(*temp_ptr);
2135 char* get_attr_info_from_dc(
char*temp_pointer,DAS *das,AttrTable *at_par) {
2140 flag = *((uint8_t*)(temp_pointer));
2141 BESDEBUG(HDF5_NAME,
"Build DAS from the disk cache file flag: "
2142 <<
" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2143 <<
" flag = 3; the initial value to get the attribute retrieval process started."
2144 <<
" The flag value is "
2145 << (
int)flag <<endl);
2149 string container_name;
2150 temp_pointer = obtain_str(temp_pointer,container_name);
2151 BESDEBUG(HDF5_NAME,
"DAS from the disk cache, container name is " << container_name << endl);
2154 AttrTable*temp_at_par = at_par;
2156 at_par = das->add_table(container_name,
new AttrTable);
2158 at_par = at_par->append_container(container_name);
2160 temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2162 at_par = temp_at_par;
2165 else if(flag == 0) {
2168 throw BESInternalError(
"The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2172 temp_pointer = obtain_str(temp_pointer,attr_name);
2173 BESDEBUG(HDF5_NAME,
"DAS from the disk cache, attr name is: " << attr_name << endl);
2177 temp_pointer = obtain_str(temp_pointer,attr_type);
2178 BESDEBUG(HDF5_NAME,
"DAS from the disk cache, attr type is: " << attr_type << endl);
2181 unsigned int num_values = *((
unsigned int*)(temp_pointer));
2182 BESDEBUG(HDF5_NAME,
"DAS from the disk cache, number of attribute values is: " << num_values << endl);
2183 temp_pointer+=
sizeof(
unsigned int);
2185 vector <string> attr_values;
2187 for(
unsigned int i = 0; i<num_values; i++) {
2189 temp_pointer = obtain_str(temp_pointer,attr_value);
2190 attr_values.push_back(attr_value);
2191 BESDEBUG(HDF5_NAME,
"DAS from the disk cache, attribute value is: " << attr_value << endl);
2194 at_par->append_attr(attr_name,attr_type,&attr_values);
2198 return temp_pointer;
2203 void get_attr_contents(AttrTable*temp_table) {
2204 if(temp_table !=NULL) {
2205 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2206 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2207 AttrTable::Attr_iter top_it = top_startit;
2208 while(top_it !=top_endit) {
2209 AttrType atype = temp_table->get_attr_type(top_it);
2210 if(atype == Attr_unknown)
2211 cerr<<
"unsupported DAS attributes" <<endl;
2212 else if(atype!=Attr_container) {
2214 cerr<<
"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2215 cerr<<
"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2216 unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2217 cerr<<
"Attribute values are "<<endl;
2218 for (
unsigned int i = 0; i <num_attrs;i++)
2219 cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<
" ";
2223 cerr<<
"Coming to the attribute container. "<<endl;
2224 cerr<<
"container name is "<<(*top_it)->name <<endl;
2225 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2226 cerr<<
"container table name is "<<sub_table->get_name() <<endl;
2227 get_attr_contents(sub_table);
2243 DDS *dds = bdds->get_dds();
2247 if (das_cache && (das =
static_cast<DAS*
>(das_cache->
get(filename)))) {
2248 BESDEBUG(HDF5_NAME,
"DAS Cached hit for : " << filename << endl);
2249 dds->transfer_attributes(das);
2255 if (!container_name.empty()) das->container_name(container_name);
2258 if (
true == _usecf) {
2260 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2262 read_cfdas( *das,filename,h5_fd);
2274 Ancillary::read_ancillary_das(*das, filename);
2276 dds->transfer_attributes(das);
2281 BESDEBUG(HDF5_NAME,
"DAS added to the cache for : " << filename << endl);
2282 das_cache->
add(das, filename);
2288 BESDEBUG(HDF5_NAME,
"Data ACCESS in add_attributes(): set the including attribute flag to true: "<<filename << endl);
2289 bdds->set_ia_flag(
true);