29 #include <sys/types.h>
38 #include <D4BaseTypeFactory.h>
39 #include <BESDMRResponse.h>
40 #include <mime_util.h>
41 #include <InternalErr.h>
42 #include <Ancillary.h>
45 #include "HDF4RequestHandler.h"
46 #include <BESResponseNames.h>
47 #include <BESDapNames.h>
48 #include <BESDASResponse.h>
49 #include <BESDDSResponse.h>
50 #include <BESDataDDSResponse.h>
52 #include <BESResponseHandler.h>
53 #include <BESVersionInfo.h>
54 #include <BESServiceRegistry.h>
56 #include <TheBESKeys.h>
57 #include <InternalErr.h>
58 #include <BESInternalError.h>
59 #include <BESDapError.h>
60 #include <BESStopWatch.h>
62 #include "BESDataNames.h"
64 #include <Ancillary.h>
65 #include "config_hdf.h"
67 #define HDF4_NAME "h4"
74 #include "HDFFloat32.h"
75 #include "HDFSPArray_RealField.h"
85 bool check_beskeys(
const string);
86 bool get_beskeys(
const string,
string &);
88 extern void read_das(DAS & das,
const string & filename);
89 extern void read_dds(DDS & dds,
const string & filename);
91 extern bool read_dds_hdfsp(DDS & dds,
const string & filename,int32 sdfd, int32 fileid,
HDFSP::File*h4file);
93 extern bool read_das_hdfsp(DAS & das,
const string & filename,int32 sdfd, int32 fileid,
HDFSP::File**h4fileptr);
95 extern void read_das_sds(DAS & das,
const string & filename,int32 sdfd,
bool ecs_metadata,
HDFSP::File**h4fileptr);
96 extern void read_dds_sds(DDS &dds,
const string & filename,int32 sdfd,
HDFSP::File*h4file,
bool dds_set_cache);
98 #ifdef USE_HDFEOS2_LIB
100 void read_das_use_eos2lib(DAS & das,
const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,
bool ecs_metadata,
HDFSP::File**h4file,HDFEOS2::File**eosfile);
101 void read_dds_use_eos2lib(DDS & dds,
const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,
HDFSP::File*h4file,HDFEOS2::File*eosfile);
102 void close_fileid(
const int sdfd,
const int fileid,
const int gridfd,
const int swathfd,
HDFSP::File*h4file,HDFEOS2::File*eosfile);
106 void close_hdf4_fileid(
const int sdfd,
const int fileid,
HDFSP::File*h4file);
107 bool rw_das_cache_file(
const string & filename, DAS *das_ptr,
bool rw_flag);
108 bool r_dds_cache_file(
const string & cache_filename, DDS *dds_ptr,
const string & hdf4_filename);
111 bool HDF4RequestHandler::_usecf =
false;
114 bool HDF4RequestHandler::_pass_fileid =
false;
115 bool HDF4RequestHandler::_disable_structmeta =
false;
116 bool HDF4RequestHandler::_enable_special_eos =
false;
117 bool HDF4RequestHandler::_disable_scaleoffset_comp =
false;
118 bool HDF4RequestHandler::_disable_ecsmetadata_min =
false;
119 bool HDF4RequestHandler::_disable_ecsmetadata_all =
false;
123 bool HDF4RequestHandler::_enable_eosgeo_cachefile =
false;
124 bool HDF4RequestHandler::_enable_data_cachefile =
false;
125 bool HDF4RequestHandler::_enable_metadata_cachefile=
false;
128 bool HDF4RequestHandler::_enable_hybrid_vdata =
false;
129 bool HDF4RequestHandler::_enable_ceres_vdata =
false;
130 bool HDF4RequestHandler::_enable_vdata_attr =
false;
131 bool HDF4RequestHandler::_enable_vdata_desc_attr =
false;
132 bool HDF4RequestHandler::_disable_vdata_nameclashing_check =
false;
133 bool HDF4RequestHandler::_enable_vgroup_attr =
false;
136 bool HDF4RequestHandler::_enable_check_modis_geo_file =
false;
137 bool HDF4RequestHandler::_enable_swath_grid_attr =
false;
138 bool HDF4RequestHandler::_enable_ceres_merra_short_name =
false;
139 bool HDF4RequestHandler::_enable_check_scale_offset_type =
false;
142 bool HDF4RequestHandler::_cache_latlon_path_exist =
false;
143 string HDF4RequestHandler::_cache_latlon_path =
"";
144 bool HDF4RequestHandler::_cache_latlon_prefix_exist =
false;
145 string HDF4RequestHandler::_cache_latlon_prefix =
"";
146 bool HDF4RequestHandler::_cache_latlon_size_exist =
false;
147 long HDF4RequestHandler::_cache_latlon_size =0;
148 bool HDF4RequestHandler::_cache_metadata_path_exist =
false;
149 string HDF4RequestHandler::_cache_metadata_path =
"";
151 HDF4RequestHandler::HDF4RequestHandler(
const string & name) :
153 add_handler(DAS_RESPONSE, HDF4RequestHandler::hdf4_build_das);
154 add_handler(DDS_RESPONSE, HDF4RequestHandler::hdf4_build_dds);
155 add_handler(DATA_RESPONSE, HDF4RequestHandler::hdf4_build_data);
156 add_handler(DMR_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
157 add_handler(DAP4DATA_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
158 add_handler(HELP_RESPONSE, HDF4RequestHandler::hdf4_build_help);
159 add_handler(VERS_RESPONSE, HDF4RequestHandler::hdf4_build_version);
161 _usecf = check_beskeys(
"H4.EnableCF");
165 _pass_fileid = check_beskeys(
"H4.EnablePassFileID");
166 _disable_structmeta = check_beskeys(
"H4.DisableStructMetaAttr");
167 _enable_special_eos = check_beskeys(
"H4.EnableSpecialEOS");
168 _disable_scaleoffset_comp = check_beskeys(
"H4.DisableScaleOffsetComp");
169 _disable_ecsmetadata_min = check_beskeys(
"H4.DisableECSMetaDataMin");
170 _disable_ecsmetadata_all = check_beskeys(
"H4.DisableECSMetaDataAll");
174 _enable_eosgeo_cachefile = check_beskeys(
"H4.EnableEOSGeoCacheFile");
175 _enable_data_cachefile = check_beskeys(
"H4.EnableDataCacheFile");
176 _enable_metadata_cachefile = check_beskeys(
"H4.EnableMetaDataCacheFile");
179 _enable_hybrid_vdata = check_beskeys(
"H4.EnableHybridVdata");
180 _enable_ceres_vdata = check_beskeys(
"H4.EnableCERESVdata");
181 _enable_vdata_attr = check_beskeys(
"H4.EnableVdata_to_Attr");
182 _enable_vdata_desc_attr = check_beskeys(
"H4.EnableVdataDescAttr");
183 _disable_vdata_nameclashing_check = check_beskeys(
"H4.DisableVdataNameclashingCheck");
184 _enable_vgroup_attr = check_beskeys(
"H4.EnableVgroupAttr");
187 _enable_check_modis_geo_file = check_beskeys(
"H4.EnableCheckMODISGeoFile");
188 _enable_swath_grid_attr = check_beskeys(
"H4.EnableSwathGridAttr");
189 _enable_ceres_merra_short_name = check_beskeys(
"H4.EnableCERESMERRAShortName");
190 _enable_check_scale_offset_type = check_beskeys(
"H4.EnableCheckScaleOffsetType");
193 _cache_latlon_path_exist =get_beskeys(
"HDF4.Cache.latlon.path",_cache_latlon_path);
194 _cache_latlon_prefix_exist =get_beskeys(
"HDF4.Cache.latlon.prefix",_cache_latlon_prefix);
195 string temp_cache_latlon_size;
196 _cache_latlon_size_exist =get_beskeys(
"HDF4.Cache.latlon.size",temp_cache_latlon_size);
197 if(_cache_latlon_size_exist ==
true) {
198 istringstream iss(temp_cache_latlon_size);
199 iss >> _cache_latlon_size;
202 _cache_metadata_path_exist =get_beskeys(
"H4.Cache.metadata.path",_cache_metadata_path);
206 HDF4RequestHandler::~HDF4RequestHandler() {
213 if (BESISDEBUG( TIMING_LOG ))
214 sw.
start(
"HDF4RequestHandler::hdf4_build_das", dhi.
data[REQUEST_ID]);
232 if((base_file_name.size() >12) && (base_file_name.compare(0,4,
"AIRS") == 0)
233 && ((base_file_name.find(
".L3.")!=string::npos) || (base_file_name.find(
".L2.")!=string::npos))
234 && (base_file_name.find(
".v6.")!=string::npos)) {
235 return hdf4_build_das_cf_sds(dhi);
247 DAS *das = bdas->get_das();
253 if (
true == _usecf) {
262 sdfd = SDstart (
const_cast < char *
>(accessed.c_str()), DFACC_READ);
264 string invalid_file_msg=
"HDF4 SDstart error for the file ";
265 invalid_file_msg +=accessed;
266 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
271 fileid = Hopen(
const_cast<char *
>(accessed.c_str()), DFACC_READ,0);
274 string invalid_file_msg=
"HDF4 Hopen error for the file ";
275 invalid_file_msg +=accessed;
276 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
280 #ifdef USE_HDFEOS2_LIB
284 HDFEOS2::File *eosfile = NULL;
288 gridfd = GDopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
292 string invalid_file_msg=
"HDF-EOS GDopen error for the file ";
293 invalid_file_msg +=accessed;
294 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
299 swathfd = SWopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
304 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
305 invalid_file_msg +=accessed;
306 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
311 bool ecs_metadata = !(_disable_ecsmetadata_all);
313 if(ecs_metadata ==
true)
314 cerr<<
"output ecs metadata "<<endl;
316 cerr<<
"Don't output ecs metadata "<<endl;
318 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
321 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
334 read_das_hdfsp(*das,accessed,sdfd,fileid,&h4file);
337 close_hdf4_fileid(sdfd,fileid,h4file);
342 close_hdf4_fileid(sdfd,fileid,h4file);
345 read_das(*das,accessed);
347 Ancillary::read_ancillary_das(*das, accessed);
354 catch (InternalErr & e) {
355 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
359 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
363 string s =
"unknown exception caught building HDF4 DAS";
364 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
375 if (BESISDEBUG( TIMING_LOG ))
376 sw.
start(
"HDF4RequestHandler::hdf4_build_das", dhi.
data[REQUEST_ID]);
394 if((base_file_name.size() >12) && (base_file_name.compare(0,4,
"AIRS") == 0)
395 && ((base_file_name.find(
".L3.")!=string::npos) || (base_file_name.find(
".L2.")!=string::npos))
396 && (base_file_name.find(
".v6.")!=string::npos)) {
397 return hdf4_build_dds_cf_sds(dhi);
404 struct timeval start_time,end_time;
405 gettimeofday(&start_time,NULL);
421 dds->filename(accessed);
427 if (
true == _usecf) {
435 sdfd = SDstart (
const_cast < char *
>(accessed.c_str()), DFACC_READ);
437 string invalid_file_msg=
"HDF4 SDstart error for the file ";
438 invalid_file_msg +=accessed;
439 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
444 fileid = Hopen(
const_cast<char *
>(accessed.c_str()), DFACC_READ,0);
447 string invalid_file_msg=
"HDF4 Hopen error for the file ";
448 invalid_file_msg +=accessed;
449 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
453 #ifdef USE_HDFEOS2_LIB
458 HDFEOS2::File *eosfile = NULL;
462 gridfd = GDopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
466 string invalid_file_msg=
"HDF-EOS GDopen error for the file ";
467 invalid_file_msg +=accessed;
468 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
473 swathfd = SWopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
478 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
479 invalid_file_msg +=accessed;
480 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
485 bool ecs_metadata = !(_disable_ecsmetadata_all);
486 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
487 Ancillary::read_ancillary_das(*das, accessed);
490 read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
493 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
505 read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
506 Ancillary::read_ancillary_das(*das, accessed);
509 read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
512 close_hdf4_fileid(sdfd,fileid,h4file);
517 close_hdf4_fileid(sdfd,fileid,h4file);
520 read_das(*das, accessed);
521 Ancillary::read_ancillary_das(*das, accessed);
522 read_dds(*dds, accessed);
527 gettimeofday(&end_time,NULL);
528 int total_time_spent = (end_time.tv_sec - start_time.tv_sec)*1000000 +end_time.tv_usec-start_time.tv_usec;
529 cerr<<
"total time spent for DDS buld is "<<total_time_spent<<
"micro seconds "<<endl;
532 Ancillary::read_ancillary_dds(*dds, accessed);
534 dds->transfer_attributes(das);
543 catch (InternalErr & e) {
544 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
548 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
552 string s =
"unknown exception caught building HDF4 DDS";
553 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
563 if (BESISDEBUG( TIMING_LOG ))
564 sw.
start(
"HDF4RequestHandler::hdf4_build_data", dhi.
data[REQUEST_ID]);
587 if((base_file_name.size() >12) && (base_file_name.compare(0,4,
"AIRS") == 0)
588 && ((base_file_name.find(
".L3.")!=string::npos) || (base_file_name.find(
".L2.")!=string::npos))
589 && (base_file_name.find(
".v6.")!=string::npos)) {
591 BESDEBUG(
"h4",
"Coming to read the data of AIRS level 3 or level 2 products." << endl);
593 if(
true == _pass_fileid)
594 return hdf4_build_data_cf_sds_with_IDs(dhi);
596 return hdf4_build_data_cf_sds(dhi);
600 if(
true == _pass_fileid)
601 return hdf4_build_data_with_IDs(dhi);
614 DDS *dds = bdds->get_dds();
620 dds->filename(accessed);
626 if (
true == _usecf) {
632 sdfd = SDstart (
const_cast < char *
>(accessed.c_str()), DFACC_READ);
634 string invalid_file_msg=
"HDF4 SDstart error for the file ";
635 invalid_file_msg +=accessed;
636 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
641 fileid = Hopen(
const_cast<char *
>(accessed.c_str()), DFACC_READ,0);
644 string invalid_file_msg=
"HDF4 Hopen error for the file ";
645 invalid_file_msg +=accessed;
646 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
651 #ifdef USE_HDFEOS2_LIB
655 HDFEOS2::File *eosfile = NULL;
659 gridfd = GDopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
663 string invalid_file_msg=
"HDF-EOS GDopen error for the file ";
664 invalid_file_msg +=accessed;
665 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
670 swathfd = SWopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
675 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
676 invalid_file_msg +=accessed;
677 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
687 bool ecs_metadata =
true;
689 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
690 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
691 ecs_metadata =
false;
693 if((
true == _disable_ecsmetadata_min)
694 || (
true == _disable_ecsmetadata_all))
695 ecs_metadata =
false;
697 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
698 Ancillary::read_ancillary_das(*das, accessed);
701 read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
704 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
715 read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
716 Ancillary::read_ancillary_das(*das, accessed);
719 read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
722 close_hdf4_fileid(sdfd,fileid,h4file);
726 close_hdf4_fileid(sdfd,fileid,h4file);
729 read_das(*das, accessed);
730 Ancillary::read_ancillary_das(*das, accessed);
731 read_dds(*dds, accessed);
734 Ancillary::read_ancillary_dds(*dds, accessed);
735 dds->transfer_attributes(das);
744 catch (InternalErr & e) {
745 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
749 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
753 string s =
"unknown exception caught building DAP2 Data Response from an HDF4 data resource";
754 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
763 if (BESISDEBUG( TIMING_LOG ))
764 sw.
start(
"HDF4RequestHandler::hdf4_build_data_with_IDs", dhi.
data[REQUEST_ID]);
769 #ifdef USE_HDFEOS2_LIB
772 HDFEOS2::File *eosfile = NULL;
788 delete bdds->get_dds();
797 hdds->filename(accessed);
806 sdfd = SDstart (
const_cast < char *
>(accessed.c_str()), DFACC_READ);
808 string invalid_file_msg=
"HDF4 SDstart error for the file ";
809 invalid_file_msg +=accessed;
810 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
814 fileid = Hopen(
const_cast<char *
>(accessed.c_str()), DFACC_READ,0);
817 string invalid_file_msg=
"HDF4 Hopen error for the file ";
818 invalid_file_msg +=accessed;
819 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
823 #ifdef USE_HDFEOS2_LIB
829 gridfd = GDopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
833 string invalid_file_msg=
"HDF-EOS GDopen error for the file ";
834 invalid_file_msg +=accessed;
835 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
840 swathfd = SWopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
845 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
846 invalid_file_msg +=accessed;
847 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
851 hdds->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
856 bool ecs_metadata =
true;
858 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
859 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
861 if((
true == _disable_ecsmetadata_min)
862 || (
true == _disable_ecsmetadata_all))
863 ecs_metadata =
false;
865 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
871 Ancillary::read_ancillary_das(*das, accessed);
874 read_dds_use_eos2lib(*hdds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
880 hdds->setHDF4Dataset(sdfd,fileid);
881 read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
882 Ancillary::read_ancillary_das(*das, accessed);
885 read_dds_hdfsp(*hdds, accessed,sdfd,fileid,h4file);
890 Ancillary::read_ancillary_dds(*hdds, accessed);
892 hdds->transfer_attributes(das);
902 #ifdef USE_HDFEOS2_LIB
913 catch (InternalErr & e) {
914 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
918 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
922 #ifdef USE_HDFEOS2_LIB
923 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
925 close_hdf4_fileid(sdfd,fileid,h4file);
927 string s =
"unknown exception caught building DAP2 data response from an HDF4 data resource";
928 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
958 dds->filename(accessed);
965 bool das_set_cache =
false;
966 bool dds_set_cache =
false;
967 bool dds_das_get_cache =
false;
973 string check_enable_mcache_key=
"H4.EnableMetaDataCacheFile";
974 bool turn_on_enable_mcache_key=
false;
975 turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
976 if(
true == turn_on_enable_mcache_key) {
978 if(
true == _enable_metadata_cachefile) {
984 string key =
"H4.Cache.metadata.path";
987 if(
true == _cache_metadata_path_exist) {
990 das_filename = _cache_metadata_path +
"/" + base_file_name +
"_das";
991 dds_filename = _cache_metadata_path +
"/" + base_file_name +
"_dds";
995 das_set_cache = rw_das_cache_file(das_filename,das,
false);
999 dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1004 if((
false == das_set_cache)&&(
false == dds_set_cache))
1005 dds_das_get_cache =
true;
1011 if(
false == dds_das_get_cache) {
1014 sdfd = SDstart (
const_cast < char *
>(accessed.c_str()), DFACC_READ);
1017 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1018 invalid_file_msg +=accessed;
1019 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1026 bool ecs_metadata = !(_disable_ecsmetadata_all);
1028 bool ecs_metadata =
true;
1029 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1030 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1031 ecs_metadata =
false;
1034 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1036 Ancillary::read_ancillary_das(*das, accessed);
1040 read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1043 if(
true == das_set_cache) {
1044 if(das_filename ==
"")
1045 throw InternalErr(__FILE__,__LINE__,
"DAS cache file name should be set ");
1046 rw_das_cache_file(das_filename,das,
true);
1051 Ancillary::read_ancillary_dds(*dds, accessed);
1052 dds->transfer_attributes(das);
1073 catch (InternalErr & e) {
1074 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1075 __FILE__, __LINE__);
1078 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1079 __FILE__, __LINE__);
1086 string s =
"unknown exception caught building HDF4 DataDDS";
1087 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
1109 DAS *das = bdas->get_das();
1115 bool das_set_cache =
false;
1116 bool das_get_cache =
false;
1117 string das_filename;
1119 string check_enable_mcache_key=
"H4.EnableMetaDataCacheFile";
1120 bool turn_on_enable_mcache_key=
false;
1121 turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1122 if(
true == turn_on_enable_mcache_key) {
1124 if(
true == _enable_metadata_cachefile) {
1126 string md_cache_dir;
1127 string key =
"H4.Cache.metadata.path";
1132 if(
true == _cache_metadata_path_exist) {
1135 das_filename = _cache_metadata_path +
"/" + base_file_name +
"_das";
1139 das_set_cache = rw_das_cache_file(das_filename,das,
false);
1142 if(
false == das_set_cache)
1143 das_get_cache =
true;
1148 if(
false == das_get_cache) {
1150 sdfd = SDstart (
const_cast < char *
>(accessed.c_str()), DFACC_READ);
1152 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1153 invalid_file_msg +=accessed;
1154 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1160 bool ecs_metadata =
true;
1161 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1162 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1163 ecs_metadata =
false;
1167 bool ecs_metadata = !(_disable_ecsmetadata_all);
1169 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1170 Ancillary::read_ancillary_das(*das, accessed);
1173 if(
true == das_set_cache)
1174 rw_das_cache_file(das_filename,das,
true);
1191 catch (InternalErr & e) {
1192 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1193 __FILE__, __LINE__);
1196 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1197 __FILE__, __LINE__);
1204 string s =
"unknown exception caught building HDF4 DataDDS";
1205 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
1225 DDS *dds = bdds->get_dds();
1228 dds->filename(accessed);
1241 bool das_set_cache =
false;
1242 bool dds_set_cache =
false;
1243 bool dds_das_get_cache =
false;
1244 string das_filename;
1245 string dds_filename;
1248 string check_enable_mcache_key=
"H4.EnableMetaDataCacheFile";
1249 bool _disable_ecsmetadata_allturn_on_enable_mcache_key=
false;
1250 turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1251 if(
true == turn_on_enable_mcache_key) {
1253 if(
true == _enable_metadata_cachefile) {
1255 if(
true == _cache_metadata_path_exist) {
1256 BESDEBUG(
"h4",
"H4.Cache.metadata.path key is set and metadata cache key is set." << endl);
1261 das_filename = _cache_metadata_path +
"/" + base_file_name +
"_das_dd";
1262 dds_filename = _cache_metadata_path +
"/" + base_file_name +
"_dds";
1265 das_set_cache = rw_das_cache_file(das_filename,das,
false);
1268 dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1270 if((
false == das_set_cache)&&(
false == dds_set_cache))
1271 dds_das_get_cache =
true;
1274 if(
false == dds_das_get_cache) {
1277 sdfd = SDstart (
const_cast < char *
>(accessed.c_str()), DFACC_READ);
1279 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1280 invalid_file_msg +=accessed;
1281 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1289 bool ecs_metadata =
true;
1291 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1292 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1294 if((
true == _disable_ecsmetadata_min)
1295 || (
true == _disable_ecsmetadata_all))
1296 ecs_metadata =
false;
1298 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1300 Ancillary::read_ancillary_das(*das, accessed);
1303 if(
true == das_set_cache) {
1304 rw_das_cache_file(das_filename,das,
true);
1309 read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1313 Ancillary::read_ancillary_dds(*dds, accessed);
1315 dds->transfer_attributes(das);
1331 catch (InternalErr & e) {
1332 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1333 __FILE__, __LINE__);
1336 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1337 __FILE__, __LINE__);
1344 string s =
"unknown exception caught building HDF4 DataDDS";
1345 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
1373 delete bdds->get_dds();
1379 hdds->filename(accessed);
1386 sdfd = SDstart (
const_cast < char *
>(accessed.c_str()), DFACC_READ);
1388 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1389 invalid_file_msg +=accessed;
1390 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1394 hdds->setHDF4Dataset(sdfd,-1);
1399 bool ecs_metadata =
true;
1401 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1402 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1404 if((
true == _disable_ecsmetadata_min)
1405 || (
true == _disable_ecsmetadata_all))
1406 ecs_metadata =
false;
1408 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1413 Ancillary::read_ancillary_das(*das, accessed);
1416 read_dds_sds(*hdds, accessed,sdfd,h4file,
false);
1421 Ancillary::read_ancillary_dds(*hdds, accessed);
1423 hdds->transfer_attributes(das);
1434 catch (InternalErr & e) {
1435 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1436 __FILE__, __LINE__);
1439 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1440 __FILE__, __LINE__);
1447 string s =
"unknown exception caught building HDF4 DataDDS";
1448 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
1458 if (BESISDEBUG( TIMING_LOG ))
1459 sw.
start(
"HDF4RequestHandler::hdf4_build_dmr", dhi.
data[REQUEST_ID]);
1466 BaseTypeFactory factory;
1467 DDS dds(&factory, name_path(data_path),
"3.2");
1468 dds.filename(data_path);
1478 if(
true == _usecf) {
1480 if(
true == _pass_fileid)
1481 return hdf4_build_dmr_with_IDs(dhi);
1488 if (
true == _usecf) {
1494 sdfd = SDstart (
const_cast < char *
>(data_path.c_str()), DFACC_READ);
1496 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1497 invalid_file_msg +=data_path;
1498 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1503 fileid = Hopen(
const_cast<char *
>(data_path.c_str()), DFACC_READ,0);
1506 string invalid_file_msg=
"HDF4 Hopen error for the file ";
1507 invalid_file_msg +=data_path;
1508 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1512 #ifdef USE_HDFEOS2_LIB
1517 HDFEOS2::File *eosfile = NULL;
1521 gridfd = GDopen(
const_cast < char *
>(data_path.c_str()), DFACC_READ);
1525 string invalid_file_msg=
"HDF-EOS GDopen error for the file ";
1526 invalid_file_msg +=data_path;
1527 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
1532 swathfd = SWopen(
const_cast < char *
>(data_path.c_str()), DFACC_READ);
1533 if (-1 == swathfd) {
1537 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
1538 invalid_file_msg +=data_path;
1539 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
1550 bool ecs_metadata =
true;
1552 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1553 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1555 if((
true == _disable_ecsmetadata_min)
1556 || (
true == _disable_ecsmetadata_all))
1557 ecs_metadata =
false;
1561 read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1565 Ancillary::read_ancillary_das(das, data_path);
1568 read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1572 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1582 read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1583 Ancillary::read_ancillary_das(das, data_path);
1586 read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1589 close_hdf4_fileid(sdfd,fileid,h4file);
1594 close_hdf4_fileid(sdfd,fileid,h4file);
1598 read_das(das, data_path);
1599 Ancillary::read_ancillary_das(das, data_path);
1600 read_dds(dds, data_path);
1604 Ancillary::read_ancillary_dds(dds, data_path);
1606 dds.transfer_attributes(&das);
1610 catch (InternalErr & e) {
1611 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1612 __FILE__, __LINE__);
1615 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1616 __FILE__, __LINE__);
1619 string s =
"unknown exception caught building HDF4 DataDDS";
1620 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
1633 DMR *dmr = bes_dmr.get_dmr();
1635 D4BaseTypeFactory MyD4TypeFactory;
1636 dmr->set_factory(&MyD4TypeFactory);
1640 dmr->build_using_dds(dds);
1653 dmr->set_factory(0);
1661 if (BESISDEBUG( TIMING_LOG ))
1662 sw.
start(
"HDF4RequestHandler::hdf4_build_dmr_with_IDs", dhi.
data[REQUEST_ID]);
1669 BaseTypeFactory factory;
1670 DDS dds(&factory, name_path(data_path),
"3.2");
1671 dds.filename(data_path);
1682 sdfd = SDstart (
const_cast < char *
>(data_path.c_str()), DFACC_READ);
1684 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1685 invalid_file_msg +=data_path;
1686 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1691 fileid = Hopen(
const_cast<char *
>(data_path.c_str()), DFACC_READ,0);
1694 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1695 invalid_file_msg +=data_path;
1696 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1701 #ifdef USE_HDFEOS2_LIB
1706 HDFEOS2::File *eosfile = NULL;
1709 gridfd = GDopen(
const_cast < char *
>(data_path.c_str()), DFACC_READ);
1713 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1714 invalid_file_msg +=data_path;
1715 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1721 swathfd = SWopen(
const_cast < char *
>(data_path.c_str()), DFACC_READ);
1722 if (-1 == swathfd) {
1726 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
1727 invalid_file_msg +=data_path;
1728 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
1739 bool ecs_metadata =
true;
1741 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1742 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1744 if((
true == _disable_ecsmetadata_min)
1745 || (
true == _disable_ecsmetadata_all))
1746 ecs_metadata =
false;
1750 read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1754 Ancillary::read_ancillary_das(das, data_path);
1757 read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1761 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1769 read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1770 Ancillary::read_ancillary_das(das, data_path);
1773 read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1776 close_hdf4_fileid(sdfd,fileid,h4file);
1783 Ancillary::read_ancillary_dds(dds, data_path);
1785 dds.transfer_attributes(&das);
1791 #ifdef USE_HDFEOS2_LIB
1812 DMR *dmr = bes_dmr.get_dmr();
1813 D4BaseTypeFactory MyD4TypeFactory;
1814 dmr->set_factory(&MyD4TypeFactory);
1815 dmr->build_using_dds(dds);
1817 #ifdef USE_HDFEOS2_LIB
1818 hdf4_dmr->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
1820 hdf4_dmr->setHDF4Dataset(sdfd,fileid);
1823 bes_dmr.set_dmr(hdf4_dmr);
1832 hdf4_dmr->set_factory(0);
1844 map < string, string > attrs;
1845 attrs[
"name"] = MODULE_NAME ;
1846 attrs[
"version"] = MODULE_VERSION ;
1847 list < string > services;
1849 if (services.size() > 0) {
1851 attrs[
"handles"] = handles;
1853 info->begin_tag(
"module", &attrs);
1854 info->end_tag(
"module");
1865 info->add_module(MODULE_NAME, MODULE_VERSION);
1870 #ifdef USE_HDFEOS2_LIB
1889 void close_hdf4_fileid(
int sdfd,
int fileid,
HDFSP::File*h4file) {
1903 bool rw_das_cache_file(
const string & filename, DAS *das_ptr,
bool w_flag) {
1905 bool das_set_cache =
false;
1906 FILE *das_file = NULL;
1909 das_file = fopen(filename.c_str(),
"r");
1911 das_file = fopen(filename.c_str(),
"w");
1913 if(NULL == das_file) {
1914 if(ENOENT == errno) {
1917 if(
false == w_flag) {
1918 BESDEBUG(
"h4",
"DAS set cache key is true." << endl);
1919 das_set_cache =
true;
1923 throw BESInternalError(
"An error occurred trying to open a das cache file " + get_errno(), __FILE__, __LINE__);
1927 int fd_das = fileno(das_file);
1930 struct flock *l_das;
1932 l_das = lock(F_RDLCK);
1934 l_das = lock(F_WRLCK);
1937 if(fcntl(fd_das,F_SETLKW,l_das) == -1) {
1940 oss <<
"cache process: " << l_das->l_pid <<
" triggered a locking error: " << get_errno();
1944 if(
false == w_flag){
1946 BESDEBUG(
"h4",
"Obtaining DAS from the cache file" << endl);
1948 das_ptr->parse(das_file);
1951 if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1953 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1956 throw InternalErr(__FILE__,__LINE__,
"Fail to parse the das from a das file.");
1961 BESDEBUG(
"h4",
"write DAS to a cache file" << endl);
1963 das_ptr->print(das_file);
1966 if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1968 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1971 throw InternalErr(__FILE__,__LINE__,
"Fail to generate a das cache file.");
1977 if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1979 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1985 return das_set_cache;
1990 bool r_dds_cache_file(
const string & cache_filename, DDS *dds_ptr,
const string & hdf4_filename) {
1992 bool dds_set_cache =
false;
1993 FILE *dds_file = NULL;
1994 dds_file = fopen(cache_filename.c_str(),
"rb");
1996 if(NULL == dds_file) {
1997 if(ENOENT == errno) {
2001 dds_set_cache =
true;
2004 throw BESInternalError(
"An error occurred trying to open a dds cache file " + get_errno(), __FILE__, __LINE__);
2008 int fd_dds = fileno(dds_file);
2009 struct flock *l_dds;
2010 l_dds = lock(F_RDLCK);
2013 if(fcntl(fd_dds,F_SETLKW,l_dds) == -1) {
2016 oss <<
"cache process: " << l_dds->l_pid <<
" triggered a locking error: " << get_errno();
2021 HDFCFUtil::read_sp_sds_dds_cache(dds_file,dds_ptr,cache_filename,hdf4_filename);
2024 if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2026 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2030 throw InternalErr(__FILE__,__LINE__,
"Fail to generate a dds cache file.");
2033 if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2035 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2042 return dds_set_cache;
2046 bool check_beskeys(
const string key) {
2050 const string dosettrue =
"true";
2051 const string dosetyes =
"yes";
2054 if(
true == found ) {
2056 if( dosettrue == doset || dosetyes == doset )
2063 bool get_beskeys(
const string key,
string &key_value) {
2073 int get_cachekey_int(
const string key) {
2080 if(
true == found ) {
2081 istringstream iss(doset);
2090 cerr<<
"OK to pass pointer of a NULL pointer "<<endl;