bes  Updated for version 3.20.6
HDF4RequestHandler.cc
1 
2 // -*- mode: c++; c-basic-offset:4 -*-
3 
4 // This file is part of hdf4_handler, a data handler for the OPeNDAP data
5 // server.
6 
7 // Copyright (c) 2002,2003 OPeNDAP, Inc.
8 // Author: James Gallagher <jgallagher@opendap.org>
9 // Author: Muqun Yang <myang6@opendap.org>
10 // This is free software; you can redistribute it and/or modify it under the
11 // terms of the GNU Lesser General Public License as published by the Free
12 // Software Foundation; either version 2.1 of the License, or (at your
13 // option) any later version.
14 //
15 // This software is distributed in the hope that it will be useful, but
16 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
17 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
18 // License for more details.
19 //
20 // You should have received a copy of the GNU Lesser General Public
21 // License along with this library; if not, write to the Free Software
22 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 //
24 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
25 
26 // HDF4RequestHandler.cc
27 
28 #include <cstdlib>
29 #include <sys/types.h>
30 #include <sys/stat.h>
31 #include <fcntl.h>
32 #include <cerrno>
33 #include <unistd.h>
34 #include <iostream>
35 #include <sstream>
36 
37 #include <DMR.h>
38 #include <D4BaseTypeFactory.h>
39 #include <BESDMRResponse.h>
40 #include <mime_util.h>
41 #include <InternalErr.h>
42 #include <Ancillary.h>
43 #include <debug.h>
44 
45 #include "HDF4RequestHandler.h"
46 #include <BESResponseNames.h>
47 #include <BESDapNames.h>
48 #include <BESDASResponse.h>
49 #include <BESDDSResponse.h>
50 #include <BESDataDDSResponse.h>
51 #include <BESInfo.h>
52 #include <BESResponseHandler.h>
53 #include <BESVersionInfo.h>
54 #include <BESServiceRegistry.h>
55 #include <BESUtil.h>
56 #include <TheBESKeys.h>
57 #include <InternalErr.h>
58 #include <BESInternalError.h>
59 #include <BESDapError.h>
60 #include <BESStopWatch.h>
61 #include <BESDebug.h>
62 #include "BESDataNames.h"
63 //#include <ConstraintEvaluator.h>
64 #include <Ancillary.h>
65 #include "config_hdf.h"
66 
67 #define HDF4_NAME "h4"
68 #include "HE2CF.h"
69 #include "HDF4_DDS.h"
70 
71 #include "HDF4_DMR.h"
72 
73 //#include "HDFCFUtil.h"
74 #include "HDFFloat32.h"
75 #include "HDFSPArray_RealField.h"
76 
77 #include "dodsutil.h"
78 //#if 0
79 #include <sys/time.h>
80 //#endif
81 
82 using namespace std;
83 using namespace libdap;
84 
85 bool check_beskeys(const string);
86 bool get_beskeys(const string,string &);
87 
88 extern void read_das(DAS & das, const string & filename);
89 extern void read_dds(DDS & dds, const string & filename);
90 
91 extern bool read_dds_hdfsp(DDS & dds, const string & filename,int32 sdfd, int32 fileid,HDFSP::File*h4file);
92 
93 extern bool read_das_hdfsp(DAS & das, const string & filename,int32 sdfd, int32 fileid,HDFSP::File**h4fileptr);
94 
95 extern void read_das_sds(DAS & das, const string & filename,int32 sdfd, bool ecs_metadata,HDFSP::File**h4fileptr);
96 extern void read_dds_sds(DDS &dds, const string & filename,int32 sdfd, HDFSP::File*h4file,bool dds_set_cache);
97 
98 #ifdef USE_HDFEOS2_LIB
99 
100 void read_das_use_eos2lib(DAS & das, const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,bool ecs_metadata,HDFSP::File**h4file,HDFEOS2::File**eosfile);
101 void read_dds_use_eos2lib(DDS & dds, const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,HDFSP::File*h4file,HDFEOS2::File*eosfile);
102 void close_fileid(const int sdfd, const int fileid,const int gridfd, const int swathfd,HDFSP::File*h4file,HDFEOS2::File*eosfile);
103 
104 #endif
105 
106 void close_hdf4_fileid(const int sdfd,const int fileid,HDFSP::File*h4file);
107 bool rw_das_cache_file(const string & filename, DAS *das_ptr,bool rw_flag);
108 bool r_dds_cache_file(const string & cache_filename, DDS *dds_ptr,const string & hdf4_filename);
109 
110 // CF key
111 bool HDF4RequestHandler::_usecf = false;
112 
113 // Keys to tune the performance -general
114 bool HDF4RequestHandler::_pass_fileid = false;
115 bool HDF4RequestHandler::_disable_structmeta = false;
116 bool HDF4RequestHandler::_enable_special_eos = false;
117 bool HDF4RequestHandler::_disable_scaleoffset_comp = false;
118 bool HDF4RequestHandler::_disable_ecsmetadata_min = false;
119 bool HDF4RequestHandler::_disable_ecsmetadata_all = false;
120 
121 
122 // Keys to tune the performance - cache
123 bool HDF4RequestHandler::_enable_eosgeo_cachefile = false;
124 bool HDF4RequestHandler::_enable_data_cachefile = false;
125 bool HDF4RequestHandler::_enable_metadata_cachefile= false;
126 
127 // Keys to handle vdata and vgroups
128 bool HDF4RequestHandler::_enable_hybrid_vdata = false;
129 bool HDF4RequestHandler::_enable_ceres_vdata = false;
130 bool HDF4RequestHandler::_enable_vdata_attr = false;
131 bool HDF4RequestHandler::_enable_vdata_desc_attr = false;
132 bool HDF4RequestHandler::_disable_vdata_nameclashing_check = false;
133 bool HDF4RequestHandler::_enable_vgroup_attr = false;
134 
135 // Misc. keys
136 bool HDF4RequestHandler::_enable_check_modis_geo_file = false;
137 bool HDF4RequestHandler::_enable_swath_grid_attr = false;
138 bool HDF4RequestHandler::_enable_ceres_merra_short_name = false;
139 bool HDF4RequestHandler::_enable_check_scale_offset_type = false;
140 
141 // Cache path,prefix and size
142 bool HDF4RequestHandler::_cache_latlon_path_exist =false;
143 string HDF4RequestHandler::_cache_latlon_path ="";
144 bool HDF4RequestHandler::_cache_latlon_prefix_exist =false;
145 string HDF4RequestHandler::_cache_latlon_prefix ="";
146 bool HDF4RequestHandler::_cache_latlon_size_exist =false;
147 long HDF4RequestHandler::_cache_latlon_size =0;
148 bool HDF4RequestHandler::_cache_metadata_path_exist =false;
149 string HDF4RequestHandler::_cache_metadata_path ="";
150 
151 HDF4RequestHandler::HDF4RequestHandler(const string & name) :
152  BESRequestHandler(name) {
153  add_handler(DAS_RESPONSE, HDF4RequestHandler::hdf4_build_das);
154  add_handler(DDS_RESPONSE, HDF4RequestHandler::hdf4_build_dds);
155  add_handler(DATA_RESPONSE, HDF4RequestHandler::hdf4_build_data);
156  add_handler(DMR_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
157  add_handler(DAP4DATA_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
158  add_handler(HELP_RESPONSE, HDF4RequestHandler::hdf4_build_help);
159  add_handler(VERS_RESPONSE, HDF4RequestHandler::hdf4_build_version);
160 
161  _usecf = check_beskeys("H4.EnableCF");
162 
163  // The following keys are only effective when usecf is true.
164  // Keys to tune the performance -general
165  _pass_fileid = check_beskeys("H4.EnablePassFileID");
166  _disable_structmeta = check_beskeys("H4.DisableStructMetaAttr");
167  _enable_special_eos = check_beskeys("H4.EnableSpecialEOS");
168  _disable_scaleoffset_comp = check_beskeys("H4.DisableScaleOffsetComp");
169  _disable_ecsmetadata_min = check_beskeys("H4.DisableECSMetaDataMin");
170  _disable_ecsmetadata_all = check_beskeys("H4.DisableECSMetaDataAll");
171 
172 
173  // Keys to tune the performance - cache
174  _enable_eosgeo_cachefile = check_beskeys("H4.EnableEOSGeoCacheFile");
175  _enable_data_cachefile = check_beskeys("H4.EnableDataCacheFile");
176  _enable_metadata_cachefile = check_beskeys("H4.EnableMetaDataCacheFile");
177 
178  // Keys to handle vdata and vgroups
179  _enable_hybrid_vdata = check_beskeys("H4.EnableHybridVdata");
180  _enable_ceres_vdata = check_beskeys("H4.EnableCERESVdata");
181  _enable_vdata_attr = check_beskeys("H4.EnableVdata_to_Attr");
182  _enable_vdata_desc_attr = check_beskeys("H4.EnableVdataDescAttr");
183  _disable_vdata_nameclashing_check = check_beskeys("H4.DisableVdataNameclashingCheck");
184  _enable_vgroup_attr = check_beskeys("H4.EnableVgroupAttr");
185 
186  // Misc. keys
187  _enable_check_modis_geo_file = check_beskeys("H4.EnableCheckMODISGeoFile");
188  _enable_swath_grid_attr = check_beskeys("H4.EnableSwathGridAttr");
189  _enable_ceres_merra_short_name = check_beskeys("H4.EnableCERESMERRAShortName");
190  _enable_check_scale_offset_type = check_beskeys("H4.EnableCheckScaleOffsetType");
191 
192  // Cache path etc.
193  _cache_latlon_path_exist =get_beskeys("HDF4.Cache.latlon.path",_cache_latlon_path);
194  _cache_latlon_prefix_exist =get_beskeys("HDF4.Cache.latlon.prefix",_cache_latlon_prefix);
195  string temp_cache_latlon_size;
196  _cache_latlon_size_exist =get_beskeys("HDF4.Cache.latlon.size",temp_cache_latlon_size);
197  if(_cache_latlon_size_exist == true) {
198  istringstream iss(temp_cache_latlon_size);
199  iss >> _cache_latlon_size;
200  }
201 
202  _cache_metadata_path_exist =get_beskeys("H4.Cache.metadata.path",_cache_metadata_path);
203 
204 }
205 
206 HDF4RequestHandler::~HDF4RequestHandler() {
207 }
208 
209 bool HDF4RequestHandler::hdf4_build_das(BESDataHandlerInterface & dhi) {
210 
211 
212  BESStopWatch sw;
213  if (BESISDEBUG( TIMING_LOG ))
214  sw.start("HDF4RequestHandler::hdf4_build_das", dhi.data[REQUEST_ID]);
215 
216  if(true == _usecf) {
217 
218  // Build DAP response only based on the HDF4 SD interfaces. Doing this
219  // way will save the use of other library open calls. Other library open
220  // calls may be expensive
221  // for an HDF4 file that only has variables created by SD interfaces.
222  // This optimization may be very useful for the aggreagation case that
223  // has many variables.
224  // Currently we only handle AIRS version 6 products. AIRS products
225  // are identified by their file names.
226  // We only obtain the filename. The path is stripped off.
227 
228  string base_file_name = basename(dhi.container->access());
229 
230  //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
231  // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
232  if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
233  && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
234  && (base_file_name.find(".v6.")!=string::npos)) {
235  return hdf4_build_das_cf_sds(dhi);
236 
237  }
238  }
239 
240  BESResponseObject *response = dhi.response_handler->get_response_object();
241  BESDASResponse *bdas = dynamic_cast<BESDASResponse *> (response);
242  if (!bdas)
243  throw BESInternalError("cast error", __FILE__, __LINE__);
244 
245  try {
247  DAS *das = bdas->get_das();
248 
249  string base_file_name = basename(dhi.container->access());
250 
251  string accessed = dhi.container->access();
252 
253  if (true == _usecf) {
254 
255  int32 sdfd = -1;
256  int32 fileid = -1;
257 
258  HDFSP::File *h4file = NULL;
259 
260  // Obtain HDF4 file IDs
261  //SDstart
262  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
263  if( -1 == sdfd){
264  string invalid_file_msg="HDF4 SDstart error for the file ";
265  invalid_file_msg +=accessed;
266  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
267  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
268  }
269 
270  // H open
271  fileid = Hopen(const_cast<char *>(accessed.c_str()), DFACC_READ,0);
272  if (-1 == fileid) {
273  SDend(sdfd);
274  string invalid_file_msg="HDF4 Hopen error for the file ";
275  invalid_file_msg +=accessed;
276  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
277  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
278  }
279 
280 #ifdef USE_HDFEOS2_LIB
281 
282  int32 gridfd = -1;
283  int32 swathfd = -1;
284  HDFEOS2::File *eosfile = NULL;
285  // Obtain HDF-EOS2 file IDs with the file open APIs.
286 
287  // Grid open
288  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
289  if (-1 == gridfd) {
290  SDend(sdfd);
291  Hclose(fileid);
292  string invalid_file_msg="HDF-EOS GDopen error for the file ";
293  invalid_file_msg +=accessed;
294  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
295  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
296  }
297 
298  // Swath open
299  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
300  if (-1 == swathfd) {
301  SDend(sdfd);
302  Hclose(fileid);
303  GDclose(gridfd);
304  string invalid_file_msg="HDF-EOS SWopen error for the file ";
305  invalid_file_msg +=accessed;
306  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
307  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
308  }
309 
310  try {
311  bool ecs_metadata = !(_disable_ecsmetadata_all);
312 #if 0
313 if(ecs_metadata == true)
314 cerr<<"output ecs metadata "<<endl;
315 else
316 cerr<<"Don't output ecs metadata "<<endl;
317 #endif
318  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
319  }
320  catch(...) {
321  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
322  throw;
323 #if 0
324  //throw InternalErr(__FILE__,__LINE__,"read_das_use_eos2lib error");
325 #endif
326  }
327  if(eosfile != NULL)
328  delete eosfile;
329  GDclose(gridfd);
330  SWclose(swathfd);
331 
332 #else
333  try {
334  read_das_hdfsp(*das,accessed,sdfd,fileid,&h4file);
335  }
336  catch(...) {
337  close_hdf4_fileid(sdfd,fileid,h4file);
338  throw;
339  //throw InternalErr(__FILE__,__LINE__,"read_das_hdfsp error");
340  }
341 #endif
342  close_hdf4_fileid(sdfd,fileid,h4file);
343  }
344  else
345  read_das(*das,accessed);
346 
347  Ancillary::read_ancillary_das(*das, accessed);
348  bdas->clear_container();
349  }
350 
351  catch (BESError & e) {
352  throw;
353  }
354  catch (InternalErr & e) {
355  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
356  __FILE__, __LINE__);
357  }
358  catch (Error & e) {
359  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
360  __FILE__, __LINE__);
361  }
362  catch (...) {
363  string s = "unknown exception caught building HDF4 DAS";
364  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
365  }
366 
367  return true;
368 }
369 
370 bool HDF4RequestHandler::hdf4_build_dds(BESDataHandlerInterface & dhi) {
371 
372 
373 
374  BESStopWatch sw;
375  if (BESISDEBUG( TIMING_LOG ))
376  sw.start("HDF4RequestHandler::hdf4_build_das", dhi.data[REQUEST_ID]);
377 
378 
379  if(true == _usecf) {
380  // Build DAP response only based on the HDF4 SD interfaces. Doing this
381  // way will save the use of other library open calls. Other library open
382  // calls may be expensive
383  // for an HDF4 file that only has variables created by SD interfaces.
384  // This optimization may be very useful for the aggreagation case that
385  // has many variables.
386  // Currently we only handle AIRS version 6 products. AIRS products
387  // are identified by their file names.
388  // We only obtain the filename. The path is stripped off.
389 
390  string base_file_name = basename(dhi.container->access());
391 
392  //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
393  // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
394  if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
395  && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
396  && (base_file_name.find(".v6.")!=string::npos)) {
397  return hdf4_build_dds_cf_sds(dhi);
398 
399  }
400  }
401 
402 // This is for the performance check. Leave it now for future use. KY 2014-10-23
403 #ifdef KENT2
404 struct timeval start_time,end_time;
405 gettimeofday(&start_time,NULL);
406 #endif
407 
408  BESResponseObject *response = dhi.response_handler->get_response_object();
409  BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *> (response);
410  if (!bdds)
411  throw BESInternalError("cast error", __FILE__, __LINE__);
412 
413  try {
415  DDS *dds = bdds->get_dds();
416 #if 0
417  //ConstraintEvaluator & ce = bdds->get_ce();
418 #endif
419 
420  string accessed = dhi.container->access();
421  dds->filename(accessed);
422 
423  DAS *das = new DAS;
424  BESDASResponse bdas(das);
426 
427  if (true == _usecf) {
428 
429  int32 sdfd = -1;
430  int32 fileid = -1;
431  HDFSP::File *h4file = NULL;
432 
433  // Obtain HDF4 file IDs
434  //SDstart
435  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
436  if( -1 == sdfd){
437  string invalid_file_msg="HDF4 SDstart error for the file ";
438  invalid_file_msg +=accessed;
439  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
440  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
441  }
442 
443  // H open
444  fileid = Hopen(const_cast<char *>(accessed.c_str()), DFACC_READ,0);
445  if (-1 == fileid) {
446  SDend(sdfd);
447  string invalid_file_msg="HDF4 Hopen error for the file ";
448  invalid_file_msg +=accessed;
449  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
450  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
451  }
452 
453 #ifdef USE_HDFEOS2_LIB
454 
455  int32 gridfd = -1;
456  int32 swathfd = -1;
457 
458  HDFEOS2::File *eosfile = NULL;
459 
460  // Obtain HDF-EOS2 file IDs with the file open APIs.
461  // Grid open
462  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
463  if (-1 == gridfd) {
464  SDend(sdfd);
465  Hclose(fileid);
466  string invalid_file_msg="HDF-EOS GDopen error for the file ";
467  invalid_file_msg +=accessed;
468  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
469  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
470  }
471 
472  // Swath open
473  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
474  if (-1 == swathfd) {
475  SDend(sdfd);
476  Hclose(fileid);
477  GDclose(gridfd);
478  string invalid_file_msg="HDF-EOS SWopen error for the file ";
479  invalid_file_msg +=accessed;
480  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
481  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
482  }
483 
484  try {
485  bool ecs_metadata = !(_disable_ecsmetadata_all);
486  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
487  Ancillary::read_ancillary_das(*das, accessed);
488 
489  // Pass file pointer(h4file, eosfile) from DAS to DDS.
490  read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
491  }
492  catch(...) {
493  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
494  throw;
495  }
496 
497  if(eosfile != NULL)
498  delete eosfile;
499 
500  GDclose(gridfd);
501  SWclose(swathfd);
502 
503 #else
504  try {
505  read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
506  Ancillary::read_ancillary_das(*das, accessed);
507 
508  // Pass file pointer(h4file) from DAS to DDS.
509  read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
510  }
511  catch(...) {
512  close_hdf4_fileid(sdfd,fileid,h4file);
513  throw;
514  }
515 
516 #endif
517  close_hdf4_fileid(sdfd,fileid,h4file);
518  }
519  else {
520  read_das(*das, accessed);
521  Ancillary::read_ancillary_das(*das, accessed);
522  read_dds(*dds, accessed);
523  }
524 
525 // Leave it for future performance tests. KY 2014-10-23
526 #ifdef KENT2
527 gettimeofday(&end_time,NULL);
528 int total_time_spent = (end_time.tv_sec - start_time.tv_sec)*1000000 +end_time.tv_usec-start_time.tv_usec;
529 cerr<<"total time spent for DDS buld is "<<total_time_spent<< "micro seconds "<<endl;
530 #endif
531 
532  Ancillary::read_ancillary_dds(*dds, accessed);
533 
534  dds->transfer_attributes(das);
535 
536  bdds->set_constraint(dhi);
537 
538  bdds->clear_container();
539  }
540  catch (BESError & e) {
541  throw;
542  }
543  catch (InternalErr & e) {
544  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
545  __FILE__, __LINE__);
546  }
547  catch (Error & e) {
548  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
549  __FILE__, __LINE__);
550  }
551  catch (...) {
552  string s = "unknown exception caught building HDF4 DDS";
553  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
554  }
555 
556  return true;
557 }
558 
559 bool HDF4RequestHandler::hdf4_build_data(BESDataHandlerInterface & dhi) {
560 
561 
562  BESStopWatch sw;
563  if (BESISDEBUG( TIMING_LOG ))
564  sw.start("HDF4RequestHandler::hdf4_build_data", dhi.data[REQUEST_ID]);
565 
566 
567  int32 sdfd = -1;
568  int32 fileid = -1;
569 
570  // Since passing file IDs requires to use the derived class and it
571  // causes the management of code structure messy, we first handle this with
572  // another method.
573  if(true == _usecf) {
574  // Build DAP response only based on the HDF4 SD interfaces. Doing this
575  // way will save the use of other library open calls. Other library open
576  // calls may be expensive
577  // for an HDF4 file that only has variables created by SD interfaces.
578  // This optimization may be very useful for the aggreagation case that
579  // has many variables.
580  // Currently we only handle AIRS version 6 products. AIRS products
581  // are identified by their file names.
582  // We only obtain the filename. The path is stripped off.
583 
584  string base_file_name = basename(dhi.container->access());
585  //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
586  // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
587  if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
588  && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
589  && (base_file_name.find(".v6.")!=string::npos)) {
590 
591  BESDEBUG("h4", "Coming to read the data of AIRS level 3 or level 2 products." << endl);
592 
593  if(true == _pass_fileid)
594  return hdf4_build_data_cf_sds_with_IDs(dhi);
595  else
596  return hdf4_build_data_cf_sds(dhi);
597 
598  }
599 
600  if(true == _pass_fileid)
601  return hdf4_build_data_with_IDs(dhi);
602 
603  }
604 
605  BESResponseObject *response = dhi.response_handler->get_response_object();
606  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *> (response);
607 
608  if (!bdds)
609  throw BESInternalError("cast error", __FILE__, __LINE__);
610 
611  try {
613 
614  DDS *dds = bdds->get_dds();
615 
616  // Not sure why keep the following line, it is not used.
617  //ConstraintEvaluator & ce = bdds->get_ce();
618 
619  string accessed = dhi.container->access();
620  dds->filename(accessed);
621 
622  DAS *das = new DAS;
623  BESDASResponse bdas(das);
625 
626  if (true == _usecf) {
627 
628  HDFSP::File *h4file = NULL;
629 
630  // Obtain HDF4 file IDs
631  //SDstart
632  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
633  if( -1 == sdfd) {
634  string invalid_file_msg="HDF4 SDstart error for the file ";
635  invalid_file_msg +=accessed;
636  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
637  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
638  }
639 
640  // H open
641  fileid = Hopen(const_cast<char *>(accessed.c_str()), DFACC_READ,0);
642  if (-1 == fileid) {
643  SDend(sdfd);
644  string invalid_file_msg="HDF4 Hopen error for the file ";
645  invalid_file_msg +=accessed;
646  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
647  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
648  }
649 
650 
651 #ifdef USE_HDFEOS2_LIB
652 
653  int32 gridfd = -1;
654  int32 swathfd = -1;
655  HDFEOS2::File *eosfile = NULL;
656  // Obtain HDF-EOS2 file IDs with the file open APIs.
657 
658  // Grid open
659  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
660  if (-1 == gridfd) {
661  SDend(sdfd);
662  Hclose(fileid);
663  string invalid_file_msg="HDF-EOS GDopen error for the file ";
664  invalid_file_msg +=accessed;
665  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
666  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
667  }
668 
669  // Swath open
670  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
671  if (-1 == swathfd) {
672  SDend(sdfd);
673  Hclose(fileid);
674  GDclose(gridfd);
675  string invalid_file_msg="HDF-EOS SWopen error for the file ";
676  invalid_file_msg +=accessed;
677  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
678  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
679 
680  }
681 
682  try {
683 
684  // Here we will check if ECS_Metadata key if set. For DataDDS,
685  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
686  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
687  bool ecs_metadata = true;
688 #if 0
689  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
690  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
691  ecs_metadata = false;
692 #endif
693  if((true == _disable_ecsmetadata_min)
694  || (true == _disable_ecsmetadata_all))
695  ecs_metadata = false;
696 
697  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
698  Ancillary::read_ancillary_das(*das, accessed);
699 
700  // Pass file pointer(h4file, eosfile) from DAS to DDS.
701  read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
702  }
703  catch(...) {
704  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
705  throw;
706  }
707 
708  if(eosfile != NULL)
709  delete eosfile;
710  GDclose(gridfd);
711  SWclose(swathfd);
712 
713 #else
714  try {
715  read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
716  Ancillary::read_ancillary_das(*das, accessed);
717 
718  // Pass file pointer(h4file) from DAS to DDS.
719  read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
720  }
721  catch(...) {
722  close_hdf4_fileid(sdfd,fileid,h4file);
723  throw;
724  }
725 #endif
726  close_hdf4_fileid(sdfd,fileid,h4file);
727  }
728  else {
729  read_das(*das, accessed);
730  Ancillary::read_ancillary_das(*das, accessed);
731  read_dds(*dds, accessed);
732  }
733 
734  Ancillary::read_ancillary_dds(*dds, accessed);
735  dds->transfer_attributes(das);
736  bdds->set_constraint(dhi);
737  bdds->clear_container();
738 
739  }
740 
741  catch (BESError & e) {
742  throw;
743  }
744  catch (InternalErr & e) {
745  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
746  __FILE__, __LINE__);
747  }
748  catch (Error & e) {
749  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
750  __FILE__, __LINE__);
751  }
752  catch (...) {
753  string s = "unknown exception caught building DAP2 Data Response from an HDF4 data resource";
754  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
755  }
756 
757  return true;
758 }
759 
760 bool HDF4RequestHandler::hdf4_build_data_with_IDs(BESDataHandlerInterface & dhi) {
761 
762  BESStopWatch sw;
763  if (BESISDEBUG( TIMING_LOG ))
764  sw.start("HDF4RequestHandler::hdf4_build_data_with_IDs", dhi.data[REQUEST_ID]);
765 
766  int32 sdfd = -1;
767  int32 fileid = -1;
768  HDFSP::File *h4file = NULL;
769 #ifdef USE_HDFEOS2_LIB
770  int32 gridfd = -1;
771  int32 swathfd = -1;
772  HDFEOS2::File *eosfile = NULL;
773 #endif
774 
775  BESResponseObject *response = dhi.response_handler->get_response_object();
776  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *> (response);
777 
778  if (!bdds)
779  throw BESInternalError("cast error", __FILE__, __LINE__);
780 
781  try {
783 
784  // Create a new HDF4DDS object.
785  HDF4DDS *hdds = new HDF4DDS(bdds->get_dds());
786 
787  // delete the old object.
788  delete bdds->get_dds();
789 
790  bdds->set_dds(hdds);
791 
792 #if 0
793  //ConstraintEvaluator & ce = bdds->get_ce();
794 #endif
795 
796  string accessed = dhi.container->access();
797  hdds->filename(accessed);
798 
799  DAS *das = new DAS;
800  BESDASResponse bdas(das);
802 
803 
804  // Obtain HDF4 file IDs
805  //SDstart
806  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
807  if( -1 == sdfd) {
808  string invalid_file_msg="HDF4 SDstart error for the file ";
809  invalid_file_msg +=accessed;
810  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
811  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
812  }
813  // H open
814  fileid = Hopen(const_cast<char *>(accessed.c_str()), DFACC_READ,0);
815  if (-1 == fileid) {
816  SDend(sdfd);
817  string invalid_file_msg="HDF4 Hopen error for the file ";
818  invalid_file_msg +=accessed;
819  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
820  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
821  }
822 
823 #ifdef USE_HDFEOS2_LIB
824 
825 
826  // Obtain HDF-EOS2 file IDs with the file open APIs.
827 
828  // Grid open
829  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
830  if (-1 == gridfd) {
831  SDend(sdfd);
832  Hclose(fileid);
833  string invalid_file_msg="HDF-EOS GDopen error for the file ";
834  invalid_file_msg +=accessed;
835  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
836  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
837  }
838 
839  // Swath open
840  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
841  if (-1 == swathfd) {
842  SDend(sdfd);
843  Hclose(fileid);
844  GDclose(gridfd);
845  string invalid_file_msg="HDF-EOS SWopen error for the file ";
846  invalid_file_msg +=accessed;
847  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
848  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
849  }
850 
851  hdds->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
852 
853  // Here we will check if ECS_Metadata key if set. For DataDDS,
854  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
855  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
856  bool ecs_metadata = true;
857 #if 0
858  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
859  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
860 #endif
861  if((true == _disable_ecsmetadata_min)
862  || (true == _disable_ecsmetadata_all))
863  ecs_metadata = false;
864 
865  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
866 
867 #if 0
868  //read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,true);
869 #endif
870 
871  Ancillary::read_ancillary_das(*das, accessed);
872 
873  // Pass file pointer(h4file, eosfile) from DAS to DDS.
874  read_dds_use_eos2lib(*hdds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
875 
876  if(eosfile != NULL)
877  delete eosfile;
878 
879 #else
880  hdds->setHDF4Dataset(sdfd,fileid);
881  read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
882  Ancillary::read_ancillary_das(*das, accessed);
883 
884  // Pass file pointer(h4file) from DAS to DDS.
885  read_dds_hdfsp(*hdds, accessed,sdfd,fileid,h4file);
886 #endif
887  if(h4file != NULL)
888  delete h4file;
889 
890  Ancillary::read_ancillary_dds(*hdds, accessed);
891 
892  hdds->transfer_attributes(das);
893 
894  bdds->set_constraint(dhi);
895 
896  bdds->clear_container();
897 
898 
899 // File IDs are closed by the derived class.
900 #if 0
901  if(true == usecf) {
902 #ifdef USE_HDFEOS2_LIB
903  GDclose(gridfd);
904  SWclose(swathfd);
905 
906 #endif
907  SDend(sdfd);
908  Hclose(fileid);
909  }
910 #endif
911  }
912 
913  catch (InternalErr & e) {
914  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
915  __FILE__, __LINE__);
916  }
917  catch (Error & e) {
918  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
919  __FILE__, __LINE__);
920  }
921  catch (...) {
922 #ifdef USE_HDFEOS2_LIB
923  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
924 #else
925  close_hdf4_fileid(sdfd,fileid,h4file);
926 #endif
927  string s = "unknown exception caught building DAP2 data response from an HDF4 data resource";
928  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
929  }
930 
931  return true;
932 }
933 
934 // Special function to build DDS for HDF4 SDS-only DDS. One can turn
935 // on cache for this function. Currently only AIRS version 6 is supported.
936 bool HDF4RequestHandler::hdf4_build_dds_cf_sds(BESDataHandlerInterface &dhi){
937 
938  int32 sdfd = -1;
939  HDFSP::File *h4file = NULL;
940 
941  BESResponseObject *response = dhi.response_handler->get_response_object();
942  BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *> (response);
943 
944  if (!bdds)
945  throw BESInternalError("cast error", __FILE__, __LINE__);
946 
947  try {
949 
950  DDS *dds = bdds->get_dds();
951 
952 #if 0
953  //BaseTypeFactory* factory = new BaseTypeFactory ;
954  //dds->set_factory( factory ) ;
955 #endif
956 
957  string accessed = dhi.container->access();
958  dds->filename(accessed);
959 
960  DAS *das = new DAS;
961  BESDASResponse bdas(das);
962 
963  // Check and set up dds and das cache files.
964  string base_file_name = basename(dhi.container->access());
965  bool das_set_cache = false;
966  bool dds_set_cache = false;
967  bool dds_das_get_cache = false;
968  string das_filename;
969  string dds_filename;
970 
971  // Check if the EnableMetaData key is set.
972 #if 0
973  string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
974  bool turn_on_enable_mcache_key= false;
975  turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
976  if(true == turn_on_enable_mcache_key) {// the EnableMetaData key is set
977 #endif
978  if(true == _enable_metadata_cachefile) {// the EnableMetaData key is set
979 
980 
981 #if 0
982  string md_cache_dir;
983  bool found = false;
984  string key = "H4.Cache.metadata.path";
985  TheBESKeys::TheKeys()->get_value(key,md_cache_dir,found);
986 #endif
987  if(true == _cache_metadata_path_exist) {// the metadata.path key is set
988 
989  // Create DAS and DDS cache file names
990  das_filename = _cache_metadata_path + "/" + base_file_name +"_das";
991  dds_filename = _cache_metadata_path + "/" + base_file_name +"_dds";
992 
993  // When the returned value das_set_cache is false, read DAS from the cache file.
994  // Otherwise, do nothing.
995  das_set_cache = rw_das_cache_file(das_filename,das,false);
996 
997  // When the returned value dds_set_cache is false, read DDS from the cache file.
998  // Otherwise, do nothing.
999  dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1000 
1001  // Set the flag to obtain DDS and DAS from the cache files.
1002  // Here, we require that both DDS and DAS should be cached.
1003  // We don't support caching one and not caching another.
1004  if((false == das_set_cache)&&(false == dds_set_cache))
1005  dds_das_get_cache = true;
1006  }
1007  }
1008 
1009 
1010  // We need to go back to retrieve DDS, DAS from the HDF file.
1011  if(false == dds_das_get_cache) {
1012 
1013  // Obtain SD ID, this is the only ID we need to use.
1014  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
1015  if( -1 == sdfd) {
1016 
1017  string invalid_file_msg="HDF4 SDstart error for the file ";
1018  invalid_file_msg +=accessed;
1019  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1020  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1021  }
1022 
1023 
1024  // Here we will check if ECS_Metadata key if set. For DDS and DAS,
1025  // only when the DisableECSMetaDataAll key is set, ecs_metadata is off.
1026  bool ecs_metadata = !(_disable_ecsmetadata_all);
1027 #if 0
1028  bool ecs_metadata = true;
1029  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1030  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1031  ecs_metadata = false;
1032 #endif
1033 
1034  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1035 
1036  Ancillary::read_ancillary_das(*das, accessed);
1037 
1038  // Pass file pointer(h4file) from DAS to DDS, also pass the flag
1039  // dds_set_cache to indicate if the handler needs to cache DDS.
1040  read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1041 
1042  // We also need to cache DAS if das_set_cache is set.
1043  if(true == das_set_cache) {
1044  if(das_filename =="")
1045  throw InternalErr(__FILE__,__LINE__,"DAS cache file name should be set ");
1046  rw_das_cache_file(das_filename,das,true);
1047  }
1048 
1049  }
1050 
1051  Ancillary::read_ancillary_dds(*dds, accessed);
1052  dds->transfer_attributes(das);
1053  bdds->set_constraint(dhi);
1054 
1055  bdds->clear_container();
1056 
1057  if(h4file != NULL)
1058  delete h4file;
1059 
1060  if(sdfd != -1)
1061  SDend(sdfd);
1062 #if 0
1063  // Cannot(should not) delete factory here. factory will be released by DAP? or errors occur.
1064  //if(factory != NULL)
1065  // delete factory;
1066 #endif
1067 
1068  }
1069 
1070  catch (BESError & e) {
1071  throw;
1072  }
1073  catch (InternalErr & e) {
1074  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1075  __FILE__, __LINE__);
1076  }
1077  catch (Error & e) {
1078  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1079  __FILE__, __LINE__);
1080  }
1081  catch (...) {
1082  if(sdfd != -1)
1083  SDend(sdfd);
1084  if(h4file != NULL)
1085  delete h4file;
1086  string s = "unknown exception caught building HDF4 DataDDS";
1087  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1088  }
1089 
1090  return true;
1091 }
1092 
1093 // Special function to build DAS for HDF4 SDS-only DDS. One can turn
1094 // on cache for this function. Currently only AIRS version 6 is supported.
1095 bool HDF4RequestHandler::hdf4_build_das_cf_sds(BESDataHandlerInterface &dhi){
1096 
1097  int32 sdfd = -1;
1098  HDFSP::File *h4file = NULL;
1099 
1100  BESResponseObject *response = dhi.response_handler->get_response_object();
1101  BESDASResponse *bdas = dynamic_cast<BESDASResponse *> (response);
1102 
1103  if (!bdas)
1104  throw BESInternalError("cast error", __FILE__, __LINE__);
1105 
1106  try {
1108 
1109  DAS *das = bdas->get_das();
1110  string base_file_name = basename(dhi.container->access());
1111 
1112  string accessed = dhi.container->access();
1113 
1114  // Check if the enable metadata cache key is set and set the key appropriate.
1115  bool das_set_cache = false;
1116  bool das_get_cache = false;
1117  string das_filename;
1118 #if 0
1119  string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
1120  bool turn_on_enable_mcache_key= false;
1121  turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1122  if(true == turn_on_enable_mcache_key) { // find the metadata cache key.
1123 #endif
1124  if(true == _enable_metadata_cachefile) { // find the metadata cache key.
1125 #if 0
1126  string md_cache_dir;
1127  string key = "H4.Cache.metadata.path";
1128  bool found = false;
1129  TheBESKeys::TheKeys()->get_value(key,md_cache_dir,found);
1130  if(true == found) { // Also find the metadata cache.
1131 #endif
1132  if(true == _cache_metadata_path_exist) {
1133 
1134  // Create the DAS cache file name.
1135  das_filename = _cache_metadata_path + "/" + base_file_name +"_das";
1136 
1137  // Read the DAS from the cached file, if das_set_cache is false.
1138  // When the das_set_cache is true, need to create a das cache file.
1139  das_set_cache = rw_das_cache_file(das_filename,das,false);
1140 
1141  // Not set cache, must get the das from cache, so das_get_cache should be true.
1142  if(false == das_set_cache)
1143  das_get_cache = true;
1144  }
1145  }
1146 
1147  // Need to retrieve DAS from the HDF4 file.
1148  if(false == das_get_cache) {
1149  // Obtain SD ID.
1150  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
1151  if( -1 == sdfd){
1152  string invalid_file_msg="HDF4 SDstart error for the file ";
1153  invalid_file_msg +=accessed;
1154  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1155  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1156  }
1157 
1158 
1159 #if 0
1160  bool ecs_metadata = true;
1161  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1162  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1163  ecs_metadata = false;
1164 #endif
1165  // Here we will check if ECS_Metadata key if set. For DDS and DAS,
1166  // only when the DisableECSMetaDataAll key is set, ecs_metadata is off.
1167  bool ecs_metadata = !(_disable_ecsmetadata_all);
1168 
1169  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1170  Ancillary::read_ancillary_das(*das, accessed);
1171 
1172  // Generate DAS cache file if the cache flag is set.
1173  if(true == das_set_cache)
1174  rw_das_cache_file(das_filename,das,true);
1175 
1176  }
1177 
1178  bdas->clear_container();
1179 
1180  if(h4file != NULL)
1181  delete h4file;
1182 
1183  if(sdfd != -1)
1184  SDend(sdfd);
1185 
1186  }
1187 
1188  catch (BESError & e) {
1189  throw;
1190  }
1191  catch (InternalErr & e) {
1192  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1193  __FILE__, __LINE__);
1194  }
1195  catch (Error & e) {
1196  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1197  __FILE__, __LINE__);
1198  }
1199  catch (...) {
1200  if(sdfd != -1)
1201  SDend(sdfd);
1202  if(h4file != NULL)
1203  delete h4file;
1204  string s = "unknown exception caught building HDF4 DataDDS";
1205  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1206  }
1207  return true;
1208 }
1209 // Special function to build data for HDF4 SDS-only DDS. One can turn
1210 // on cache for this function. Currently only AIRS version 6 is supported.
1211 bool HDF4RequestHandler::hdf4_build_data_cf_sds(BESDataHandlerInterface &dhi){
1212 
1213  int32 sdfd = -1;
1214  HDFSP::File *h4file = NULL;
1215 
1216  BESResponseObject *response = dhi.response_handler->get_response_object();
1217  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *> (response);
1218 
1219  if (!bdds)
1220  throw BESInternalError("cast error", __FILE__, __LINE__);
1221 
1222  try {
1224 
1225  DDS *dds = bdds->get_dds();
1226 
1227  string accessed = dhi.container->access();
1228  dds->filename(accessed);
1229 
1230  DAS *das = new DAS;
1231  BESDASResponse bdas(das);
1233 
1234 #if 0
1235  //BaseTypeFactory* factory = new BaseTypeFactory ;
1236  //dds->set_factory( factory ) ;
1237 #endif
1238 
1239  string base_file_name = basename(dhi.container->access());
1240 
1241  bool das_set_cache = false;
1242  bool dds_set_cache = false;
1243  bool dds_das_get_cache = false;
1244  string das_filename;
1245  string dds_filename;
1246 
1247 #if 0
1248  string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
1249  bool _disable_ecsmetadata_allturn_on_enable_mcache_key= false;
1250  turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1251  if(true == turn_on_enable_mcache_key) {
1252 #endif
1253  if(true == _enable_metadata_cachefile) {
1254 
1255  if(true == _cache_metadata_path_exist) {
1256  BESDEBUG("h4", "H4.Cache.metadata.path key is set and metadata cache key is set." << endl);
1257 
1258  // Notice, since the DAS output may be different between DAS/DDS service and DataDDS service.
1259  // See comments about ecs_metadata below.
1260  // So we create a different DAS file. This can be optimized in the future if necessary.
1261  das_filename = _cache_metadata_path + "/" + base_file_name +"_das_dd";
1262  dds_filename = _cache_metadata_path + "/" + base_file_name +"_dds";
1263 
1264  // If das_set_cache is true, data is read from the DAS cache.
1265  das_set_cache = rw_das_cache_file(das_filename,das,false);
1266 
1267  // If dds_set_cache is true, data is read from the DDS cache.
1268  dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1269  // Need to set a flag to generate DAS and DDS cache files
1270  if((false == das_set_cache)&&(false == dds_set_cache))
1271  dds_das_get_cache = true;
1272  }
1273  }
1274  if(false == dds_das_get_cache) {
1275 
1276  // Obtain HDF4 SD ID
1277  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
1278  if( -1 == sdfd){
1279  string invalid_file_msg="HDF4 SDstart error for the file ";
1280  invalid_file_msg +=accessed;
1281  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1282  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1283  }
1284 
1285 
1286  // Here we will check if ECS_Metadata key if set. For DataDDS,
1287  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1288  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1289  bool ecs_metadata = true;
1290 #if 0
1291  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1292  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1293 #endif
1294  if((true == _disable_ecsmetadata_min)
1295  || (true == _disable_ecsmetadata_all))
1296  ecs_metadata = false;
1297 
1298  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1299 
1300  Ancillary::read_ancillary_das(*das, accessed);
1301 
1302  // Need to write DAS to a cache file.
1303  if(true == das_set_cache) {
1304  rw_das_cache_file(das_filename,das,true);
1305  }
1306 
1307  // Pass file pointer h4file from DAS to DDS. Also need to pass the
1308  // flag that indicates if a DDS file needs to be created.
1309  read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1310 
1311  }
1312 
1313  Ancillary::read_ancillary_dds(*dds, accessed);
1314 
1315  dds->transfer_attributes(das);
1316 
1317  bdds->set_constraint(dhi);
1318 
1319  bdds->clear_container();
1320 
1321  if(h4file != NULL)
1322  delete h4file;
1323 
1324  if(sdfd != -1)
1325  SDend(sdfd);
1326  }
1327 
1328  catch (BESError & e) {
1329  throw;
1330  }
1331  catch (InternalErr & e) {
1332  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1333  __FILE__, __LINE__);
1334  }
1335  catch (Error & e) {
1336  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1337  __FILE__, __LINE__);
1338  }
1339  catch (...) {
1340  if(sdfd != -1)
1341  SDend(sdfd);
1342  if(h4file != NULL)
1343  delete h4file;
1344  string s = "unknown exception caught building HDF4 DataDDS";
1345  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1346  }
1347 
1348 
1349  return true;
1350 }
1351 
1352 // Notice the metadata cache doesn't apply when the EnableFileID key is set.
1353 // This is because the file ID generated by creating DDS/DAS WO cache is needed to access the data.
1354 // So to make this work at any time, we have to create SDS ID even when the EnableFileID key is set.
1355 // This is against the purpose of EnableFileID key. To acheieve the same purpose for AIRS,
1356 // one can set the DataCache key, the performance is similar or even better than just using the EnableFileID key.
1357 bool HDF4RequestHandler::hdf4_build_data_cf_sds_with_IDs(BESDataHandlerInterface &dhi){
1358 
1359  int32 sdfd = -1;
1360  HDFSP::File *h4file = NULL;
1361 
1362  BESResponseObject *response = dhi.response_handler->get_response_object();
1363  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *> (response);
1364 
1365  if (!bdds)
1366  throw BESInternalError("cast error", __FILE__, __LINE__);
1367 
1368  try {
1370 
1371  HDF4DDS *hdds = new HDF4DDS(bdds->get_dds());
1372 
1373  delete bdds->get_dds();
1374 
1375  bdds->set_dds(hdds);
1376 
1377 
1378  string accessed = dhi.container->access();
1379  hdds->filename(accessed);
1380 
1381  DAS *das = new DAS;
1382  BESDASResponse bdas(das);
1384 
1385  //Obtain SD ID.
1386  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
1387  if( -1 == sdfd) {
1388  string invalid_file_msg="HDF4 SDstart error for the file ";
1389  invalid_file_msg +=accessed;
1390  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1391  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1392  }
1393 
1394  hdds->setHDF4Dataset(sdfd,-1);
1395 
1396  // Here we will check if ECS_Metadata key if set. For DataDDS,
1397  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1398  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1399  bool ecs_metadata = true;
1400 #if 0
1401  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1402  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1403 #endif
1404  if((true == _disable_ecsmetadata_min)
1405  || (true == _disable_ecsmetadata_all))
1406  ecs_metadata = false;
1407 
1408  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1409 #if 0
1410  //read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,true);
1411 #endif
1412 
1413  Ancillary::read_ancillary_das(*das, accessed);
1414 
1415  // Pass file pointer(h4file, eosfile) from DAS to DDS.
1416  read_dds_sds(*hdds, accessed,sdfd,h4file,false);
1417 
1418  if(h4file != NULL)
1419  delete h4file;
1420 
1421  Ancillary::read_ancillary_dds(*hdds, accessed);
1422 
1423  hdds->transfer_attributes(das);
1424 
1425  bdds->set_constraint(dhi);
1426 
1427  bdds->clear_container();
1428 
1429  }
1430 
1431  catch (BESError & e) {
1432  throw;
1433  }
1434  catch (InternalErr & e) {
1435  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1436  __FILE__, __LINE__);
1437  }
1438  catch (Error & e) {
1439  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1440  __FILE__, __LINE__);
1441  }
1442  catch (...) {
1443  if(sdfd != -1)
1444  SDend(sdfd);
1445  if(h4file != NULL)
1446  delete h4file;
1447  string s = "unknown exception caught building HDF4 DataDDS";
1448  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1449  }
1450 
1451 
1452  return true;
1453 }
1454 bool HDF4RequestHandler::hdf4_build_dmr(BESDataHandlerInterface &dhi)
1455 {
1456 
1457  BESStopWatch sw;
1458  if (BESISDEBUG( TIMING_LOG ))
1459  sw.start("HDF4RequestHandler::hdf4_build_dmr", dhi.data[REQUEST_ID]);
1460 
1461  // Because this code does not yet know how to build a DMR directly, use
1462  // the DMR ctor that builds a DMR using a 'full DDS' (a DDS with attributes).
1463  // First step, build the 'full DDS'
1464  string data_path = dhi.container->access();
1465 
1466  BaseTypeFactory factory;
1467  DDS dds(&factory, name_path(data_path), "3.2");
1468  dds.filename(data_path);
1469 
1470  DAS das;
1471 
1472  int32 sdfd = -1;
1473  int32 fileid = -1;
1474 
1475  // Since passing file IDs requires to use the derived class and it
1476  // causes the management of code structure messy, we first handle this with
1477  // another method.
1478  if(true == _usecf) {
1479 
1480  if(true == _pass_fileid)
1481  return hdf4_build_dmr_with_IDs(dhi);
1482 
1483  }
1484 
1485 
1486  try {
1487 
1488  if (true == _usecf) {
1489 
1490  HDFSP::File *h4file = NULL;
1491 
1492  // Obtain HDF4 file IDs
1493  //SDstart
1494  sdfd = SDstart (const_cast < char *>(data_path.c_str()), DFACC_READ);
1495  if( -1 == sdfd){
1496  string invalid_file_msg="HDF4 SDstart error for the file ";
1497  invalid_file_msg +=data_path;
1498  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1499  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1500  }
1501 
1502  // H open
1503  fileid = Hopen(const_cast<char *>(data_path.c_str()), DFACC_READ,0);
1504  if (-1 == fileid) {
1505  SDend(sdfd);
1506  string invalid_file_msg="HDF4 Hopen error for the file ";
1507  invalid_file_msg +=data_path;
1508  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1509  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1510  }
1511 
1512 #ifdef USE_HDFEOS2_LIB
1513 
1514  int32 gridfd = -1;
1515  int32 swathfd = -1;
1516 
1517  HDFEOS2::File *eosfile = NULL;
1518 
1519  // Obtain HDF-EOS2 file IDs with the file open APIs.
1520  // Grid open
1521  gridfd = GDopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1522  if (-1 == gridfd) {
1523  SDend(sdfd);
1524  Hclose(fileid);
1525  string invalid_file_msg="HDF-EOS GDopen error for the file ";
1526  invalid_file_msg +=data_path;
1527  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1528  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1529  }
1530 
1531  // Swath open
1532  swathfd = SWopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1533  if (-1 == swathfd) {
1534  SDend(sdfd);
1535  Hclose(fileid);
1536  GDclose(gridfd);
1537  string invalid_file_msg="HDF-EOS SWopen error for the file ";
1538  invalid_file_msg +=data_path;
1539  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1540  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1541  }
1542 
1543 
1544  // Here we will check if ECS_Metadata key if set. For DAP4's DMR,
1545  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1546  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1547  // This is one difference between DAP2 and DAP4 mapping. Since
1548  // people can use BES key to turn on the ECS metadata, so this is okay.
1549  // KY 2014-10-23
1550  bool ecs_metadata = true;
1551 #if 0
1552  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1553  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1554 #endif
1555  if((true == _disable_ecsmetadata_min)
1556  || (true == _disable_ecsmetadata_all))
1557  ecs_metadata = false;
1558 
1559  try {
1560 
1561  read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1562 #if 0
1563  //read_das_use_eos2lib(*das, data_path,sdfd,fileid,gridfd,swathfd,true);
1564 #endif
1565  Ancillary::read_ancillary_das(das, data_path);
1566 
1567  // Pass file pointer(h4file, eosfile) from DAS to DDS
1568  read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1569  }
1570 
1571  catch(...) {
1572  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1573  throw;
1574  }
1575  if(eosfile != NULL)
1576  delete eosfile;
1577  GDclose(gridfd);
1578  SWclose(swathfd);
1579 
1580 #else
1581  try {
1582  read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1583  Ancillary::read_ancillary_das(das, data_path);
1584 
1585  // Pass file pointer(h4file) from DAS to DDS.
1586  read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1587  }
1588  catch(...) {
1589  close_hdf4_fileid(sdfd,fileid,h4file);
1590  throw;
1591  //throw InternalErr(__FILE__,__LINE__,"read_dds_hdfsp error");
1592  }
1593 #endif
1594  close_hdf4_fileid(sdfd,fileid,h4file);
1595 
1596  }
1597  else {
1598  read_das(das, data_path);
1599  Ancillary::read_ancillary_das(das, data_path);
1600  read_dds(dds, data_path);
1601  }
1602 
1603 
1604  Ancillary::read_ancillary_dds(dds, data_path);
1605 
1606  dds.transfer_attributes(&das);
1607 
1608  }
1609 
1610  catch (InternalErr & e) {
1611  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1612  __FILE__, __LINE__);
1613  }
1614  catch (Error & e) {
1615  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1616  __FILE__, __LINE__);
1617  }
1618  catch (...) {
1619  string s = "unknown exception caught building HDF4 DataDDS";
1620  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1621  }
1622 
1623 #if 0
1624  //dds.print(cout);
1625  //dds.print_das(cout);
1626 #endif
1627  //
1628  // Extract the DMR Response object - this holds the DMR used by the
1629  // other parts of the framework.
1630  BESResponseObject *response = dhi.response_handler->get_response_object();
1631  BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1632 
1633  DMR *dmr = bes_dmr.get_dmr();
1634 
1635  D4BaseTypeFactory MyD4TypeFactory;
1636  dmr->set_factory(&MyD4TypeFactory);
1637 #if 0
1638  //dmr->set_factory(new D4BaseTypeFactory);
1639 #endif
1640  dmr->build_using_dds(dds);
1641 
1642 #if 0
1643  //dmr->print(cout);
1644 #endif
1645 
1646  // Instead of fiddling with the internal storage of the DHI object,
1647  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1648  // methods to set the constraints. But, why? Ans: from Patrick is that
1649  // in the 'container' mode of BES each container can have a different
1650  // CE.
1651  bes_dmr.set_dap4_constraint(dhi);
1652  bes_dmr.set_dap4_function(dhi);
1653  dmr->set_factory(0);
1654 
1655  return true;
1656 }
1657 
1658 bool HDF4RequestHandler::hdf4_build_dmr_with_IDs(BESDataHandlerInterface & dhi) {
1659 
1660  BESStopWatch sw;
1661  if (BESISDEBUG( TIMING_LOG ))
1662  sw.start("HDF4RequestHandler::hdf4_build_dmr_with_IDs", dhi.data[REQUEST_ID]);
1663 
1664  // Because this code does not yet know how to build a DMR directly, use
1665  // the DMR ctor that builds a DMR using a 'full DDS' (a DDS with attributes).
1666  // First step, build the 'full DDS'
1667  string data_path = dhi.container->access();
1668 
1669  BaseTypeFactory factory;
1670  DDS dds(&factory, name_path(data_path), "3.2");
1671  dds.filename(data_path);
1672 
1673  DAS das;
1674 
1675  int32 sdfd = -1;
1676  int32 fileid = -1;
1677 
1678  HDFSP::File *h4file = NULL;
1679 
1680  // Obtain HDF4 file IDs
1681  //SDstart
1682  sdfd = SDstart (const_cast < char *>(data_path.c_str()), DFACC_READ);
1683  if( -1 == sdfd){
1684  string invalid_file_msg="HDF4 SDstart error for the file ";
1685  invalid_file_msg +=data_path;
1686  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1687  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1688  }
1689 
1690  // H open
1691  fileid = Hopen(const_cast<char *>(data_path.c_str()), DFACC_READ,0);
1692  if (-1 == fileid) {
1693  SDend(sdfd);
1694  string invalid_file_msg="HDF4 SDstart error for the file ";
1695  invalid_file_msg +=data_path;
1696  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1697  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1698  }
1699 
1700 
1701 #ifdef USE_HDFEOS2_LIB
1702 
1703  int32 gridfd = -1;
1704  int32 swathfd = -1;
1705 
1706  HDFEOS2::File *eosfile = NULL;
1707  // Obtain HDF-EOS2 file IDs with the file open APIs.
1708  // Grid open
1709  gridfd = GDopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1710  if (-1 == gridfd) {
1711  SDend(sdfd);
1712  Hclose(fileid);
1713  string invalid_file_msg="HDF4 SDstart error for the file ";
1714  invalid_file_msg +=data_path;
1715  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1716  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1717 
1718  }
1719 
1720  // Swath open
1721  swathfd = SWopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1722  if (-1 == swathfd) {
1723  SDend(sdfd);
1724  Hclose(fileid);
1725  GDclose(gridfd);
1726  string invalid_file_msg="HDF-EOS SWopen error for the file ";
1727  invalid_file_msg +=data_path;
1728  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1729  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1730 
1731  }
1732 
1733  // Here we will check if ECS_Metadata key if set. For DAP4's DMR,
1734  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1735  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1736  // This is one difference between DAP2 and DAP4 mapping. Since
1737  // people can use BES key to turn on the ECS metadata, so this is okay.
1738  // KY 2014-10-23
1739  bool ecs_metadata = true;
1740 #if 0
1741  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1742  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1743 #endif
1744  if((true == _disable_ecsmetadata_min)
1745  || (true == _disable_ecsmetadata_all))
1746  ecs_metadata = false;
1747 
1748  try {
1749 
1750  read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1751 #if 0
1752  //read_das_use_eos2lib(*das, data_path,sdfd,fileid,gridfd,swathfd,true);
1753 #endif
1754  Ancillary::read_ancillary_das(das, data_path);
1755 
1756  // Pass file pointer(h4file, eosfile) from DAS to DDS.
1757  read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1758  }
1759 
1760  catch(...) {
1761  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1762  throw;
1763  }
1764  if(eosfile != NULL)
1765  delete eosfile;
1766 
1767 #else
1768  try {
1769  read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1770  Ancillary::read_ancillary_das(das, data_path);
1771 
1772  // Pass file pointer(h4file) from DAS to DDS.
1773  read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1774  }
1775  catch(...) {
1776  close_hdf4_fileid(sdfd,fileid,h4file);
1777  throw;
1778  }
1779 #endif
1780  if(h4file != NULL)
1781  delete h4file;
1782 
1783  Ancillary::read_ancillary_dds(dds, data_path);
1784 
1785  dds.transfer_attributes(&das);
1786 
1787 
1788 // File IDs are closed by the derived class.
1789 #if 0
1790  if(true == usecf) {
1791 #ifdef USE_HDFEOS2_LIB
1792  GDclose(gridfd);
1793  SWclose(swathfd);
1794 
1795 #endif
1796  SDend(sdfd);
1797  Hclose(fileid);
1798  }
1799 
1800  //dds.print(cout);
1801  //dds.print_das(cout);
1802 #endif
1803  // Extract the DMR Response object - this holds the DMR used by the
1804  // other parts of the framework.
1805  BESResponseObject *response = dhi.response_handler->get_response_object();
1806  BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1807 
1808  // In this handler we use a different pattern since the handler specializes the DDS/DMR.
1809  // First, build the DMR adding the open handle to the HDF4 dataset, then free the DMR
1810  // the BES built and add this one. The HDF4DMR object will close the open dataset when
1811  // the BES runs the DMR's destructor.
1812  DMR *dmr = bes_dmr.get_dmr();
1813  D4BaseTypeFactory MyD4TypeFactory;
1814  dmr->set_factory(&MyD4TypeFactory);
1815  dmr->build_using_dds(dds);
1816  HDF4DMR *hdf4_dmr = new HDF4DMR(dmr);
1817 #ifdef USE_HDFEOS2_LIB
1818  hdf4_dmr->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
1819 #else
1820  hdf4_dmr->setHDF4Dataset(sdfd,fileid);
1821 #endif
1822  delete dmr; // The call below will make 'dmr' unreachable; delete it now to avoid a leak.
1823  bes_dmr.set_dmr(hdf4_dmr); // BESDMRResponse will delete hdf4_dmr
1824 
1825  // Instead of fiddling with the internal storage of the DHI object,
1826  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1827  // methods to set the constraints. But, why? Ans: from Patrick is that
1828  // in the 'container' mode of BES each container can have a different
1829  // CE.
1830  bes_dmr.set_dap4_constraint(dhi);
1831  bes_dmr.set_dap4_function(dhi);
1832  hdf4_dmr->set_factory(0);
1833 
1834  return true;
1835 
1836 }
1837 
1838 bool HDF4RequestHandler::hdf4_build_help(BESDataHandlerInterface & dhi) {
1839  BESResponseObject *response = dhi.response_handler->get_response_object();
1840  BESInfo *info = dynamic_cast<BESInfo *> (response);
1841  if (!info)
1842  throw BESInternalError("cast error", __FILE__, __LINE__);
1843 
1844  map < string, string > attrs;
1845  attrs["name"] = MODULE_NAME ;
1846  attrs["version"] = MODULE_VERSION ;
1847  list < string > services;
1848  BESServiceRegistry::TheRegistry()->services_handled(HDF4_NAME, services);
1849  if (services.size() > 0) {
1850  string handles = BESUtil::implode(services, ',');
1851  attrs["handles"] = handles;
1852  }
1853  info->begin_tag("module", &attrs);
1854  info->end_tag("module");
1855 
1856  return true;
1857 }
1858 
1859 bool HDF4RequestHandler::hdf4_build_version(BESDataHandlerInterface & dhi) {
1860  BESResponseObject *response = dhi.response_handler->get_response_object();
1861  BESVersionInfo *info = dynamic_cast<BESVersionInfo *> (response);
1862  if (!info)
1863  throw BESInternalError("cast error", __FILE__, __LINE__);
1864 
1865  info->add_module(MODULE_NAME, MODULE_VERSION);
1866 
1867  return true;
1868 }
1869 
1870 #ifdef USE_HDFEOS2_LIB
1871 void close_fileid(int sdfd, int fileid,int gridfd, int swathfd,HDFSP:: File* h4file, HDFEOS2::File*eosfile) {
1872  if(h4file !=NULL)
1873  delete h4file;
1874  if(sdfd != -1)
1875  SDend(sdfd);
1876  if(fileid != -1)
1877  Hclose(fileid);
1878 
1879 
1880  if(eosfile !=NULL)
1881  delete eosfile;
1882  if(gridfd != -1)
1883  GDclose(gridfd);
1884  if(swathfd != -1)
1885  SWclose(swathfd);
1886 
1887 }
1888 #endif
1889 void close_hdf4_fileid(int sdfd, int fileid,HDFSP::File*h4file) {
1890 
1891  if(h4file !=NULL)
1892  delete h4file;
1893 
1894  if(sdfd != -1)
1895  SDend(sdfd);
1896  if(fileid != -1)
1897  Hclose(fileid);
1898 
1899 }
1900 
1901 // Handling DAS and DDS cache. Currently we only apply the cache to special products(AIRS version 6 level 3 or 2).
1902 // read or write DAS from/to a cache file.
1903 bool rw_das_cache_file(const string & filename, DAS *das_ptr,bool w_flag) {
1904 
1905  bool das_set_cache = false;
1906  FILE *das_file = NULL;
1907 
1908  if(false == w_flag) // open a cache file for reading.
1909  das_file = fopen(filename.c_str(),"r");
1910  else
1911  das_file = fopen(filename.c_str(),"w");
1912 
1913  if(NULL == das_file) {
1914  if(ENOENT == errno) {
1915  // Since the das service always tries to read the data from a cache and if the cache file doesn't exist,
1916  // it will generates a cache file, so here we set a flag to indicate if a cache file needs to be generated.
1917  if(false == w_flag) {
1918  BESDEBUG("h4", "DAS set cache key is true." << endl);
1919  das_set_cache = true;
1920  }
1921  }
1922  else
1923  throw BESInternalError( "An error occurred trying to open a das cache file " + get_errno(), __FILE__, __LINE__);
1924  }
1925  else {
1926 
1927  int fd_das = fileno(das_file);
1928 
1929  // Set a corresponding read(shared) or write(exclusive) lock.
1930  struct flock *l_das;
1931  if(false == w_flag)
1932  l_das = lock(F_RDLCK);
1933  else
1934  l_das = lock(F_WRLCK);
1935 
1936  // Hold a lock.
1937  if(fcntl(fd_das,F_SETLKW,l_das) == -1) {
1938  fclose(das_file);
1939  ostringstream oss;
1940  oss << "cache process: " << l_das->l_pid << " triggered a locking error: " << get_errno();
1941  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1942  }
1943 
1944  if(false == w_flag){
1945  // Read DAS from a cache file
1946  BESDEBUG("h4", "Obtaining DAS from the cache file" << endl);
1947  try {
1948  das_ptr->parse(das_file);
1949  }
1950  catch(...) {
1951  if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1952  fclose(das_file);
1953  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1954  }
1955  fclose(das_file);
1956  throw InternalErr(__FILE__,__LINE__,"Fail to parse the das from a das file.");
1957  }
1958  }
1959  else {
1960  // Write DAS to a cache file
1961  BESDEBUG("h4", "write DAS to a cache file" << endl);
1962  try {
1963  das_ptr->print(das_file);
1964  }
1965  catch(...) {
1966  if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1967  fclose(das_file);
1968  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1969  }
1970  fclose(das_file);
1971  throw InternalErr(__FILE__,__LINE__,"Fail to generate a das cache file.");
1972  }
1973 
1974  }
1975 
1976  // Unlock the cache file
1977  if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1978  fclose(das_file);
1979  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1980  }
1981  fclose(das_file);
1982 
1983  }
1984 
1985  return das_set_cache;
1986 
1987 }
1988 
1989 // Read dds from a cache file.
1990 bool r_dds_cache_file(const string & cache_filename, DDS *dds_ptr,const string & hdf4_filename) {
1991 
1992  bool dds_set_cache = false;
1993  FILE *dds_file = NULL;
1994  dds_file = fopen(cache_filename.c_str(),"rb");
1995 
1996  if(NULL == dds_file) {
1997  if(ENOENT == errno) {
1998  // Since the das service always tries to read the data from a cache and if the cache file doesn't exist,
1999  // it is supposed that the handler should generate a cache file,
2000  // so set a flag to indicate if a cache file needs to be generated.
2001  dds_set_cache = true;
2002  }
2003  else
2004  throw BESInternalError( "An error occurred trying to open a dds cache file " + get_errno(), __FILE__, __LINE__);
2005  }
2006  else {
2007 
2008  int fd_dds = fileno(dds_file);
2009  struct flock *l_dds;
2010  l_dds = lock(F_RDLCK);
2011 
2012  // hold a read(shared) lock to read dds from a file.
2013  if(fcntl(fd_dds,F_SETLKW,l_dds) == -1) {
2014  fclose(dds_file);
2015  ostringstream oss;
2016  oss << "cache process: " << l_dds->l_pid << " triggered a locking error: " << get_errno();
2017  throw BESInternalError( oss.str(), __FILE__, __LINE__);
2018  }
2019 
2020  try {
2021  HDFCFUtil::read_sp_sds_dds_cache(dds_file,dds_ptr,cache_filename,hdf4_filename);
2022  }
2023  catch(...) {
2024  if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2025  fclose(dds_file);
2026  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2027  }
2028 
2029  fclose(dds_file);
2030  throw InternalErr(__FILE__,__LINE__,"Fail to generate a dds cache file.");
2031  }
2032 
2033  if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2034  fclose(dds_file);
2035  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2036  }
2037 
2038  fclose(dds_file);
2039 
2040  }
2041 
2042  return dds_set_cache;
2043 
2044 }
2045 
2046 bool check_beskeys(const string key) {
2047 
2048  bool found = false;
2049  string doset ="";
2050  const string dosettrue ="true";
2051  const string dosetyes = "yes";
2052 
2053  TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2054  if( true == found ) {
2055  doset = BESUtil::lowercase( doset ) ;
2056  if( dosettrue == doset || dosetyes == doset )
2057  return true;
2058  }
2059  return false;
2060 
2061 }
2062 
2063 bool get_beskeys(const string key,string &key_value) {
2064 
2065  bool found = false;
2066 
2067  TheBESKeys::TheKeys()->get_value( key, key_value, found ) ;
2068  return found;
2069 
2070 }
2071 
2072 #if 0
2073 int get_cachekey_int(const string key) {
2074 
2075  bool found = false;
2076  int ret_value = 0;
2077  string doset ="";
2078 
2079  TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2080  if( true == found ) {
2081  istringstream iss(doset);
2082  iss >>ret_value;
2083  }
2084  return ret_value;
2085 
2086 }
2087 #endif
2088 #if 0
2089 void test_func(HDFSP::File**h4file) {
2090 cerr<<"OK to pass pointer of a NULL pointer "<<endl;
2091 
2092 }
2093 #endif
BESRequestHandler
Represents a specific data type request handler.
Definition: BESRequestHandler.h:74
BESDataHandlerInterface::container
BESContainer * container
pointer to current container in this interface
Definition: BESDataHandlerInterface.h:75
BESServiceRegistry::services_handled
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
Definition: BESServiceRegistry.cc:334
BESDapResponse::set_dap4_constraint
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:137
BESStopWatch::start
virtual bool start(std::string name)
Definition: BESStopWatch.cc:58
BESDDSResponse::set_container
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Definition: BESDDSResponse.cc:64
BESDataDDSResponse::set_container
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Definition: BESDataDDSResponse.cc:50
close_fileid
void close_fileid(hid_t fid)
Definition: h5get.cc:414
BESDapResponse::set_dap4_function
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:154
BESDataDDSResponse::set_dds
void set_dds(libdap::DDS *ddsIn)
Definition: BESDataDDSResponse.h:73
BESContainer::get_symbolic_name
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
HDF4DDS
Definition: HDF4_DDS.h:39
BESDDSResponse::clear_container
virtual void clear_container()
clear the container in the DAP response object
Definition: BESDDSResponse.cc:73
BESDDSResponse::get_dds
libdap::DDS * get_dds()
Definition: BESDDSResponse.h:80
BESDASResponse
Represents an OPeNDAP DAS DAP2 data object within the BES.
Definition: BESDASResponse.h:44
BESInfo
informational response object
Definition: BESInfo.h:63
BESDASResponse::set_container
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Definition: BESDASResponse.cc:49
libdap
Definition: BESDapFunctionResponseCache.h:35
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:62
BESResponseHandler::get_response_object
virtual BESResponseObject * get_response_object()
return the current response object
Definition: BESResponseHandler.cc:82
BESVersionInfo
Definition: BESVersionInfo.h:47
BESContainer::access
virtual std::string access()=0
returns the true name of this container
BESDataHandlerInterface::data
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
Definition: BESDataHandlerInterface.h:90
HDFSP::File
Definition: HDFSP.h:726
BESDataDDSResponse::clear_container
virtual void clear_container()
clear the container in the DAP response object
Definition: BESDataDDSResponse.cc:59
HDF4DMR
Definition: HDF4_DMR.h:39
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
BESStopWatch
Definition: BESStopWatch.h:55
TheBESKeys::get_value
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:272
BESDDSResponse
Holds a DDS object within the BES.
Definition: BESDDSResponse.h:50
BESDASResponse::clear_container
virtual void clear_container()
clear the container in the DAP response object
Definition: BESDASResponse.cc:58
BESDapResponse::set_constraint
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:115
BESDataDDSResponse
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Definition: BESDataDDSResponse.h:46
Error
BESUtil::lowercase
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:200
BESUtil::implode
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:638
BESDapError
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
BESDataHandlerInterface
Structure storing information used by the BES to handle the request.
Definition: BESDataHandlerInterface.h:56
BESError
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
BESResponseObject
Abstract base class representing a specific set of information in response to a request to the BES.
Definition: BESResponseObject.h:45
BESDMRResponse
Represents an OPeNDAP DMR DAP4 data object within the BES.
Definition: BESDMRResponse.h:39