bes  Updated for version 3.20.6
h5gmcfdap.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler: an HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
22 // Suite 203, Champaign, IL 61820
23 
32 
33 #include <sys/types.h>
34 #include <sys/stat.h>
35 #include <fcntl.h>
36 #include <unistd.h>
37 #include <iostream>
38 #include <sstream>
39 
40 #include <BESDebug.h>
41 #include <InternalErr.h>
42 
43 #include "HDF5RequestHandler.h"
44 #include "h5cfdaputil.h"
45 #include "h5gmcfdap.h"
46 #include "HDF5CFByte.h"
47 #include "HDF5CFUInt16.h"
48 #include "HDF5CFInt16.h"
49 #include "HDF5CFUInt32.h"
50 #include "HDF5CFInt32.h"
51 #include "HDF5CFFloat32.h"
52 #include "HDF5CFFloat64.h"
53 #include "HDF5CFStr.h"
54 #include "HDF5CFArray.h"
55 #include "HDF5GMCFMissLLArray.h"
56 #include "HDF5GMCFFillIndexArray.h"
58 #include "HDF5GMCFSpecialCVArray.h"
59 #include "HDF5GMSPCFArray.h"
60 
61 using namespace std;
62 using namespace libdap;
63 using namespace HDF5CF;
64 
65 // Map general HDF5 products to DAP DDS
66 void map_gmh5_cfdds(DDS &dds, hid_t file_id, const string& filename){
67 
68  BESDEBUG("h5","Coming to GM products DDS mapping function map_gmh5_cfdds() "<<endl);
69 
70  H5GCFProduct product_type = check_product(file_id);
71 
72  GMPattern gproduct_pattern = OTHERGMS;
73 
74  GMFile * f = NULL;
75 
76  try {
77  f = new GMFile(filename.c_str(),file_id,product_type,gproduct_pattern);
78  }
79  catch(...) {
80  throw InternalErr(__FILE__,__LINE__,"Cannot allocate memory for GMFile ");
81  }
82  // Generally don't need to handle attributes when handling DDS.
83  bool include_attr = false;
84  try {
85  // Retrieve all HDF5 info(Not the values)
86  f->Retrieve_H5_Info(filename.c_str(),file_id,include_attr);
87 
88  // Update product type
89  // Newer version of a product may have different layout and the
90  // product type needs to be changed to reflect it. We also want
91  // to support the older version in case people still use them.
92  // This routine will check if newer layout can be applied. If yes,
93  // update the product type.
95 
97 
98  // Need to add dimension names.
99  f->Add_Dim_Name();
100 
101  // Handle coordinate variables
102  f->Handle_CVar();
103 #if 0
104  // We need to retrieve coordinate variable attributes for memory cache use.
105  //f->Retrieve_H5_CVar_Supported_Attr_Values();
106  //if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
107  // (HDF5RequestHandler::get_srdata_mem_cache() != NULL)){
108  // f->Retrieve_H5_Supported_Attr_Values();
109 #endif
110 
111 
112  // Handle special variables
113  f->Handle_SpVar();
114 
115  // When cv memory cache is on, the unit attributes are needed to
116  // distinguish whether this is lat/lon. Generally, memory cache
117  // is not used. This snipnet will not be accessed.
118  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
119  (HDF5RequestHandler::get_srdata_mem_cache() != NULL)){
120 
121  // Handle unsupported datatypes including the attributes
122  f->Handle_Unsupported_Dtype(true);
123 
124  // Handle unsupported dataspaces including the attributes
125  f->Handle_Unsupported_Dspace(true);
126 
127  // We need to retrieve coordinate variable attributes for memory cache use.
129 
130  }
131  else {
132 
133  // Handle unsupported datatypes
134  f->Handle_Unsupported_Dtype(include_attr);
135 
136  // Handle unsupported dataspaces
137  f->Handle_Unsupported_Dspace(include_attr);
138 
139  }
140 
141  // Need to handle the "coordinate" attributes when memory cache is turned on.
142  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
143  (HDF5RequestHandler::get_srdata_mem_cache() != NULL))
144  f->Add_Supplement_Attrs(HDF5RequestHandler::get_add_path_attrs());
145 
146  // Adjust object names(may remove redundant paths)
147  f->Adjust_Obj_Name();
148 
149  // Flatten the object names
150  f->Flatten_Obj_Name(include_attr);
151 
152  // Handle Object name clashings
153  // Only when the check_nameclashing key is turned on or
154  // general product.
155  if(General_Product == product_type ||
156  true == HDF5RequestHandler::get_check_name_clashing())
157  f->Handle_Obj_NameClashing(include_attr);
158 
159  // Adjust Dimension name
160  f->Adjust_Dim_Name();
161  if(General_Product == product_type ||
162  true == HDF5RequestHandler::get_check_name_clashing())
164 
165  f->Handle_Hybrid_EOS5();
166  if(true == f->Have_Grid_Mapping_Attrs())
168  // Need to handle the "coordinate" attributes when memory cache is turned on.
169  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
170  (HDF5RequestHandler::get_srdata_mem_cache() != NULL))
171  f->Handle_Coor_Attr();
172 
175  }
176  catch (HDF5CF::Exception &e){
177  if (f != NULL)
178  delete f;
179  throw InternalErr(e.what());
180  }
181 
182  // generate DDS.
183  try {
184  gen_gmh5_cfdds(dds,f);
185  }
186  catch(...) {
187  if (f != NULL)
188  delete f;
189  throw;
190  }
191 
192  if (f != NULL)
193  delete f;
194 }
195 
196 // Map general HDF5 products to DAP DAS
197 void map_gmh5_cfdas(DAS &das, hid_t file_id, const string& filename){
198 
199  BESDEBUG("h5","Coming to GM products DAS mapping function map_gmh5_cfdas() "<<endl);
200 
201  H5GCFProduct product_type = check_product(file_id);
202  GMPattern gproduct_pattern = OTHERGMS;
203 
204  GMFile *f = NULL;
205 
206  try {
207  f = new GMFile(filename.c_str(),file_id,product_type,gproduct_pattern);
208  }
209  catch(...) {
210  throw InternalErr(__FILE__,__LINE__,"Cannot allocate memory for GMFile ");
211  }
212 
213  bool include_attr = true;
214  try {
215  f->Retrieve_H5_Info(filename.c_str(),file_id,include_attr);
216 
217  // Update product type(check comments of map_gmh5_cfdds)
218  f->Update_Product_Type();
219 
221 
222  f->Add_Dim_Name();
223  f->Handle_CVar();
224  f->Handle_SpVar();
225  f->Handle_Unsupported_Dtype(include_attr);
226 
227  // Remove unsupported dataspace
228  f->Handle_Unsupported_Dspace(include_attr);
229 
230  // Need to retrieve the attribute values to feed DAS
232 
233  // Handle other unsupported objects,
234  // currently it mainly generates the info. for the
235  // unsupported objects other than datatype, dataspace,links and named datatype
236  // One area is maybe very long string. So we retrieve the attribute
237  // values before calling this function.
238  f->Handle_Unsupported_Others(include_attr);
239 
240 
241  // Need to add original variable name and path
242  // and other special attributes
243  // Can be turned on/off by using the check_path_attrs keys.
244  f->Add_Supplement_Attrs(HDF5RequestHandler::get_add_path_attrs());
245  f->Adjust_Obj_Name();
246  f->Flatten_Obj_Name(include_attr);
247  if(General_Product == product_type ||
248  true == HDF5RequestHandler::get_check_name_clashing())
249  f->Handle_Obj_NameClashing(include_attr);
250  if(f->HaveUnlimitedDim() == true)
251  f->Adjust_Dim_Name();
252  // Handle the "coordinate" attributes.
253  f->Handle_Coor_Attr();
254 
255  f->Handle_Hybrid_EOS5();
256  if(true == f->Have_Grid_Mapping_Attrs())
258 
260 
262  }
263  catch (HDF5CF::Exception &e){
264  if (f!= NULL)
265  delete f;
266  throw InternalErr(e.what());
267  }
268 
269  // Generate the DAS attributes.
270  try {
271  gen_gmh5_cfdas(das,f);
272  }
273  catch (...) {
274  if (f!= NULL)
275  delete f;
276  throw;
277 
278  }
279 
280  if (f != NULL)
281  delete f;
282 }
283 
284 // Generate DDS mapped from general HDF5 products
285 void gen_gmh5_cfdds( DDS & dds, HDF5CF:: GMFile *f) {
286 
287  BESDEBUG("h5","Coming to GM DDS generation function gen_gmh5_cfdds() "<<endl);
288 
289  const vector<HDF5CF::Var *>& vars = f->getVars();
290  const vector<HDF5CF::GMCVar *>& cvars = f->getCVars();
291  const vector<HDF5CF::GMSPVar *>& spvars = f->getSPVars();
292  const string filename = f->getPath();
293  const hid_t fileid = f->getFileID();
294 
295  // Read Variable info.
296 
297  vector<HDF5CF::Var *>::const_iterator it_v;
298  vector<HDF5CF::GMCVar *>::const_iterator it_cv;
299  vector<HDF5CF::GMSPVar *>::const_iterator it_spv;
300 
301  for (it_v = vars.begin(); it_v !=vars.end();++it_v) {
302  BESDEBUG("h5","variable full path= "<< (*it_v)->getFullPath() <<endl);
303  // Handle 64-integer DAP4 CF mapping
304  if(need_attr_values_for_dap4(*it_v) == true)
305  f->Retrieve_H5_Var_Attr_Values(*it_v);
306  gen_dap_onevar_dds(dds,*it_v,fileid, filename);
307  }
308  for (it_cv = cvars.begin(); it_cv !=cvars.end();++it_cv) {
309  BESDEBUG("h5","variable full path= "<< (*it_cv)->getFullPath() <<endl);
310  gen_dap_onegmcvar_dds(dds,*it_cv,fileid, filename);
311  }
312 
313  for (it_spv = spvars.begin(); it_spv !=spvars.end();it_spv++) {
314  BESDEBUG("h5","variable full path= "<< (*it_spv)->getFullPath() <<endl);
315  gen_dap_onegmspvar_dds(dds,*it_spv,fileid, filename);
316  }
317 
318 }
319 
320 // Generate DAS mapped from general HDF5 products
321 void gen_gmh5_cfdas( DAS & das, HDF5CF:: GMFile *f) {
322 
323  BESDEBUG("h5","Coming to GM DAS generation function gen_gmh5_cfdas() "<<endl);
324 
325  // First check if this is for generating the ignored object info.
326  if(true == f->Get_IgnoredInfo_Flag()) {
327  gen_gmh5_cf_ignored_obj_info(das, f);
328  return;
329  }
330 
331  const vector<HDF5CF::Var *>& vars = f->getVars();
332  const vector<HDF5CF::GMCVar *>& cvars = f->getCVars();
333  const vector<HDF5CF::GMSPVar *>& spvars = f->getSPVars();
334  const vector<HDF5CF::Group *>& grps = f->getGroups();
335  const vector<HDF5CF::Attribute *>& root_attrs = f->getAttributes();
336 
337 
338  vector<HDF5CF::Var *>::const_iterator it_v;
339  vector<HDF5CF::GMCVar *>::const_iterator it_cv;
340  vector<HDF5CF::GMSPVar *>::const_iterator it_spv;
341  vector<HDF5CF::Group *>::const_iterator it_g;
342  vector<HDF5CF::Attribute *>::const_iterator it_ra;
343 
344  // Handling the file attributes(attributes under the root group)
345  // The table name is "HDF_GLOBAL".
346 
347  if (false == root_attrs.empty()) {
348 
349  AttrTable *at = das.get_table(FILE_ATTR_TABLE_NAME);
350  if (NULL == at)
351  at = das.add_table(FILE_ATTR_TABLE_NAME, new AttrTable);
352 
353  for (it_ra = root_attrs.begin(); it_ra != root_attrs.end(); ++it_ra) {
354  // Check and may update the 64-bit integer attributes in DAP4.
355  check_update_int64_attr("",*it_ra);
356  gen_dap_oneobj_das(at,*it_ra,NULL);
357  }
358  }
359 
360  if (false == grps.empty()) {
361  for (it_g = grps.begin();
362  it_g != grps.end(); ++it_g) {
363  AttrTable *at = das.get_table((*it_g)->getNewName());
364  if (NULL == at)
365  at = das.add_table((*it_g)->getNewName(), new AttrTable);
366 
367  for (it_ra = (*it_g)->getAttributes().begin();
368  it_ra != (*it_g)->getAttributes().end(); ++it_ra) {
369  check_update_int64_attr((*it_g)->getNewName(),*it_ra);
370  gen_dap_oneobj_das(at,*it_ra,NULL);
371  }
372  }
373  }
374 
375  for (it_v = vars.begin();
376  it_v != vars.end(); ++it_v) {
377  if (false == ((*it_v)->getAttributes().empty())) {
378 
379  // Skip the 64-bit integer variables. The attribute mapping of
380  // DAP4 CF 64-bit integer variable support
381  // has been taken care at the routine gen_dap_onevar_dds()
382  // defined at h5commoncfdap.cc
383  if(H5INT64 == (*it_v)->getType() || H5UINT64 == (*it_v)->getType()){
384  continue;
385  }
386 
387  AttrTable *at = das.get_table((*it_v)->getNewName());
388  if (NULL == at)
389  at = das.add_table((*it_v)->getNewName(), new AttrTable);
390 
391  for (it_ra = (*it_v)->getAttributes().begin();
392  it_ra != (*it_v)->getAttributes().end(); ++it_ra) {
393  gen_dap_oneobj_das(at,*it_ra,*it_v);
394  }
395  // TODO: If a var has integer-64 bit datatype attributes, maybe
396  // we can just keep that attributes(not consistent but
397  // easy to implement) or we have to duplicate all
398  // the var in dmr and delete this var from dds.
399 
400  }
401 
402  // GPM needs to be handled in a special way(mostly _FillValue)
403  if(GPMS_L3 == f->getProductType() || GPMM_L3 == f->getProductType()
404  || GPM_L1 == f->getProductType())
405  update_GPM_special_attrs(das,*it_v,false);
406 
407  }
408 
409  for (it_cv = cvars.begin();
410  it_cv != cvars.end(); ++it_cv) {
411  if (false == ((*it_cv)->getAttributes().empty())) {
412 
413  // TODO: Add 64-bit int support for coordinates, this has not been tackled.
414  if(H5INT64 == (*it_cv)->getType() || H5UINT64 == (*it_cv)->getType()){
415  continue;
416  }
417 
418  AttrTable *at = das.get_table((*it_cv)->getNewName());
419  if (NULL == at)
420  at = das.add_table((*it_cv)->getNewName(), new AttrTable);
421 
422  for (it_ra = (*it_cv)->getAttributes().begin();
423  it_ra != (*it_cv)->getAttributes().end(); ++it_ra){
424  gen_dap_oneobj_das(at,*it_ra,*it_cv);
425  }
426 
427  }
428  // Though CF doesn't allow _FillValue, still keep it to keep the original form.
429  if(GPMS_L3 == f->getProductType() || GPMM_L3 == f->getProductType()
430  || GPM_L1 == f->getProductType())
431  update_GPM_special_attrs(das,*it_cv,true);
432 
433  }
434 
435  // Currently the special variables are only limited to the ACOS/OCO2 64-bit integer variables
436  for (it_spv = spvars.begin();
437  it_spv != spvars.end(); ++it_spv) {
438  if (false == ((*it_spv)->getAttributes().empty())) {
439 
440  AttrTable *at = das.get_table((*it_spv)->getNewName());
441  if (NULL == at)
442  at = das.add_table((*it_spv)->getNewName(), new AttrTable);
443 #if 0
444  // cerr<<"spv coordinate variable name "<<(*it_spv)->getNewName() <<endl;
445 #endif
446 
447  for (it_ra = (*it_spv)->getAttributes().begin();
448  it_ra != (*it_spv)->getAttributes().end(); ++it_ra)
449  gen_dap_oneobj_das(at,*it_ra,*it_spv);
450  }
451  }
452 
453  // CHECK ALL UNLIMITED DIMENSIONS from the coordinate variables based on the names.
454  if(f->HaveUnlimitedDim() == true) {
455 
456  BESDEBUG("h5","Find unlimited dimension in the GM DAS generation function gen_gmh5_cfdas() "<<endl);
457 
458  // Currently there is no way for DAP to present the unlimited dimension info.
459  // when there are no dimension names. So don't create DODS_EXTRA even if
460  // there is an unlimited dimension in the file. KY 2016-02-18
461  if(cvars.size() >0){
462 
463  // First check if we do have unlimited dimension in the coordinate variables.
464  // Since unsupported fakedims are removed, we may not have unlimited dimensions.
465  bool still_has_unlimited = false;
466  for (it_cv = cvars.begin();
467  it_cv != cvars.end(); ++it_cv) {
468  // Check unlimited dimension names.
469  for (vector<Dimension*>::const_iterator ird = (*it_cv)->getDimensions().begin();
470  ird != (*it_cv)->getDimensions().end(); ++ird) {
471 
472  // Currently we only check one unlimited dimension, which is the most
473  // common case. When receiving the conventions from JG, will add
474  // the support of multi-unlimited dimension. KY 2016-02-09
475  if((*ird)->HaveUnlimitedDim() == true) {
476  still_has_unlimited = true;
477  break;
478  }// if((*ird) is HaveUnlimitedDim()
479  }// for (vector<Dimension*>::
480  if(true == still_has_unlimited)
481  break;
482  }// for (it_cv=cvars.begin();
483 
484  if(true == still_has_unlimited) {
485  AttrTable* at = das.get_table("DODS_EXTRA");
486  if (NULL == at)
487  at = das.add_table("DODS_EXTRA", new AttrTable);
488 
489  string unlimited_names;
490 
491  for (it_cv = cvars.begin();
492  it_cv != cvars.end(); ++it_cv) {
493 #if 0
494  bool has_unlimited_dim = false;
495 #endif
496  // Check unlimited dimension names.
497  for (vector<Dimension*>::const_iterator ird = (*it_cv)->getDimensions().begin();
498  ird != (*it_cv)->getDimensions().end(); ++ird) {
499 
500  // Currently we only check one unlimited dimension, which is the most
501  // common case. When receiving the conventions from JG, will add
502  // the support of multi-unlimited dimension. KY 2016-02-09
503  if((*ird)->HaveUnlimitedDim() == true) {
504  if(unlimited_names=="") {
505  unlimited_names = (*ird)->getNewName();
506  if(at !=NULL)
507  at->append_attr("Unlimited_Dimension","String",unlimited_names);
508  }
509  else {
510  if(unlimited_names.rfind((*ird)->getNewName()) == string::npos) {
511  unlimited_names = unlimited_names+" "+(*ird)->getNewName();
512  if(at !=NULL)
513  at->append_attr("Unlimited_Dimension","String",(*ird)->getNewName());
514  }
515  }
516  }// if((*ird)->HaveUnlimitedDim()
517  }// for (vector<Dimension*>::
518  }// for (it_cv=cvars.begin();
519  }// if(true == still_has_unlimited)
520 
521  }//if(cvars.size()>0)
522 #if 0
523  // The following line will generate the string like "Band1 str1 str2".
524  //if(unlimited_names!="")
525  // // at->append_attr("Unlimited_Dimension","String",unlimited_names);
526 #endif
527  }
528 }
529 
530 // Generate the ignored object info. for the CF option of the general products
531 void gen_gmh5_cf_ignored_obj_info(DAS &das, HDF5CF::GMFile *f) {
532 
533  BESDEBUG("h5","Coming to gen_gmh5_cf_ignored_obj_info() "<<endl);
534  AttrTable *at = das.get_table("Ignored_Object_Info");
535  if (NULL == at)
536  at = das.add_table("Ignored_Object_Info", new AttrTable);
537 
538  at->append_attr("Message","String",f->Get_Ignored_Msg());
539 
540 }
541 
542 // Generate the DDS for a coordinate variable of the General products
543 void gen_dap_onegmcvar_dds(DDS &dds,const HDF5CF::GMCVar* cvar, const hid_t file_id, const string & filename) {
544 
545  BESDEBUG("h5","Coming to gen_dap_onegmcvar_dds() "<<endl);
546 
547  if(cvar->getType() == H5INT64 || cvar->getType() == H5UINT64)
548  return;
549  BaseType *bt = NULL;
550 
551  switch(cvar->getType()) {
552 #define HANDLE_CASE(tid,type) \
553  case tid: \
554  bt = new (type)(cvar->getNewName(),cvar->getFullPath()); \
555  break;
556 
557  HANDLE_CASE(H5FLOAT32, HDF5CFFloat32);
558  HANDLE_CASE(H5FLOAT64, HDF5CFFloat64);
559  HANDLE_CASE(H5CHAR,HDF5CFInt16);
560  HANDLE_CASE(H5UCHAR, HDF5CFByte);
561  HANDLE_CASE(H5INT16, HDF5CFInt16);
562  HANDLE_CASE(H5UINT16, HDF5CFUInt16);
563  HANDLE_CASE(H5INT32, HDF5CFInt32);
564  HANDLE_CASE(H5UINT32, HDF5CFUInt32);
565  HANDLE_CASE(H5FSTRING, Str);
566  HANDLE_CASE(H5VSTRING, Str);
567 
568  default:
569  throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
570 #undef HANDLE_CASE
571  }
572 
573  if (bt) {
574 
575  const vector<HDF5CF::Dimension *>& dims = cvar->getDimensions();
576  vector <HDF5CF::Dimension*>:: const_iterator it_d;
577  vector <size_t> dimsizes;
578  dimsizes.resize(cvar->getRank());
579  for(int i = 0; i <cvar->getRank();i++)
580  dimsizes[i] = (dims[i])->getSize();
581 
582 
583  if(dims.size() == 0)
584  throw InternalErr(__FILE__,__LINE__,"the coordinate variable cannot be a scalar");
585 
586  switch(cvar->getCVType()) {
587 
588  case CV_EXIST:
589  {
590  HDF5CFArray *ar = NULL;
591 
592  // Need to check if this CV is lat/lon. This is necessary when data memory cache is turned on.
593  bool is_latlon = cvar->isLatLon();
594 
595  try {
596  ar = new HDF5CFArray (
597  cvar->getRank(),
598  file_id,
599  filename,
600  cvar->getType(),
601  dimsizes,
602  cvar->getFullPath(),
603  cvar->getTotalElems(),
604  CV_EXIST,
605  is_latlon,
606  cvar->getCompRatio(),
607  cvar->getNewName(),
608  bt);
609  }
610  catch(...) {
611  delete bt;
612  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5CFArray. ");
613  }
614 
615  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
616  if (""==(*it_d)->getNewName())
617  ar->append_dim((*it_d)->getSize());
618  else
619  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
620  }
621 
622  dds.add_var(ar);
623  delete bt;
624  delete ar;
625  }
626  break;
627 
628  case CV_LAT_MISS:
629  case CV_LON_MISS:
630  {
631  // Using HDF5GMCFMissLLArray
632  HDF5GMCFMissLLArray *ar = NULL;
633  try {
634  ar = new HDF5GMCFMissLLArray (
635  cvar->getRank(),
636  filename,
637  file_id,
638  cvar->getType(),
639  cvar->getFullPath(),
640  cvar->getPtType(),
641  cvar->getCVType(),
642  cvar->getNewName(),
643  bt);
644  }
645  catch(...) {
646  delete bt;
647  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissLLArray. ");
648  }
649 
650 
651  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
652  if (""==(*it_d)->getNewName())
653  ar->append_dim((*it_d)->getSize());
654  else
655  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
656  }
657 
658  dds.add_var(ar);
659  delete bt;
660  delete ar;
661  }
662  break;
663 
664  case CV_NONLATLON_MISS:
665  {
666 
667  if (cvar->getRank() !=1) {
668  delete bt;
669  throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
670  }
671  int nelem = (cvar->getDimensions()[0])->getSize();
672 
673  HDF5GMCFMissNonLLCVArray *ar = NULL;
674 
675  try {
676  ar = new HDF5GMCFMissNonLLCVArray(
677  cvar->getRank(),
678  nelem,
679  cvar->getNewName(),
680  bt);
681  }
682  catch(...) {
683  delete bt;
684  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
685  }
686 
687 
688  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
689  if (""==(*it_d)->getNewName())
690  ar->append_dim((*it_d)->getSize());
691  else
692  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
693  }
694  dds.add_var(ar);
695  delete bt;
696  delete ar;
697  }
698  break;
699 
700  case CV_FILLINDEX:
701  {
702 
703  if (cvar->getRank() !=1) {
704  delete bt;
705  throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
706  }
707 
708  HDF5GMCFFillIndexArray *ar = NULL;
709 
710  try {
711  ar = new HDF5GMCFFillIndexArray(
712  cvar->getRank(),
713  cvar->getType(),
714  cvar->getNewName(),
715  bt);
716  }
717  catch(...) {
718  delete bt;
719  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
720  }
721 
722 
723  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
724  if (""==(*it_d)->getNewName())
725  ar->append_dim((*it_d)->getSize());
726  else
727  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
728  }
729  dds.add_var(ar);
730  delete bt;
731  delete ar;
732  }
733  break;
734 
735 
736  case CV_SPECIAL:
737  {
738  // Currently only handle 1-D special CV.
739  if (cvar->getRank() !=1) {
740  delete bt;
741  throw InternalErr(__FILE__, __LINE__, "The rank of special coordinate variable must be 1");
742  }
743  int nelem = (cvar->getDimensions()[0])->getSize();
744 
745  HDF5GMCFSpecialCVArray * ar = NULL;
746  ar = new HDF5GMCFSpecialCVArray(
747  cvar->getType(),
748  nelem,
749  cvar->getFullPath(),
750  cvar->getPtType(),
751  cvar->getNewName(),
752  bt);
753 
754  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
755  if (""==(*it_d)->getNewName())
756  ar->append_dim((*it_d)->getSize());
757  else
758  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
759  }
760 
761  dds.add_var(ar);
762  delete bt;
763  delete ar;
764 
765  }
766  break;
767  case CV_MODIFY:
768  default:
769  delete bt;
770  throw InternalErr(__FILE__,__LINE__,"Coordinate variable type is not supported.");
771  }
772  }
773 }
774 
775 // Generate DDS for special variable in a general product
776 void gen_dap_onegmspvar_dds(DDS &dds,const HDF5CF::GMSPVar* spvar, const hid_t fileid, const string & filename) {
777 
778  BESDEBUG("h5","Coming to gen_dap_onegmspvar_dds() "<<endl);
779  BaseType *bt = NULL;
780 
781  switch(spvar->getType()) {
782 #define HANDLE_CASE(tid,type) \
783  case tid: \
784  bt = new (type)(spvar->getNewName(),spvar->getFullPath()); \
785  break;
786 
787  HANDLE_CASE(H5FLOAT32, HDF5CFFloat32);
788  HANDLE_CASE(H5FLOAT64, HDF5CFFloat64);
789  HANDLE_CASE(H5CHAR,HDF5CFInt16);
790  HANDLE_CASE(H5UCHAR, HDF5CFByte);
791  HANDLE_CASE(H5INT16, HDF5CFInt16);
792  HANDLE_CASE(H5UINT16, HDF5CFUInt16);
793  HANDLE_CASE(H5INT32, HDF5CFInt32);
794  HANDLE_CASE(H5UINT32, HDF5CFUInt32);
795  HANDLE_CASE(H5FSTRING, Str);
796  HANDLE_CASE(H5VSTRING, Str);
797  default:
798  throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
799 #undef HANDLE_CASE
800  }
801 
802  if (bt) {
803 
804  const vector<HDF5CF::Dimension *>& dims = spvar->getDimensions();
805  vector <HDF5CF::Dimension*>:: const_iterator it_d;
806 
807  if(dims.size() == 0)
808  throw InternalErr(__FILE__,__LINE__,"Currently don't support scalar special variables. ");
809 
810  HDF5GMSPCFArray *ar = NULL;
811 
812  try {
813  ar = new HDF5GMSPCFArray (
814  spvar->getRank(),
815  filename,
816  fileid,
817  spvar->getType(),
818  spvar->getFullPath(),
819  spvar->getOriginalType(),
820  spvar->getStartBit(),
821  spvar->getBitNum(),
822  spvar->getNewName(),
823  bt);
824  }
825  catch(...) {
826  delete bt;
827  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
828  }
829 
830 
831  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
832  if (""==(*it_d)->getNewName())
833  ar->append_dim((*it_d)->getSize());
834  else
835  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
836  }
837 
838  dds.add_var(ar);
839  delete bt;
840  delete ar;
841  }
842 
843 }
844 
845 // When we add floating point fill value at HDF5CF.cc, the value will be changed
846 // a little bit when it changes to string representation.
847 // For example, -9999.9 becomes -9999.9000123. To reduce the misunderstanding,we
848 // just add fillvalue in the string type here. KY 2014-04-02
849 void update_GPM_special_attrs(DAS& das, const HDF5CF::Var *var,bool is_cvar) {
850 
851  BESDEBUG("h5","Coming to update_GPM_special_attrs() "<<endl);
852  if(H5FLOAT64 == var->getType() ||
853  H5FLOAT32 == var->getType() ||
854  H5INT16 == var->getType() ||
855  H5CHAR == var->getType()) {
856 
857  AttrTable *at = das.get_table(var->getNewName());
858  if (NULL == at)
859  at = das.add_table(var->getNewName(), new AttrTable);
860  bool has_fillvalue = false;
861  AttrTable::Attr_iter it = at->attr_begin();
862  while (it!=at->attr_end() && false==has_fillvalue) {
863  if (at->get_name(it) =="_FillValue")
864  {
865  has_fillvalue = true;
866  string fillvalue ="";
867  if(H5FLOAT32 == var->getType()) {
868  const string cor_fill_value = "-9999.9";
869  fillvalue = (*at->get_attr_vector(it)->begin());
870  if((fillvalue.find(cor_fill_value) == 0) && (fillvalue!= cor_fill_value)) {
871  at->del_attr("_FillValue");
872  at->append_attr("_FillValue","Float32",cor_fill_value);
873  }
874  }
875  else if(H5FLOAT64 == var->getType()) {
876  const string cor_fill_value = "-9999.9";
877  const string exist_fill_value_substr = "-9999.8999";
878  fillvalue = (*at->get_attr_vector(it)->begin());
879  if((fillvalue.find(exist_fill_value_substr) == 0) && (fillvalue!= cor_fill_value)) {
880  at->del_attr("_FillValue");
881  at->append_attr("_FillValue","Float64",cor_fill_value);
882  }
883 
884  }
885  }
886  it++;
887  }
888 
889  // Add the fill value
890  if(false == is_cvar ) {
891 
892  // Current versions of GPM don't add fillvalues. We add the fillvalue according to the document.
893  if (has_fillvalue != true ) {
894 
895  if(H5FLOAT32 == var->getType())
896  at->append_attr("_FillValue","Float32","-9999.9");
897  else if(H5FLOAT64 == var->getType())
898  at->append_attr("_FillValue","Float64","-9999.9");
899  else if (H5INT16 == var->getType())
900  at->append_attr("_FillValue","Int16","-9999");
901  else if (H5CHAR == var->getType())// H5CHAR maps to DAP int16
902  at->append_attr("_FillValue","Int16","-99");
903 
904  }
905  }
906  }
907 }
908 
909 
HDF5CF::GMFile::Get_IgnoredInfo_Flag
bool Get_IgnoredInfo_Flag()
Obtain ignored info. flag.
Definition: HDF5CF.h:916
HDF5CF::File::getGroups
const std::vector< Group * > & getGroups() const
Public interface to obtain all the group info.
Definition: HDF5CF.h:670
HDF5CF::GMFile::Handle_Unsupported_Others
void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes for general HDF5 products.
Definition: HDF5GMCF.cc:674
HDF5CFStr.h
This class provides a way to map HDF5 Str to DAP Str for the CF option.
HDF5CF::Var::getRank
int getRank() const
Get the dimension rank of this variable.
Definition: HDF5CF.h:295
HDF5CF::GMFile::Get_Ignored_Msg
const std::string & Get_Ignored_Msg()
Get the message that contains the ignored obj. info.
Definition: HDF5CF.h:922
HDF5CF::GMCVar
This class is a derived class of CVar. It represents a coordinate variable for general HDF5 files.
Definition: HDF5CF.h:415
HDF5CF::GMFile::Handle_Unsupported_Dtype
void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:384
HDF5CF::GMFile::Update_Product_Type
void Update_Product_Type()
Update "product type" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:235
HDF5GMCFFillIndexArray
Definition: HDF5GMCFFillIndexArray.h:44
HDF5CFInt16.h
This class provides a way to map HDF5 int16 to DAP int16 for the CF option.
HDF5CFUInt32.h
This class provides a way to map HDF5 unsigned 32-bit integer to DAP uint32 for the CF option.
HDF5CF::File::getAttributes
const std::vector< Attribute * > & getAttributes() const
Public interface to obtain information of all attributes under the root group.
Definition: HDF5CF.h:664
HDF5CF::Var
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:259
HDF5CF::GMFile::Handle_Obj_NameClashing
void Handle_Obj_NameClashing(bool)
Handle object name clashing for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4873
HDF5CFByte
Definition: HDF5CFByte.h:43
HDF5CF::GMFile::Handle_CVar
void Handle_CVar()
Handle coordinate variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:2768
HDF5CF::GMFile::Handle_DimNameClashing
void Handle_DimNameClashing()
Definition: HDF5GMCF.cc:4978
HDF5CF::File::getFileID
hid_t getFileID() const
Obtain the HDF5 file ID.
Definition: HDF5CF.h:646
HDF5CF::GMFile::Handle_Grid_Mapping_Vars
void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5GMCF.cc:6749
HDF5CF::Var::getType
H5DataType getType() const
Get the data type of this variable(Not HDF5 datatype id)
Definition: HDF5CF.h:301
HDF5GMSPCFArray.h
This class specifies the retrieval of data values for special HDF5 products Currently this only appli...
HDF5CF::GMFile::Rename_NC4_NonCoordVars
void Rename_NC4_NonCoordVars()
Remove the _nc4_non_coord from the variable new names.
Definition: HDF5GMCF.cc:6801
HDF5GMCFFillIndexArray.h
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
HDF5CFUInt16.h
This class provides a way to map HDF5 unsigned 16-bit integer to DAP uint16 for the CF option.
h5gmcfdap.h
Map and generate DDS and DAS for the CF option for generic HDF5 products.
HDF5CF::Var::getNewName
const std::string & getNewName() const
Get the new name of this variable.
Definition: HDF5CF.h:277
HDF5CF::File::HaveUnlimitedDim
bool HaveUnlimitedDim() const
Has unlimited dimensions.
Definition: HDF5CF.h:676
HDF5CF::GMFile::Handle_Coor_Attr
void Handle_Coor_Attr()
Handle "coordinates" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:5820
HDF5CF::GMFile::Add_Dim_Name
void Add_Dim_Name()
Add dimension name.
Definition: HDF5GMCF.cc:803
HDF5CF::GMFile
This class is a derived class of File. It includes methods applied to general HDF5 files only.
Definition: HDF5CF.h:821
HDF5CFFloat64
Definition: HDF5CFFloat64.h:41
HDF5CFInt16
Definition: HDF5CFInt16.h:40
HDF5CFByte.h
This class provides a way to map HDF5 byte to DAP byte for the CF option.
HDF5CFFloat32
Definition: HDF5CFFloat32.h:42
HDF5CFUInt32
Definition: HDF5CFUInt32.h:42
HDF5CF::GMFile::Remove_Unused_FakeDimVars
void Remove_Unused_FakeDimVars()
Unsupported datatype array may generate FakeDim. Remove them.
Definition: HDF5GMCF.cc:6753
libdap
Definition: BESDapFunctionResponseCache.h:35
HDF5RequestHandler.h
include the entry functions to execute the handlers
HDF5CF::File::Retrieve_H5_Var_Attr_Values
virtual void Retrieve_H5_Var_Attr_Values(Var *var)
Retrieve attribute values for a variable.
Definition: HDF5CF.cc:741
HDF5CFArray.h
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
HDF5CF::GMFile::Retrieve_H5_Supported_Attr_Values
void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:332
HDF5GMCFMissLLArray.h
This class specifies the retrieval of the missing lat/lon values for general HDF5 products.
HDF5GMCFMissLLArray
Definition: HDF5GMCFMissLLArray.h:45
HDF5GMCFSpecialCVArray
Definition: HDF5GMCFSpecialCVArray.h:43
HDF5GMCFMissNonLLCVArray
Definition: HDF5GMCFMissNonLLCVArray.h:42
HDF5CFInt32
Definition: HDF5CFInt32.h:41
HDF5CF::GMFile::Handle_Unsupported_Dspace
void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for general HDF5 products.
Definition: HDF5GMCF.cc:577
HDF5CF::CVar::getCVType
CVType getCVType() const
Get the coordinate variable type of this variable.
Definition: HDF5CF.h:360
h5cfdaputil.h
Helper functions for generating DAS attributes and a function to check BES Key.
HDF5CF::Var::getCompRatio
int getCompRatio() const
Get the compression ratio of this dataset.
Definition: HDF5CF.h:318
HDF5CFInt32.h
This class provides a way to map HDF5 32-bit integer to DAP Int32 for the CF option.
HDF5CF::GMFile::Retrieve_H5_Info
void Retrieve_H5_Info(const char *path, hid_t file_id, bool include_attr)
Retrieve DDS information from the HDF5 file; real implementation for general HDF5 products.
Definition: HDF5GMCF.cc:216
HDF5CFArray
Definition: HDF5CFArray.h:47
HDF5CF::File::getVars
const std::vector< Var * > & getVars() const
Public interface to obtain information of all variables.
Definition: HDF5CF.h:658
HDF5CFFloat64.h
This class provides a way to map HDF5 64-bit floating-point(double) to DAP 64-bit floating-point for ...
HDF5CF::GMFile::Adjust_Obj_Name
void Adjust_Obj_Name()
Adjust object names based on different general NASA HDF5 products.
Definition: HDF5GMCF.cc:4730
HDF5CF::GMFile::Flatten_Obj_Name
void Flatten_Obj_Name(bool include_attr)
Flatten the object name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4815
HDF5CFFloat32.h
This class provides a way to map HDF5 float to DAP float for the CF option.
HDF5GMCFMissNonLLCVArray.h
This class specifies the retrieval of the values of non-lat/lon coordinate variables for general HDF5...
HDF5GMSPCFArray
Definition: HDF5GMSPCFArray.h:45
HDF5CF::GMFile::Add_Supplement_Attrs
void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5105
HDF5CF::GMCVar::getPtType
H5GCFProduct getPtType() const
Get the data type of this variable.
Definition: HDF5CF.h:427
HDF5CF::GMFile::Handle_SpVar
void Handle_SpVar()
Handle special variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4640
HDF5CF::Exception
Definition: HDF5CF.h:70
HDF5CF::File::getPath
const std::string & getPath() const
Obtain the path of the file.
Definition: HDF5CF.h:652
HDF5CFUInt16
Definition: HDF5CFUInt16.h:40
HDF5GMCFSpecialCVArray.h
This class specifies the retrieval of the missing lat/lon values for general HDF5 products.
HDF5CF::Var::getDimensions
const std::vector< Dimension * > & getDimensions() const
Get the list of the dimensions.
Definition: HDF5CF.h:312
HDF5CF::GMFile::Retrieve_H5_CVar_Supported_Attr_Values
void Retrieve_H5_CVar_Supported_Attr_Values()
Retrieve coordinate variable attributes.
Definition: HDF5GMCF.cc:317
HDF5CF::GMFile::Have_Grid_Mapping_Attrs
bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5GMCF.cc:6745
HDF5CF::GMFile::Adjust_Dim_Name
void Adjust_Dim_Name()
Adjust dimension name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5043
HDF5CF::GMFile::Remove_Unneeded_Objects
void Remove_Unneeded_Objects()
Remove unneeded objects.
Definition: HDF5GMCF.cc:257
HDF5CF::GMSPVar
This class is a derived class of Var. It represents a special general HDF5 product(currently ACOS and...
Definition: HDF5CF.h:380
HDF5CF::Var::getFullPath
const std::string & getFullPath() const
Get the full path of this variable.
Definition: HDF5CF.h:283