bes  Updated for version 3.20.5
h5gmcfdap.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler: an HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
22 // Suite 203, Champaign, IL 61820
23 
32 
33 #include <sys/types.h>
34 #include <sys/stat.h>
35 #include <fcntl.h>
36 #include <unistd.h>
37 #include <iostream>
38 #include <sstream>
39 
40 #include <BESDebug.h>
41 #include <InternalErr.h>
42 
43 #include "HDF5RequestHandler.h"
44 #include "h5cfdaputil.h"
45 #include "h5gmcfdap.h"
46 #include "HDF5CFByte.h"
47 #include "HDF5CFUInt16.h"
48 #include "HDF5CFInt16.h"
49 #include "HDF5CFUInt32.h"
50 #include "HDF5CFInt32.h"
51 #include "HDF5CFFloat32.h"
52 #include "HDF5CFFloat64.h"
53 #include "HDF5CFStr.h"
54 #include "HDF5CFArray.h"
55 #include "HDF5GMCFMissLLArray.h"
56 #include "HDF5GMCFFillIndexArray.h"
58 #include "HDF5GMCFSpecialCVArray.h"
59 #include "HDF5GMSPCFArray.h"
60 
61 using namespace std;
62 using namespace libdap;
63 using namespace HDF5CF;
64 
65 // Map general HDF5 products to DAP DDS
66 void map_gmh5_cfdds(DDS &dds, hid_t file_id, const string& filename){
67 
68  BESDEBUG("h5","Coming to GM products DDS mapping function map_gmh5_cfdds() "<<endl);
69 
70  H5GCFProduct product_type = check_product(file_id);
71 
72  GMPattern gproduct_pattern = OTHERGMS;
73 
74  GMFile * f = NULL;
75 
76  try {
77  f = new GMFile(filename.c_str(),file_id,product_type,gproduct_pattern);
78  }
79  catch(...) {
80  throw InternalErr(__FILE__,__LINE__,"Cannot allocate memory for GMFile ");
81  }
82  // Generally don't need to handle attributes when handling DDS.
83  bool include_attr = false;
84  try {
85  // Retrieve all HDF5 info(Not the values)
86  f->Retrieve_H5_Info(filename.c_str(),file_id,include_attr);
87 
88  // Update product type
89  // Newer version of a product may have different layout and the
90  // product type needs to be changed to reflect it. We also want
91  // to support the older version in case people still use them.
92  // This routine will check if newer layout can be applied. If yes,
93  // update the product type.
95 
97 
98  // Need to add dimension names.
99  f->Add_Dim_Name();
100 
101  // Handle coordinate variables
102  f->Handle_CVar();
103 #if 0
104  // We need to retrieve coordinate variable attributes for memory cache use.
105  //f->Retrieve_H5_CVar_Supported_Attr_Values();
106  //if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
107  // (HDF5RequestHandler::get_srdata_mem_cache() != NULL)){
108  // f->Retrieve_H5_Supported_Attr_Values();
109 #endif
110 
111 
112  // Handle special variables
113  f->Handle_SpVar();
114 
115  // When cv memory cache is on, the unit attributes are needed to
116  // distinguish whether this is lat/lon. Generally, memory cache
117  // is not used. This snipnet will not be accessed.
118  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
119  (HDF5RequestHandler::get_srdata_mem_cache() != NULL)){
120 
121  // Handle unsupported datatypes including the attributes
122  f->Handle_Unsupported_Dtype(true);
123 
124  // Handle unsupported dataspaces including the attributes
125  f->Handle_Unsupported_Dspace(true);
126 
127  // We need to retrieve coordinate variable attributes for memory cache use.
129 
130  }
131  else {
132 
133  // Handle unsupported datatypes
134  f->Handle_Unsupported_Dtype(include_attr);
135 
136  // Handle unsupported dataspaces
137  f->Handle_Unsupported_Dspace(include_attr);
138 
139  }
140 
141  // Need to handle the "coordinate" attributes when memory cache is turned on.
142  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
143  (HDF5RequestHandler::get_srdata_mem_cache() != NULL))
144  f->Add_Supplement_Attrs(HDF5RequestHandler::get_add_path_attrs());
145 
146  // Adjust object names(may remove redundant paths)
147  f->Adjust_Obj_Name();
148 
149  // Flatten the object names
150  f->Flatten_Obj_Name(include_attr);
151 
152  // Handle Object name clashings
153  // Only when the check_nameclashing key is turned on or
154  // general product.
155  if(General_Product == product_type ||
156  true == HDF5RequestHandler::get_check_name_clashing())
157  f->Handle_Obj_NameClashing(include_attr);
158 
159  // Adjust Dimension name
160  f->Adjust_Dim_Name();
161  if(General_Product == product_type ||
162  true == HDF5RequestHandler::get_check_name_clashing())
164 
165  f->Handle_Hybrid_EOS5();
166  if(true == f->Have_Grid_Mapping_Attrs())
168  // Need to handle the "coordinate" attributes when memory cache is turned on.
169  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
170  (HDF5RequestHandler::get_srdata_mem_cache() != NULL))
171  f->Handle_Coor_Attr();
172 
175  }
176  catch (HDF5CF::Exception &e){
177  if (f != NULL)
178  delete f;
179  throw InternalErr(e.what());
180  }
181 
182  // generate DDS.
183  try {
184  gen_gmh5_cfdds(dds,f);
185  }
186  catch(...) {
187  if (f != NULL)
188  delete f;
189  throw;
190  }
191 
192  if (f != NULL)
193  delete f;
194 }
195 
196 // Map general HDF5 products to DAP DAS
197 void map_gmh5_cfdas(DAS &das, hid_t file_id, const string& filename){
198 
199  BESDEBUG("h5","Coming to GM products DAS mapping function map_gmh5_cfdas() "<<endl);
200 
201  H5GCFProduct product_type = check_product(file_id);
202  GMPattern gproduct_pattern = OTHERGMS;
203 
204  GMFile *f = NULL;
205 
206  try {
207  f = new GMFile(filename.c_str(),file_id,product_type,gproduct_pattern);
208  }
209  catch(...) {
210  throw InternalErr(__FILE__,__LINE__,"Cannot allocate memory for GMFile ");
211  }
212 
213  bool include_attr = true;
214  try {
215  f->Retrieve_H5_Info(filename.c_str(),file_id,include_attr);
216 
217  // Update product type(check comments of map_gmh5_cfdds)
218  f->Update_Product_Type();
219 
221 
222  f->Add_Dim_Name();
223  f->Handle_CVar();
224  f->Handle_SpVar();
225  f->Handle_Unsupported_Dtype(include_attr);
226 
227  // Remove unsupported dataspace
228  f->Handle_Unsupported_Dspace(include_attr);
229 
230  // Need to retrieve the attribute values to feed DAS
232 
233  // Handle other unsupported objects,
234  // currently it mainly generates the info. for the
235  // unsupported objects other than datatype, dataspace,links and named datatype
236  // One area is maybe very long string. So we retrieve the attribute
237  // values before calling this function.
238  f->Handle_Unsupported_Others(include_attr);
239 
240 
241  // Need to add original variable name and path
242  // and other special attributes
243  // Can be turned on/off by using the check_path_attrs keys.
244  f->Add_Supplement_Attrs(HDF5RequestHandler::get_add_path_attrs());
245  f->Adjust_Obj_Name();
246  f->Flatten_Obj_Name(include_attr);
247  if(General_Product == product_type ||
248  true == HDF5RequestHandler::get_check_name_clashing())
249  f->Handle_Obj_NameClashing(include_attr);
250 
251  // Handle the "coordinate" attributes.
252  f->Handle_Coor_Attr();
253 
254  f->Handle_Hybrid_EOS5();
255  if(true == f->Have_Grid_Mapping_Attrs())
257 
259 
261  }
262  catch (HDF5CF::Exception &e){
263  if (f!= NULL)
264  delete f;
265  throw InternalErr(e.what());
266  }
267 
268  // Generate the DAS attributes.
269  try {
270  gen_gmh5_cfdas(das,f);
271  }
272  catch (...) {
273  if (f!= NULL)
274  delete f;
275  throw;
276 
277  }
278 
279  if (f != NULL)
280  delete f;
281 }
282 
283 // Generate DDS mapped from general HDF5 products
284 void gen_gmh5_cfdds( DDS & dds, HDF5CF:: GMFile *f) {
285 
286  BESDEBUG("h5","Coming to GM DDS generation function gen_gmh5_cfdds() "<<endl);
287 
288  const vector<HDF5CF::Var *>& vars = f->getVars();
289  const vector<HDF5CF::GMCVar *>& cvars = f->getCVars();
290  const vector<HDF5CF::GMSPVar *>& spvars = f->getSPVars();
291  const string filename = f->getPath();
292  const hid_t fileid = f->getFileID();
293 
294  // Read Variable info.
295 
296  vector<HDF5CF::Var *>::const_iterator it_v;
297  vector<HDF5CF::GMCVar *>::const_iterator it_cv;
298  vector<HDF5CF::GMSPVar *>::const_iterator it_spv;
299 
300  for (it_v = vars.begin(); it_v !=vars.end();++it_v) {
301  BESDEBUG("h5","variable full path= "<< (*it_v)->getFullPath() <<endl);
302  // Handle 64-integer DAP4 CF mapping
303  if(need_attr_values_for_dap4(*it_v) == true)
304  f->Retrieve_H5_Var_Attr_Values(*it_v);
305  gen_dap_onevar_dds(dds,*it_v,fileid, filename);
306  }
307  for (it_cv = cvars.begin(); it_cv !=cvars.end();++it_cv) {
308  BESDEBUG("h5","variable full path= "<< (*it_cv)->getFullPath() <<endl);
309  gen_dap_onegmcvar_dds(dds,*it_cv,fileid, filename);
310  }
311 
312  for (it_spv = spvars.begin(); it_spv !=spvars.end();it_spv++) {
313  BESDEBUG("h5","variable full path= "<< (*it_spv)->getFullPath() <<endl);
314  gen_dap_onegmspvar_dds(dds,*it_spv,fileid, filename);
315  }
316 
317 }
318 
319 // Generate DAS mapped from general HDF5 products
320 void gen_gmh5_cfdas( DAS & das, HDF5CF:: GMFile *f) {
321 
322  BESDEBUG("h5","Coming to GM DAS generation function gen_gmh5_cfdas() "<<endl);
323 
324  // First check if this is for generating the ignored object info.
325  if(true == f->Get_IgnoredInfo_Flag()) {
326  gen_gmh5_cf_ignored_obj_info(das, f);
327  return;
328  }
329 
330  const vector<HDF5CF::Var *>& vars = f->getVars();
331  const vector<HDF5CF::GMCVar *>& cvars = f->getCVars();
332  const vector<HDF5CF::GMSPVar *>& spvars = f->getSPVars();
333  const vector<HDF5CF::Group *>& grps = f->getGroups();
334  const vector<HDF5CF::Attribute *>& root_attrs = f->getAttributes();
335 
336 
337  vector<HDF5CF::Var *>::const_iterator it_v;
338  vector<HDF5CF::GMCVar *>::const_iterator it_cv;
339  vector<HDF5CF::GMSPVar *>::const_iterator it_spv;
340  vector<HDF5CF::Group *>::const_iterator it_g;
341  vector<HDF5CF::Attribute *>::const_iterator it_ra;
342 
343  // Handling the file attributes(attributes under the root group)
344  // The table name is "HDF_GLOBAL".
345 
346  if (false == root_attrs.empty()) {
347 
348  AttrTable *at = das.get_table(FILE_ATTR_TABLE_NAME);
349  if (NULL == at)
350  at = das.add_table(FILE_ATTR_TABLE_NAME, new AttrTable);
351 
352  for (it_ra = root_attrs.begin(); it_ra != root_attrs.end(); ++it_ra) {
353  // Check and may update the 64-bit integer attributes in DAP4.
354  check_update_int64_attr("",*it_ra);
355  gen_dap_oneobj_das(at,*it_ra,NULL);
356  }
357  }
358 
359  if (false == grps.empty()) {
360  for (it_g = grps.begin();
361  it_g != grps.end(); ++it_g) {
362  AttrTable *at = das.get_table((*it_g)->getNewName());
363  if (NULL == at)
364  at = das.add_table((*it_g)->getNewName(), new AttrTable);
365 
366  for (it_ra = (*it_g)->getAttributes().begin();
367  it_ra != (*it_g)->getAttributes().end(); ++it_ra) {
368  check_update_int64_attr((*it_g)->getNewName(),*it_ra);
369  gen_dap_oneobj_das(at,*it_ra,NULL);
370  }
371  }
372  }
373 
374  for (it_v = vars.begin();
375  it_v != vars.end(); ++it_v) {
376  if (false == ((*it_v)->getAttributes().empty())) {
377 
378  // Skip the 64-bit integer variables. The attribute mapping of
379  // DAP4 CF 64-bit integer variable support
380  // has been taken care at the routine gen_dap_onevar_dds()
381  // defined at h5commoncfdap.cc
382  if(H5INT64 == (*it_v)->getType() || H5UINT64 == (*it_v)->getType()){
383  continue;
384  }
385 
386  AttrTable *at = das.get_table((*it_v)->getNewName());
387  if (NULL == at)
388  at = das.add_table((*it_v)->getNewName(), new AttrTable);
389 
390  for (it_ra = (*it_v)->getAttributes().begin();
391  it_ra != (*it_v)->getAttributes().end(); ++it_ra) {
392  gen_dap_oneobj_das(at,*it_ra,*it_v);
393  }
394  // TODO: If a var has integer-64 bit datatype attributes, maybe
395  // we can just keep that attributes(not consistent but
396  // easy to implement) or we have to duplicate all
397  // the var in dmr and delete this var from dds.
398 
399  }
400 
401  // GPM needs to be handled in a special way(mostly _FillValue)
402  if(GPMS_L3 == f->getProductType() || GPMM_L3 == f->getProductType()
403  || GPM_L1 == f->getProductType())
404  update_GPM_special_attrs(das,*it_v,false);
405 
406  }
407 
408  for (it_cv = cvars.begin();
409  it_cv != cvars.end(); ++it_cv) {
410  if (false == ((*it_cv)->getAttributes().empty())) {
411 
412  // TODO: Add 64-bit int support for coordinates, this has not been tackled.
413  if(H5INT64 == (*it_cv)->getType() || H5UINT64 == (*it_cv)->getType()){
414  continue;
415  }
416 
417  AttrTable *at = das.get_table((*it_cv)->getNewName());
418  if (NULL == at)
419  at = das.add_table((*it_cv)->getNewName(), new AttrTable);
420 
421  for (it_ra = (*it_cv)->getAttributes().begin();
422  it_ra != (*it_cv)->getAttributes().end(); ++it_ra){
423  gen_dap_oneobj_das(at,*it_ra,*it_cv);
424  }
425 
426  }
427  // Though CF doesn't allow _FillValue, still keep it to keep the original form.
428  if(GPMS_L3 == f->getProductType() || GPMM_L3 == f->getProductType()
429  || GPM_L1 == f->getProductType())
430  update_GPM_special_attrs(das,*it_cv,true);
431 
432  }
433 
434  // Currently the special variables are only limited to the ACOS/OCO2 64-bit integer variables
435  for (it_spv = spvars.begin();
436  it_spv != spvars.end(); ++it_spv) {
437  if (false == ((*it_spv)->getAttributes().empty())) {
438 
439  AttrTable *at = das.get_table((*it_spv)->getNewName());
440  if (NULL == at)
441  at = das.add_table((*it_spv)->getNewName(), new AttrTable);
442 #if 0
443  // cerr<<"spv coordinate variable name "<<(*it_spv)->getNewName() <<endl;
444 #endif
445 
446  for (it_ra = (*it_spv)->getAttributes().begin();
447  it_ra != (*it_spv)->getAttributes().end(); ++it_ra)
448  gen_dap_oneobj_das(at,*it_ra,*it_spv);
449  }
450  }
451 
452  // CHECK ALL UNLIMITED DIMENSIONS from the coordinate variables based on the names.
453  if(f->HaveUnlimitedDim() == true) {
454 
455  BESDEBUG("h5","Find unlimited dimension in the GM DAS generation function gen_gmh5_cfdas() "<<endl);
456 
457  // Currently there is no way for DAP to present the unlimited dimension info.
458  // when there are no dimension names. So don't create DODS_EXTRA even if
459  // there is an unlimited dimension in the file. KY 2016-02-18
460  if(cvars.size() >0){
461 
462  // First check if we do have unlimited dimension in the coordinate variables.
463  // Since unsupported fakedims are removed, we may not have unlimited dimensions.
464  bool still_has_unlimited = false;
465  for (it_cv = cvars.begin();
466  it_cv != cvars.end(); ++it_cv) {
467  // Check unlimited dimension names.
468  for (vector<Dimension*>::const_iterator ird = (*it_cv)->getDimensions().begin();
469  ird != (*it_cv)->getDimensions().end(); ++ird) {
470 
471  // Currently we only check one unlimited dimension, which is the most
472  // common case. When receiving the conventions from JG, will add
473  // the support of multi-unlimited dimension. KY 2016-02-09
474  if((*ird)->HaveUnlimitedDim() == true) {
475  still_has_unlimited = true;
476  break;
477  }// if((*ird) is HaveUnlimitedDim()
478  }// for (vector<Dimension*>::
479  if(true == still_has_unlimited)
480  break;
481  }// for (it_cv=cvars.begin();
482 
483  if(true == still_has_unlimited) {
484  AttrTable* at = das.get_table("DODS_EXTRA");
485  if (NULL == at)
486  at = das.add_table("DODS_EXTRA", new AttrTable);
487 
488  string unlimited_names;
489 
490  for (it_cv = cvars.begin();
491  it_cv != cvars.end(); ++it_cv) {
492 #if 0
493  bool has_unlimited_dim = false;
494 #endif
495  // Check unlimited dimension names.
496  for (vector<Dimension*>::const_iterator ird = (*it_cv)->getDimensions().begin();
497  ird != (*it_cv)->getDimensions().end(); ++ird) {
498 
499  // Currently we only check one unlimited dimension, which is the most
500  // common case. When receiving the conventions from JG, will add
501  // the support of multi-unlimited dimension. KY 2016-02-09
502  if((*ird)->HaveUnlimitedDim() == true) {
503  if(unlimited_names=="") {
504  unlimited_names = (*ird)->getNewName();
505  if(at !=NULL)
506  at->append_attr("Unlimited_Dimension","String",unlimited_names);
507  }
508  else {
509  if(unlimited_names.rfind((*ird)->getNewName()) == string::npos) {
510  unlimited_names = unlimited_names+" "+(*ird)->getNewName();
511  if(at !=NULL)
512  at->append_attr("Unlimited_Dimension","String",(*ird)->getNewName());
513  }
514  }
515  }// if((*ird)->HaveUnlimitedDim()
516  }// for (vector<Dimension*>::
517  }// for (it_cv=cvars.begin();
518  }// if(true == still_has_unlimited)
519 
520  }//if(cvars.size()>0)
521 #if 0
522  // The following line will generate the string like "Band1 str1 str2".
523  //if(unlimited_names!="")
524  // // at->append_attr("Unlimited_Dimension","String",unlimited_names);
525 #endif
526  }
527 }
528 
529 // Generate the ignored object info. for the CF option of the general products
530 void gen_gmh5_cf_ignored_obj_info(DAS &das, HDF5CF::GMFile *f) {
531 
532  BESDEBUG("h5","Coming to gen_gmh5_cf_ignored_obj_info() "<<endl);
533  AttrTable *at = das.get_table("Ignored_Object_Info");
534  if (NULL == at)
535  at = das.add_table("Ignored_Object_Info", new AttrTable);
536 
537  at->append_attr("Message","String",f->Get_Ignored_Msg());
538 
539 }
540 
541 // Generate the DDS for a coordinate variable of the General products
542 void gen_dap_onegmcvar_dds(DDS &dds,const HDF5CF::GMCVar* cvar, const hid_t file_id, const string & filename) {
543 
544  BESDEBUG("h5","Coming to gen_dap_onegmcvar_dds() "<<endl);
545 
546  if(cvar->getType() == H5INT64 || cvar->getType() == H5UINT64)
547  return;
548  BaseType *bt = NULL;
549 
550  switch(cvar->getType()) {
551 #define HANDLE_CASE(tid,type) \
552  case tid: \
553  bt = new (type)(cvar->getNewName(),cvar->getFullPath()); \
554  break;
555 
556  HANDLE_CASE(H5FLOAT32, HDF5CFFloat32);
557  HANDLE_CASE(H5FLOAT64, HDF5CFFloat64);
558  HANDLE_CASE(H5CHAR,HDF5CFInt16);
559  HANDLE_CASE(H5UCHAR, HDF5CFByte);
560  HANDLE_CASE(H5INT16, HDF5CFInt16);
561  HANDLE_CASE(H5UINT16, HDF5CFUInt16);
562  HANDLE_CASE(H5INT32, HDF5CFInt32);
563  HANDLE_CASE(H5UINT32, HDF5CFUInt32);
564  HANDLE_CASE(H5FSTRING, Str);
565  HANDLE_CASE(H5VSTRING, Str);
566 
567  default:
568  throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
569 #undef HANDLE_CASE
570  }
571 
572  if (bt) {
573 
574  const vector<HDF5CF::Dimension *>& dims = cvar->getDimensions();
575  vector <HDF5CF::Dimension*>:: const_iterator it_d;
576  vector <size_t> dimsizes;
577  dimsizes.resize(cvar->getRank());
578  for(int i = 0; i <cvar->getRank();i++)
579  dimsizes[i] = (dims[i])->getSize();
580 
581 
582  if(dims.size() == 0)
583  throw InternalErr(__FILE__,__LINE__,"the coordinate variable cannot be a scalar");
584 
585  switch(cvar->getCVType()) {
586 
587  case CV_EXIST:
588  {
589  HDF5CFArray *ar = NULL;
590 
591  // Need to check if this CV is lat/lon. This is necessary when data memory cache is turned on.
592  bool is_latlon = cvar->isLatLon();
593 
594  try {
595  ar = new HDF5CFArray (
596  cvar->getRank(),
597  file_id,
598  filename,
599  cvar->getType(),
600  dimsizes,
601  cvar->getFullPath(),
602  cvar->getTotalElems(),
603  CV_EXIST,
604  is_latlon,
605  cvar->getCompRatio(),
606  cvar->getNewName(),
607  bt);
608  }
609  catch(...) {
610  delete bt;
611  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5CFArray. ");
612  }
613 
614  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
615  if (""==(*it_d)->getNewName())
616  ar->append_dim((*it_d)->getSize());
617  else
618  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
619  }
620 
621  dds.add_var(ar);
622  delete bt;
623  delete ar;
624  }
625  break;
626 
627  case CV_LAT_MISS:
628  case CV_LON_MISS:
629  {
630  // Using HDF5GMCFMissLLArray
631  HDF5GMCFMissLLArray *ar = NULL;
632  try {
633  ar = new HDF5GMCFMissLLArray (
634  cvar->getRank(),
635  filename,
636  file_id,
637  cvar->getType(),
638  cvar->getFullPath(),
639  cvar->getPtType(),
640  cvar->getCVType(),
641  cvar->getNewName(),
642  bt);
643  }
644  catch(...) {
645  delete bt;
646  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissLLArray. ");
647  }
648 
649 
650  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
651  if (""==(*it_d)->getNewName())
652  ar->append_dim((*it_d)->getSize());
653  else
654  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
655  }
656 
657  dds.add_var(ar);
658  delete bt;
659  delete ar;
660  }
661  break;
662 
663  case CV_NONLATLON_MISS:
664  {
665 
666  if (cvar->getRank() !=1) {
667  delete bt;
668  throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
669  }
670  int nelem = (cvar->getDimensions()[0])->getSize();
671 
672  HDF5GMCFMissNonLLCVArray *ar = NULL;
673 
674  try {
675  ar = new HDF5GMCFMissNonLLCVArray(
676  cvar->getRank(),
677  nelem,
678  cvar->getNewName(),
679  bt);
680  }
681  catch(...) {
682  delete bt;
683  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
684  }
685 
686 
687  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
688  if (""==(*it_d)->getNewName())
689  ar->append_dim((*it_d)->getSize());
690  else
691  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
692  }
693  dds.add_var(ar);
694  delete bt;
695  delete ar;
696  }
697  break;
698 
699  case CV_FILLINDEX:
700  {
701 
702  if (cvar->getRank() !=1) {
703  delete bt;
704  throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
705  }
706 
707  HDF5GMCFFillIndexArray *ar = NULL;
708 
709  try {
710  ar = new HDF5GMCFFillIndexArray(
711  cvar->getRank(),
712  cvar->getType(),
713  cvar->getNewName(),
714  bt);
715  }
716  catch(...) {
717  delete bt;
718  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
719  }
720 
721 
722  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
723  if (""==(*it_d)->getNewName())
724  ar->append_dim((*it_d)->getSize());
725  else
726  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
727  }
728  dds.add_var(ar);
729  delete bt;
730  delete ar;
731  }
732  break;
733 
734 
735  case CV_SPECIAL:
736  {
737  // Currently only handle 1-D special CV.
738  if (cvar->getRank() !=1) {
739  delete bt;
740  throw InternalErr(__FILE__, __LINE__, "The rank of special coordinate variable must be 1");
741  }
742  int nelem = (cvar->getDimensions()[0])->getSize();
743 
744  HDF5GMCFSpecialCVArray * ar = NULL;
745  ar = new HDF5GMCFSpecialCVArray(
746  cvar->getType(),
747  nelem,
748  cvar->getFullPath(),
749  cvar->getPtType(),
750  cvar->getNewName(),
751  bt);
752 
753  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
754  if (""==(*it_d)->getNewName())
755  ar->append_dim((*it_d)->getSize());
756  else
757  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
758  }
759 
760  dds.add_var(ar);
761  delete bt;
762  delete ar;
763 
764  }
765  break;
766  case CV_MODIFY:
767  default:
768  delete bt;
769  throw InternalErr(__FILE__,__LINE__,"Coordinate variable type is not supported.");
770  }
771  }
772 }
773 
774 // Generate DDS for special variable in a general product
775 void gen_dap_onegmspvar_dds(DDS &dds,const HDF5CF::GMSPVar* spvar, const hid_t fileid, const string & filename) {
776 
777  BESDEBUG("h5","Coming to gen_dap_onegmspvar_dds() "<<endl);
778  BaseType *bt = NULL;
779 
780  switch(spvar->getType()) {
781 #define HANDLE_CASE(tid,type) \
782  case tid: \
783  bt = new (type)(spvar->getNewName(),spvar->getFullPath()); \
784  break;
785 
786  HANDLE_CASE(H5FLOAT32, HDF5CFFloat32);
787  HANDLE_CASE(H5FLOAT64, HDF5CFFloat64);
788  HANDLE_CASE(H5CHAR,HDF5CFInt16);
789  HANDLE_CASE(H5UCHAR, HDF5CFByte);
790  HANDLE_CASE(H5INT16, HDF5CFInt16);
791  HANDLE_CASE(H5UINT16, HDF5CFUInt16);
792  HANDLE_CASE(H5INT32, HDF5CFInt32);
793  HANDLE_CASE(H5UINT32, HDF5CFUInt32);
794  HANDLE_CASE(H5FSTRING, Str);
795  HANDLE_CASE(H5VSTRING, Str);
796  default:
797  throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
798 #undef HANDLE_CASE
799  }
800 
801  if (bt) {
802 
803  const vector<HDF5CF::Dimension *>& dims = spvar->getDimensions();
804  vector <HDF5CF::Dimension*>:: const_iterator it_d;
805 
806  if(dims.size() == 0)
807  throw InternalErr(__FILE__,__LINE__,"Currently don't support scalar special variables. ");
808 
809  HDF5GMSPCFArray *ar = NULL;
810 
811  try {
812  ar = new HDF5GMSPCFArray (
813  spvar->getRank(),
814  filename,
815  fileid,
816  spvar->getType(),
817  spvar->getFullPath(),
818  spvar->getOriginalType(),
819  spvar->getStartBit(),
820  spvar->getBitNum(),
821  spvar->getNewName(),
822  bt);
823  }
824  catch(...) {
825  delete bt;
826  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
827  }
828 
829 
830  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
831  if (""==(*it_d)->getNewName())
832  ar->append_dim((*it_d)->getSize());
833  else
834  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
835  }
836 
837  dds.add_var(ar);
838  delete bt;
839  delete ar;
840  }
841 
842 }
843 
844 // When we add floating point fill value at HDF5CF.cc, the value will be changed
845 // a little bit when it changes to string representation.
846 // For example, -9999.9 becomes -9999.9000123. To reduce the misunderstanding,we
847 // just add fillvalue in the string type here. KY 2014-04-02
848 void update_GPM_special_attrs(DAS& das, const HDF5CF::Var *var,bool is_cvar) {
849 
850  BESDEBUG("h5","Coming to update_GPM_special_attrs() "<<endl);
851  if(H5FLOAT64 == var->getType() ||
852  H5FLOAT32 == var->getType() ||
853  H5INT16 == var->getType() ||
854  H5CHAR == var->getType()) {
855 
856  AttrTable *at = das.get_table(var->getNewName());
857  if (NULL == at)
858  at = das.add_table(var->getNewName(), new AttrTable);
859  bool has_fillvalue = false;
860  AttrTable::Attr_iter it = at->attr_begin();
861  while (it!=at->attr_end() && false==has_fillvalue) {
862  if (at->get_name(it) =="_FillValue")
863  {
864  has_fillvalue = true;
865  string fillvalue ="";
866  if(H5FLOAT32 == var->getType()) {
867  const string cor_fill_value = "-9999.9";
868  fillvalue = (*at->get_attr_vector(it)->begin());
869  if((fillvalue.find(cor_fill_value) == 0) && (fillvalue!= cor_fill_value)) {
870  at->del_attr("_FillValue");
871  at->append_attr("_FillValue","Float32",cor_fill_value);
872  }
873  }
874  else if(H5FLOAT64 == var->getType()) {
875  const string cor_fill_value = "-9999.9";
876  const string exist_fill_value_substr = "-9999.8999";
877  fillvalue = (*at->get_attr_vector(it)->begin());
878  if((fillvalue.find(exist_fill_value_substr) == 0) && (fillvalue!= cor_fill_value)) {
879  at->del_attr("_FillValue");
880  at->append_attr("_FillValue","Float64",cor_fill_value);
881  }
882 
883  }
884  }
885  it++;
886  }
887 
888  // Add the fill value
889  if(false == is_cvar ) {
890 
891  // Current versions of GPM don't add fillvalues. We add the fillvalue according to the document.
892  if (has_fillvalue != true ) {
893 
894  if(H5FLOAT32 == var->getType())
895  at->append_attr("_FillValue","Float32","-9999.9");
896  else if(H5FLOAT64 == var->getType())
897  at->append_attr("_FillValue","Float64","-9999.9");
898  else if (H5INT16 == var->getType())
899  at->append_attr("_FillValue","Int16","-9999");
900  else if (H5CHAR == var->getType())// H5CHAR maps to DAP int16
901  at->append_attr("_FillValue","Int16","-99");
902 
903  }
904  }
905  }
906 }
907 
908 
This class provides a way to map HDF5 Str to DAP Str for the CF option.
void Flatten_Obj_Name(bool include_attr)
Flatten the object name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4806
This class specifies the retrieval of the missing lat/lon values for general HDF5 products.
void Handle_SpVar()
Handle special variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4631
H5GCFProduct getPtType() const
Get the data type of this variable.
Definition: HDF5CF.h:427
const std::string & getFullPath() const
Get the full path of this variable.
Definition: HDF5CF.h:283
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:259
const std::string & Get_Ignored_Msg()
Get the message that contains the ignored obj. info.
Definition: HDF5CF.h:922
void Handle_Coor_Attr()
Handle "coordinates" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:5811
void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes for general HDF5 products.
Definition: HDF5GMCF.cc:674
void Handle_CVar()
Handle coordinate variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:2762
void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5GMCF.cc:6740
const std::string & getNewName() const
Get the new name of this variable.
Definition: HDF5CF.h:277
void Handle_DimNameClashing()
Definition: HDF5GMCF.cc:4969
void Add_Dim_Name()
Add dimension name.
Definition: HDF5GMCF.cc:797
This class provides a way to map HDF5 unsigned 16-bit integer to DAP uint16 for the CF option.
const std::vector< Attribute * > & getAttributes() const
Public interface to obtain information of all attributes under the root group.
Definition: HDF5CF.h:664
void Handle_Obj_NameClashing(bool)
Handle object name clashing for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4864
This class provides a way to map HDF5 float to DAP float for the CF option.
This class is a derived class of Var. It represents a special general HDF5 product(currently ACOS and...
Definition: HDF5CF.h:380
int getCompRatio() const
Get the compression ratio of this dataset.
Definition: HDF5CF.h:318
This class is a derived class of CVar. It represents a coordinate variable for general HDF5 files.
Definition: HDF5CF.h:415
int getRank() const
Get the dimension rank of this variable.
Definition: HDF5CF.h:295
hid_t getFileID() const
Obtain the HDF5 file ID.
Definition: HDF5CF.h:646
This class specifies the retrieval of the missing lat/lon values for general HDF5 products.
This class provides a way to map HDF5 int16 to DAP int16 for the CF option.
const std::vector< Var * > & getVars() const
Public interface to obtain information of all variables.
Definition: HDF5CF.h:658
void Remove_Unused_FakeDimVars()
Unsupported datatype array may generate FakeDim. Remove them.
Definition: HDF5GMCF.cc:6744
bool HaveUnlimitedDim() const
Has unlimited dimensions.
Definition: HDF5CF.h:676
Map and generate DDS and DAS for the CF option for generic HDF5 products.
void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for general HDF5 products.
Definition: HDF5GMCF.cc:577
This class provides a way to map HDF5 64-bit floating-point(double) to DAP 64-bit floating-point for ...
bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5GMCF.cc:6736
This class is a derived class of File. It includes methods applied to general HDF5 files only.
Definition: HDF5CF.h:821
CVType getCVType() const
Get the coordinate variable type of this variable.
Definition: HDF5CF.h:360
void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:332
void Adjust_Dim_Name()
Adjust dimension name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5034
include the entry functions to execute the handlers
This class specifies the retrieval of data values for special HDF5 products Currently this only appli...
void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:384
const std::vector< Dimension * > & getDimensions() const
Get the list of the dimensions.
Definition: HDF5CF.h:312
This class provides a way to map HDF5 32-bit integer to DAP Int32 for the CF option.
H5DataType getType() const
Get the data type of this variable(Not HDF5 datatype id)
Definition: HDF5CF.h:301
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
virtual void Retrieve_H5_Var_Attr_Values(Var *var)
Retrieve attribute values for a variable.
Definition: HDF5CF.cc:741
void Remove_Unneeded_Objects()
Remove unneeded objects.
Definition: HDF5GMCF.cc:257
void Update_Product_Type()
Update "product type" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:235
Helper functions for generating DAS attributes and a function to check BES Key.
This class provides a way to map HDF5 unsigned 32-bit integer to DAP uint32 for the CF option.
bool Get_IgnoredInfo_Flag()
Obtain ignored info. flag.
Definition: HDF5CF.h:916
const std::vector< Group * > & getGroups() const
Public interface to obtain all the group info.
Definition: HDF5CF.h:670
void Adjust_Obj_Name()
Adjust object names based on different general NASA HDF5 products.
Definition: HDF5GMCF.cc:4721
This class specifies the retrieval of the values of non-lat/lon coordinate variables for general HDF5...
This class provides a way to map HDF5 byte to DAP byte for the CF option.
void Retrieve_H5_CVar_Supported_Attr_Values()
Retrieve coordinate variable attributes.
Definition: HDF5GMCF.cc:317
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
const std::string & getPath() const
Obtain the path of the file.
Definition: HDF5CF.h:652
void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5096
void Rename_NC4_NonCoordVars()
Remove the _nc4_non_coord from the variable new names.
Definition: HDF5GMCF.cc:6792
void Retrieve_H5_Info(const char *path, hid_t file_id, bool include_attr)
Retrieve DDS information from the HDF5 file; real implementation for general HDF5 products.
Definition: HDF5GMCF.cc:216