bes  Updated for version 3.20.6
HDFCFUtil.h
1 // Helper functions for handling dimension maps,clear memories and separating namelist
3 //
4 // Authors: MuQun Yang <myang6@hdfgroup.org> Eunsoo Seo
5 // Copyright (c) 2010-2012 The HDF Group
7 #ifndef HDFCFUTIL_H
8 #define HDFCFUTIL_H
9 
10 #include <stdlib.h>
11 #include <string>
12 #include <iostream>
13 #include <sstream>
14 #include <vector>
15 #include <map>
16 #include <set>
17 #include <assert.h>
18 #include <iomanip>
19 #include <dirent.h>
20 #include <libgen.h>
21 #include <unistd.h>
22 #include <cerrno>
23 
24 #include "mfhdf.h"
25 #include "hdf.h"
26 
27 #ifdef USE_HDFEOS2_LIB
28 #include "HdfEosDef.h"
29 #include "HDFEOS2.h"
30 #endif
31 #include "HDFSP.h"
32 
33 #include <TheBESKeys.h>
34 #include <BESUtil.h>
35 #include <DDS.h>
36 #include <DAS.h>
37 #include <InternalErr.h>
38 
39 #include "escaping.h" // for escattr
40 
41 // This is the maximum number of MODIS special values.
42 const int MAX_NON_SCALE_SPECIAL_VALUE=65535;
43 
44 // This is the minimum number of MODIS special values.
45 const int MIN_NON_SCALE_SPECIAL_VALUE=65500;
46 
47 
48 
49 // This is used to retrieve dimension map info. when retrieving the data values of HDF-EOS swath that
50 // has swath dimension maps.
52 {
53  // geo dimension name
54  std::string geodim;
55 
56  // data dimension name
57  std::string datadim;
58 
59  // offset and increment of a dimension map
60  int32 offset, inc;
61 };
62 
63 
64 struct HDFCFUtil
65 {
66 
69  static void close_fileid(int32 sdfd,int32 file_id,int32 gridfd,int32 swathfd,bool pass_fileid_key);
70 
71 
78  static std::string escattr(std::string s);
79 
80 
83  static void Split (const char *s, int len, char sep,
84  std::vector < std::string > &names);
85 
87  static void Split (const char *sz, char sep,
88  std::vector < std::string > &names);
89 
93  static bool insert_map(std::map<std::string,std::string>& m, std::string key, std::string val);
94 
96  static std::string get_CF_string(std::string s);
97 
99  static void gen_unique_name(std::string &str,std::set<std::string>& namelist, int&clash_index);
100 
102  static void Handle_NameClashing(std::vector<std::string>&newobjnamelist);
103  static void Handle_NameClashing(std::vector<std::string>&newobjnamelist,std::set<std::string>&objnameset);
104 
106  static std::string print_attr(int32, int, void*);
107 
109  static std::string print_type(int32);
110 
112  static short obtain_type_size(int32);
113 
114 
115  // Subsetting the 2-D fields
116  template <typename T> static void LatLon2DSubset (T* outlatlon, int ydim, int xdim, T* latlon, int32 * offset, int32 * count, int32 * step);
117 
120  static void correct_fvalue_type(libdap::AttrTable *at,int32 dtype);
121 
122 
126  static void correct_scale_offset_type(libdap::AttrTable *at);
127 
128 
129 #ifdef USE_HDFEOS2_LIB
130 
133 
142  static bool change_data_type(libdap::DAS & das, SOType scaletype, const std::string & new_field_name);
143 
148  static bool is_special_value(int32 dtype,float fillvalue, float realvalue);
149 
151  static int check_geofile_dimmap(const std::string & geofilename);
152 
156  static bool is_modis_dimmap_nonll_field(std::string & fieldname);
157 
159  static void obtain_dimmap_info(const std::string& filename, HDFEOS2::Dataset*dataset,std::vector<struct dimmap_entry>& dimmaps, std::string & modis_geofilename,bool &geofile_nas_dimmap);
160 
162  //Scale and offset values with other types will be converted to float and their input values must also be in float.
163  //If the add_offset_found is false, the add_offset attribute will not be added. add_offset information will not be used.
164  static void add_scale_offset_attrs(libdap::AttrTable* at, const std::string& s_type, float svalue_f, double svalue_d, bool add_offset_found,
165  const std::string& o_type, float ovalue_f, double ovalue_d);
166 
167  static void add_scale_str_offset_attrs(libdap::AttrTable* at, const std::string& s_type, const std::string& s_value_str, bool add_offset_found,
168  const std::string& o_type, float ovalue_f, double ovalue_d);
169 
170 
171 
173  static void handle_modis_special_attrs_disable_scale_comp(libdap::AttrTable *at,const string &filename, bool is_grid, const std::string &newfname, SOType scaletype);
174 
178  static void handle_modis_special_attrs(libdap::AttrTable *at,const std::string &filename, bool is_grid, const std::string & newfname, SOType scaletype, bool gridname_change_valid_range, bool changedtype, bool &change_fvtype);
179 
181  static void handle_modis_vip_special_attrs(const std::string& valid_range_value,const std::string& scale_factor_value, float& valid_min, float & valid_max);
182 
184  static void handle_amsr_attrs(libdap::AttrTable *at);
185 
186  // The following four functions is to handle 1-D CF CV variables required by CF projection conventions for grid.
187  // Obtains the latitude and longitude dimension info. of an HDF-EOS2 grid.
188  static void obtain_grid_latlon_dim_info(HDFEOS2::GridDataset*,string &,int32 &,string &,int32 &);
189 
190  // Adds the 1-D cf grid projection mapping attribute to data variables
191  // It is called by the function add_cf_grid_attrs.
192  static void add_cf_grid_mapping_attr(libdap::DAS &das, HDFEOS2::GridDataset*gdset,const string& cf_projection,
193  const string & dim0name,int32 dim0size,const string &dim1name,int32 dim1size);
194 
195  // This function adds the 1-D horizontal coordinate variables as well as the dummy projection variable to the grid.
196  static void add_cf_grid_cvs(libdap::DDS & dds, HDFEOS2::GridDataset *gdset);
197 
198  //Adds 1D grid mapping CF attributes to CV and data variables.
199  static void add_cf_grid_cv_attrs(libdap::DAS &das, HDFEOS2::GridDataset *gdset);
200 
201 #endif
202 
203  // Check OBPG attributes. Specifically, check if slope and intercept can be obtained from the file level.
204  // If having global slope and intercept, obtain OBPG scaling, slope and intercept values.
205  static void check_obpg_global_attrs(HDFSP::File*f,std::string & scaling, float & slope,bool &global_slope_flag,float & intercept, bool & global_intercept_flag);
206 
207  // For some OBPG files that only provide slope and intercept at the file level,
208  // global slope and intercept are needed to add to all fields and their names are needed to be changed to scale_factor and add_offset.
209  // For OBPG files that provide slope and intercept at the field level, slope and intercept are needed to rename to scale_factor and add_offset.
210  static void add_obpg_special_attrs(HDFSP::File*f,libdap::DAS &das, HDFSP::SDField* spsds,std::string & scaling, float&slope,bool &global_slope_flag,float& intercept,bool &global_intercept_flag);
211 
212  // Handle HDF4 OTHERHDF products that follow SDS dimension scale model.
213  // The special handling of AVHRR data is also included.
214  static void handle_otherhdf_special_attrs(HDFSP::File *f, libdap::DAS &das);
215 
216  // Add missing CF attributes for non-CV varibles
217  static void add_missing_cf_attrs(HDFSP::File*f,libdap::DAS &das);
218 
219  // Handle Merra and CERES attributes with the BES key EnableCERESMERRAShortName.
220  static void handle_merra_ceres_attrs_with_bes_keys(HDFSP::File*f, libdap::DAS &das,const std::string& filename);
221 
222  // Handle the attributes with the BES key EnableVdataDescAttr.
223  static void handle_vdata_attrs_with_desc_key(HDFSP::File*f,libdap::DAS &das);
224 
225  // Map AIRS version 6 or MOD08 HDF-EOS object attributes to DAP2, here we simply call Hopen and just retrieve the vgroup info. for the performance reason.
226  static void map_eos2_objects_attrs(libdap::DAS &das,const string &filename);
227  static void map_eos2_one_object_attrs_wrapper(libdap::DAS &das,int32 file_id,int32 vgroup_id, const string &vgroup_name, bool is_grid);
228  static void map_eos2_one_object_attrs(libdap::DAS &das,int32 file_id,int32 obj_attr_group_id, const string &vgroup_name);
229 
230  // Parse TRMM V7 GridHeaders
231  static void parser_trmm_v7_gridheader(const std:: vector<char>&value, int& latsize, int&lonsize, float& lat_start, float& lon_start, float& lat_res, float& lon_res, bool check_reg_orig);
232 
233  // Use to generate cache file name.
234  // Reverse the char array order
235  static void rev_str(char *str, int len);
236 
237  // Return the index of the char array for the integer part
238  static int int_to_str(int,char str[],int);
239 
240  // Convert a double number to char array
241  static void dtoa(double,char *,int);
242 
243  // Obtain the double number in the string format
244  static std::string get_double_str(double,int,int);
245 
246  // Obtain the integer in the string format
247  static std::string get_int_str(int);
248 
249 
250 #if 0
251  static size_t write_vector_to_file(const std::string &,const vector<double> &,size_t);
252  static ssize_t write_vector_to_file2(const std::string &,const vector<double> &,size_t);
253 #endif
254  // Read double-type data from a file to a vector of double type
255  static ssize_t read_vector_from_file(int fd,vector<double> &,size_t);
256 
257  // wrap function of Unix read to a buffer. Memory for the buffer should be allocated.
258  static ssize_t read_buffer_from_file(int fd,void*buf,size_t);
259  static std::string obtain_cache_fname(const std::string & fprefix, const std::string & fname, const std::string &vname);
260  static size_t obtain_dds_cache_size(HDFSP::File*);
261  static void write_sp_sds_dds_cache(HDFSP::File*,FILE*,size_t,const std::string & fname);
262  static void read_sp_sds_dds_cache(FILE*,libdap::DDS * dds_ptr,const std::string &filename, const std::string &hdf_filename);
263 
264 };
265 
267 inline int32
268 INDEX_nD_TO_1D (const std::vector < int32 > &dims,
269  const std::vector < int32 > &pos)
270 {
271  /*
272  int a[10][20][30]; // & a[1][2][3] == a + (20*30+1 + 30*2 + 1 *3);
273  int b[10][2]; // &b[1][2] == b + (20*1 + 2);
274  */
275  assert (dims.size () == pos.size ());
276  int32 sum = 0;
277  int32 start = 1;
278 
279  for (unsigned int p = 0; p < pos.size (); p++) {
280  int32 m = 1;
281 
282  for (unsigned int j = start; j < dims.size (); j++)
283  m *= dims[j];
284  sum += m * pos[p];
285  start++;
286  }
287  return sum;
288 }
289 
290 // Build a lock of a certain type.
291 static inline struct flock *lock(int type) {
292  static struct flock lock;
293  lock.l_type = type;
294  lock.l_whence = SEEK_SET;
295  lock.l_start = 0;
296  lock.l_len = 0;
297  lock.l_pid = getpid();
298 
299  return &lock;
300 }
301 
302 static inline string get_errno() {
303  char *s_err = strerror(errno);
304  if (s_err)
305  return s_err;
306  else
307  return "Unknown error.";
308 }
309 
311 //
312 // \param input Input variable
313 // \param dim dimension info of the input
314 // \param start start indexes of each dim
315 // \param stride stride of each dim
316 // \param edge count of each dim
317 // \param poutput output variable
318 // \parrm index dimension index
319 // \return 0 if successful. -1 otherwise.
320 //
321 #if 0
322 template<typename T>
323 int subset(
324  const T input[],
325  int rank,
326  vector<int> & dim,
327  vector<int> & start,
328  vector<int> & stride,
329  vector<int> & edge,
330  std::vector<T> *poutput,
331  vector<int>& pos,
332  int index)
333 {
334  for(int k=0; k<edge[index]; k++)
335  {
336  pos[index] = start[index] + k*stride[index];
337  if(index+1<rank)
338  subset(input, rank, dim, start, stride, edge, poutput,pos,index+1);
339  if(index==rank-1)
340  {
341  poutput->push_back(input[INDEX_nD_TO_1D( dim, pos)]);
342  }
343  } // end of for
344  return 0;
345 } // end of template<typename T> static int subset
346 #endif
347 
348 #endif
HDFCFUtil::print_attr
static std::string print_attr(int32, int, void *)
Print attribute values in string.
Definition: HDFCFUtil.cc:265
HDFSP::SDField
One instance of this class represents one SDS object.
Definition: HDFSP.h:345
HDFCFUtil::Split
static void Split(const char *s, int len, char sep, std::vector< std::string > &names)
Definition: HDFCFUtil.cc:77
HDFCFUtil::escattr
static std::string escattr(std::string s)
Definition: HDFCFUtil.cc:3275
HDFCFUtil::obtain_type_size
static short obtain_type_size(int32)
Obtain datatype size.
Definition: HDFCFUtil.cc:447
HDFCFUtil::insert_map
static bool insert_map(std::map< std::string, std::string > &m, std::string key, std::string val)
Definition: HDFCFUtil.cc:145
HDFCFUtil::get_CF_string
static std::string get_CF_string(std::string s)
Change special characters to "_".
Definition: HDFCFUtil.cc:161
HDFCFUtil::correct_scale_offset_type
static void correct_scale_offset_type(libdap::AttrTable *at)
Definition: HDFCFUtil.cc:611
HDFSP::File
Definition: HDFSP.h:726
dimmap_entry
Definition: HDFCFUtil.h:51
HDFCFUtil::Handle_NameClashing
static void Handle_NameClashing(std::vector< std::string > &newobjnamelist)
General routines to handle name clashings.
Definition: HDFCFUtil.cc:257
HDFCFUtil::close_fileid
static void close_fileid(int32 sdfd, int32 file_id, int32 gridfd, int32 swathfd, bool pass_fileid_key)
Definition: HDFCFUtil.cc:3685
HDFCFUtil
Definition: HDFCFUtil.h:64
HDFCFUtil::correct_fvalue_type
static void correct_fvalue_type(libdap::AttrTable *at, int32 dtype)
Definition: HDFCFUtil.cc:544
HDFCFUtil::gen_unique_name
static void gen_unique_name(std::string &str, std::set< std::string > &namelist, int &clash_index)
Obtain the unique name for the clashed names and save it to set namelist.
Definition: HDFCFUtil.cc:191
HDFCFUtil::print_type
static std::string print_type(int32)
Print datatype in string.
Definition: HDFCFUtil.cc:386