bes  Updated for version 3.20.6
HE2CF.cc
1 // This file is part of the hdf4 data handler for the OPeNDAP data server.
3 // Copyright (c) 2010-2012 The HDF Group
4 //
5 // Author: Hyo-Kyung Lee <hyoklee@hdfgroup.org>
6 //
7 // This is free software; you can redistribute it and/or modify it under the
8 // terms of the GNU Lesser General Public License as published by the Free
9 // Software Foundation; either version 2.1 of the License, or (at your
10 // option) any later version.
11 //
12 // This software is distributed in the hope that it will be useful, but
13 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
15 // License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with this software; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 //
21 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23 #include "HE2CF.h"
24 #include "escaping.h"
25 #include "InternalErr.h"
26 #include "HDFCFUtil.h"
27 #include <iomanip>
28 
29 using namespace libdap;
30 using namespace std;
31 
32 // Private member functions
33 bool
34 HE2CF::get_vgroup_field_refids(const string& _gname,
35  int32* _ref_df,
36  int32* _ref_gf)
37 {
38 
39  int32 vrefid = Vfind(file_id, (char*)_gname.c_str());
40  if (FAIL == vrefid) {
41  Vend(file_id);
42  ostringstream error;
43  error <<"cannot obtain the reference number for vgroup "<<_gname;
44  throw_error(error.str());
45  return false;
46  }
47 
48  int32 vgroup_id = Vattach(file_id, vrefid, "r");
49  if (FAIL == vgroup_id) {
50  Vend(file_id);
51  ostringstream error;
52  error <<"cannot obtain the group id for vgroup "<<_gname;
53  throw_error(error.str());
54  return false;
55  }
56 
57  int32 npairs = Vntagrefs(vgroup_id);
58  int32 ref_df = -1;
59  int32 ref_gf = -1;
60 
61  if(npairs < 0){
62  Vdetach(vgroup_id);
63  Vend(file_id);
64  ostringstream error;
65  error << "Got " << npairs
66  << " npairs for " << _gname;
67  throw_error(error.str());
68  return false;
69  }
70 
71  for (int i = 0; i < npairs; ++i) {
72 
73  int32 tag = 0;
74  int32 ref = 0;
75 
76  if (Vgettagref(vgroup_id, i, &tag, &ref) < 0){
77 
78  // Previously just output stderr, it doesn't throw an internal error.
79  // I believe this is wrong. Use DAP's internal error to throw an error.KY 2011-4-26
80  Vdetach(vgroup_id);
81  Vend(file_id);
82  ostringstream error;
83  error << "failed to get tag / ref";
84  throw_error(error.str());
85  return false;
86  }
87 
88  if(Visvg(vgroup_id, ref)){
89 
90  char cvgroup_name[VGNAMELENMAX*4]; // the child vgroup name
91  int32 istat = 0;
92  int32 vgroup_cid = 0; // Vgroup id
93 
94  vgroup_cid = Vattach(file_id, ref,"r");
95  if (FAIL == vgroup_cid) {
96  Vdetach(vgroup_id);
97  Vend(file_id);
98  ostringstream error;
99  error << "cannot obtain the vgroup id";
100  throw_error(error.str());
101  return false;
102  }
103 
104  istat = Vgetname(vgroup_cid,cvgroup_name);
105  if (FAIL == istat) {
106  Vdetach(vgroup_cid);
107  Vdetach(vgroup_id);
108  Vend(file_id);
109  ostringstream error;
110  error << "cannot obtain the vgroup id";
111  throw_error(error.str());
112  return false;
113  }
114 
115  if(strncmp(cvgroup_name, "Data Fields", 11) == 0){
116  ref_df = ref;
117  }
118 
119  if(strncmp(cvgroup_name, "Geolocation Fields", 18) == 0){
120  ref_gf = ref;
121  }
122 
123  if (FAIL == Vdetach(vgroup_cid)) {
124  Vdetach(vgroup_id);
125  Vend(file_id);
126  ostringstream error;
127  error << "cannot close the vgroup "<< cvgroup_name <<"Successfully";
128  throw_error(error.str());
129  return false;
130  }
131 
132  }
133  }
134  *_ref_df = ref_df;
135  *_ref_gf = ref_gf;
136 
137  if (FAIL == Vdetach(vgroup_id)) {
138  Vend(file_id);
139  ostringstream error;
140  error << "cannot close the vgroup "<< _gname <<"Successfully";
141  throw_error(error.str());
142  return false;
143  }
144  return true;
145 }
146 
147 bool
148 HE2CF::open_sd(const string& _filename,const int sd_id_in)
149 {
150  int32 num_datasets = -1;
151  sd_id = sd_id_in;
152  if(SDfileinfo(sd_id, &num_datasets, &num_global_attributes)
153  == FAIL){
154  if(file_id != -1)
155  Vend(file_id);
156  ostringstream error;
157  error << "Failed to call SDfileinfo() on "
158  << _filename
159  << " file.";
160  throw_error(error.str());
161  return false;
162  }
163  return true;
164 }
165 
166 bool
167 HE2CF::open_vgroup(const string& _filename,const int file_id_in)
168 {
169 
170  file_id = file_id_in;
171  if (Vstart(file_id) < 0){
172  ostringstream error;
173  error << "Failed to call Vstart on " << _filename << endl;
174  throw_error(error.str());
175  return false;
176  }
177  return true;
178 }
179 
180 
181 void HE2CF::set_DAS(DAS* _das)
182 {
183  das = _das;
184 }
185 
186 bool HE2CF::set_non_ecsmetadata_attrs() {
187 
188  for(int i = 0; i < num_global_attributes; i++){
189 
190  // H4_MAX_NC_NAME is from the user guide example. It's 256.
191  char temp_name[H4_MAX_NC_NAME];
192  int32 attr_type = 0;
193  int32 attr_count = 0;
194  if(SDattrinfo(sd_id, i, temp_name, &attr_type, &attr_count) == FAIL) {
195  Vend(file_id);
196  ostringstream error;
197  error << "Fail to obtain SDS global attribute info." << endl;
198  throw_error(error.str());
199  }
200 
201  string attr_namestr(temp_name);
202  // Check if this attribute is an HDF-EOS2 metadata(coremeta etc. ) attribute
203  // If yes, ignore this attribute.
204  if (true == is_eosmetadata(attr_namestr))
205  continue;
206 
207  // When DisableStructMetaAttr key is true, StructMetadata.0 is not in the
208  // ECS metadata list. So this routine will pick up this attribute and generate
209  // the DAP output here. Anyhow,
210  // StructMetadata attribute should not be generated here. We will turn it off.
211  if (attr_namestr.compare(0,14, "StructMetadata" )== 0)
212  continue;
213 
214  // When DisableECSMetaDataAll key is true, All ECS attributes(Coremetadata etc.)
215  // should not be in the
216  // ECS metadata list. But this routine will pick up those attributes and generate
217  // the DAP output here. Anyhow,
218  // these attributes should not be generated here. We will turn it off.
219  if (attr_namestr.compare(0,12, "CoreMetadata" )== 0)
220  continue;
221  if (attr_namestr.compare(0,12, "coremetadata" )== 0)
222  continue;
223  if (attr_namestr.compare(0,15, "ArchiveMetadata" )== 0)
224  continue;
225  if (attr_namestr.compare(0,15, "archivemetadata" )== 0)
226  continue;
227  if (attr_namestr.compare(0,15, "Productmetadata" )== 0)
228  continue;
229  if (attr_namestr.compare(0,15, "productmetadata" )== 0)
230  continue;
231 
232  // USE VECTOR
233  vector<char>attr_data;
234  attr_data.resize((attr_count+1) *DFKNTsize(attr_type));
235 
236  if(SDreadattr(sd_id, i, &attr_data[0]) == FAIL){
237  Vend(file_id);
238  //delete[] attr_data;
239  ostringstream error;
240  error << "Fail to read SDS global attributes" << endl;
241  throw_error(error.str());
242 
243  }
244 
245  // Handle character type attribute as a string.
246  if (attr_type == DFNT_CHAR || attr_type == DFNT_UCHAR) {
247  attr_data[attr_count] = '\0';
248  attr_count = 1;
249  }
250 
251  AttrTable *at = das->get_table("HDF_GLOBAL");
252  if (!at)
253  at = das->add_table("HDF_GLOBAL", new AttrTable);
254 
255  attr_namestr = HDFCFUtil::get_CF_string(attr_namestr);
256 
257 #if 0
258  if(attr_type == DFNT_UCHAR || attr_type == DFNT_CHAR){
259  string tempstring2(attr_data);
260  string tempfinalstr= string(tempstring2.c_str());
261 
262  // Using the customized escattr. Don't escape \n,\r and \t. KY 2013-10-14
263  //tempfinalstr=escattr(tempfinalstr);
264  at->append_attr(attr_namestr, "String" , HDFCFUtil::escattr(tempfinalstr));
265  }
266 
267 #endif
268 
269  for (int loc=0; loc < attr_count ; loc++) {
270  string print_rep = HDFCFUtil::print_attr(attr_type, loc, (void*)&attr_data[0] );
271  at->append_attr(attr_namestr, HDFCFUtil::print_type(attr_type), print_rep);
272  }
273 
274  }
275 
276  return true;
277 }
278 
279 
280 
281 // Combine ECS metadata coremetadata.0,coremetadata.1 etc. into one string.
282 bool HE2CF::set_metadata(const string& metadata_basename,vector<string>& non_number_names, vector<string>& no_num_data)
283 {
284  bool suffix_is_num_or_null = true;
285  metadata.clear();
286 
287  // Metadata like coremetadata,coremetadata.0,coremetadata.1
288  list<string> one_dot_names;
289 
290  // Metadata like coremetadata, coremetadata.0, coremetadata.0.1
291  list<string> two_dots_names;
292 
293  // Metadata with the suffix as .s, .t, like productmetadta.s, productmetadata.t
294  // is non_number_names passed from the function
295 
296  int list_flag = -1;
297 
298  for(int i = 0; i < num_global_attributes; i++){
299 
300  // H4_MAX_NC_NAME is from the user guide example. It's 256.
301  char temp_name[H4_MAX_NC_NAME];
302  int32 attr_type=0;
303  int32 attr_count = 0;
304  if(SDattrinfo(sd_id, i, temp_name, &attr_type, &attr_count) == FAIL) {
305  Vend(file_id);
306  ostringstream error;
307  error << "Fail to obtain SDS global attribute info." << endl;
308  throw_error(error.str());
309  }
310 
311  string temp_name_str(temp_name);
312 
313  // Find the basename, arrange the metadata name list.
314  if(temp_name_str.find(metadata_basename)==0) {
315  arrange_list(one_dot_names,two_dots_names,non_number_names,temp_name_str,list_flag);
316  }
317  }
318 
319  list<string>::const_iterator lit;
320 
321  // list_flag = 0, no suffix
322  // list_flag = 1, only .0, coremetadata.0
323  // list_flag = 2, coremetadata.0, coremetadata.1 etc
324  // list_flag = 3, coremeatadata.0, coremetadata.0.1 etc
325 //cerr<<"list_flag "<<list_flag <<endl;
326  if ( list_flag >= 0 && list_flag <=2) {
327  for (lit = one_dot_names.begin();lit!=one_dot_names.end();++lit) {
328  set_eosmetadata_namelist(*lit);
329  string cur_data;
330  obtain_SD_attr_value(*lit,cur_data);
331 //cerr<<"metadata "<<cur_data <<endl;
332  metadata.append(cur_data);
333  }
334  }
335 
336  if( 3== list_flag) {
337  for (lit = two_dots_names.begin();lit!=two_dots_names.end();++lit){
338  set_eosmetadata_namelist(*lit);
339  string cur_data;
340  obtain_SD_attr_value(*lit,cur_data);
341  metadata.append(cur_data);
342  }
343  }
344 
345  if(non_number_names.size() >0) {
346  suffix_is_num_or_null = false;
347  no_num_data.resize(non_number_names.size());
348  }
349 
350  for (unsigned int i =0; i<non_number_names.size();i++) {
351  set_eosmetadata_namelist(non_number_names[i]);
352  obtain_SD_attr_value(non_number_names[i],no_num_data[i]);
353  }
354 
355  return suffix_is_num_or_null;
356 
357 }
358 
359 // This routine will generate three ECS metadata lists. Note in theory list sl1 and sl2 should be sorted.
360 // Since the ECS metadata is always written(sorted) in increasing numeric order, we don't perform this now.
361 // Should watch if there are any outliers. KY 2012-08-31
362 void HE2CF::arrange_list(list<string> & sl1, list<string>&sl2,vector<string>&v1,string name,int& flag) {
363 
364  // No dot in the ECS name
365  if(name.find(".") == string::npos) {
366  sl1.push_front(name);
367  sl2.push_front(name);
368  flag = 0;
369  }
370  else if (name.find_first_of(".") == name.find_last_of(".")) {
371 
372  size_t dot_pos = name.find_first_of(".");
373 
374  if((dot_pos+1)==name.size())
375  throw InternalErr(__FILE__, __LINE__,"Should have characters or numbers after ." );
376 
377  string str_after_dot = name.substr(dot_pos+1);
378  stringstream sstr(str_after_dot);
379 
380  int number_after_dot = 0;
381  sstr >> number_after_dot;
382 
383  // No dot after ECS metadata
384  if (!sstr)
385  v1.push_back(name);
386  // .0 after the main name of ECS metadata
387  else if(0 == number_after_dot) {
388  sl1.push_back(name);
389  sl2.push_back(name);
390  // For only .0 case, set flag to 1.
391  if(flag!=1)
392  flag =1;
393  }
394  else {// .1 or .2 etc. after the main ECS metadata name
395  sl1.push_back(name);
396  if (3 == flag)
397  throw InternalErr(__FILE__, __LINE__,
398  "ecs metadata suffix .1 and .0.1 cannot exist at the same file" );
399  if (flag !=2)
400  flag = 2;
401  }
402  }
403  else {// We don't distinguish if .0.1 and .0.0.1 will appear.
404  // have two dots in the ECS name.
405  sl2.push_back(name);
406  if (2 == flag)
407  throw InternalErr(__FILE__, __LINE__,"ecs metadata suffix .1 and .0.1 cannot exist at the same file" );
408  if (flag !=3)
409  flag = 3;
410  }
411 
412 }
413 
414 // Obtain SD attribute value
415 void HE2CF::obtain_SD_attr_value(const string& attrname, string &cur_data) {
416 
417  int32 sds_index = SDfindattr(sd_id, attrname.c_str());
418  if(sds_index == FAIL){
419  Vend(file_id);
420  ostringstream error;
421  error << "Failed to obtain the SDS global attribute" << attrname << endl;
422  throw InternalErr(__FILE__, __LINE__,error.str());
423  }
424 
425  // H4_MAX_NC_NAME is from the user guide example. It's 256.
426  char temp_name[H4_MAX_NC_NAME];
427  int32 type = 0;
428  int32 count = 0;
429 
430  if(SDattrinfo(sd_id, sds_index, temp_name, &type, &count) == FAIL) {
431  Vend(file_id);
432  ostringstream error;
433  error << "Failed to obtain the SDS global attribute" << attrname << "information" << endl;
434  throw InternalErr(__FILE__, __LINE__,error.str());
435  }
436 
437  vector<char> attrvalue;
438  attrvalue.resize((count+1)*DFKNTsize(type));
439 
440  if(SDreadattr(sd_id, sds_index, &attrvalue[0]) == FAIL){
441  Vend(file_id);
442  ostringstream error;
443  error << "Failed to read the SDS global attribute" << attrname << endl;
444  throw InternalErr(__FILE__, __LINE__,error.str());
445 
446  }
447  // Remove the last NULL character
448  // string temp_data(attrvalue.begin(),attrvalue.end()-1);
449  // cur_data = temp_data;
450 
451  if(attrvalue[count] != '\0')
452  throw InternalErr(__FILE__,__LINE__,"the last character of the attribute buffer should be NULL");
453 
454  // No need to escape the special characters since they are ECS metadata. Will see. KY 2013-10-14
455  cur_data.resize(attrvalue.size()-1);
456  copy(attrvalue.begin(),attrvalue.end()-1,cur_data.begin());
457 }
458 
459 
460 bool HE2CF::set_vgroup_map(int32 _refid,bool isgeo)
461 {
462  // Clear existing maps first.
463  // Note: It should only clear the corresponding groups: Geolocation or Data.
464  if(false == isgeo) {
465  vg_dsd_map.clear();
466  vg_dvd_map.clear();
467  }
468  else {
469  vg_gsd_map.clear();
470  vg_gvd_map.clear();
471  }
472 
473  int32 vgroup_id = Vattach(file_id, _refid, "r");
474  if (FAIL == vgroup_id) {
475  Vend(file_id);
476  ostringstream error;
477  error << "Fail to attach the vgroup " ;
478  throw_error(error.str());
479  return false;
480  }
481 
482  int32 npairs = Vntagrefs(vgroup_id);
483  if (FAIL == npairs) {
484  Vdetach(vgroup_id);
485  Vend(file_id);
486  ostringstream error;
487  error << "Fail to obtain the number of objects in a group " ;
488  throw_error(error.str());
489  return false;
490  }
491 
492  for (int i = 0; i < npairs; ++i) {
493 
494  int32 tag2 = 0;
495  int32 ref2 = 0;
496  char buf[H4_MAX_NC_NAME];
497 
498  if (Vgettagref(vgroup_id, i, &tag2, &ref2) < 0){
499  Vdetach(vgroup_id);
500  Vend(file_id);
501  ostringstream error;
502  error << "Vgettagref failed for vgroup_id=." << vgroup_id;
503  throw_error(error.str());
504  return false;
505  }
506 
507  if(tag2 == DFTAG_NDG){
508 
509  int32 sds_index = SDreftoindex(sd_id, ref2); // index
510  if (FAIL == sds_index) {
511  Vdetach(vgroup_id);
512  Vend(file_id);
513  ostringstream error;
514  error << "Cannot obtain the SDS index ";
515  throw_error(error.str());
516  return false;
517  }
518 
519  int32 sds_id = SDselect(sd_id, sds_index); // sds_id
520  if (FAIL == sds_id) {
521  Vdetach(vgroup_id);
522  Vend(file_id);
523  ostringstream error;
524  error << "Cannot obtain the SDS ID ";
525  throw_error(error.str());
526  return false;
527  }
528 
529  int32 rank;
530  int32 dimsizes[H4_MAX_VAR_DIMS];
531  int32 datatype;
532  int32 num_attrs;
533 
534  if(FAIL == SDgetinfo(sds_id, buf, &rank, dimsizes, &datatype, &num_attrs)) {
535  Vdetach(vgroup_id);
536  Vend(file_id);
537  ostringstream error;
538  error << "Cannot obtain the SDS info.";
539  throw_error(error.str());
540  return false;
541  }
542  if(false == isgeo)
543  vg_dsd_map[string(buf)] = sds_id;
544  else
545  vg_gsd_map[string(buf)] = sds_id;
546 
547  }
548 
549  if(tag2 == DFTAG_VH){
550 
551  int vid;
552  if ((vid = VSattach(file_id, ref2, "r")) < 0) {
553 
554  Vdetach(vgroup_id);
555  Vend(file_id);
556  ostringstream error;
557  error << "VSattach failed for file_id=." << file_id;
558  throw_error(error.str());
559  }
560  if (FAIL == VSgetname(vid, buf)) {
561 
562  Vdetach(vgroup_id);
563  Vend(file_id);
564  ostringstream error;
565  error << "VSgetname failed for file_id=." << file_id;
566  throw_error(error.str());
567  }
568  if(false == isgeo)
569  vg_dvd_map[string(buf)] = ref2;
570  else
571  vg_gvd_map[string(buf)] = ref2;
572 
573  if (FAIL == VSdetach(vid)) {
574 
575  Vdetach(vgroup_id);
576  Vend(file_id);
577  ostringstream error;
578  error << "VSdetach failed for file_id=." << file_id;
579  throw_error(error.str());
580 
581  }
582  }// if
583  } // for
584 
585  if (FAIL == Vdetach(vgroup_id)){
586  Vend(file_id);
587  ostringstream error;
588  error << "VSdetach failed for file_id=." << file_id;
589  throw_error(error.str());
590  }
591 
592  return true;
593 }
594 
595 bool HE2CF::write_attr_long_name(const string& _long_name,
596  const string& _varname,
597  int _fieldtype)
598 {
599  AttrTable *at = das->get_table(_varname);
600  if (!at){
601  at = das->add_table(_varname, new AttrTable);
602  }
603  if(_fieldtype > 3){
604  at->append_attr("long_name", "String", _long_name + "(fake)");
605  }
606  else{
607  at->append_attr("long_name", "String", _long_name);
608  }
609  return true;
610 }
611 bool HE2CF::write_attr_long_name(const string& _group_name,
612  const string& _long_name,
613  const string& _varname,
614  int _fieldtype)
615 {
616  AttrTable *at = das->get_table(_varname);
617  if (!at){
618  at = das->add_table(_varname, new AttrTable);
619  }
620  if(_fieldtype > 3){
621  at->append_attr("long_name", "String",
622  _group_name + ":" + _long_name + "(fake)");
623  }
624  else{
625  at->append_attr("long_name", "String",
626  _group_name + ":" + _long_name);
627  }
628  return true;
629 }
630 
631 
632 bool
633 HE2CF::write_attr_sd(int32 _sds_id, const string& _newfname,int _fieldtype)
634 {
635  char buf_var[H4_MAX_NC_NAME];
636  char buf_attr[H4_MAX_NC_NAME];
637  int32 rank;
638  int32 dimsizes[H4_MAX_VAR_DIMS];
639  int32 datatype;
640  int32 num_attrs;
641  int32 n_values;
642  intn status;
643 
644  status = SDgetinfo(_sds_id, buf_var,
645  &rank, dimsizes, &datatype, &num_attrs);
646  if (FAIL == status) {
647  Vend(file_id);
648  SDendaccess(_sds_id);
649  ostringstream error;
650  error << "Cannot obtain the SDS info. ";
651  throw_error(error.str());
652 
653  }
654  AttrTable *at = das->get_table(_newfname);
655  if (!at){
656  at = das->add_table(_newfname, new AttrTable);
657  }
658 
659  //Check if having the "coordinates" attribute when fieldtype is 0
660  bool v_has_coordinates = false;
661  if(0 == _fieldtype) {
662  if(at->simple_find("coordinates")!= at->attr_end())
663  v_has_coordinates = true;
664  }
665 
666  //Check if having the "units" attributes when fieldtype is 1(latitude) or 2(longitude)
667  bool llcv_has_units = false;
668 
669  // See if we can ignore scale and offset.
670  short llcv_ignore_scale = 0;
671  short llcv_ignore_offset = 0;
672  if(1 == _fieldtype || 2 == _fieldtype) {
673  if(at->simple_find("units")!= at->attr_end())
674  llcv_has_units = true;
675  try {
676  llcv_ignore_scale = check_scale_offset(_sds_id,true);
677  }
678  catch(...) {
679  SDendaccess(_sds_id);
680  Vend(file_id);
681  throw;
682  }
683  try {
684  llcv_ignore_offset = check_scale_offset(_sds_id,false);
685  }
686  catch(...) {
687  SDendaccess(_sds_id);
688  Vend(file_id);
689  throw;
690  }
691 
692  // We need to check if we ignore scale_factor and the scale_factor type is an integer.
693  // If that's the case, we need to throw an exception since we cannot retrieve the
694  // integer type scale_factor added by the HDF4 API for the latitude and longitude.
695  // We don't think such a case exists. If any, we throw an exception and hope the
696  // user to report this case to us.
697 
698  // Find a case that the variable type is integer,
699  // that the scale_factor and add_offset attributes exist
700  // that the add_offset is not 0.
701  // We don't support this case and highly suspect that such a case doesn't exist.
702  if(-1 == llcv_ignore_offset && 2 == llcv_ignore_scale){
703  SDendaccess(_sds_id);
704  Vend(file_id);
705  ostringstream error;
706  error <<"The latitude or longitude has <scale_factor> and <add_offset> attributes, "
707  <<" the latitude or longitude have integer type and <add_offset> is not 0, "
708  <<" we don't support such a case in the current implementation, "
709  <<" please report to eoshelp@hdfgroup.org if you encounter this situation.";
710  throw_error(error.str());
711  }
712  }
713 
714  for (int j=0; j < num_attrs; j++){
715 
716  status = SDattrinfo(_sds_id, j, buf_attr, &datatype, &n_values);
717  if (status < 0){
718  Vend(file_id);
719  SDendaccess(_sds_id);
720  ostringstream error;
721  error << "SDattrinfo() failed on " << buf_attr;
722  throw_error(error.str());
723  }
724 
725  if(true == v_has_coordinates) {
726  if(!strncmp(buf_attr, "coordinates", H4_MAX_NC_NAME))
727  continue;
728  }
729 
730  if(true == llcv_has_units) {
731  if(!strncmp(buf_attr, "units", H4_MAX_NC_NAME))
732  continue;
733  }
734 
735  if( 1 == _fieldtype || 2 == _fieldtype) {
736  if(!strncmp(buf_attr, "scale_factor", H4_MAX_NC_NAME))
737  if((1 == llcv_ignore_scale) || (2==llcv_ignore_scale) )
738  continue;
739  if(!strncmp(buf_attr, "add_offset", H4_MAX_NC_NAME))
740  if(1 == llcv_ignore_offset)
741  continue;
742  }
743 
744  vector<char> value;
745  value.resize((n_values+1) * DFKNTsize(datatype));
746  status = SDreadattr(_sds_id, j, &value[0]);
747  if (status < 0){
748  Vend(file_id);
749  SDendaccess(_sds_id);
750  ostringstream error;
751  error << "SDreadattr() failed on " << buf_attr << endl;
752  throw_error(error.str());
753  }
754 
755  // Handle character type attribute as a string.
756  if (datatype == DFNT_CHAR || datatype == DFNT_UCHAR) {
757  //*(value + n_values) = '\0';
758  value[n_values] = '\0';
759  n_values = 1;
760  }
761 
762  // Need to check and change if attribute names contain special characters. KY 2012-6-8
763  string attr_cf_name = string(buf_attr,strlen(buf_attr));
764  attr_cf_name = HDFCFUtil::get_CF_string(attr_cf_name);
765  for (int loc=0; loc < n_values ; loc++) {
766  string print_rep = HDFCFUtil::print_attr(datatype, loc, (void *)&value[0]);
767 
768  // Override any existing _FillValue attribute.
769  if (attr_cf_name == "_FillValue") {
770  at->del_attr(attr_cf_name);
771  }
772  // Override any existing long_name attribute.
773  if (attr_cf_name == "long_name") {
774  at->del_attr(attr_cf_name);
775  }
776 
777  // No need to escape special characters since print_rep already does that.
778  at->append_attr(attr_cf_name, HDFCFUtil::print_type(datatype), print_rep);
779 
780  }
781  }
782 
783  status = SDendaccess(_sds_id);
784  if(status < 0) {
785  ostringstream error;
786  error << "SDendaccess failed on variable " << _newfname;
787  throw_error(error.str());
788  }
789 
790  return true;
791 }
792 
793 bool HE2CF::write_attr_vdata(int32 _vd_id, const string& _newfname, int _fieldtype)
794 {
795  int32 number_type = 0;
796  int32 count = 0;
797  int32 size = 0;
798  char buf[H4_MAX_NC_NAME];
799 
800  int vid = 0;
801 
802  if ((vid = VSattach(file_id, _vd_id, "r")) < 0) {
803 
804  Vend(file_id);
805 
806  ostringstream error;
807  error << "VSattach failed.";
808  throw_error(error.str());
809  }
810 
811  // Don't use VSnattrs - it returns the TOTAL number of attributes
812  // of a vdata and its fields. We should use VSfnattrs.
813  count = VSfnattrs(vid, _HDF_VDATA);
814  if (FAIL == count) {
815 
816  VSdetach(vid);
817  Vend(file_id);
818  ostringstream error;
819  error << "VSfnattrs failed.";
820  throw_error(error.str());
821  }
822 
823  AttrTable *at = das->get_table(_newfname);
824  if (!at)
825  at = das->add_table(_newfname, new AttrTable);
826 
827 
828  //Check if having the "coordinates" attribute when fieldtype is 0
829  bool v_has_coordinates = false;
830  if(0 == _fieldtype) {
831  if(at->simple_find("coordinates")!= at->attr_end())
832  v_has_coordinates = true;
833  }
834 
835  //Check if having the "units" attributes when fieldtype is 1 or 2
836  bool llcv_has_units = false;
837  if(1 == _fieldtype || 2 == _fieldtype) {
838  if(at->simple_find("units")!= at->attr_end())
839  llcv_has_units = true;
840  }
841 
842 
843  for(int i=0; i < count; i++){
844  int32 count_v = 0;
845  if (VSattrinfo(vid, _HDF_VDATA, i, buf,
846  &number_type, &count_v, &size) < 0) {
847  VSdetach(vid);
848  Vend(file_id);
849  ostringstream error;
850  error << "VSattrinfo failed.";
851  throw_error(error.str());
852  }
853 
854  if(true == v_has_coordinates) {
855  if(!strncmp(buf, "coordinates", H4_MAX_NC_NAME))
856  continue;
857  }
858 
859  else if(true == llcv_has_units) {
860  if(!strncmp(buf, "units", H4_MAX_NC_NAME))
861  continue;
862  }
863 
864  vector<char> data;
865  data.resize((count_v+1) * DFKNTsize(number_type));
866  if (VSgetattr(vid, _HDF_VDATA, i, &data[0]) < 0) {
867 
868  // problem: clean up and throw an exception
869  VSdetach(vid);
870  Vend(file_id);
871  ostringstream error;
872  error << "VSgetattr failed.";
873  throw_error(error.str());
874  }
875  // Handle character type attribute as a string.
876  if (number_type == DFNT_CHAR || number_type == DFNT_UCHAR8) {
877  data[count_v] ='\0';
878  count_v = 1;
879  }
880 
881  for(int j=0; j < count_v ; j++){
882 
883  string print_rep = HDFCFUtil::print_attr(number_type, j, (void *)&data[0]);
884 
885  // Override any existing _FillValue attribute.
886  if(!strncmp(buf, "_FillValue", H4_MAX_NC_NAME)){
887  at->del_attr(buf);
888  }
889 
890  // Override any existing long_name attribute.
891  if(!strncmp(buf, "long_name", H4_MAX_NC_NAME)){
892  at->del_attr(buf);
893  }
894 
895  string vdataname(buf);
896  at->append_attr(HDFCFUtil::get_CF_string(buf), HDFCFUtil::print_type(number_type), print_rep);
897 
898  }
899  } // for
900  // We need to call VFnfields to process fields as well in near future.
901  VSdetach(vid);
902 
903  return true;
904 }
905 
906 void
907 HE2CF::throw_error(string _error)
908 {
909  throw InternalErr(__FILE__, __LINE__,
910  _error);
911 }
912 
913 
914 // Public member functions
915 HE2CF::HE2CF()
916 {
917  num_global_attributes = -1;
918  file_id = -1;
919  sd_id = -1;
920  metadata = "";
921  gname = "";
922  das = NULL;
923 }
924 
925 HE2CF::~HE2CF()
926 {
927  // Actually this is not necessary since C++ will clean up the string.
928  metadata.clear();
929 }
930 
931 bool
933 {
934 
935  // Close Vgroup interface.
936  int istat = Vend(file_id);
937  if(istat == FAIL){
938  ostringstream error;
939  error << "Failed to call Vend in HE2CF::close.";
940  throw_error(error.str());
941  return false;
942  }
943 
944  return true;
945 }
946 
947 string
948 HE2CF::get_metadata(const string& _name, bool& suffix_is_number,vector<string>&meta_nonnum_names, vector<string>& meta_nonum_data)
949 {
950  suffix_is_number = set_metadata(_name,meta_nonnum_names,meta_nonum_data);
951  return metadata;
952 }
953 
954 bool
955 HE2CF::open(const string& _filename,const int hc_sd_id, const int hc_file_id)
956 {
957  if(_filename == ""){
958  ostringstream error;
959  error << "=open(): filename is empty.";
960  throw_error(error.str());
961  return false;
962  }
963 
964  if(!open_vgroup(_filename,hc_file_id)){
965  ostringstream error;
966  error << "=open(): failed to open vgroup.";
967  throw_error(error.str());
968  return false;
969  }
970 
971  if(!open_sd(_filename,hc_sd_id)){
972  ostringstream error;
973  error << "=open(): failed to open sd.";
974  throw_error(error.str());
975  return false;
976  }
977 
978  return true;
979 }
980 
981 
982 
983 
984 bool
985 HE2CF::write_attribute(const string& _gname,
986  const string& _fname,
987  const string& _newfname,
988  int _n_groups,
989  int _fieldtype)
990 {
991 
992  if(_n_groups > 1){
993  write_attr_long_name(_gname, _fname, _newfname, _fieldtype);
994  }
995  else{
996  write_attr_long_name(_fname, _newfname, _fieldtype);
997  }
998  int32 ref_df = -1; // ref id for /Data Fields/
999  int32 ref_gf = -1; // ref id for /Geolocaion Fields/
1000 
1001  // This if block effectively constructs the vg_dsd_map and vg_dvd_map at most once
1002  // since starting from the second time, gname will be equal to _gname.
1003  if(gname != _gname){
1004  // Construct field:(SD|Vdata)ref table for faster lookup.
1005  gname = _gname;
1006  get_vgroup_field_refids(_gname, &ref_df, &ref_gf);
1007  if (ref_gf !=-1)
1008  set_vgroup_map(ref_gf,true);
1009 
1010  if (ref_df != 1)
1011  set_vgroup_map(ref_df,false);
1012  }
1013 
1014  // Use a cached table.
1015  int32 id = -1;
1016 
1017  // Using SDS name to SDS id table is fine for the HDF-EOS2 case since under one grid/swath,
1018  // the data field name is unique. Generally SDS name cannot be used as a key since one may have
1019  // the same SDS name for two different SDS objects. But in this context(HDF-EOS2 objects), it is okay.
1020  // KY 2012-7-3
1021  // Retrive sds_id for SDS
1022  // Go over all possible maps (data fields and geolocation fields)
1023  id = vg_dsd_map[_fname];
1024  if(id > 0){
1025  write_attr_sd(id, _newfname,_fieldtype);
1026  }
1027  // Retrieve ref_id for Vdata
1028  // The Vdata we retrieve here are HDF-EOS2 objects(Swath 1-D data)
1029  // Not all Vdata from the HDF file.
1030  id = vg_dvd_map[_fname];
1031  if(id > 0){
1032  write_attr_vdata(id, _newfname,_fieldtype);
1033  }
1034 
1035  id = vg_gsd_map[_fname];
1036  if(id > 0){
1037  write_attr_sd(id, _newfname,_fieldtype);
1038  }
1039  // Retrieve ref_id for Vdata
1040  // The Vdata we retrieve here are HDF-EOS2 objects(Swath 1-D data)
1041  // Not all Vdata from the HDF file.
1042  id = vg_gvd_map[_fname];
1043  if(id > 0){
1044  write_attr_vdata(id, _newfname,_fieldtype);
1045  }
1046 
1047  return true;
1048 }
1049 
1050 // The application will make sure the fill value falls in the valid range of the datatype values.
1051 bool
1052 HE2CF::write_attribute_FillValue(const string& _varname,
1053  int _type,
1054  float value)
1055 {
1056  void* v_ptr = NULL;
1057  // Casting between pointers of different types may generate unexpected behavior.
1058  // gcc 4.8.2 reveals this problem in one test(SwathFile.HDF).
1059  // So what we should do is to use char* or vector<char> in this case and memcpy the value to the vector<char>.
1060  // Casting the vector pointer to void *. The compiler can correctly interpret the data based on the data type.
1061  vector<char>v_val;
1062 
1063  // Cast the value depending on the type.
1064  switch (_type) {
1065 
1066  case DFNT_UINT8:
1067  {
1068  uint8 val = (uint8) value;
1069  v_val.resize(1);
1070  memcpy(&v_val[0],&val,1);
1071  v_ptr = (void*)&v_val[0];
1072  }
1073 
1074  break;
1075  case DFNT_INT8:
1076  {
1077  int8 val = (int8) value;
1078  v_val.resize(1);
1079  memcpy(&v_val[0],&val,1);
1080  v_ptr = (void*)&v_val[0];
1081  }
1082  break;
1083  case DFNT_INT16:
1084  {
1085  int16 val = (int16) value;
1086  v_val.resize(sizeof(short));
1087  memcpy(&v_val[0],&val,sizeof(short));
1088  v_ptr = (void*)&v_val[0];
1089  }
1090  break;
1091 
1092  case DFNT_UINT16:
1093  {
1094  uint16 val = (uint16) value;
1095  v_val.resize(sizeof(unsigned short));
1096  memcpy(&v_val[0],&val,sizeof(unsigned short));
1097  v_ptr = (void*)&v_val[0];
1098  }
1099  break;
1100 
1101  case DFNT_INT32:
1102  {
1103  int32 val = (int32) value;
1104  v_val.resize(sizeof(int));
1105  memcpy(&v_val[0],&val,sizeof(int));
1106  v_ptr = (void*)&v_val[0];
1107  }
1108  break;
1109  case DFNT_UINT32:
1110  {
1111  uint32 val = (uint32) value;
1112  v_val.resize(sizeof(unsigned int));
1113  memcpy(&v_val[0],&val,sizeof(int));
1114  v_ptr = (void*)&v_val[0];
1115  }
1116  break;
1117  case DFNT_FLOAT:
1118  {
1119  v_ptr = (void*)&value;
1120  }
1121  break;
1122  case DFNT_DOUBLE:
1123  {
1124  float64 val = (float64) value;
1125  v_val.resize(sizeof(double));
1126  memcpy(&v_val[0],&val,sizeof(double));
1127  v_ptr = (void*)&v_val[0];
1128  }
1129  break;
1130  default:
1131  throw_error("Invalid FillValue Type - ");
1132  break;
1133  }
1134 
1135  AttrTable *at = das->get_table(_varname);
1136  if (!at){
1137  at = das->add_table(_varname, new AttrTable);
1138  }
1139  string print_rep = HDFCFUtil::print_attr(_type, 0, v_ptr);
1140  at->append_attr("_FillValue", HDFCFUtil::print_type(_type), print_rep);
1141 
1142  return true;
1143 }
1144 
1145 bool
1146 HE2CF::write_attribute_coordinates(const string& _varname, string _coordinates)
1147 {
1148 
1149  AttrTable *at = das->get_table(_varname);
1150  if (!at){
1151  at = das->add_table(_varname, new AttrTable);
1152  }
1153  at->append_attr("coordinates", "String", _coordinates);
1154 
1155  return true;
1156 }
1157 
1158 bool
1159 HE2CF::write_attribute_units(const string& _varname, string _units)
1160 {
1161 
1162  AttrTable *at = das->get_table(_varname);
1163  if (!at){
1164  at = das->add_table(_varname, new AttrTable);
1165  }
1166  at->del_attr("units"); // Override any existing units attribute.
1167  at->append_attr("units", "String", _units);
1168 
1169  return true;
1170 }
1171 
1172 
1173 // Check if the scale_factor or the add_offset can be ignored.
1174 // _sds_id is the SDS ID of the variable.
1175 // is_scale is true, the attribute is scale_factor.
1176 // is_scale is false, the attribute is add_offset.
1177 // When the attribute is scale_factor,
1178 // the meaning of the returned value:
1179 // 0: the scale_factor won't be ignored if exists.
1180 // 1 or 2: the scale_factor can be ignored. 2 is the case when the variable type is integer. 1 is the case when the variable type is float.
1181 // When the attribute is add_offset,
1182 // the meaning of the returned value:
1183 // 0: the add_offset won't be ignored if exists. It doesn't check if add_offset attribute exists.
1184 // -1: the add_offset won't be ignored and the add_offset definitely exists,the value is not 0.
1185 // 1: the add_offset will be ignored.
1186 short
1187 HE2CF::check_scale_offset(int32 _sds_id, bool is_scale) {
1188 
1189  char buf_var[H4_MAX_NC_NAME];
1190  char buf_attr[H4_MAX_NC_NAME];
1191  int32 rank;
1192  int32 dimsizes[H4_MAX_VAR_DIMS];
1193  int32 datatype;
1194  int32 num_attrs;
1195  int32 n_values;
1196  intn status;
1197 
1198  short ignore_so = 0;
1199  status = SDgetinfo(_sds_id, buf_var,
1200  &rank, dimsizes, &datatype, &num_attrs);
1201  if (FAIL == status) {
1202  SDendaccess(_sds_id);
1203  ostringstream error;
1204  error << "Cannot obtain the SDS info. ";
1205  throw_error(error.str());
1206  }
1207 
1208  bool has_so = false;
1209  int so_index = -1;
1210  int32 so_dtype = -1;
1211  int32 var_dtype = datatype;
1212  string so_name =((true==is_scale)?"scale_factor":"add_offset");
1213 
1214  for (int j=0; j < num_attrs; j++){
1215 
1216  status = SDattrinfo(_sds_id, j, buf_attr, &datatype, &n_values);
1217  if (status < 0){
1218  SDendaccess(_sds_id);
1219  ostringstream error;
1220  error << "SDattrinfo() failed on " << buf_attr;
1221  throw_error(error.str());
1222  }
1223 
1224  if(!strncmp(buf_attr, so_name.c_str(), H4_MAX_NC_NAME)) {
1225  if(1 == n_values) {
1226  has_so = true;
1227  so_index = j;
1228  so_dtype = datatype;
1229  break;
1230  }
1231  }
1232  }
1233 
1234  // We find the attribute, now check if we can ignore this attribute.
1235  if(true == has_so) {
1236  vector<char> value;
1237  // We have already checked the number of attribute values. The number is 1.
1238  value.resize(DFKNTsize(so_dtype));
1239  status = SDreadattr(_sds_id, so_index, &value[0]);
1240  if (status < 0){
1241  SDendaccess(_sds_id);
1242  ostringstream error;
1243  error << "SDreadattr() failed on the attribute scale_factor." << endl;
1244  throw_error(error.str());
1245  }
1246  // If this attribute is "scale_factor",
1247  if(true == is_scale) {
1248  if(DFNT_FLOAT32 == so_dtype) {
1249  float final_scale_value = *((float*)((void*)(&value[0])));
1250  if(final_scale_value == 1.0)
1251  ignore_so = 1;
1252  // This is the ugly case that the variable type is integer, the
1253  // scale_factor is handled by the handler reading function.
1254  // However, the add_offset for this case cannot be 0.
1255  // So we need to check this case. We may also need to
1256  // check the scale_factor value.
1257  else if(var_dtype !=DFNT_FLOAT32 && var_dtype != DFNT_FLOAT64)
1258  ignore_so = 2;
1259  }
1260  else if(DFNT_FLOAT64 == so_dtype) {
1261  double final_scale_value = *((double*)((void*)(&value[0])));
1262  if(final_scale_value == 1.0)
1263  ignore_so = 1;
1264  else if(var_dtype !=DFNT_FLOAT32 && var_dtype != DFNT_FLOAT64)
1265  ignore_so = 2;
1266  }
1267  }
1268  else {
1269  // has offset, we need to make a mark for the case when scale_factor applies to a varialbe that has an integer type
1270  ignore_so = -1;
1271  string print_rep = HDFCFUtil::print_attr(so_dtype, 0, (void *)&value[0]);
1272  if(DFNT_FLOAT32 == so_dtype || DFNT_FLOAT64 == so_dtype) {
1273  if(atof(print_rep.c_str()) == 0.0)
1274  ignore_so = 1;
1275  }
1276  else {
1277  if(atoi(print_rep.c_str()) == 0)
1278  ignore_so = 1;
1279  }
1280  }
1281  }
1282  return ignore_so;
1283 }
1284 
HDFCFUtil::print_attr
static std::string print_attr(int32, int, void *)
Print attribute values in string.
Definition: HDFCFUtil.cc:265
HE2CF::open
bool open(const std::string &filename, const int sd_id, const int file_id)
openes \afilename HDF4 file.
Definition: HE2CF.cc:955
HDFCFUtil::escattr
static std::string escattr(std::string s)
Definition: HDFCFUtil.cc:3275
libdap
Definition: BESDapFunctionResponseCache.h:35
HE2CF::write_attribute_FillValue
bool write_attribute_FillValue(const std::string &varname, int type, float val)
Definition: HE2CF.cc:1052
HDFCFUtil::get_CF_string
static std::string get_CF_string(std::string s)
Change special characters to "_".
Definition: HDFCFUtil.cc:161
HE2CF::set_DAS
void set_DAS(libdap::DAS *das)
sets DAS pointer so that we can bulid attribute tables.
Definition: HE2CF.cc:181
HE2CF::write_attribute
bool write_attribute(const std::string &gname, const std::string &fname, const std::string &newfname, int n_groups, int fieldtype)
Definition: HE2CF.cc:985
HE2CF::write_attribute_units
bool write_attribute_units(const std::string &varname, std::string units)
Definition: HE2CF.cc:1159
HE2CF::write_attribute_coordinates
bool write_attribute_coordinates(const std::string &varname, std::string coord)
Definition: HE2CF.cc:1146
HE2CF::close
bool close()
closes the opened file.
Definition: HE2CF.cc:932
HE2CF::get_metadata
string get_metadata(const std::string &metadataname, bool &suffix_is_num, std::vector< std::string > &non_num_names, std::vector< std::string > &non_num_data)
retrieves the merged metadata.
Definition: HE2CF.cc:948
HDFCFUtil::print_type
static std::string print_type(int32)
Print datatype in string.
Definition: HDFCFUtil.cc:386