bes  Updated for version 3.20.6
vdata.cc
1 // This file is part of the hdf4 data handler for the OPeNDAP data server.
2 
3 // Copyright (c) 2005 OPeNDAP, Inc.
4 // Author: James Gallagher <jgallagher@opendap.org>
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public License
17 // along with this software; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 
23 // Copyright 1996, by the California Institute of Technology.
24 // ALL RIGHTS RESERVED. United States Government Sponsorship
25 // acknowledged. Any commercial use must be negotiated with the
26 // Office of Technology Transfer at the California Institute of
27 // Technology. This software may be subject to U.S. export control
28 // laws and regulations. By accepting this software, the user
29 // agrees to comply with all applicable U.S. export laws and
30 // regulations. User has the responsibility to obtain export
31 // licenses, or other export authority as may be required before
32 // exporting such information to foreign countries or providing
33 // access to foreign persons.
34 
35 // U.S. Government Sponsorship under NASA Contract
36 // NAS7-1260 is acknowledged.
37 //
38 // Author: Todd.K.Karakashian@jpl.nasa.gov
39 //
40 // $RCSfile: vdata.cc,v $ - classes for HDF VDATA
41 //
43 
44 #include "config_hdf.h"
45 
46 #include <mfhdf.h>
47 
48 #ifdef __POWERPC__
49 #undef isascii
50 #endif
51 
52 #include <string>
53 #include <vector>
54 #include <set>
55 #include <algorithm>
56 
57 using std::set;
58 using std::less;
59 
60 // libdap
61 #include <InternalErr.h>
62 #include <util.h>
63 #include <debug.h>
64 
65 // bes
66 #include <BESDebug.h>
67 
68 #include <hcstream.h>
69 #include <hdfclass.h>
70 
71 static void LoadField(int32 vid, int index, int32 begin, int32 end,
72  hdf_field & f);
73 static bool IsInternalVdata(int32 fid, int32 ref);
74 
75 //
76 // hdfistream_vdata -- protected member functions
77 //
78 
79 // initialize hdfistream_vdata
80 void hdfistream_vdata::_init(void) {
81  _vdata_id = _index = _attr_index = _nattrs = 0;
82  _meta = false;
83  _vdata_refs.clear();
84  _recs.set = false;
85  return;
86 }
87 
88 void hdfistream_vdata::_get_fileinfo(void) {
89  // build list ref numbers of all Vdata's in the file
90  int32 ref = -1;
91  while ((ref = VSgetid(_file_id, ref)) != -1) {
92  if (!IsInternalVdata(_file_id, ref))
93  _vdata_refs.push_back(ref);
94  }
95  return;
96 }
97 
98 void hdfistream_vdata::_seek_next(void) {
99  _index++;
100  if (!eos())
101  _seek(_vdata_refs[_index]);
102  return;
103 }
104 
105 void hdfistream_vdata::_seek(const char *name) {
106  int32 ref = VSfind(_file_id, name);
107  if (ref < 0)
108  THROW(hcerr_vdatafind);
109  else
110  _seek(ref);
111 
112  return;
113 }
114 
115 void hdfistream_vdata::_seek(int32 ref) {
116  if (_vdata_id != 0)
117  VSdetach(_vdata_id);
118  vector<int32>::iterator r = find(_vdata_refs.begin(), _vdata_refs.end(),
119  ref);
120  if (r == _vdata_refs.end())
121  THROW(hcerr_vdatafind);
122  _index = r - _vdata_refs.begin();
123  if ((_vdata_id = VSattach(_file_id, ref, "r")) < 0) {
124  _vdata_id = 0;
125  THROW(hcerr_vdataopen);
126  }
127  _attr_index = 0;
128  _nattrs = VSfnattrs(_vdata_id, _HDF_VDATA);
129  return;
130 }
131 
132 //
133 // hdfistream_vdata -- public member functions
134 //
135 
136 hdfistream_vdata::hdfistream_vdata(const string filename) :
137  hdfistream_obj(filename) {
138  _init();
139  if (_filename.length() != 0) // if ctor specified a null filename
140  open(_filename.c_str());
141  return;
142 }
143 
144 void hdfistream_vdata::open(const string & filename) {
145  open(filename.c_str());
146  return;
147 }
148 
149 void hdfistream_vdata::open(const char *filename) {
150  if (_file_id != 0)
151  close();
152  if ((_file_id = Hopen(filename, DFACC_RDONLY, 0)) < 0)
153  THROW(hcerr_openfile);
154  if (Vstart(_file_id) < 0) // Vstart is a macro for Vinitialize
155  THROW(hcerr_openfile);
156 
157  BESDEBUG("h4", "vdata file opened: id=" << _file_id << endl);
158 
159  _filename = filename;
160  _get_fileinfo();
161  rewind();
162  return;
163 }
164 
165 void hdfistream_vdata::close(void) {
166  BESDEBUG("h4",
167  "vdata file closed: id=" << _file_id << ", this: " << this << endl);
168 
169  if (_vdata_id != 0)
170  VSdetach(_vdata_id);
171  if (_file_id != 0) {
172  int status = Vend(_file_id); // Vend is a macro for Vfinish
173  BESDEBUG("h4",
174  "vdata Vend status: " << status << ", this: " << this << endl);
175 
176  status = Hclose(_file_id);
177  BESDEBUG("h4",
178  "vdata HClose status: " << status << ", this: " << this << endl);
179  }
180  _vdata_id = _file_id = _index = _attr_index = _nattrs = 0;
181  _vdata_refs.clear(); // clear refs
182  _recs.set = false;
183  return;
184 }
185 
186 void hdfistream_vdata::seek(int index) {
187  if (index < 0 || index >= (int) _vdata_refs.size())
188  THROW(hcerr_range);
189  _seek(_vdata_refs[index]);
190  _index = index;
191  return;
192 }
193 
194 void hdfistream_vdata::seek_ref(int ref) {
195  _seek(ref); // _seek() sets _index
196  return;
197 }
198 
199 void hdfistream_vdata::seek(const string & name) {
200  seek(name.c_str());
201 }
202 
203 void hdfistream_vdata::seek(const char *name) {
204  _seek(name);
205  return;
206 }
207 
208 bool hdfistream_vdata::setrecs(int32 begin, int32 end) {
209  if (_vdata_id != 0) {
210  int32 il;
211  VSQueryinterlace(_vdata_id, &il);
212  if (il != FULL_INTERLACE)
213  return false;
214  else {
215  int32 cnt;
216  VSQuerycount(_vdata_id, &cnt);
217  if (begin < 0 || end >= cnt)
218  return false;
219  else {
220  _recs.begin = begin;
221  _recs.end = end;
222  _recs.set = true;
223  }
224  }
225  }
226  return true;
227 }
228 
229 // check to see if stream is positioned past the last attribute in the
230 // currently open Vdata
231 bool hdfistream_vdata::eo_attr(void) const {
232  if (_filename.length() == 0) // no file open
233  THROW(hcerr_invstream);
234  if (eos() && !bos()) // if eos(), then always eo_attr()
235  return true;
236  else {
237  return (_attr_index >= _nattrs); // or positioned after last Vdata attr?
238  }
239 }
240 
241 // Read all attributes in the stream
242 hdfistream_vdata & hdfistream_vdata::operator>>(vector<hdf_attr> &hav) {
243  // hav = vector<hdf_attr>0; // reset vector
244  for (hdf_attr att; !eo_attr();) {
245  *this >> att;
246  hav.push_back(att);
247  }
248  return *this;
249 }
250 
251 // read all Vdata's in the stream
252 hdfistream_vdata & hdfistream_vdata::operator>>(vector<hdf_vdata> &hvv) {
253  for (hdf_vdata hv; !eos();) {
254  *this >> hv;
255  hvv.push_back(hv);
256  }
257  return *this;
258 }
259 
260 // read an attribute from the stream
261 hdfistream_vdata & hdfistream_vdata::operator>>(hdf_attr & ha) {
262  // delete any previous data in ha
263  ha.name = string();
264  ha.values = hdf_genvec();
265 
266  if (_filename.length() == 0) // no file open
267  THROW(hcerr_invstream);
268  if (eo_attr()) // if positioned past last attr, do nothing
269  return *this;
270 
271  char name[hdfclass::MAXSTR];
272  int32 number_type, count, size;
273  if (VSattrinfo(_vdata_id, _HDF_VDATA, _attr_index, name, &number_type,
274  &count, &size) < 0)
275  THROW(hcerr_vdatainfo);
276 
277  // allocate a temporary C array to hold data from VSgetattr()
278  char *data;
279  data = new char[count * DFKNTsize(number_type)];
280  if (data == 0)
281  THROW(hcerr_nomemory);
282 
283  // read attribute values and store them in an hdf_genvec
284  if (VSgetattr(_vdata_id, _HDF_VDATA, _attr_index, data) < 0) {
285  delete[] data; // problem: clean up and throw an exception
286  THROW(hcerr_vdatainfo);
287  }
288  // try { // try to allocate an hdf_genvec
289  if (count > 0) {
290  ha.values = hdf_genvec(number_type, data, count);
291  // }
292  // catch(...) { // problem allocating hdf_genvec: clean up and rethrow
293  // delete []data;
294  // throw;
295  // }
296  }
297  delete[] data; // deallocate temporary C array
298 
299  // increment attribute index to next attribute
300  ++_attr_index;
301  ha.name = name; // assign attribute name
302  return *this;
303 }
304 
305 // read a Vdata from the stream
306 hdfistream_vdata & hdfistream_vdata::operator>>(hdf_vdata & hv) {
307 
308  // delete any previous data in hv
309  hv.fields.clear();
310  hv.vclass = hv.name = string();
311 
312  if (_vdata_id == 0)
313  THROW(hcerr_invstream); // no vdata open!
314  if (eos())
315  return *this;
316 
317  // assign Vdata ref
318  hv.ref = _vdata_refs[_index];
319  // retrieve Vdata attributes
320  *this >> hv.attrs;
321  // retrieve Vdata name, class, number of records
322  char name[hdfclass::MAXSTR];
323  char vclass[hdfclass::MAXSTR];
324  int32 nrecs;
325  if (VSinquire(_vdata_id, &nrecs, (int32 *) 0, (char *) 0, (int32 *) 0, name)
326  < 0)
327  THROW(hcerr_vdatainfo);
328  hv.name = string(name);
329  if (VSgetclass(_vdata_id, vclass) < 0)
330  THROW(hcerr_vdatainfo);
331  hv.vclass = string(vclass);
332 
333  // retrieve number of fields
334  int nfields = VFnfields(_vdata_id);
335  if (nfields < 0)
336  THROW(hcerr_vdatainfo);
337 
338  // retrieve field information
339  hv.fields = vector<hdf_field> ();
340  for (int i = 0; i < nfields; ++i) {
341  hv.fields.push_back(hdf_field());
342  if (_meta)
343  LoadField(_vdata_id, i, 0, 0, hv.fields[i]);
344  else if (_recs.set)
345  LoadField(_vdata_id, i, _recs.begin, _recs.end, hv.fields[i]);
346  else
347  LoadField(_vdata_id, i, 0, nrecs - 1, hv.fields[i]);
348  }
349  _seek_next();
350  return *this;
351 }
352 
353 // The following code causes memory leaking when called in vgroup.cc.
354 // Since it is only used in vgroup.cc, we move the code to vgroup.cc.
355 // The memory leaking is gone.
356 #if 0
357 bool hdfistream_vdata::isInternalVdata(int ref) const {
358  set<string, less<string> > reserved_names;
359  reserved_names.insert("RIATTR0.0N");
360 
361  set<string, less<string> > reserved_classes;
362  reserved_classes.insert("Attr0.0");
363  reserved_classes.insert("RIATTR0.0C");
364  reserved_classes.insert("DimVal0.0");
365  reserved_classes.insert("DimVal0.1");
366  reserved_classes.insert("_HDF_CHK_TBL_0");
367 
368  // get name, class of vdata
369  int vid;
370  if ((vid = VSattach(_file_id, ref, "r")) < 0) {
371  THROW(hcerr_vdataopen);
372  }
373  char name[hdfclass::MAXSTR];
374  char vclass[hdfclass::MAXSTR];
375  if (VSgetname(vid, name) < 0) {
376  VSdetach(vid);
377  THROW(hcerr_vdatainfo);
378  }
379  if (reserved_names.find(string(name)) != reserved_names.end()) {
380  VSdetach(vid);
381  return true;
382  }
383 
384  if (VSgetclass(vid, vclass) < 0) {
385  VSdetach(vid);
386  THROW(hcerr_vdatainfo);
387  }
388 
389  VSdetach(vid);
390 
391  if (reserved_classes.find(string(vclass)) != reserved_classes.end())
392  return true;
393 
394  return false;
395 }
396 #endif
397 
398 static void LoadField(int32 vid, int index, int32 begin, int32 end,
399  hdf_field & f) {
400  DBG(cerr << "LoadField - vid: " << vid << endl);
401 
402  // position to first record too read
403  if (VSseek(vid, begin) < 0)
404  THROW(hcerr_vdataseek);
405  int32 nrecs = end - begin + 1;
406 
407  // retrieve name of field
408  DBG(cerr << "vid: " << vid << ", index: " << index << endl);
409  char *fieldname = VFfieldname(vid, index);
410  if (fieldname == 0)
411  THROW(hcerr_vdatainfo);
412  f.name = string(fieldname);
413 
414  // retrieve order of field
415  int32 fieldorder = VFfieldorder(vid, index);
416  if (fieldorder < 0)
417  THROW(hcerr_vdatainfo);
418 
419  // retrieve size of the field in memory
420  int32 fieldsize = VFfieldisize(vid, index);
421  if (fieldsize < 0)
422  THROW(hcerr_vdatainfo);
423 
424  // retrieve HDF data type of field
425  int32 fieldtype = VFfieldtype(vid, index);
426  if (fieldtype < 0)
427  THROW(hcerr_vdatainfo);
428 
429  // for each component, set type and optionally load data
430  hdf_genvec gv;
431  vector<char> data;
432  // char *data = 0;
433  if (nrecs > 0) { // if nrecs > 0 then load data for field
434  // data = new char[fieldsize * nrecs];
435  data.resize(fieldsize * nrecs);
436  DBG(cerr << "LoadField: vid=" << vid << ", fieldname=" << fieldname << endl);
437  // TODO: Is this correct?
438  // This code originally treated a negative return as a failure and
439  // threw an exception. I'm still waiting on a verdict, but it seems
440  // that the negative return may have been indicating an empty vdata
441  // (or SDS dataset?) instead. For now, simply calling return here
442  // addresses a problem where some NASA data files cannot otherwise be
443  // read. See Trac ticket http://scm.opendap.org/trac/ticket/1793.
444  // jhrg 8/17/11
445  if (VSsetfields(vid, fieldname) < 0) {
446  return;
447  // throw InternalErr(__FILE__, __LINE__, "VSsetfields");
448  }
449 
450  if (VSread(vid, (uchar8 *)&data[0], nrecs, FULL_INTERLACE) < 0) {
451  throw InternalErr(__FILE__, __LINE__, "VSread error with the field: " + f.name + " (" + long_to_string(vid) + ").");
452  }
453 #if 0
454  if ((VSsetfields(vid, fieldname) < 0) || (VSread(vid, (uchar8 *) data,
455  nrecs, FULL_INTERLACE) < 0)) {
456  delete[] data;
457  throw hcerr_vdataread(__FILE__, __LINE__);
458  }
459 #endif
460  }
461  int stride = fieldorder;
462  for (int i = 0; i < fieldorder; ++i) {
463  if (nrecs == 0)
464  gv = hdf_genvec(fieldtype, 0, 0, 0, 0);
465  else
466  gv = hdf_genvec(fieldtype, &data[0], i, (nrecs * fieldorder) - 1, stride);
467  f.vals.push_back(gv);
468  }
469  // delete[] data;
470 }
471 
472 //
473 // hdf_vdata related member functions
474 //
475 
476 // verify that the hdf_field class is in an OK state.
477 bool hdf_field::_ok(void) const {
478 
479  // make sure that there is some data stored in the hdf_field
480  if (vals.size() == 0)
481  return false;
482 
483  // if the field has order > 1, check to make sure that the number types of all
484  // the columns in the field are the same and are not 0
485  if (vals.size() > 1) {
486  int32 nt = vals[0].number_type();
487  if (nt == 0)
488  return false;
489  for (int i = 1; i < (int) vals.size(); ++i)
490  if (vals[i].number_type() != nt || vals[i].number_type() == 0)
491  return false;
492  }
493 
494  return true; // passed all the tests
495 }
496 
497 bool hdf_vdata::_ok(void) const {
498 
499  // make sure there are fields stored in this vdata
500  if (fields.size() == 0)
501  return false;
502 
503  // make sure the fields are themselves OK
504  for (int i = 0; i < (int) fields.size(); ++i)
505  if (!fields[i])
506  return false;
507 
508  return true; // passed all the tests
509 }
510 
511 bool IsInternalVdata(int32 fid, int32 ref) {
512  set<string, less<string> > reserved_names;
513  reserved_names.insert("RIATTR0.0N");
514 
515  set<string, less<string> > reserved_classes;
516  reserved_classes.insert("Attr0.0");
517  reserved_classes.insert("RIATTR0.0C");
518  reserved_classes.insert("DimVal0.0");
519  reserved_classes.insert("DimVal0.1");
520  reserved_classes.insert("_HDF_CHK_TBL_0");
521 
522  // get name, class of vdata
523  int vid;
524  if ((vid = VSattach(fid, ref, "r")) < 0) {
525  THROW(hcerr_vdataopen);
526  }
527  char name[hdfclass::MAXSTR];
528  char vclass[hdfclass::MAXSTR];
529  if (VSgetname(vid, name) < 0) {
530  VSdetach(vid);
531  THROW(hcerr_vdatainfo);
532  }
533  if (reserved_names.find(string(name)) != reserved_names.end()) {
534  VSdetach(vid);
535  return true;
536  }
537 
538  if (VSgetclass(vid, vclass) < 0) {
539  VSdetach(vid);
540  THROW(hcerr_vdatainfo);
541  }
542 
543  VSdetach(vid);
544 
545  if (reserved_classes.find(string(vclass)) != reserved_classes.end())
546  return true;
547 
548  return false;
549 }
hdf_field
Definition: hdfclass.h:193
hcerr_invstream
Definition: hcerr.h:111
hdfistream_obj
Definition: hcstream.h:51
hcerr_openfile
Definition: hcerr.h:125
hcerr_vdatafind
Definition: hcerr.h:261
hcerr_vdataseek
Definition: hcerr.h:275
hcerr_range
Definition: hcerr.h:104
hcerr_nomemory
Definition: hcerr.h:97
hcerr_vdatainfo
Definition: hcerr.h:254
hcerr_vdataopen
Definition: hcerr.h:247
hdf_vdata
Definition: hdfclass.h:204
hdfistream_vdata
Definition: hcstream.h:245
hdf_genvec
Definition: hdfclass.h:71
hdf_attr
Definition: hdfclass.h:149
hcerr_vdataread
Definition: hcerr.h:268