bes  Updated for version 3.20.6
HDF5BaseArray.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler an HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Author: Muqun Yang <myang6@hdfgroup.org>
5 
6 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
7 //
8 // This is free software; you can redistribute it and/or modify it under the
9 // terms of the GNU Lesser General Public License as published by the Free
10 // Software Foundation; either version 2.1 of the License, or (at your
11 // option) any later version.
12 //
13 // This software is distributed in the hope that it will be useful, but
14 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
16 // License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
24 // Suite 203, Champaign, IL 61820
41 
42 #include <iostream>
43 #include <sstream>
44 #include <cassert>
45 #include <algorithm>
46 #include <BESDebug.h>
47 #include "InternalErr.h"
48 
49 #include "HDF5BaseArray.h"
50 #include "HDF5RequestHandler.h"
51 #include "ObjMemCache.h"
52 
53 using namespace std;
54 using namespace libdap;
55 #if 0
56 BaseType *HDF5BaseArray::ptr_duplicate()
57 {
58  return new HDF5BaseArray(*this);
59 }
60 
61 // Always return true.
62 // Data will be read from the missing coordinate variable class(HDF5GMCFMissNonLLCVArray etc.)
63 bool HDF5BaseArray::read()
64 {
65  BESDEBUG("h5","Coming to HDF5BaseArray read "<<endl);
66  return true;
67 }
68 
69 #endif
70 
71 // parse constraint expr. and make hdf5 coordinate point location.
72 // return number of elements to read.
73 int
74 HDF5BaseArray::format_constraint (int *offset, int *step, int *count)
75 {
76  long nels = 1;
77  int id = 0;
78 
79  Dim_iter p = dim_begin ();
80 
81  while (p != dim_end ()) {
82 
83  int start = dimension_start (p, true);
84  int stride = dimension_stride (p, true);
85  int stop = dimension_stop (p, true);
86 
87  // Check for illegal constraint
88  if (start > stop) {
89  ostringstream oss;
90  oss << "Array/Grid hyperslab start point "<< start <<
91  " is greater than stop point " << stop <<".";
92  throw Error(malformed_expr, oss.str());
93  }
94 
95  offset[id] = start;
96  step[id] = stride;
97  count[id] = ((stop - start) / stride) + 1; // count of elements
98  nels *= count[id]; // total number of values for variable
99 
100  BESDEBUG ("h5",
101  "=format_constraint():"
102  << "id=" << id << " offset=" << offset[id]
103  << " step=" << step[id]
104  << " count=" << count[id]
105  << endl);
106 
107  id++;
108  p++;
109  }// "while (p != dim_end ())"
110 
111  return nels;
112 }
113 
114 void HDF5BaseArray::write_nature_number_buffer(int rank, int tnumelm) {
115 
116  if (rank != 1)
117  throw InternalErr(__FILE__, __LINE__, "Currently the rank of the missing field should be 1");
118 
119  vector<int>offset;
120  vector<int>count;
121  vector<int>step;
122  offset.resize(rank);
123  count.resize(rank);
124  step.resize(rank);
125 
126 
127  int nelms = format_constraint(&offset[0], &step[0], &count[0]);
128 
129  // Since we always assign the the missing Z dimension as 32-bit
130  // integer, so no need to check the type. The missing Z-dim is always
131  // 1-D with natural number 1,2,3,....
132  vector<int>val;
133  val.resize(nelms);
134 
135  if (nelms == tnumelm) {
136  for (int i = 0; i < nelms; i++)
137  val[i] = i;
138  set_value((dods_int32 *) &val[0], nelms);
139  }
140  else {
141  for (int i = 0; i < count[0]; i++)
142  val[i] = offset[0] + step[0] * i;
143  set_value((dods_int32 *) &val[0], nelms);
144  }
145 }
146 
147 //#if 0
148 void HDF5BaseArray::read_data_from_mem_cache(H5DataType h5type, const vector<size_t> &h5_dimsizes,void* buf) {
149 
150  BESDEBUG("h5", "Coming to read_data_from_mem_cache"<<endl);
151  vector<int>offset;
152  vector<int>count;
153  vector<int>step;
154 
155  int ndims = h5_dimsizes.size();
156  if(ndims == 0)
157  throw InternalErr(__FILE__, __LINE__, "Currently we only support array numeric data in the cache, the number of dimension for this file is 0");
158 
159 
160  offset.resize(ndims);
161  count.resize(ndims);
162  step.resize(ndims);
163  int nelms = format_constraint (&offset[0], &step[0], &count[0]);
164 
165  // set the original position to the starting point
166  vector<size_t>pos(ndims,0);
167  for (int i = 0; i< ndims; i++)
168  pos[i] = offset[i];
169 
170 
171  switch (h5type) {
172 
173  case H5UCHAR:
174 
175  {
176  vector<unsigned char> val;
177  subset<unsigned char>(
178  buf,
179  ndims,
180  h5_dimsizes,
181  &offset[0],
182  &step[0],
183  &count[0],
184  &val,
185  pos,
186  0
187  );
188 
189  set_value ((dods_byte *) &val[0], nelms);
190  } // case H5UCHAR
191  break;
192 
193  case H5CHAR:
194  {
195 
196  vector<char>val;
197  subset<char>(
198  buf,
199  ndims,
200  h5_dimsizes,
201  &offset[0],
202  &step[0],
203  &count[0],
204  &val,
205  pos,
206  0
207  );
208 
209  vector<short>newval;
210  newval.resize(nelms);
211 
212  for (int counter = 0; counter < nelms; counter++)
213  newval[counter] = (short) (val[counter]);
214  set_value ((dods_int16 *) &val[0], nelms);
215 
216  } // case H5CHAR
217  break;
218 
219  case H5INT16:
220  {
221  vector<short> val;
222  subset<short>(
223  buf,
224  ndims,
225  h5_dimsizes,
226  &offset[0],
227  &step[0],
228  &count[0],
229  &val,
230  pos,
231  0
232  );
233 
234 
235  set_value ((dods_int16 *) &val[0], nelms);
236  }// H5INT16
237  break;
238 
239 
240  case H5UINT16:
241  {
242  vector<unsigned short> val;
243  subset<unsigned short>(
244  buf,
245  ndims,
246  h5_dimsizes,
247  &offset[0],
248  &step[0],
249  &count[0],
250  &val,
251  pos,
252  0
253  );
254 
255 
256  set_value ((dods_uint16 *) &val[0], nelms);
257  } // H5UINT16
258  break;
259 
260  case H5INT32:
261  {
262  vector<int>val;
263  subset<int>(
264  buf,
265  ndims,
266  h5_dimsizes,
267  &offset[0],
268  &step[0],
269  &count[0],
270  &val,
271  pos,
272  0
273  );
274 
275  set_value ((dods_int32 *) &val[0], nelms);
276  } // case H5INT32
277  break;
278 
279  case H5UINT32:
280  {
281  vector<unsigned int>val;
282  subset<unsigned int>(
283  buf,
284  ndims,
285  h5_dimsizes,
286  &offset[0],
287  &step[0],
288  &count[0],
289  &val,
290  pos,
291  0
292  );
293 
294  set_value ((dods_uint32 *) &val[0], nelms);
295  }
296  break;
297  // Add the code for the CF option DAP4 support
298  // For the CF option DAP2 support, the code will
299  // not come here since 64-integer will be ignored
300  // in DAP2.
301  case H5INT64:
302  {
303  vector<long long>val;
304  subset<long long>(
305  buf,
306  ndims,
307  h5_dimsizes,
308  &offset[0],
309  &step[0],
310  &count[0],
311  &val,
312  pos,
313  0
314  );
315 
316  set_value ((dods_int64 *) &val[0], nelms);
317  } // case H5INT64
318  break;
319 
320  case H5UINT64:
321  {
322  vector<unsigned long long>val;
323  subset<unsigned long long>(
324  buf,
325  ndims,
326  h5_dimsizes,
327  &offset[0],
328  &step[0],
329  &count[0],
330  &val,
331  pos,
332  0
333  );
334 
335  set_value ((dods_uint64 *) &val[0], nelms);
336  }
337  break;
338 
339 
340  case H5FLOAT32:
341  {
342  vector<float>val;
343  subset<float>(
344  buf,
345  ndims,
346  h5_dimsizes,
347  &offset[0],
348  &step[0],
349  &count[0],
350  &val,
351  pos,
352  0
353  );
354  set_value ((dods_float32 *) &val[0], nelms);
355  }
356  break;
357 
358 
359  case H5FLOAT64:
360  {
361 
362  vector<double>val;
363  subset<double>(
364  buf,
365  ndims,
366  h5_dimsizes,
367  &offset[0],
368  &step[0],
369  &count[0],
370  &val,
371  pos,
372  0
373  );
374  set_value ((dods_float64 *) &val[0], nelms);
375  } // case H5FLOAT64
376  break;
377 
378  default:
379  throw InternalErr(__FILE__,__LINE__,"Non-supported datatype");
380 
381  }
382 }
383 
385 //
386 // \param input Input variable
387 // \param dim dimension info of the input
388 // \param start start indexes of each dim
389 // \param stride stride of each dim
390 // \param edge count of each dim
391 // \param poutput output variable
392 // \parrm index dimension index
393 // \return 0 if successful. -1 otherwise.
394 //
395 template<typename T>
397  void* input,
398  int rank,
399  const vector<size_t> & dim,
400  int start[],
401  int stride[],
402  int edge[],
403  vector<T> *poutput,
404  vector<size_t>& pos,
405  int index)
406 {
407  for(int k=0; k<edge[index]; k++)
408  {
409  pos[index] = start[index] + k*stride[index];
410  if(index+1<rank)
411  subset(input, rank, dim, start, stride, edge, poutput,pos,index+1);
412  if(index==rank-1)
413  {
414  size_t cur_pos = INDEX_nD_TO_1D( dim, pos);
415  void* tempbuf = (void*)((char*)input+cur_pos*sizeof(T));
416  poutput->push_back(*(static_cast<T*>(tempbuf)));
417  //"poutput->push_back(input[HDF5CFUtil::INDEX_nD_TO_1D( dim, pos)]);"
418  }
419  } // end of for
420  return 0;
421 } // end of template<typename T> static int subset
422 
423 size_t HDF5BaseArray::INDEX_nD_TO_1D (const std::vector < size_t > &dims,
424  const std::vector < size_t > &pos){
425  //
426  // "int a[10][20][30] // & a[1][2][3] == a + (20*30+1 + 30*2 + 1 *3)"
427  // "int b[10][2] // &b[1][1] == b + (2*1 + 1)"
428  //
429  if(dims.size () != pos.size ())
430  throw InternalErr(__FILE__,__LINE__,"dimension error in INDEX_nD_TO_1D routine.");
431  size_t sum = 0;
432  size_t start = 1;
433 
434  for (size_t p = 0; p < pos.size (); p++) {
435  size_t m = 1;
436 
437  for (size_t j = start; j < dims.size (); j++)
438  m *= dims[j];
439  sum += m * pos[p];
440  start++;
441  }
442  return sum;
443 }
444 
445 // This routine will check if any section(separated by sep) of string cur_str is inside the vector str_list.
446 // The first found string will be returned or empty string will return if not found in the whole cur_str.
447 string HDF5BaseArray::
448 check_str_sect_in_list(const vector<string>&str_list, const string &cur_str,const char sep) {
449 
450  string ret_str;
451  string::size_type start = 0;
452  string::size_type end = 0;
453  // Obtain the ret_str value
454  // The cur_str will be chopped into tokens separated by sep.
455  while ((end = cur_str.find(sep, start)) != string::npos) {
456  if(std::find(str_list.begin(),str_list.end(),cur_str.substr(start,end-start))!=
457  str_list.end()) {
458  ret_str = cur_str.substr(start,end-start);
459  break;
460  }
461  start = end + 1;
462  }
463 
464  // We will not include the last sect (rightmost sect) of cur_str.
465 #if 0
466  //if(ret_str != "") {
467  // if(ret_str == cur_str.substr(cur_str.find_last_of(sep)+1))
468  // ret_str ="";
469  //}
470  //
471 #endif
472 
473  return ret_str;
474 
475 }
476 
477 // This routine will check if there is any sub-string of the fullpath(fname+varname) that is exactly the subset of the fullpath with the same ending
478 // of the fullpath is contained in the slist.
479 // Examples: slist contains { /foo1/foovar foovar2 } fname is /temp/myfile/foo1/ varname is foovar. The rotuine will return true.
480 // fname is /myfile/foo2/ varname is foovar. The routine will return false.
481 bool HDF5BaseArray::
482 check_var_cache_files(const vector<string>&slist, const string &fname,const string &varname) {
483 
484  bool ret_value = false;
485  if(fname=="" || varname=="")
486  return ret_value;
487 
488  string fullpath;
489 
490  if(fname[fname.size()-1] == '/') {
491  if(varname[0]!='/')
492  fullpath = fname+varname;
493  else
494  fullpath = fname.substr(0,fname.size()-1)+varname;
495  }
496  else {
497  if(varname[0]!='/')
498  fullpath = fname+'/'+varname;
499  else
500  fullpath = fname+varname;
501  }
502 
503 
504  for(unsigned int i = 0; i<slist.size();i++) {
505 #if 0
506 //cerr<<"fullpath is "<<fullpath <<endl;
507 //cerr<<"slist[i] is "<<slist[i] <<endl;
508 //cerr<<"fullpath - slist size"<<fullpath.size() -slist[i].size()<<endl;
509 //cerr<<"fullpath.rfind(slist[i] is "<<fullpath.rfind(slist[i]) <<endl;
510 #endif
511  if(fullpath.rfind(slist[i])==(fullpath.size()-slist[i].size())){
512  ret_value = true;
513  break;
514  }
515  }
516  return ret_value;
517 }
518 
519 // Handle data when memory cache is turned on.
520 void HDF5BaseArray::
521 handle_data_with_mem_cache(H5DataType h5_dtype, size_t total_elems,const short cache_flag, const string & cache_key) {
522 
523  //
524  ObjMemCache * mem_data_cache= NULL;
525  if(1 == cache_flag)
526  mem_data_cache = HDF5RequestHandler::get_srdata_mem_cache();
527  else if(cache_flag > 1) {
528  mem_data_cache = HDF5RequestHandler::get_lrdata_mem_cache();
529 
530 #if 0
531 //cerr<<"coming to the large metadata cache "<<endl;
532 //cerr<<"The cache key is "<<cache_key <<endl;
533 
534 // dump the values in the cache,keep this line to check if memory cache works.
535 //mem_data_cache->dump(cerr);
536 #endif
537 
538  }
539 
540 
541  if(mem_data_cache == NULL)
542  throw InternalErr(__FILE__,__LINE__,"The memory data cache should NOT be NULL.");
543 
544  HDF5DataMemCache* mem_cache_ptr = static_cast<HDF5DataMemCache*>(mem_data_cache->get(cache_key));
545  if(mem_cache_ptr) {
546 
547  BESDEBUG("h5","Cache flag: 1 small data cache, 2 large data cache genenral"
548  <<" 3 large data cache common dir, 4 large data cache real var" <<endl);
549 
550  BESDEBUG("h5","Data Memory Cache hit, the variable name is "<<name() <<". The cache flag is "<< cache_flag<<endl);
551 
552  //const string var_name = mem_cache_ptr->get_varname();
553 
554  // Obtain the buffer and do subsetting
555  const size_t var_size = mem_cache_ptr->get_var_buf_size();
556  if(!var_size)
557  throw InternalErr(__FILE__,__LINE__,"The cached data buffer size is 0.");
558  else {
559 
560  void *buf = mem_cache_ptr->get_var_buf();
561 
562  // Obtain dimension size info.
563  vector<size_t> dim_sizes;
564  Dim_iter i_dim = dim_begin();
565  Dim_iter i_enddim = dim_end();
566  while (i_dim != i_enddim) {
567  dim_sizes.push_back(dimension_size(i_dim));
568  ++i_dim;
569  }
570  // read data from the memory cache
571  read_data_from_mem_cache(h5_dtype,dim_sizes,buf);
572  }
573  }
574  else{
575 
576  BESDEBUG("h5","Cache flag: 1 small data cache, 2 large data cache genenral"
577  <<" 3 large data cache common dir, 4 large data cache real var" <<endl);
578 
579  BESDEBUG("h5","Data Memory added to the cache, the variable name is "<<name() <<". The cache flag is "<< cache_flag<<endl);
580 
581  vector <char> buf;
582  if(total_elems == 0)
583  throw InternalErr(__FILE__,__LINE__,"The total number of elements is 0.");
584 
585  buf.resize(total_elems*HDF5CFUtil::H5_numeric_atomic_type_size(h5_dtype));
586 
587  // This routine will read the data, send it to the DAP and save the buf to the cache.
588  read_data_NOT_from_mem_cache(true,&buf[0]);
589 
590  // Create a new cache element.
591 #if 0
592  //HDF5DataMemCache* new_mem_cache = new HDF5DataMemCache(varname);
593 #endif
594  HDF5DataMemCache* new_mem_cache_ele = new HDF5DataMemCache();
595  new_mem_cache_ele->set_databuf(buf);
596 
597  // Add this entry to the cache list
598  mem_data_cache->add(new_mem_cache_ele, cache_key);
599  }
600 
601  return;
602 }
603 
604 
605 
HDF5BaseArray.h
A helper class that aims to reduce code redundence for different special CF derived array class For e...
ObjMemCache
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:84
ObjMemCache::add
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
libdap
Definition: BESDapFunctionResponseCache.h:35
HDF5BaseArray
Definition: HDF5BaseArray.h:55
HDF5RequestHandler.h
include the entry functions to execute the handlers
HDF5DataMemCache
Definition: HDF5_DataMemCache.h:24
HDF5BaseArray::subset
int subset(void *input, int rank, const std::vector< size_t > &dim, int start[], int stride[], int edge[], std::vector< T > *poutput, std::vector< size_t > &pos, int index)
Getting a subset of a variable.
Definition: HDF5BaseArray.cc:396
Error
ObjMemCache::get
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105