bes  Updated for version 3.20.6
HDF5CFArray.cc
Go to the documentation of this file.
1 // This file is part of the hdf5_handler implementing for the CF-compliant
2 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3 //
4 // This is free software; you can redistribute it and/or modify it under the
5 // terms of the GNU Lesser General Public License as published by the Free
6 // Software Foundation; either version 2.1 of the License, or (at your
7 // option) any later version.
8 //
9 // This software is distributed in the hope that it will be useful, but
10 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12 // License for more details.
13 //
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 //
18 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
20 // Suite 203, Champaign, IL 61820
21 
30 
31 #include "config_hdf5.h"
32 #include <iostream>
33 #include <sstream>
34 #include <cassert>
35 #include <BESDebug.h>
36 #include <sys/stat.h>
37 #include "InternalErr.h"
38 
39 #include "Str.h"
40 #include "HDF5RequestHandler.h"
41 #include "HDF5CFArray.h"
42 #include "h5cfdaputil.h"
43 #include "ObjMemCache.h"
44 
45 using namespace std;
46 using namespace libdap;
47 
48 
49 BaseType *HDF5CFArray::ptr_duplicate()
50 {
51  return new HDF5CFArray(*this);
52 }
53 
54 // Read in an HDF5 Array
55 bool HDF5CFArray::read()
56 {
57 
58  BESDEBUG("h5","Coming to HDF5CFArray read "<<endl);
59  if(length() == 0)
60  return true;
61 
62  if((NULL == HDF5RequestHandler::get_lrdata_mem_cache()) &&
63  NULL == HDF5RequestHandler::get_srdata_mem_cache()){
64  read_data_NOT_from_mem_cache(false,NULL);
65  return true;
66  }
67 
68  // Flag to check if using large raw data cache or small raw data cache.
69  short use_cache_flag = 0;
70 
71  // The small data cache is checked first to reduce the resources to operate the big data cache.
72  if(HDF5RequestHandler::get_srdata_mem_cache() != NULL) {
73  if(((cvtype == CV_EXIST) && (islatlon != true)) || (cvtype == CV_NONLATLON_MISS)
74  || (cvtype == CV_FILLINDEX) ||(cvtype == CV_MODIFY) ||(cvtype == CV_SPECIAL)){
75 
76  if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==true)
77  use_cache_flag = 1;
78  }
79  }
80 
81  // If this varible doesn't fit the small data cache, let's check if it fits the large data cache.
82  if(use_cache_flag !=1) {
83 
84  if(HDF5RequestHandler::get_lrdata_mem_cache() != NULL) {
85 
86  // This is the trival case.
87  // If no information is provided in the configuration file of large data cache,
88  // just cache the lat/lon varible per file.
89  if(HDF5RequestHandler::get_common_cache_dirs() == false) {
90  if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
91  || (cvtype == CV_EXIST && islatlon == true)) {
92 #if 0
93 //cerr<<"coming to use_cache_flag =2 "<<endl;
94 #endif
95  // Only the data with the numeric datatype DAP2 and CF support are cached.
96  if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==true)
97  use_cache_flag = 2;
98  }
99  }
100  else {// Have large data cache configuration info.
101 
102  // Need to check if we don't want to cache some CVs, now
103  // this only applies to lat/lon CV.
104  if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
105  || (cvtype == CV_EXIST && islatlon == true)) {
106 
107  vector<string> cur_lrd_non_cache_dir_list;
108  HDF5RequestHandler::get_lrd_non_cache_dir_list(cur_lrd_non_cache_dir_list);
109 
110  // Check if this file is included in the non-cache directory
111  if( (cur_lrd_non_cache_dir_list.size() == 0) ||
112  ("" == check_str_sect_in_list(cur_lrd_non_cache_dir_list,filename,'/'))) {
113 
114  // Only data with the numeric datatype DAP2 and CF support are cached.
115  if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==true)
116  use_cache_flag = 3;
117  }
118  }
119  // Here we allow all the variable names to be cached.
120  // The file path that includes the variables can also included.
121  vector<string> cur_lrd_var_cache_file_list;
122  HDF5RequestHandler::get_lrd_var_cache_file_list(cur_lrd_var_cache_file_list);
123  if(cur_lrd_var_cache_file_list.size() >0){
124 #if 0
125 //cerr<<"lrd var cache is "<<cur_lrd_var_cache_file_list[i]<<endl;
127 #endif
128  if(true == check_var_cache_files(cur_lrd_var_cache_file_list,filename,varname)){
129 #if 0
130 //cerr<<"varname is "<<varname <<endl;
131 //cerr<<"have var cached "<<endl;
132 #endif
133 
134  // Only the data with the numeric datatype DAP2 and CF support are cached.
135  if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==true)
136  use_cache_flag = 4;
137  }
138  }
139  }
140  }
141  }
142 
143  if(0 == use_cache_flag)
144  read_data_NOT_from_mem_cache(false,NULL);
145  else {// memory cache cases
146 
147  string cache_key;
148 
149  // Possibly we have common lat/lon dirs,so check here.
150  if( 3 == use_cache_flag){
151  vector<string> cur_cache_dlist;
152  HDF5RequestHandler::get_lrd_cache_dir_list(cur_cache_dlist);
153  string cache_dir = check_str_sect_in_list(cur_cache_dlist,filename,'/');
154  if(cache_dir != "")
155  cache_key = cache_dir + varname;
156  else {
157  cache_key = filename + varname;
158  // If this lat/lon is not in the common dir. list, it is still cached as a general lat/lon.
159  // Change the flag to 2.
160  use_cache_flag = 2;
161  }
162 
163  }
164  else
165  cache_key = filename + varname;
166 
167  handle_data_with_mem_cache(dtype,total_elems,use_cache_flag,cache_key);
168 
169  }
170 
171  return true;
172 }
173 
174 // Reading data not from memory cache: The data can be read from the disk cache or can be read via the HDF5 APIs
175 void HDF5CFArray::read_data_NOT_from_mem_cache(bool add_mem_cache,void*buf) {
176 
177  vector<int>offset;
178  vector<int>count;
179  vector<int>step;
180  vector<hsize_t> hoffset;
181  vector<hsize_t>hcount;
182  vector<hsize_t>hstep;
183  int nelms = 1;
184 
185  if (rank <= 0)
186  throw InternalErr (__FILE__, __LINE__,
187  "The number of dimension of the variable is <=0 for an array.");
188  else {
189 
190  offset.resize(rank);
191  count.resize(rank);
192  step.resize(rank);
193  hoffset.resize(rank);
194  hcount.resize(rank);
195  hstep.resize(rank);
196  nelms = format_constraint (&offset[0], &step[0], &count[0]);
197  for (int i = 0; i <rank; i++) {
198  hoffset[i] = (hsize_t) offset[i];
199  hcount[i] = (hsize_t) count[i];
200  hstep[i] = (hsize_t) step[i];
201  }
202  }
203 
204  hid_t dsetid = -1;
205  hid_t dspace = -1;
206  hid_t mspace = -1;
207  hid_t dtypeid = -1;
208  hid_t memtype = -1;
209 
210  bool data_from_disk_cache = false;
211  bool data_to_disk_cache = false;
212 
213  // Check if the disk cache can be applied.
214  bool use_disk_cache = valid_disk_cache();
215 
216  string cache_fpath;
217 
218  if(true == use_disk_cache) {
219 
220  BESDEBUG("h5","Coming to use disk cache "<<endl);
221 
222  unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
223  string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
224  string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
225 
226  string cache_fname=HDF5CFUtil::obtain_cache_fname(diskcache_prefix,filename,varname);
227  cache_fpath = diskcache_dir + "/"+ cache_fname;
228 
229  int temp_total_elems = 1;
230  for (unsigned int i = 0; i <dimsizes.size();i++)
231  temp_total_elems = temp_total_elems*dimsizes[i];
232  short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
233 
234  int expected_file_size = dtype_size *temp_total_elems;
235  int fd = 0;
236  HDF5DiskCache *disk_cache = HDF5DiskCache::get_instance(disk_cache_size,diskcache_dir,diskcache_prefix);
237  if( true == disk_cache->get_data_from_cache(cache_fpath, expected_file_size,fd)) {
238 
239  vector<size_t> offset_size_t;
240  offset_size_t.resize(rank);
241  for(int i = 0; i <rank;i++)
242  offset_size_t[i] = (size_t)offset[i];
243  size_t offset_1st = INDEX_nD_TO_1D(dimsizes,offset_size_t);
244  vector<size_t>end;
245  end.resize(rank);
246  for (int i = 0; i < rank; i++)
247  end[i] = offset[i] +(count[i]-1)*step[i];
248  size_t offset_last = INDEX_nD_TO_1D(dimsizes,end);
249 #if 0
250 //cerr<<"offset_1d is "<<offset_1st <<endl;
251 //cerr<<"offset_last is "<<offset_last <<endl;
252 #endif
253  size_t total_read = dtype_size*(offset_last-offset_1st+1);
254 
255  off_t fpos = lseek(fd,dtype_size*offset_1st,SEEK_SET);
256  if (-1 == fpos) {
257  disk_cache->unlock_and_close(cache_fpath);
258  disk_cache->purge_file(cache_fpath);
259  }
260 
262  else
263  data_from_disk_cache = obtain_cached_data(disk_cache,cache_fpath,fd, step,count,total_read,dtype_size);
264 
265  }
266 
267  if(true == data_from_disk_cache)
268  return;
269  else
270  data_to_disk_cache = true;
271 
272  }
273 
274 // END CACHE
275 
276  bool pass_fileid = HDF5RequestHandler::get_pass_fileid();
277  if(false == pass_fileid) {
278  if ((fileid = H5Fopen(filename.c_str(),H5F_ACC_RDONLY,H5P_DEFAULT))<0) {
279  ostringstream eherr;
280  eherr << "HDF5 File " << filename
281  << " cannot be opened. "<<endl;
282  throw InternalErr (__FILE__, __LINE__, eherr.str ());
283  }
284  }
285 
286  if ((dsetid = H5Dopen(fileid,varname.c_str(),H5P_DEFAULT))<0) {
287  HDF5CFUtil::close_fileid(fileid,pass_fileid);
288  ostringstream eherr;
289  eherr << "HDF5 dataset " << varname
290  << " cannot be opened. "<<endl;
291  throw InternalErr (__FILE__, __LINE__, eherr.str ());
292  }
293 
294  if ((dspace = H5Dget_space(dsetid))<0) {
295 
296  H5Dclose(dsetid);
297  HDF5CFUtil::close_fileid(fileid,pass_fileid);
298  ostringstream eherr;
299  eherr << "Space id of the HDF5 dataset " << varname
300  << " cannot be obtained. "<<endl;
301  throw InternalErr (__FILE__, __LINE__, eherr.str ());
302  }
303 
304  if (H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
305  &hoffset[0], &hstep[0],
306  &hcount[0], NULL) < 0) {
307 
308  H5Sclose(dspace);
309  H5Dclose(dsetid);
310  HDF5CFUtil::close_fileid(fileid,pass_fileid);
311  ostringstream eherr;
312  eherr << "The selection of hyperslab of the HDF5 dataset " << varname
313  << " fails. "<<endl;
314  throw InternalErr (__FILE__, __LINE__, eherr.str ());
315  }
316 
317  mspace = H5Screate_simple(rank, &hcount[0],NULL);
318  if (mspace < 0) {
319  H5Sclose(dspace);
320  H5Dclose(dsetid);
321  HDF5CFUtil::close_fileid(fileid,pass_fileid);
322  ostringstream eherr;
323  eherr << "The creation of the memory space of the HDF5 dataset " << varname
324  << " fails. "<<endl;
325  throw InternalErr (__FILE__, __LINE__, eherr.str ());
326  }
327 
328 
329  if ((dtypeid = H5Dget_type(dsetid)) < 0) {
330 
331  H5Sclose(mspace);
332  H5Sclose(dspace);
333  H5Dclose(dsetid);
334  HDF5CFUtil::close_fileid(fileid,pass_fileid);
335  ostringstream eherr;
336  eherr << "Obtaining the datatype of the HDF5 dataset " << varname
337  << " fails. "<<endl;
338  throw InternalErr (__FILE__, __LINE__, eherr.str ());
339 
340  }
341 
342  if ((memtype = H5Tget_native_type(dtypeid, H5T_DIR_ASCEND))<0) {
343 
344  H5Sclose(mspace);
345  H5Tclose(dtypeid);
346  H5Sclose(dspace);
347  H5Dclose(dsetid);
348  HDF5CFUtil::close_fileid(fileid,pass_fileid);
349  ostringstream eherr;
350  eherr << "Obtaining the memory type of the HDF5 dataset " << varname
351  << " fails. "<<endl;
352  throw InternalErr (__FILE__, __LINE__, eherr.str ());
353 
354  }
355 
356  hid_t read_ret = -1;
357 
358  // Before reading the data, we will check if the memory cache is turned on,
359  // The add_mem_cache is only true when the data memory cache keys are on and used.
360  if(true == add_mem_cache) {
361  if(buf== NULL) {
362  H5Sclose(mspace);
363  H5Tclose(dtypeid);
364  H5Sclose(dspace);
365  H5Dclose(dsetid);
366  HDF5CFUtil::close_fileid(fileid,pass_fileid);
367  throw InternalErr(__FILE__,__LINE__,"The memory data cache buffer needs to be set");
368  }
369  read_ret= H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf);
370  if(read_ret <0){
371  H5Sclose(mspace);
372  H5Tclose(dtypeid);
373  H5Sclose(dspace);
374  H5Dclose(dsetid);
375  HDF5CFUtil::close_fileid(fileid,pass_fileid);
376  throw InternalErr(__FILE__,__LINE__,"Cannot read the data to the buffer.");
377  }
378  }
379 
380 
381  // Now reading the data, note dtype is not dtypeid.
382  // dtype is an enum defined by the handler.
383 
384  switch (dtype) {
385 
386  case H5CHAR:
387  {
388 
389  vector<char> val;
390  val.resize(nelms);
391 
392  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
393  if (read_ret < 0) {
394 
395  H5Sclose(mspace);
396  H5Tclose(memtype);
397  H5Tclose(dtypeid);
398  H5Sclose(dspace);
399  H5Dclose(dsetid);
400  HDF5CFUtil::close_fileid(fileid,pass_fileid);
401  ostringstream eherr;
402  eherr << "Cannot read the HDF5 dataset " << varname
403  << " with the type of H5T_NATIVE_CHAR "<<endl;
404  throw InternalErr (__FILE__, __LINE__, eherr.str ());
405 
406  }
407 
408  vector<short>newval;
409  newval.resize(nelms);
410 
411  for (int counter = 0; counter < nelms; counter++)
412  newval[counter] = (short) (val[counter]);
413 
414  set_value ((dods_int16 *) &newval[0], nelms);
415 
416  if(true == data_to_disk_cache) {
417  try {
418  BESDEBUG("h5","writing data to disk cache "<<endl);
419  write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,2,val,nelms);
420  }
421  catch(...) {
422  H5Sclose(mspace);
423  H5Tclose(memtype);
424  H5Tclose(dtypeid);
425  H5Sclose(dspace);
426  H5Dclose(dsetid);
427  HDF5CFUtil::close_fileid(fileid,pass_fileid);
428  ostringstream eherr;
429  eherr << "write data to cache failed.";
430  throw InternalErr (__FILE__, __LINE__, eherr.str ());
431 
432  }
433  }
434 
435  } // case H5CHAR
436  break;
437 
438  // Note: for DAP2, H5INT64,H5UINT64 will be ignored.
439  case H5UCHAR:
440  case H5UINT16:
441  case H5INT16:
442  case H5INT32:
443  case H5UINT32:
444  case H5INT64:
445  case H5UINT64:
446  case H5FLOAT32:
447  case H5FLOAT64:
448 
449 
450  {
451  size_t dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
452  vector<char> val;
453  val.resize(nelms*dtype_size);
454 
455  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
456  if (read_ret < 0) {
457  H5Sclose(mspace);
458  H5Tclose(memtype);
459  H5Tclose(dtypeid);
460  H5Sclose(dspace);
461  H5Dclose(dsetid);
462  HDF5CFUtil::close_fileid(fileid,pass_fileid);
463  ostringstream eherr;
464  eherr << "Cannot read the HDF5 dataset " << varname
465  << " with the type of H5T_NATIVE_UCHAR "<<endl;
466  throw InternalErr (__FILE__, __LINE__, eherr.str ());
467 
468  }
469  // Not sure if "set_value ((dods_byte *) &val[0], nelms);" works.
470  val2buf(&val[0]);
471  set_read_p(true);
472 
473  if(true == data_to_disk_cache) {
474  BESDEBUG("h5","writing data to disk cache "<<endl);
475  try {
476  write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,dtype_size,val,nelms);
477  }
478  catch(...) {
479  H5Sclose(mspace);
480  H5Tclose(memtype);
481  H5Tclose(dtypeid);
482  H5Sclose(dspace);
483  H5Dclose(dsetid);
484  HDF5CFUtil::close_fileid(fileid,pass_fileid);
485  ostringstream eherr;
486  eherr << "write data to cache failed.";
487  throw InternalErr (__FILE__, __LINE__, eherr.str ());
488 
489  }
490 
491  }
492  } // case H5UCHAR...
493  break;
494 
495 
496 
497 #if 0
498  case H5INT16:
499  {
500  vector<short>val;
501  val.resize(nelms);
502 
503  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
504  if (read_ret < 0) {
505 
506  H5Sclose(mspace);
507  H5Tclose(memtype);
508  H5Tclose(dtypeid);
509  H5Sclose(dspace);
510  H5Dclose(dsetid);
511  HDF5CFUtil::close_fileid(fileid,pass_fileid);
512  //H5Fclose(fileid);
513  ostringstream eherr;
514  eherr << "Cannot read the HDF5 dataset " << varname
515  << " with the type of H5T_NATIVE_SHORT "<<endl;
516  throw InternalErr (__FILE__, __LINE__, eherr.str ());
517 
518  }
519  set_value ((dods_int16 *) &val[0], nelms);
520  }// H5INT16
521  break;
522 
523 
524  case H5UINT16:
525  {
526  vector<unsigned short> val;
527  val.resize(nelms);
528  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
529  if (read_ret < 0) {
530 
531  H5Sclose(mspace);
532  H5Tclose(memtype);
533  H5Tclose(dtypeid);
534  H5Sclose(dspace);
535  H5Dclose(dsetid);
536  HDF5CFUtil::close_fileid(fileid,pass_fileid);
537  ostringstream eherr;
538  eherr << "Cannot read the HDF5 dataset " << varname
539  << " with the type of H5T_NATIVE_USHORT "<<endl;
540  throw InternalErr (__FILE__, __LINE__, eherr.str ());
541 
542  }
543  set_value ((dods_uint16 *) &val[0], nelms);
544  } // H5UINT16
545  break;
546 
547 
548  case H5INT32:
549  {
550  vector<int>val;
551  val.resize(nelms);
552  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
553  if (read_ret < 0) {
554  H5Sclose(mspace);
555  H5Tclose(memtype);
556  H5Tclose(dtypeid);
557  H5Sclose(dspace);
558  H5Dclose(dsetid);
559  HDF5CFUtil::close_fileid(fileid,pass_fileid);
560  ostringstream eherr;
561  eherr << "Cannot read the HDF5 dataset " << varname
562  << " with the type of H5T_NATIVE_INT "<<endl;
563  throw InternalErr (__FILE__, __LINE__, eherr.str ());
564 
565  }
566  set_value ((dods_int32 *) &val[0], nelms);
567  } // case H5INT32
568  break;
569 
570  case H5UINT32:
571  {
572  vector<unsigned int>val;
573  val.resize(nelms);
574  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
575  if (read_ret < 0) {
576  H5Sclose(mspace);
577  H5Tclose(memtype);
578  H5Tclose(dtypeid);
579  H5Sclose(dspace);
580  H5Dclose(dsetid);
581  HDF5CFUtil::close_fileid(fileid,pass_fileid);
582  ostringstream eherr;
583  eherr << "Cannot read the HDF5 dataset " << varname
584  << " with the type of H5T_NATIVE_UINT "<<endl;
585  throw InternalErr (__FILE__, __LINE__, eherr.str ());
586 
587  }
588  set_value ((dods_uint32 *) &val[0], nelms);
589  }
590  break;
591 
592  case H5FLOAT32:
593  {
594 
595  vector<float>val;
596  val.resize(nelms);
597 
598  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
599  if (read_ret < 0) {
600  H5Sclose(mspace);
601  H5Tclose(memtype);
602  H5Tclose(dtypeid);
603  H5Sclose(dspace);
604  H5Dclose(dsetid);
605  HDF5CFUtil::close_fileid(fileid,pass_fileid);
606  ostringstream eherr;
607  eherr << "Cannot read the HDF5 dataset " << varname
608  << " with the type of H5T_NATIVE_FLOAT "<<endl;
609  throw InternalErr (__FILE__, __LINE__, eherr.str ());
610 
611  }
612  set_value ((dods_float32 *) &val[0], nelms);
613  }
614  break;
615 
616 
617  case H5FLOAT64:
618  {
619 
620  vector<double>val;
621  val.resize(nelms);
622  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
623 
624  if (read_ret < 0) {
625  H5Sclose(mspace);
626  H5Tclose(memtype);
627  H5Tclose(dtypeid);
628  H5Sclose(dspace);
629  H5Dclose(dsetid);
630  HDF5CFUtil::close_fileid(fileid,pass_fileid);
631  ostringstream eherr;
632  eherr << "Cannot read the HDF5 dataset " << varname
633  << " with the type of H5T_NATIVE_DOUBLE "<<endl;
634  throw InternalErr (__FILE__, __LINE__, eherr.str ());
635 
636  }
637  set_value ((dods_float64 *) &val[0], nelms);
638  } // case H5FLOAT64
639  break;
640 
641 #endif
642 
643  case H5FSTRING:
644  {
645  size_t ty_size = H5Tget_size(dtypeid);
646  if (ty_size == 0) {
647  H5Sclose(mspace);
648  H5Tclose(memtype);
649  H5Tclose(dtypeid);
650  H5Sclose(dspace);
651  H5Dclose(dsetid);
652  HDF5CFUtil::close_fileid(fileid,pass_fileid);
653  ostringstream eherr;
654  eherr << "Cannot obtain the size of the fixed size HDF5 string of the dataset "
655  << varname <<endl;
656  throw InternalErr (__FILE__, __LINE__, eherr.str ());
657  }
658 
659  vector <char> strval;
660  strval.resize(nelms*ty_size);
661  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(void*)&strval[0]);
662 
663  if (read_ret < 0) {
664  H5Sclose(mspace);
665  H5Tclose(memtype);
666  H5Tclose(dtypeid);
667  H5Sclose(dspace);
668  H5Dclose(dsetid);
669  HDF5CFUtil::close_fileid(fileid,pass_fileid);
670  ostringstream eherr;
671  eherr << "Cannot read the HDF5 dataset " << varname
672  << " with the type of the fixed size HDF5 string "<<endl;
673  throw InternalErr (__FILE__, __LINE__, eherr.str ());
674  }
675 
676  string total_string(strval.begin(),strval.end());
677  strval.clear();//save some memory;
678  vector <string> finstrval;
679  finstrval.resize(nelms);
680  for (int i = 0; i<nelms; i++)
681  finstrval[i] = total_string.substr(i*ty_size,ty_size);
682 
683  // Check if we should drop the long string
684 
685  // If the size of an individual element is longer than the current netCDF JAVA
686  // string and the "EnableDropLongString" key is turned on,
687  // No string is generated.
688  if ((true == HDF5RequestHandler::get_drop_long_string()) &&
689  ty_size > NC_JAVA_STR_SIZE_LIMIT) {
690  for (int i = 0; i<nelms; i++)
691  finstrval[i] = "";
692  }
693  set_value(finstrval,nelms);
694  total_string.clear();
695  }
696  break;
697 
698 
699  case H5VSTRING:
700  {
701  size_t ty_size = H5Tget_size(memtype);
702  if (ty_size == 0) {
703  H5Sclose(mspace);
704  H5Tclose(memtype);
705  H5Tclose(dtypeid);
706  H5Sclose(dspace);
707  H5Dclose(dsetid);
708  HDF5CFUtil::close_fileid(fileid,pass_fileid);
709  ostringstream eherr;
710  eherr << "Cannot obtain the size of the fixed size HDF5 string of the dataset "
711  << varname <<endl;
712  throw InternalErr (__FILE__, __LINE__, eherr.str ());
713  }
714  vector <char> strval;
715  strval.resize(nelms*ty_size);
716  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(void*)&strval[0]);
717 
718  if (read_ret < 0) {
719  H5Sclose(mspace);
720  H5Tclose(memtype);
721  H5Tclose(dtypeid);
722  H5Sclose(dspace);
723  H5Dclose(dsetid);
724  HDF5CFUtil::close_fileid(fileid,pass_fileid);
725  ostringstream eherr;
726  eherr << "Cannot read the HDF5 dataset " << varname
727  << " with the type of the HDF5 variable length string "<<endl;
728  throw InternalErr (__FILE__, __LINE__, eherr.str ());
729  }
730 
731  vector<string>finstrval;
732  finstrval.resize(nelms);
733  char*temp_bp = &strval[0];
734  char*onestring = NULL;
735  for (int i =0;i<nelms;i++) {
736  onestring = *(char**)temp_bp;
737  if(onestring!=NULL )
738  finstrval[i] =string(onestring);
739 
740  else // We will add a NULL if onestring is NULL.
741  finstrval[i]="";
742  temp_bp +=ty_size;
743  }
744 
745  if (false == strval.empty()) {
746  herr_t ret_vlen_claim;
747  ret_vlen_claim = H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(void*)&strval[0]);
748  if (ret_vlen_claim < 0){
749  H5Sclose(mspace);
750  H5Tclose(memtype);
751  H5Tclose(dtypeid);
752  H5Sclose(dspace);
753  H5Dclose(dsetid);
754  HDF5CFUtil::close_fileid(fileid,pass_fileid);
755  ostringstream eherr;
756  eherr << "Cannot reclaim the memory buffer of the HDF5 variable length string of the dataset "
757  << varname <<endl;
758  throw InternalErr (__FILE__, __LINE__, eherr.str ());
759 
760  }
761  }
762 
763  // If the size of one string element is longer than the current netCDF JAVA
764  // string and the "EnableDropLongString" key is turned on,
765  // No string is generated.
766  if (true == HDF5RequestHandler::get_drop_long_string()) {
767  bool drop_long_str = false;
768  for (int i =0;i<nelms;i++) {
769  if(finstrval[i].size() >NC_JAVA_STR_SIZE_LIMIT){
770  drop_long_str = true;
771  break;
772  }
773  }
774  if (drop_long_str == true) {
775  for (int i =0;i<nelms;i++)
776  finstrval[i] = "";
777  }
778  }
779  set_value(finstrval,nelms);
780 
781  }
782  break;
783 
784  default:
785  {
786  H5Tclose(memtype);
787  H5Tclose(dtypeid);
788  H5Sclose(mspace);
789  H5Sclose(dspace);
790  H5Dclose(dsetid);
791  HDF5CFUtil::close_fileid(fileid,pass_fileid);
792  ostringstream eherr;
793  eherr << "Cannot read the HDF5 dataset " << varname
794  << " with the unsupported HDF5 datatype"<<endl;
795  throw InternalErr (__FILE__, __LINE__, eherr.str ());
796  }
797  }
798 
799  H5Tclose(memtype);
800  H5Tclose(dtypeid);
801  H5Sclose(mspace);
802  H5Sclose(dspace);
803  H5Dclose(dsetid);
804  HDF5CFUtil::close_fileid(fileid,pass_fileid);
805 
806  return;
807 }
808 
809 bool HDF5CFArray::valid_disk_cache() {
810 
811  bool ret_value = false;
812  if(true == HDF5RequestHandler::get_use_disk_cache()) {
813 
814  BESDEBUG("h5","Coming to disk cache "<<endl);
815  // Check if this is a valid numeric datatype we want to support
816  if(dtype == H5CHAR || dtype ==H5UCHAR || dtype==H5INT16 || dtype ==H5UINT16 ||
817  dtype == H5INT32 || dtype ==H5UINT32 || dtype ==H5FLOAT32 || dtype==H5FLOAT64 ||
818  dtype == H5INT64 || dtype ==H5UINT64){
819 
820  BESDEBUG("h5","Coming to disk cache datatype block"<<endl);
821 
822  string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
823  string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
824  long diskcache_size = HDF5RequestHandler::get_disk_cache_size();
825 
826  if(("" == diskcache_dir)||(""==diskcache_prefix)||(diskcache_size <=0))
827  throw InternalErr (__FILE__, __LINE__, "Either the cached dir is empty or the prefix is NULL or the cache size is not set.");
828  else {
829  struct stat sb;
830  if(stat(diskcache_dir.c_str(),&sb) !=0) {
831  string err_mesg="The cached directory " + diskcache_dir;
832  err_mesg = err_mesg + " doesn't exist. ";
833  throw InternalErr(__FILE__,__LINE__,err_mesg);
834  }
835  else {
836  if(true == S_ISDIR(sb.st_mode)) {
837  if(access(diskcache_dir.c_str(),R_OK|W_OK|X_OK) == -1) {
838  string err_mesg="The cached directory " + diskcache_dir;
839  err_mesg = err_mesg + " can NOT be read,written or executable.";
840  throw InternalErr(__FILE__,__LINE__,err_mesg);
841  }
842  }
843  else {
844  string err_mesg="The cached directory " + diskcache_dir;
845  err_mesg = err_mesg + " is not a directory.";
846  throw InternalErr(__FILE__,__LINE__,err_mesg);
847  }
848  }
849  }
850 
851  short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
852  // Check if we only need to cache the specific compressed dat
853  if(true == HDF5RequestHandler::get_disk_cache_comp_data()){
854  BESDEBUG("h5","Compression disk cache key is true"<<endl);
855  ret_value = valid_disk_cache_for_compressed_data(dtype_size);
856  BESDEBUG("h5","variable disk cache passes the compression parameter check"<<endl);
857  }
858  else {
859  BESDEBUG("h5","Compression disk cache key is NOT set, disk cache key is true."<<endl);
860  ret_value = true;
861  }
862 
863  }
864 
865  }
866  return ret_value;
867 }
868 
869 bool HDF5CFArray:: valid_disk_cache_for_compressed_data(short dtype_size) {
870 
871  bool ret_value = false;
872  // The compression ratio should be smaller then the threshold(hard to compress)
873  // and the total var size should be bigger than the defined size(bigger)
874 #if 0
875  size_t total_byte = total_elems*dtype_size;
876 #endif
877  if((comp_ratio < HDF5RequestHandler::get_disk_comp_threshold())
878  && (total_elems*dtype_size >= HDF5RequestHandler::get_disk_var_size())) {
879  if( true == HDF5RequestHandler::get_disk_cache_float_only_comp()) {
880  if(dtype==H5FLOAT32 || dtype == H5FLOAT64)
881  ret_value = true;
882  }
883  else
884  ret_value = true;
885  }
886  return ret_value;
887 
888 }
889 
890 bool HDF5CFArray::obtain_cached_data(HDF5DiskCache *disk_cache,const string & cache_fpath, int fd,vector<int> &cd_step, vector<int>&cd_count,size_t total_read,short dtype_size) {
891 
892  ssize_t ret_read_val = -1;
893  vector<char>buf;
894 
895  buf.resize(total_read);
896  ret_read_val = HDF5CFUtil::read_buffer_from_file(fd,(void*)&buf[0],total_read);
897  disk_cache->unlock_and_close(cache_fpath);
898  if((-1 == ret_read_val) || (ret_read_val != (ssize_t)total_read)) {
899  disk_cache->purge_file(cache_fpath);
900  return false;
901  }
902  else {
903  unsigned int nele_to_read = 1;
904  for(int i = 0; i<rank;i++)
905  nele_to_read *=cd_count[i];
906 
907  if(nele_to_read == (total_read/dtype_size)) {
908  val2buf(&buf[0]);
909  set_read_p(true);
910  }
911  else { // Need to re-assemble the buffer according to different datatype
912 
913  vector<int>cd_start(rank,0);
914  vector<size_t>cd_pos(rank,0);
915  int nelms_to_send = 1;
916  for(int i = 0; i <rank; i++)
917  nelms_to_send = nelms_to_send*cd_count[i];
918 
919  switch (dtype) {
920 
921  case H5CHAR:
922  {
923 #if 0
924  vector<int>total_val;
925  total_val.resize(total_read/dtype_size);
926  memcpy(&total_val[0],(void*)&buf[0],total_read);
927 
928  vector<int>final_val;
929  subset<int>(
930  &total_val[0],
931  rank,
932  dimsizes,
933  &cd_start[0],
934  &cd_step[0],
935  &cd_count[0],
936  &final_val,
937  cd_pos,
938  0
939  );
940 
941 #endif
942  vector<short>final_val;
943  subset<short>(
944  &buf[0],
945  rank,
946  dimsizes,
947  &cd_start[0],
948  &cd_step[0],
949  &cd_count[0],
950  &final_val,
951  cd_pos,
952  0
953  );
954  set_value((dods_int16*)&final_val[0],nelms_to_send);
955 
956  }
957 
958  break;
959  case H5UCHAR:
960  {
961 #if 0
962  vector<unsigned char>total_val;
963  total_val.resize(total_read/dtype_size);
964  memcpy(&total_val[0],(void*)&buf[0],total_read);
965 
966  vector<unsigned char>final_val;
967  subset<unsigned char>(
968  &total_val[0],
969  rank,
970  dimsizes,
971  &cd_start[0],
972  &cd_step[0],
973  &cd_count[0],
974  &final_val,
975  cd_pos,
976  0
977  );
978 
979 #endif
980  vector<unsigned char>final_val;
981  subset<unsigned char>(
982  &buf[0],
983  rank,
984  dimsizes,
985  &cd_start[0],
986  &cd_step[0],
987  &cd_count[0],
988  &final_val,
989  cd_pos,
990  0
991  );
992 
993  set_value ((dods_byte *) &final_val[0], nelms_to_send);
994  }
995  break;
996 
997  case H5INT16:
998  {
999 #if 0
1000  vector<short>total_val;
1001  total_val.resize(total_read/dtype_size);
1002  memcpy(&total_val[0],(void*)&buf[0],total_read);
1003 
1004  vector<short>final_val;
1005  subset<short>(
1006  &total_val[0],
1007  rank,
1008  dimsizes,
1009  &cd_start[0],
1010  &cd_step[0],
1011  &cd_count[0],
1012  &final_val,
1013  cd_pos,
1014  0
1015  );
1016 #endif
1017 
1018  vector<short>final_val;
1019  subset<short>(
1020  &buf[0],
1021  rank,
1022  dimsizes,
1023  &cd_start[0],
1024  &cd_step[0],
1025  &cd_count[0],
1026  &final_val,
1027  cd_pos,
1028  0
1029  );
1030 
1031  set_value ((dods_int16 *) &final_val[0], nelms_to_send);
1032  }
1033  break;
1034 
1035  case H5UINT16:
1036  {
1037 #if 0
1038  vector<unsigned short>total_val;
1039  total_val.resize(total_read/dtype_size);
1040  memcpy(&total_val[0],(void*)&buf[0],total_read);
1041 
1042  vector<unsigned short>final_val;
1043  subset<unsigned short>(
1044  &total_val[0],
1045  rank,
1046  dimsizes,
1047  &cd_start[0],
1048  &cd_step[0],
1049  &cd_count[0],
1050  &final_val,
1051  cd_pos,
1052  0
1053  );
1054 #endif
1055 
1056  vector<unsigned short>final_val;
1057  subset<unsigned short>(
1058  &buf[0],
1059  rank,
1060  dimsizes,
1061  &cd_start[0],
1062  &cd_step[0],
1063  &cd_count[0],
1064  &final_val,
1065  cd_pos,
1066  0
1067  );
1068 
1069  set_value ((dods_uint16 *) &final_val[0], nelms_to_send);
1070  }
1071  break;
1072 
1073  case H5INT32:
1074  {
1075 #if 0
1076  vector<int>total_val;
1077  total_val.resize(total_read/dtype_size);
1078  memcpy(&total_val[0],(void*)&buf[0],total_read);
1079 
1080  vector<int>final_val;
1081  subset<int>(
1082  &total_val[0],
1083  rank,
1084  dimsizes,
1085  &cd_start[0],
1086  &cd_step[0],
1087  &cd_count[0],
1088  &final_val,
1089  cd_pos,
1090  0
1091  );
1092 
1093 #endif
1094 
1095  vector<int>final_val;
1096  subset<int>(
1097  &buf[0],
1098  rank,
1099  dimsizes,
1100  &cd_start[0],
1101  &cd_step[0],
1102  &cd_count[0],
1103  &final_val,
1104  cd_pos,
1105  0
1106  );
1107 
1108 
1109  set_value ((dods_int32 *) &final_val[0], nelms_to_send);
1110  }
1111  break;
1112 
1113  case H5UINT32:
1114  {
1115 #if 0
1116  vector<unsigned int>total_val;
1117  total_val.resize(total_read/dtype_size);
1118  memcpy(&total_val[0],(void*)&buf[0],total_read);
1119 
1120  vector<unsigned int>final_val;
1121  subset<unsigned int>(
1122  &total_val[0],
1123  rank,
1124  dimsizes,
1125  &cd_start[0],
1126  &cd_step[0],
1127  &cd_count[0],
1128  &final_val,
1129  cd_pos,
1130  0
1131  );
1132 #endif
1133 
1134  vector<unsigned int>final_val;
1135  subset<unsigned int>(
1136  &buf[0],
1137  rank,
1138  dimsizes,
1139  &cd_start[0],
1140  &cd_step[0],
1141  &cd_count[0],
1142  &final_val,
1143  cd_pos,
1144  0
1145  );
1146 
1147  set_value ((dods_uint32 *) &final_val[0], nelms_to_send);
1148  }
1149  break;
1150 
1151  case H5INT64: // Only for DAP4 CF
1152  {
1153 #if 0
1154  vector<unsigned int>total_val;
1155  total_val.resize(total_read/dtype_size);
1156  memcpy(&total_val[0],(void*)&buf[0],total_read);
1157 
1158  vector<unsigned int>final_val;
1159  subset<unsigned int>(
1160  &total_val[0],
1161  rank,
1162  dimsizes,
1163  &cd_start[0],
1164  &cd_step[0],
1165  &cd_count[0],
1166  &final_val,
1167  cd_pos,
1168  0
1169  );
1170 #endif
1171 
1172  vector<long long >final_val;
1173  subset<long long >(
1174  &buf[0],
1175  rank,
1176  dimsizes,
1177  &cd_start[0],
1178  &cd_step[0],
1179  &cd_count[0],
1180  &final_val,
1181  cd_pos,
1182  0
1183  );
1184 
1185  set_value ((dods_int64 *) &final_val[0], nelms_to_send);
1186  }
1187  break;
1188 
1189 
1190 
1191  case H5UINT64: // Only for DAP4 CF
1192  {
1193 #if 0
1194  vector<unsigned int>total_val;
1195  total_val.resize(total_read/dtype_size);
1196  memcpy(&total_val[0],(void*)&buf[0],total_read);
1197 
1198  vector<unsigned int>final_val;
1199  subset<unsigned int>(
1200  &total_val[0],
1201  rank,
1202  dimsizes,
1203  &cd_start[0],
1204  &cd_step[0],
1205  &cd_count[0],
1206  &final_val,
1207  cd_pos,
1208  0
1209  );
1210 #endif
1211 
1212  vector<unsigned long long >final_val;
1213  subset<unsigned long long >(
1214  &buf[0],
1215  rank,
1216  dimsizes,
1217  &cd_start[0],
1218  &cd_step[0],
1219  &cd_count[0],
1220  &final_val,
1221  cd_pos,
1222  0
1223  );
1224 
1225  set_value ((dods_uint64 *) &final_val[0], nelms_to_send);
1226  }
1227  break;
1228 
1229 
1230  case H5FLOAT32:
1231  {
1232 #if 0
1233  vector<float>total_val;
1234  total_val.resize(total_read/dtype_size);
1235  memcpy(&total_val[0],(void*)&buf[0],total_read);
1236 
1237  vector<float>final_val;
1238  subset<float>(
1239  &total_val[0],
1240  rank,
1241  dimsizes,
1242  &cd_start[0],
1243  &cd_step[0],
1244  &cd_count[0],
1245  &final_val,
1246  cd_pos,
1247  0
1248  );
1249 #endif
1250 
1251  vector<float>final_val;
1252  subset<float>(
1253  &buf[0],
1254  rank,
1255  dimsizes,
1256  &cd_start[0],
1257  &cd_step[0],
1258  &cd_count[0],
1259  &final_val,
1260  cd_pos,
1261  0
1262  );
1263 
1264 
1265  set_value ((dods_float32 *) &final_val[0], nelms_to_send);
1266  }
1267  break;
1268  case H5FLOAT64:
1269  {
1270 #if 0
1271  vector<double>total_val;
1272  total_val.resize(total_read/dtype_size);
1273  memcpy(&total_val[0],(void*)&buf[0],total_read);
1274 
1275  vector<double>final_val;
1276  subset<double>(
1277  &total_val[0],
1278  rank,
1279  dimsizes,
1280  &cd_start[0],
1281  &cd_step[0],
1282  &cd_count[0],
1283  &final_val,
1284  cd_pos,
1285  0
1286  );
1287 #endif
1288  vector<double>final_val;
1289  subset<double>(
1290  &buf[0],
1291  rank,
1292  dimsizes,
1293  &cd_start[0],
1294  &cd_step[0],
1295  &cd_count[0],
1296  &final_val,
1297  cd_pos,
1298  0
1299  );
1300 
1301  set_value ((dods_float64 *) &final_val[0], nelms_to_send);
1302  }
1303  break;
1304  default:
1305  throw InternalErr (__FILE__, __LINE__, "unsupported data type.");
1306 
1307  }// "end switch(dtype)"
1308  }// "end else (stride is not 1)"
1309  return true;
1310  }// "end else(full_read = true)"
1311 }
1312 
1313 
1314 void
1315 HDF5CFArray::write_data_to_cache(hid_t dset_id, hid_t /*dspace_id*/, hid_t /*mspace_id*/, hid_t memtype,
1316  const string& cache_fpath, short dtype_size, const vector<char> &buf, int nelms) {
1317 
1318  unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
1319  string disk_cache_dir = HDF5RequestHandler::get_disk_cache_dir();
1320  string disk_cache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
1321  HDF5DiskCache *disk_cache = HDF5DiskCache::get_instance(disk_cache_size,disk_cache_dir,disk_cache_prefix);
1322  int total_nelem = 1;
1323  for(int i = 0; i <rank; i++)
1324  total_nelem = total_nelem*dimsizes[i];
1325 
1326  vector<char>val;
1327 
1328  if(H5CHAR == dtype) {
1329 
1330  vector<short>newval;
1331  newval.resize(total_nelem);
1332  if(total_nelem == nelms) {
1333  for (int i = 0; i < total_nelem;i++)
1334  newval[i] = (short)buf[i];
1335  disk_cache->write_cached_data2(cache_fpath,sizeof(short)*total_nelem,(const void*)&newval[0]);
1336  }
1337  else {
1338  vector<char>val2;
1339  val2.resize(total_nelem);
1340  if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val2[0])<0)
1341  throw InternalErr (__FILE__, __LINE__, "Cannot read the whole HDF5 dataset for the disk cache.");
1342  for (int i = 0; i < total_nelem;i++)
1343  newval[i] = (short)val2[i];
1344  disk_cache->write_cached_data2(cache_fpath,sizeof(short)*total_nelem,(const void*)&newval[0]);
1345  }
1346  }
1347  else {
1348  if(total_nelem == nelms) {
1349  disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(const void*)&buf[0]);
1350  }
1351  else {
1352  val.resize(dtype_size*total_nelem);
1353  if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val[0])<0)
1354  throw InternalErr (__FILE__, __LINE__, "Cannot read the whole SDS for cache.");
1355 
1356  disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(const void*)&val[0]);
1357  }
1358  }
1359 }
1360 
1361 #if 0
1362 void HDF5CFArray::read_data_from_mem_cache(void*buf) {
1363 
1364  vector<int>offset;
1365  vector<int>count;
1366  vector<int>step;
1367  int nelms = format_constraint (&offset[0], &step[0], &count[0]);
1368  // set the original position to the starting point
1369  vector<int>at_pos(at_ndims,0);
1370  for (int i = 0; i< rank; i++)
1371  at_pos[i] = at_offset[i];
1372 
1373 
1374  switch (dtype) {
1375 
1376  case H5UCHAR:
1377 
1378  {
1379  vector<unsigned char> val;
1380  val.resize(nelms);
1381  subset<unsigned char>(
1382  &total_val[0],
1383  rank,
1384  dimsizes,
1385  offset,
1386  step,
1387  count,
1388  &final_val,
1389  at_pos,
1390  0
1391  );
1392 
1393 
1394  set_value ((dods_byte *) &val[0], nelms);
1395  } // case H5UCHAR
1396  break;
1397 
1398 
1399  case H5CHAR:
1400  {
1401 
1402  vector<char> val;
1403  val.resize(nelms);
1404 
1405  if (0 == rank)
1406  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1407  else
1408  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1409 
1410  if (read_ret < 0) {
1411 
1412  if (rank > 0)
1413  H5Sclose(mspace);
1414  H5Tclose(memtype);
1415  H5Tclose(dtypeid);
1416  H5Sclose(dspace);
1417  H5Dclose(dsetid);
1418  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1419  ostringstream eherr;
1420  eherr << "Cannot read the HDF5 dataset " << varname
1421  << " with the type of H5T_NATIVE_CHAR "<<endl;
1422  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1423 
1424  }
1425 
1426  vector<short>newval;
1427  newval.resize(nelms);
1428 
1429  for (int counter = 0; counter < nelms; counter++)
1430  newval[counter] = (short) (val[counter]);
1431 
1432  set_value ((dods_int16 *) &newval[0], nelms);
1433  } // case H5CHAR
1434  break;
1435 
1436 
1437  case H5INT16:
1438  {
1439  vector<short>val;
1440  val.resize(nelms);
1441 
1442  if (0 == rank)
1443  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1444  else
1445  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1446 
1447  if (read_ret < 0) {
1448 
1449  if (rank > 0)
1450  H5Sclose(mspace);
1451  H5Tclose(memtype);
1452  H5Tclose(dtypeid);
1453  H5Sclose(dspace);
1454  H5Dclose(dsetid);
1455  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1456  //H5Fclose(fileid);
1457  ostringstream eherr;
1458  eherr << "Cannot read the HDF5 dataset " << varname
1459  << " with the type of H5T_NATIVE_SHORT "<<endl;
1460  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1461 
1462  }
1463  set_value ((dods_int16 *) &val[0], nelms);
1464  }// H5INT16
1465  break;
1466 
1467 
1468  case H5UINT16:
1469  {
1470  vector<unsigned short> val;
1471  val.resize(nelms);
1472  if (0 == rank)
1473  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1474  else
1475  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1476 
1477  if (read_ret < 0) {
1478 
1479  if (rank > 0) H5Sclose(mspace);
1480  H5Tclose(memtype);
1481  H5Tclose(dtypeid);
1482  H5Sclose(dspace);
1483  H5Dclose(dsetid);
1484  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1485  ostringstream eherr;
1486  eherr << "Cannot read the HDF5 dataset " << varname
1487  << " with the type of H5T_NATIVE_USHORT "<<endl;
1488  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1489 
1490  }
1491  set_value ((dods_uint16 *) &val[0], nelms);
1492  } // H5UINT16
1493  break;
1494 
1495 
1496  case H5INT32:
1497  {
1498  vector<int>val;
1499  val.resize(nelms);
1500  if (0 == rank)
1501  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1502  else
1503  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1504 
1505  if (read_ret < 0) {
1506  if (rank > 0)
1507  H5Sclose(mspace);
1508  H5Tclose(memtype);
1509  H5Tclose(dtypeid);
1510  H5Sclose(dspace);
1511  H5Dclose(dsetid);
1512  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1513  ostringstream eherr;
1514  eherr << "Cannot read the HDF5 dataset " << varname
1515  << " with the type of H5T_NATIVE_INT "<<endl;
1516  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1517 
1518  }
1519  set_value ((dods_int32 *) &val[0], nelms);
1520  } // case H5INT32
1521  break;
1522 
1523  case H5UINT32:
1524  {
1525  vector<unsigned int>val;
1526  val.resize(nelms);
1527  if (0 == rank)
1528  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1529  else
1530  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1531 
1532  if (read_ret < 0) {
1533 
1534  if (rank > 0)
1535  H5Sclose(mspace);
1536  H5Tclose(memtype);
1537  H5Tclose(dtypeid);
1538  H5Sclose(dspace);
1539  H5Dclose(dsetid);
1540  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1541  ostringstream eherr;
1542  eherr << "Cannot read the HDF5 dataset " << varname
1543  << " with the type of H5T_NATIVE_UINT "<<endl;
1544  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1545 
1546  }
1547  set_value ((dods_uint32 *) &val[0], nelms);
1548  }
1549  break;
1550 
1551  case H5FLOAT32:
1552  {
1553 
1554  vector<float>val;
1555  val.resize(nelms);
1556 
1557  if (0 == rank)
1558  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1559  else
1560  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1561 
1562  if (read_ret < 0) {
1563  if (rank > 0)
1564  H5Sclose(mspace);
1565  H5Tclose(memtype);
1566  H5Tclose(dtypeid);
1567  H5Sclose(dspace);
1568  H5Dclose(dsetid);
1569  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1570  ostringstream eherr;
1571  eherr << "Cannot read the HDF5 dataset " << varname
1572  << " with the type of H5T_NATIVE_FLOAT "<<endl;
1573  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1574 
1575  }
1576  set_value ((dods_float32 *) &val[0], nelms);
1577  }
1578  break;
1579 
1580 
1581  case H5FLOAT64:
1582  {
1583 
1584  vector<double>val;
1585  val.resize(nelms);
1586  if (0 == rank)
1587  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1588  else
1589  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1590 
1591  if (read_ret < 0) {
1592  if (rank > 0)
1593  H5Sclose(mspace);
1594  H5Tclose(memtype);
1595  H5Tclose(dtypeid);
1596  H5Sclose(dspace);
1597  H5Dclose(dsetid);
1598  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1599  ostringstream eherr;
1600  eherr << "Cannot read the HDF5 dataset " << varname
1601  << " with the type of H5T_NATIVE_DOUBLE "<<endl;
1602  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1603 
1604  }
1605  set_value ((dods_float64 *) &val[0], nelms);
1606  } // case H5FLOAT64
1607  break;
1608 
1609 
1610 
1611  // Just see if it works.
1612  val2buf(buf);
1613  set_read_p(true);
1614  return;
1615 }
1616 #endif
1617 // We don't inherit libdap Array Class's transform_to_dap4 method since CF option is still using it.
1618 // This function is used for 64-bit integer mapping to DAP4 for the CF option. largely borrowed from
1619 // DAP4 code.
1620 BaseType* HDF5CFArray::h5cfdims_transform_to_dap4(D4Group *grp) {
1621 
1622  if(grp == NULL)
1623  return NULL;
1624  Array *dest = static_cast<HDF5CFArray*>(ptr_duplicate());
1625 
1626  // If there is just a size, don't make
1627  // a D4Dimension (In DAP4 you cannot share a dimension unless it has
1628  // a name). jhrg 3/18/14
1629 
1630  for (Array::Dim_iter d = dest->dim_begin(), e = dest->dim_end(); d != e; ++d) {
1631  if (false == (*d).name.empty()) {
1632 
1633  D4Group *temp_grp = grp;
1634  D4Dimension *d4_dim = NULL;
1635  while(temp_grp) {
1636 
1637  D4Dimensions *temp_dims = temp_grp->dims();
1638 
1639  // Check if the dimension is defined in this group
1640  d4_dim = temp_dims->find_dim((*d).name);
1641  if(d4_dim) {
1642  (*d).dim = d4_dim;
1643  break;
1644  }
1645 
1646  if(temp_grp->get_parent())
1647  temp_grp = static_cast<D4Group*>(temp_grp->get_parent());
1648  else
1649  temp_grp = 0;
1650 
1651  }
1652 
1653  // Not find this dimension in any of the ancestor groups, add it to this group.
1654  // The following block is fine, but to avoid the complaint from sonarcloud.
1655  // Use a bool.
1656  bool d4_dim_null = ((d4_dim==NULL)?true:false);
1657 #if 0
1658  //if(d4_dim == NULL) {
1659 #endif
1660  // Not find this dimension in any of the ancestor groups, add it to this group.
1661  if(d4_dim_null == true) {
1662 
1663  d4_dim = new D4Dimension((*d).name, (*d).size);
1664  D4Dimensions * dims = grp->dims();
1665  dims->add_dim_nocopy(d4_dim);
1666  (*d).dim = d4_dim;
1667  }
1668  }
1669  }
1670 
1671  dest->set_is_dap4(true);
1672 
1673  return dest;
1674 
1675 }
1676 #if 0
1677 // parse constraint expr. and make hdf5 coordinate point location.
1678 // return number of elements to read.
1679 int
1680 HDF5CFArray::format_constraint (int *offset, int *step, int *count)
1681 {
1682 
1683  long nels = 1;
1684  int id = 0;
1685 
1686  Dim_iter p = dim_begin ();
1687 
1688  while (p != dim_end ()) {
1689 
1690  int start = dimension_start (p, true);
1691  int stride = dimension_stride (p, true);
1692  int stop = dimension_stop (p, true);
1693 
1694  // Check for illegal constraint
1695  if (start > stop) {
1696  ostringstream oss;
1697 
1698  oss << "Array/Grid hyperslab start point "<< start <<
1699  " is greater than stop point " << stop <<".";
1700  throw Error(malformed_expr, oss.str());
1701  }
1702 
1703  offset[id] = start;
1704  step[id] = stride;
1705  count[id] = ((stop - start) / stride) + 1; // count of elements
1706  nels *= count[id]; // total number of values for variable
1707 
1708  BESDEBUG ("h5",
1709  "=format_constraint():"
1710  << "id=" << id << " offset=" << offset[id]
1711  << " step=" << step[id]
1712  << " count=" << count[id]
1713  << endl);
1714 
1715  id++;
1716  p++;
1717  }
1718 
1719  return nels;
1720 }
1721 
1722 #endif
HDF5CFArray::read_data_NOT_from_mem_cache
virtual void read_data_NOT_from_mem_cache(bool add_cache, void *buf)
Definition: HDF5CFArray.cc:175
HDF5DiskCache
Definition: HDF5DiskCache.h:16
BESFileLockingCache::purge_file
virtual void purge_file(const std::string &file)
Purge a single file from the cache.
Definition: BESFileLockingCache.cc:1085
libdap
Definition: BESDapFunctionResponseCache.h:35
BESFileLockingCache::unlock_and_close
virtual void unlock_and_close(const std::string &target)
Definition: BESFileLockingCache.cc:713
HDF5RequestHandler.h
include the entry functions to execute the handlers
HDF5CFArray.h
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
HDF5CFUtil::read_buffer_from_file
static ssize_t read_buffer_from_file(int fd, void *buf, size_t)
Getting a subset of a variable.
Definition: HDF5CFUtil.cc:1181
HDF5DiskCache::get_instance
static HDF5DiskCache * get_instance(const long, const std::string &, const std::string &)
Definition: HDF5DiskCache.cc:99
h5cfdaputil.h
Helper functions for generating DAS attributes and a function to check BES Key.
HDF5CFArray
Definition: HDF5CFArray.h:47
Error