31 #include "config_hdf5.h"
37 #include "InternalErr.h"
43 #include "ObjMemCache.h"
49 BaseType *HDF5CFArray::ptr_duplicate()
55 bool HDF5CFArray::read()
58 BESDEBUG(
"h5",
"Coming to HDF5CFArray read "<<endl);
62 if((NULL == HDF5RequestHandler::get_lrdata_mem_cache()) &&
63 NULL == HDF5RequestHandler::get_srdata_mem_cache()){
64 read_data_NOT_from_mem_cache(
false,NULL);
69 short use_cache_flag = 0;
72 if(HDF5RequestHandler::get_srdata_mem_cache() != NULL) {
73 if(((cvtype == CV_EXIST) && (islatlon !=
true)) || (cvtype == CV_NONLATLON_MISS)
74 || (cvtype == CV_FILLINDEX) ||(cvtype == CV_MODIFY) ||(cvtype == CV_SPECIAL)){
76 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==
true)
82 if(use_cache_flag !=1) {
84 if(HDF5RequestHandler::get_lrdata_mem_cache() != NULL) {
89 if(HDF5RequestHandler::get_common_cache_dirs() ==
false) {
90 if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
91 || (cvtype == CV_EXIST && islatlon ==
true)) {
96 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==
true)
104 if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
105 || (cvtype == CV_EXIST && islatlon ==
true)) {
107 vector<string> cur_lrd_non_cache_dir_list;
108 HDF5RequestHandler::get_lrd_non_cache_dir_list(cur_lrd_non_cache_dir_list);
111 if( (cur_lrd_non_cache_dir_list.size() == 0) ||
112 (
"" == check_str_sect_in_list(cur_lrd_non_cache_dir_list,filename,
'/'))) {
115 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==
true)
121 vector<string> cur_lrd_var_cache_file_list;
122 HDF5RequestHandler::get_lrd_var_cache_file_list(cur_lrd_var_cache_file_list);
123 if(cur_lrd_var_cache_file_list.size() >0){
128 if(
true == check_var_cache_files(cur_lrd_var_cache_file_list,filename,varname)){
135 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==
true)
143 if(0 == use_cache_flag)
144 read_data_NOT_from_mem_cache(
false,NULL);
150 if( 3 == use_cache_flag){
151 vector<string> cur_cache_dlist;
152 HDF5RequestHandler::get_lrd_cache_dir_list(cur_cache_dlist);
153 string cache_dir = check_str_sect_in_list(cur_cache_dlist,filename,
'/');
155 cache_key = cache_dir + varname;
157 cache_key = filename + varname;
165 cache_key = filename + varname;
167 handle_data_with_mem_cache(dtype,total_elems,use_cache_flag,cache_key);
180 vector<hsize_t> hoffset;
181 vector<hsize_t>hcount;
182 vector<hsize_t>hstep;
186 throw InternalErr (__FILE__, __LINE__,
187 "The number of dimension of the variable is <=0 for an array.");
193 hoffset.resize(rank);
196 nelms = format_constraint (&offset[0], &step[0], &count[0]);
197 for (
int i = 0; i <rank; i++) {
198 hoffset[i] = (hsize_t) offset[i];
199 hcount[i] = (hsize_t) count[i];
200 hstep[i] = (hsize_t) step[i];
210 bool data_from_disk_cache =
false;
211 bool data_to_disk_cache =
false;
214 bool use_disk_cache = valid_disk_cache();
218 if(
true == use_disk_cache) {
220 BESDEBUG(
"h5",
"Coming to use disk cache "<<endl);
222 unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
223 string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
224 string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
226 string cache_fname=HDF5CFUtil::obtain_cache_fname(diskcache_prefix,filename,varname);
227 cache_fpath = diskcache_dir +
"/"+ cache_fname;
229 int temp_total_elems = 1;
230 for (
unsigned int i = 0; i <dimsizes.size();i++)
231 temp_total_elems = temp_total_elems*dimsizes[i];
232 short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
234 int expected_file_size = dtype_size *temp_total_elems;
237 if(
true == disk_cache->get_data_from_cache(cache_fpath, expected_file_size,fd)) {
239 vector<size_t> offset_size_t;
240 offset_size_t.resize(rank);
241 for(
int i = 0; i <rank;i++)
242 offset_size_t[i] = (
size_t)offset[i];
243 size_t offset_1st = INDEX_nD_TO_1D(dimsizes,offset_size_t);
246 for (
int i = 0; i < rank; i++)
247 end[i] = offset[i] +(count[i]-1)*step[i];
248 size_t offset_last = INDEX_nD_TO_1D(dimsizes,end);
253 size_t total_read = dtype_size*(offset_last-offset_1st+1);
255 off_t fpos = lseek(fd,dtype_size*offset_1st,SEEK_SET);
263 data_from_disk_cache = obtain_cached_data(disk_cache,cache_fpath,fd, step,count,total_read,dtype_size);
267 if(
true == data_from_disk_cache)
270 data_to_disk_cache =
true;
276 bool pass_fileid = HDF5RequestHandler::get_pass_fileid();
277 if(
false == pass_fileid) {
278 if ((fileid = H5Fopen(filename.c_str(),H5F_ACC_RDONLY,H5P_DEFAULT))<0) {
280 eherr <<
"HDF5 File " << filename
281 <<
" cannot be opened. "<<endl;
282 throw InternalErr (__FILE__, __LINE__, eherr.str ());
286 if ((dsetid = H5Dopen(fileid,varname.c_str(),H5P_DEFAULT))<0) {
287 HDF5CFUtil::close_fileid(fileid,pass_fileid);
289 eherr <<
"HDF5 dataset " << varname
290 <<
" cannot be opened. "<<endl;
291 throw InternalErr (__FILE__, __LINE__, eherr.str ());
294 if ((dspace = H5Dget_space(dsetid))<0) {
297 HDF5CFUtil::close_fileid(fileid,pass_fileid);
299 eherr <<
"Space id of the HDF5 dataset " << varname
300 <<
" cannot be obtained. "<<endl;
301 throw InternalErr (__FILE__, __LINE__, eherr.str ());
304 if (H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
305 &hoffset[0], &hstep[0],
306 &hcount[0], NULL) < 0) {
310 HDF5CFUtil::close_fileid(fileid,pass_fileid);
312 eherr <<
"The selection of hyperslab of the HDF5 dataset " << varname
314 throw InternalErr (__FILE__, __LINE__, eherr.str ());
317 mspace = H5Screate_simple(rank, &hcount[0],NULL);
321 HDF5CFUtil::close_fileid(fileid,pass_fileid);
323 eherr <<
"The creation of the memory space of the HDF5 dataset " << varname
325 throw InternalErr (__FILE__, __LINE__, eherr.str ());
329 if ((dtypeid = H5Dget_type(dsetid)) < 0) {
334 HDF5CFUtil::close_fileid(fileid,pass_fileid);
336 eherr <<
"Obtaining the datatype of the HDF5 dataset " << varname
338 throw InternalErr (__FILE__, __LINE__, eherr.str ());
342 if ((memtype = H5Tget_native_type(dtypeid, H5T_DIR_ASCEND))<0) {
348 HDF5CFUtil::close_fileid(fileid,pass_fileid);
350 eherr <<
"Obtaining the memory type of the HDF5 dataset " << varname
352 throw InternalErr (__FILE__, __LINE__, eherr.str ());
360 if(
true == add_mem_cache) {
366 HDF5CFUtil::close_fileid(fileid,pass_fileid);
367 throw InternalErr(__FILE__,__LINE__,
"The memory data cache buffer needs to be set");
369 read_ret= H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf);
375 HDF5CFUtil::close_fileid(fileid,pass_fileid);
376 throw InternalErr(__FILE__,__LINE__,
"Cannot read the data to the buffer.");
392 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
400 HDF5CFUtil::close_fileid(fileid,pass_fileid);
402 eherr <<
"Cannot read the HDF5 dataset " << varname
403 <<
" with the type of H5T_NATIVE_CHAR "<<endl;
404 throw InternalErr (__FILE__, __LINE__, eherr.str ());
409 newval.resize(nelms);
411 for (
int counter = 0; counter < nelms; counter++)
412 newval[counter] = (
short) (val[counter]);
414 set_value ((dods_int16 *) &newval[0], nelms);
416 if(
true == data_to_disk_cache) {
418 BESDEBUG(
"h5",
"writing data to disk cache "<<endl);
419 write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,2,val,nelms);
427 HDF5CFUtil::close_fileid(fileid,pass_fileid);
429 eherr <<
"write data to cache failed.";
430 throw InternalErr (__FILE__, __LINE__, eherr.str ());
451 size_t dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
453 val.resize(nelms*dtype_size);
455 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
462 HDF5CFUtil::close_fileid(fileid,pass_fileid);
464 eherr <<
"Cannot read the HDF5 dataset " << varname
465 <<
" with the type of H5T_NATIVE_UCHAR "<<endl;
466 throw InternalErr (__FILE__, __LINE__, eherr.str ());
473 if(
true == data_to_disk_cache) {
474 BESDEBUG(
"h5",
"writing data to disk cache "<<endl);
476 write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,dtype_size,val,nelms);
484 HDF5CFUtil::close_fileid(fileid,pass_fileid);
486 eherr <<
"write data to cache failed.";
487 throw InternalErr (__FILE__, __LINE__, eherr.str ());
503 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
511 HDF5CFUtil::close_fileid(fileid,pass_fileid);
514 eherr <<
"Cannot read the HDF5 dataset " << varname
515 <<
" with the type of H5T_NATIVE_SHORT "<<endl;
516 throw InternalErr (__FILE__, __LINE__, eherr.str ());
519 set_value ((dods_int16 *) &val[0], nelms);
526 vector<unsigned short> val;
528 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
536 HDF5CFUtil::close_fileid(fileid,pass_fileid);
538 eherr <<
"Cannot read the HDF5 dataset " << varname
539 <<
" with the type of H5T_NATIVE_USHORT "<<endl;
540 throw InternalErr (__FILE__, __LINE__, eherr.str ());
543 set_value ((dods_uint16 *) &val[0], nelms);
552 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
559 HDF5CFUtil::close_fileid(fileid,pass_fileid);
561 eherr <<
"Cannot read the HDF5 dataset " << varname
562 <<
" with the type of H5T_NATIVE_INT "<<endl;
563 throw InternalErr (__FILE__, __LINE__, eherr.str ());
566 set_value ((dods_int32 *) &val[0], nelms);
572 vector<unsigned int>val;
574 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
581 HDF5CFUtil::close_fileid(fileid,pass_fileid);
583 eherr <<
"Cannot read the HDF5 dataset " << varname
584 <<
" with the type of H5T_NATIVE_UINT "<<endl;
585 throw InternalErr (__FILE__, __LINE__, eherr.str ());
588 set_value ((dods_uint32 *) &val[0], nelms);
598 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
605 HDF5CFUtil::close_fileid(fileid,pass_fileid);
607 eherr <<
"Cannot read the HDF5 dataset " << varname
608 <<
" with the type of H5T_NATIVE_FLOAT "<<endl;
609 throw InternalErr (__FILE__, __LINE__, eherr.str ());
612 set_value ((dods_float32 *) &val[0], nelms);
622 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
630 HDF5CFUtil::close_fileid(fileid,pass_fileid);
632 eherr <<
"Cannot read the HDF5 dataset " << varname
633 <<
" with the type of H5T_NATIVE_DOUBLE "<<endl;
634 throw InternalErr (__FILE__, __LINE__, eherr.str ());
637 set_value ((dods_float64 *) &val[0], nelms);
645 size_t ty_size = H5Tget_size(dtypeid);
652 HDF5CFUtil::close_fileid(fileid,pass_fileid);
654 eherr <<
"Cannot obtain the size of the fixed size HDF5 string of the dataset "
656 throw InternalErr (__FILE__, __LINE__, eherr.str ());
659 vector <char> strval;
660 strval.resize(nelms*ty_size);
661 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(
void*)&strval[0]);
669 HDF5CFUtil::close_fileid(fileid,pass_fileid);
671 eherr <<
"Cannot read the HDF5 dataset " << varname
672 <<
" with the type of the fixed size HDF5 string "<<endl;
673 throw InternalErr (__FILE__, __LINE__, eherr.str ());
676 string total_string(strval.begin(),strval.end());
678 vector <string> finstrval;
679 finstrval.resize(nelms);
680 for (
int i = 0; i<nelms; i++)
681 finstrval[i] = total_string.substr(i*ty_size,ty_size);
688 if ((
true == HDF5RequestHandler::get_drop_long_string()) &&
689 ty_size > NC_JAVA_STR_SIZE_LIMIT) {
690 for (
int i = 0; i<nelms; i++)
693 set_value(finstrval,nelms);
694 total_string.clear();
701 size_t ty_size = H5Tget_size(memtype);
708 HDF5CFUtil::close_fileid(fileid,pass_fileid);
710 eherr <<
"Cannot obtain the size of the fixed size HDF5 string of the dataset "
712 throw InternalErr (__FILE__, __LINE__, eherr.str ());
714 vector <char> strval;
715 strval.resize(nelms*ty_size);
716 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(
void*)&strval[0]);
724 HDF5CFUtil::close_fileid(fileid,pass_fileid);
726 eherr <<
"Cannot read the HDF5 dataset " << varname
727 <<
" with the type of the HDF5 variable length string "<<endl;
728 throw InternalErr (__FILE__, __LINE__, eherr.str ());
731 vector<string>finstrval;
732 finstrval.resize(nelms);
733 char*temp_bp = &strval[0];
734 char*onestring = NULL;
735 for (
int i =0;i<nelms;i++) {
736 onestring = *(
char**)temp_bp;
738 finstrval[i] =string(onestring);
745 if (
false == strval.empty()) {
746 herr_t ret_vlen_claim;
747 ret_vlen_claim = H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(
void*)&strval[0]);
748 if (ret_vlen_claim < 0){
754 HDF5CFUtil::close_fileid(fileid,pass_fileid);
756 eherr <<
"Cannot reclaim the memory buffer of the HDF5 variable length string of the dataset "
758 throw InternalErr (__FILE__, __LINE__, eherr.str ());
766 if (
true == HDF5RequestHandler::get_drop_long_string()) {
767 bool drop_long_str =
false;
768 for (
int i =0;i<nelms;i++) {
769 if(finstrval[i].size() >NC_JAVA_STR_SIZE_LIMIT){
770 drop_long_str =
true;
774 if (drop_long_str ==
true) {
775 for (
int i =0;i<nelms;i++)
779 set_value(finstrval,nelms);
791 HDF5CFUtil::close_fileid(fileid,pass_fileid);
793 eherr <<
"Cannot read the HDF5 dataset " << varname
794 <<
" with the unsupported HDF5 datatype"<<endl;
795 throw InternalErr (__FILE__, __LINE__, eherr.str ());
804 HDF5CFUtil::close_fileid(fileid,pass_fileid);
809 bool HDF5CFArray::valid_disk_cache() {
811 bool ret_value =
false;
812 if(
true == HDF5RequestHandler::get_use_disk_cache()) {
814 BESDEBUG(
"h5",
"Coming to disk cache "<<endl);
816 if(dtype == H5CHAR || dtype ==H5UCHAR || dtype==H5INT16 || dtype ==H5UINT16 ||
817 dtype == H5INT32 || dtype ==H5UINT32 || dtype ==H5FLOAT32 || dtype==H5FLOAT64 ||
818 dtype == H5INT64 || dtype ==H5UINT64){
820 BESDEBUG(
"h5",
"Coming to disk cache datatype block"<<endl);
822 string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
823 string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
824 long diskcache_size = HDF5RequestHandler::get_disk_cache_size();
826 if((
"" == diskcache_dir)||(
""==diskcache_prefix)||(diskcache_size <=0))
827 throw InternalErr (__FILE__, __LINE__,
"Either the cached dir is empty or the prefix is NULL or the cache size is not set.");
830 if(stat(diskcache_dir.c_str(),&sb) !=0) {
831 string err_mesg=
"The cached directory " + diskcache_dir;
832 err_mesg = err_mesg +
" doesn't exist. ";
833 throw InternalErr(__FILE__,__LINE__,err_mesg);
836 if(
true == S_ISDIR(sb.st_mode)) {
837 if(access(diskcache_dir.c_str(),R_OK|W_OK|X_OK) == -1) {
838 string err_mesg=
"The cached directory " + diskcache_dir;
839 err_mesg = err_mesg +
" can NOT be read,written or executable.";
840 throw InternalErr(__FILE__,__LINE__,err_mesg);
844 string err_mesg=
"The cached directory " + diskcache_dir;
845 err_mesg = err_mesg +
" is not a directory.";
846 throw InternalErr(__FILE__,__LINE__,err_mesg);
851 short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
853 if(
true == HDF5RequestHandler::get_disk_cache_comp_data()){
854 BESDEBUG(
"h5",
"Compression disk cache key is true"<<endl);
855 ret_value = valid_disk_cache_for_compressed_data(dtype_size);
856 BESDEBUG(
"h5",
"variable disk cache passes the compression parameter check"<<endl);
859 BESDEBUG(
"h5",
"Compression disk cache key is NOT set, disk cache key is true."<<endl);
869 bool HDF5CFArray:: valid_disk_cache_for_compressed_data(
short dtype_size) {
871 bool ret_value =
false;
875 size_t total_byte = total_elems*dtype_size;
877 if((comp_ratio < HDF5RequestHandler::get_disk_comp_threshold())
878 && (total_elems*dtype_size >= HDF5RequestHandler::get_disk_var_size())) {
879 if(
true == HDF5RequestHandler::get_disk_cache_float_only_comp()) {
880 if(dtype==H5FLOAT32 || dtype == H5FLOAT64)
890 bool HDF5CFArray::obtain_cached_data(
HDF5DiskCache *disk_cache,
const string & cache_fpath,
int fd,vector<int> &cd_step, vector<int>&cd_count,
size_t total_read,
short dtype_size) {
892 ssize_t ret_read_val = -1;
895 buf.resize(total_read);
898 if((-1 == ret_read_val) || (ret_read_val != (ssize_t)total_read)) {
903 unsigned int nele_to_read = 1;
904 for(
int i = 0; i<rank;i++)
905 nele_to_read *=cd_count[i];
907 if(nele_to_read == (total_read/dtype_size)) {
913 vector<int>cd_start(rank,0);
914 vector<size_t>cd_pos(rank,0);
915 int nelms_to_send = 1;
916 for(
int i = 0; i <rank; i++)
917 nelms_to_send = nelms_to_send*cd_count[i];
924 vector<int>total_val;
925 total_val.resize(total_read/dtype_size);
926 memcpy(&total_val[0],(
void*)&buf[0],total_read);
928 vector<int>final_val;
942 vector<short>final_val;
954 set_value((dods_int16*)&final_val[0],nelms_to_send);
962 vector<unsigned char>total_val;
963 total_val.resize(total_read/dtype_size);
964 memcpy(&total_val[0],(
void*)&buf[0],total_read);
966 vector<unsigned char>final_val;
967 subset<unsigned char>(
980 vector<unsigned char>final_val;
981 subset<unsigned char>(
993 set_value ((dods_byte *) &final_val[0], nelms_to_send);
1000 vector<short>total_val;
1001 total_val.resize(total_read/dtype_size);
1002 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1004 vector<short>final_val;
1018 vector<short>final_val;
1031 set_value ((dods_int16 *) &final_val[0], nelms_to_send);
1038 vector<unsigned short>total_val;
1039 total_val.resize(total_read/dtype_size);
1040 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1042 vector<unsigned short>final_val;
1043 subset<unsigned short>(
1056 vector<unsigned short>final_val;
1057 subset<unsigned short>(
1069 set_value ((dods_uint16 *) &final_val[0], nelms_to_send);
1076 vector<int>total_val;
1077 total_val.resize(total_read/dtype_size);
1078 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1080 vector<int>final_val;
1095 vector<int>final_val;
1109 set_value ((dods_int32 *) &final_val[0], nelms_to_send);
1116 vector<unsigned int>total_val;
1117 total_val.resize(total_read/dtype_size);
1118 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1120 vector<unsigned int>final_val;
1121 subset<unsigned int>(
1134 vector<unsigned int>final_val;
1135 subset<unsigned int>(
1147 set_value ((dods_uint32 *) &final_val[0], nelms_to_send);
1154 vector<unsigned int>total_val;
1155 total_val.resize(total_read/dtype_size);
1156 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1158 vector<unsigned int>final_val;
1159 subset<unsigned int>(
1172 vector<long long >final_val;
1185 set_value ((dods_int64 *) &final_val[0], nelms_to_send);
1194 vector<unsigned int>total_val;
1195 total_val.resize(total_read/dtype_size);
1196 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1198 vector<unsigned int>final_val;
1199 subset<unsigned int>(
1212 vector<unsigned long long >final_val;
1213 subset<unsigned long long >(
1225 set_value ((dods_uint64 *) &final_val[0], nelms_to_send);
1233 vector<float>total_val;
1234 total_val.resize(total_read/dtype_size);
1235 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1237 vector<float>final_val;
1251 vector<float>final_val;
1265 set_value ((dods_float32 *) &final_val[0], nelms_to_send);
1271 vector<double>total_val;
1272 total_val.resize(total_read/dtype_size);
1273 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1275 vector<double>final_val;
1288 vector<double>final_val;
1301 set_value ((dods_float64 *) &final_val[0], nelms_to_send);
1305 throw InternalErr (__FILE__, __LINE__,
"unsupported data type.");
1315 HDF5CFArray::write_data_to_cache(hid_t dset_id, hid_t , hid_t , hid_t memtype,
1316 const string& cache_fpath,
short dtype_size,
const vector<char> &buf,
int nelms) {
1318 unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
1319 string disk_cache_dir = HDF5RequestHandler::get_disk_cache_dir();
1320 string disk_cache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
1322 int total_nelem = 1;
1323 for(
int i = 0; i <rank; i++)
1324 total_nelem = total_nelem*dimsizes[i];
1328 if(H5CHAR == dtype) {
1330 vector<short>newval;
1331 newval.resize(total_nelem);
1332 if(total_nelem == nelms) {
1333 for (
int i = 0; i < total_nelem;i++)
1334 newval[i] = (
short)buf[i];
1335 disk_cache->write_cached_data2(cache_fpath,
sizeof(
short)*total_nelem,(
const void*)&newval[0]);
1339 val2.resize(total_nelem);
1340 if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val2[0])<0)
1341 throw InternalErr (__FILE__, __LINE__,
"Cannot read the whole HDF5 dataset for the disk cache.");
1342 for (
int i = 0; i < total_nelem;i++)
1343 newval[i] = (
short)val2[i];
1344 disk_cache->write_cached_data2(cache_fpath,
sizeof(
short)*total_nelem,(
const void*)&newval[0]);
1348 if(total_nelem == nelms) {
1349 disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&buf[0]);
1352 val.resize(dtype_size*total_nelem);
1353 if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val[0])<0)
1354 throw InternalErr (__FILE__, __LINE__,
"Cannot read the whole SDS for cache.");
1356 disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&val[0]);
1362 void HDF5CFArray::read_data_from_mem_cache(
void*buf) {
1367 int nelms = format_constraint (&offset[0], &step[0], &count[0]);
1369 vector<int>at_pos(at_ndims,0);
1370 for (
int i = 0; i< rank; i++)
1371 at_pos[i] = at_offset[i];
1379 vector<unsigned char> val;
1381 subset<unsigned char>(
1394 set_value ((dods_byte *) &val[0], nelms);
1406 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1408 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1418 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1419 ostringstream eherr;
1420 eherr <<
"Cannot read the HDF5 dataset " << varname
1421 <<
" with the type of H5T_NATIVE_CHAR "<<endl;
1422 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1426 vector<short>newval;
1427 newval.resize(nelms);
1429 for (
int counter = 0; counter < nelms; counter++)
1430 newval[counter] = (
short) (val[counter]);
1432 set_value ((dods_int16 *) &newval[0], nelms);
1443 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1445 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1455 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1457 ostringstream eherr;
1458 eherr <<
"Cannot read the HDF5 dataset " << varname
1459 <<
" with the type of H5T_NATIVE_SHORT "<<endl;
1460 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1463 set_value ((dods_int16 *) &val[0], nelms);
1470 vector<unsigned short> val;
1473 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1475 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1479 if (rank > 0) H5Sclose(mspace);
1484 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1485 ostringstream eherr;
1486 eherr <<
"Cannot read the HDF5 dataset " << varname
1487 <<
" with the type of H5T_NATIVE_USHORT "<<endl;
1488 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1491 set_value ((dods_uint16 *) &val[0], nelms);
1501 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1503 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1512 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1513 ostringstream eherr;
1514 eherr <<
"Cannot read the HDF5 dataset " << varname
1515 <<
" with the type of H5T_NATIVE_INT "<<endl;
1516 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1519 set_value ((dods_int32 *) &val[0], nelms);
1525 vector<unsigned int>val;
1528 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1530 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1540 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1541 ostringstream eherr;
1542 eherr <<
"Cannot read the HDF5 dataset " << varname
1543 <<
" with the type of H5T_NATIVE_UINT "<<endl;
1544 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1547 set_value ((dods_uint32 *) &val[0], nelms);
1558 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1560 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1569 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1570 ostringstream eherr;
1571 eherr <<
"Cannot read the HDF5 dataset " << varname
1572 <<
" with the type of H5T_NATIVE_FLOAT "<<endl;
1573 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1576 set_value ((dods_float32 *) &val[0], nelms);
1587 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1589 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1598 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1599 ostringstream eherr;
1600 eherr <<
"Cannot read the HDF5 dataset " << varname
1601 <<
" with the type of H5T_NATIVE_DOUBLE "<<endl;
1602 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1605 set_value ((dods_float64 *) &val[0], nelms);
1620 BaseType* HDF5CFArray::h5cfdims_transform_to_dap4(D4Group *grp) {
1624 Array *dest =
static_cast<HDF5CFArray*
>(ptr_duplicate());
1630 for (Array::Dim_iter d = dest->dim_begin(), e = dest->dim_end(); d != e; ++d) {
1631 if (
false == (*d).name.empty()) {
1633 D4Group *temp_grp = grp;
1634 D4Dimension *d4_dim = NULL;
1637 D4Dimensions *temp_dims = temp_grp->dims();
1640 d4_dim = temp_dims->find_dim((*d).name);
1646 if(temp_grp->get_parent())
1647 temp_grp =
static_cast<D4Group*
>(temp_grp->get_parent());
1656 bool d4_dim_null = ((d4_dim==NULL)?
true:
false);
1661 if(d4_dim_null ==
true) {
1663 d4_dim =
new D4Dimension((*d).name, (*d).size);
1664 D4Dimensions * dims = grp->dims();
1665 dims->add_dim_nocopy(d4_dim);
1671 dest->set_is_dap4(
true);
1680 HDF5CFArray::format_constraint (
int *offset,
int *step,
int *count)
1686 Dim_iter p = dim_begin ();
1688 while (p != dim_end ()) {
1690 int start = dimension_start (p,
true);
1691 int stride = dimension_stride (p,
true);
1692 int stop = dimension_stop (p,
true);
1698 oss <<
"Array/Grid hyperslab start point "<< start <<
1699 " is greater than stop point " << stop <<
".";
1700 throw Error(malformed_expr, oss.str());
1705 count[id] = ((stop - start) / stride) + 1;
1709 "=format_constraint():"
1710 <<
"id=" <<
id <<
" offset=" << offset[
id]
1711 <<
" step=" << step[
id]
1712 <<
" count=" << count[
id]