7 #include "HDFSPArray_RealField.h"
14 #include "InternalErr.h"
16 #include "BESInternalError.h"
17 #include "HDFCFUtil.h"
18 #include "HDF4RequestHandler.h"
25 #define SIGNED_BYTE_TO_INT32 1
32 BESDEBUG(
"h4",
"Coming to HDFSPArray_RealField read "<<endl);
45 int nelms = format_constraint (&offset[0], &step[0], &count[0]);
51 string check_data_cache_key =
"H4.EnableDataCacheFile";
52 bool enable_check_data_cache_key =
false;
53 enable_check_data_cache_key = HDFCFUtil::check_beskeys(check_data_cache_key);
56 bool data_from_cache =
false;
57 bool data_to_cache =
false;
60 if(-1 == dtype_size) {
61 string err_mesg =
"Wrong data type size for the variable ";
63 throw InternalErr(__FILE__,__LINE__,err_mesg);
68 if(
true == HDF4RequestHandler::get_enable_data_cachefile()) {
76 string bescachedir= llcache->getCacheDirFromConfig();
77 string bescacheprefix = llcache->getCachePrefixFromConfig();
78 unsigned int cachesize = llcache->getCacheSizeFromConfig();
82 string bescachedir = HDF4RequestHandler::get_cache_latlon_path();
83 string bescacheprefix = HDF4RequestHandler::get_cache_latlon_prefix();
84 long cachesize = HDF4RequestHandler::get_cache_latlon_size();
86 if((
"" == bescachedir)||(
""==bescacheprefix)||(cachesize <=0))
87 throw InternalErr (__FILE__, __LINE__,
"Either the cached dir is empty or the prefix is NULL or the cache size is not set.");
90 if(stat(bescachedir.c_str(),&sb) !=0) {
91 string err_mesg=
"The cached directory " + bescachedir;
92 err_mesg = err_mesg +
" doesn't exist. ";
93 throw InternalErr(__FILE__,__LINE__,err_mesg);
96 if(
true == S_ISDIR(sb.st_mode)) {
97 if(access(bescachedir.c_str(),R_OK|W_OK|X_OK) == -1) {
98 string err_mesg=
"The cached directory " + bescachedir;
99 err_mesg = err_mesg +
" can NOT be read,written or executable.";
100 throw InternalErr(__FILE__,__LINE__,err_mesg);
104 string err_mesg=
"The cached directory " + bescachedir;
105 err_mesg = err_mesg +
" is not a directory.";
106 throw InternalErr(__FILE__,__LINE__,err_mesg);
112 string cache_fname=HDFCFUtil::obtain_cache_fname(bescacheprefix,filename,name());
113 cache_fpath = bescachedir +
"/"+ cache_fname;
116 for (
unsigned int i = 0; i <dimsizes.size();i++)
117 total_elems = total_elems*dimsizes[i];
119 if(-1 == dtype_size) {
120 string err_mesg =
"Wrong data type size for the variable ";
122 throw InternalErr(__FILE__,__LINE__,err_mesg);
124 int expected_file_size = dtype_size *total_elems;
126 if(
true == llcache->get_data_from_cache(cache_fpath, expected_file_size,fd)) {
128 vector<int32>offset32;
129 offset32.resize(offset.size());
130 for (
int i = 0; i< rank;i++)
131 offset32[i] = offset[i];
132 int offset_1st = INDEX_nD_TO_1D(dimsizes,offset32);
135 for (
int i = 0; i < rank; i++)
136 end[i] = offset[i] +(count[i]-1)*step[i];
137 int offset_last = INDEX_nD_TO_1D(dimsizes,end);
140 size_t total_read = dtype_size*(offset_last-offset_1st+1);
142 off_t fpos = lseek(fd,dtype_size*offset_1st,SEEK_SET);
150 data_from_cache = obtain_cached_data(llcache,cache_fpath,fd, step,count,total_read,dtype_size);
154 if(
true == data_from_cache)
157 data_to_cache =
true;
162 string check_pass_fileid_key_str=
"H4.EnablePassFileID";
163 bool check_pass_fileid_key =
false;
164 check_pass_fileid_key = HDFCFUtil::check_beskeys(check_pass_fileid_key_str);
166 bool check_pass_fileid_key = HDF4RequestHandler::get_pass_fileid();
169 vector<int32>offset32;
170 offset32.resize(rank);
171 vector<int32>count32;
172 count32.resize(rank);
177 for (
int i = 0; i < rank; i++) {
178 offset32[i] = (int32) offset[i];
179 count32[i] = (int32) count[i];
180 step32[i] = (int32) step[i];
186 if(
false == check_pass_fileid_key) {
187 sdid = SDstart (
const_cast < char *
>(filename.c_str ()), DFACC_READ);
190 eherr <<
"File " << filename.c_str () <<
" cannot be open.";
191 throw InternalErr (__FILE__, __LINE__, eherr.str ());
198 string check_enable_mcache_key=
"H4.EnableMetaDataCacheFile";
199 bool turn_on_enable_mcache_key=
false;
200 turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
204 if(
true == HDF4RequestHandler::get_enable_metadata_cachefile()) {
206 sdid = SDstart (
const_cast < char *
>(filename.c_str ()), DFACC_READ);
209 eherr <<
"File " << filename.c_str () <<
" cannot be open.";
210 throw InternalErr (__FILE__, __LINE__, eherr.str ());
213 check_pass_fileid_key =
false;
224 int32 sdsindex = SDreftoindex (sdid, (int32) fieldref);
225 if (sdsindex == -1) {
228 eherr <<
"SDS index " << sdsindex <<
" is not right.";
229 throw InternalErr (__FILE__, __LINE__, eherr.str ());
233 sdsid = SDselect (sdid, sdsindex);
237 eherr <<
"SDselect failed.";
238 throw InternalErr (__FILE__, __LINE__, eherr.str ());
251 r = SDreaddata (sdsid, &offset32[0], &step32[0], &count32[0], &buf[0]);
256 eherr <<
"SDreaddata failed.";
257 throw InternalErr (__FILE__, __LINE__, eherr.str ());
260 #ifndef SIGNED_BYTE_TO_INT32
265 newval.resize(nelms);
266 for (
int counter = 0; counter < nelms; counter++)
267 newval[counter] = (int32) (buf[counter]);
269 set_value ((dods_int32 *) &newval[0], nelms);
271 if(
true == data_to_cache) {
273 write_data_to_cache(sdsid,cache_fpath,dtype_size,buf,nelms);
279 eherr <<
"write data to cache failed.";
280 throw InternalErr (__FILE__, __LINE__, eherr.str ());
311 buf.resize(nelms*dtype_size);
313 r = SDreaddata (sdsid, &offset32[0], &step32[0], &count32[0], &buf[0]);
318 eherr <<
"SDreaddata failed";
319 throw InternalErr (__FILE__, __LINE__, eherr.str ());
326 if(
true == data_to_cache) {
328 write_data_to_cache(sdsid,cache_fpath,dtype_size,buf,nelms);
334 eherr <<
"write data to cache failed.";
335 throw InternalErr (__FILE__, __LINE__, eherr.str ());
344 throw InternalErr (__FILE__, __LINE__,
"unsupported data type.");
348 r = SDendaccess (sdsid);
351 eherr <<
"SDendaccess failed.";
352 throw InternalErr (__FILE__, __LINE__, eherr.str ());
361 eherr <<
"SDend failed.";
362 throw InternalErr (__FILE__, __LINE__, eherr.str ());
369 bool HDFSPArray_RealField::obtain_cached_data(
BESH4Cache *llcache,
const string & cache_fpath,
int fd,vector<int> &cd_step, vector<int>&cd_count,
size_t total_read,
short dtype_size) {
371 ssize_t ret_read_val = -1;
375 buf.resize(total_read);
376 ret_read_val = HDFCFUtil::read_buffer_from_file(fd,(
void*)&buf[0],total_read);
378 if((-1 == ret_read_val) || (ret_read_val != (ssize_t)total_read)) {
383 unsigned int nele_to_read = 1;
384 for(
int i = 0; i<rank;i++)
385 nele_to_read *=cd_count[i];
387 if(nele_to_read == (total_read/dtype_size)) {
394 vector<int>cd_start(rank,0);
395 vector<int32>cd_pos(rank,0);
396 int nelms_to_send = 1;
397 for(
int i = 0; i <rank; i++)
398 nelms_to_send = nelms_to_send*cd_count[i];
404 #ifndef SIGNED_BYTE_TO_INT32
405 vector<int8>total_val;
406 total_val.resize(total_read/dtype_size);
407 memcpy(&total_val[0],(
void*)&buf[0],total_read);
409 vector<int8>final_val;
422 set_value((dods_byte*)&final_val[0],nelms_to_send);
425 vector<int32>total_val2;
426 total_val2.resize(total_read/dtype_size);
427 memcpy(&total_val2[0],(
void*)&buf[0],total_read);
429 vector<int32>final_val2;
442 set_value((dods_int32*)&final_val2[0],nelms_to_send);
451 vector<uint8>total_val;
452 total_val.resize(total_read/dtype_size);
453 memcpy(&total_val[0],(
void*)&buf[0],total_read);
455 vector<uint8>final_val;
469 set_value ((dods_byte *) &final_val[0], nelms_to_send);
475 vector<int16>total_val;
476 total_val.resize(total_read/dtype_size);
477 memcpy(&total_val[0],(
void*)&buf[0],total_read);
479 vector<int16>final_val;
493 set_value ((dods_int16 *) &final_val[0], nelms_to_send);
499 vector<uint16>total_val;
500 total_val.resize(total_read/dtype_size);
501 memcpy(&total_val[0],(
void*)&buf[0],total_read);
503 vector<uint16>final_val;
517 set_value ((dods_uint16 *) &final_val[0], nelms_to_send);
523 vector<int32>total_val;
524 total_val.resize(total_read/dtype_size);
525 memcpy(&total_val[0],(
void*)&buf[0],total_read);
527 vector<int32>final_val;
541 set_value ((dods_int32 *) &final_val[0], nelms_to_send);
547 vector<uint32>total_val;
548 total_val.resize(total_read/dtype_size);
549 memcpy(&total_val[0],(
void*)&buf[0],total_read);
551 vector<uint32>final_val;
565 set_value ((dods_uint32 *) &final_val[0], nelms_to_send);
571 vector<float32>total_val;
572 total_val.resize(total_read/dtype_size);
573 memcpy(&total_val[0],(
void*)&buf[0],total_read);
575 vector<float32>final_val;
589 set_value ((dods_float32 *) &final_val[0], nelms_to_send);
594 vector<float64>total_val;
595 total_val.resize(total_read/dtype_size);
596 memcpy(&total_val[0],(
void*)&buf[0],total_read);
598 vector<float64>final_val;
611 set_value ((dods_float64 *) &final_val[0], nelms_to_send);
615 throw InternalErr (__FILE__, __LINE__,
"unsupported data type.");
624 HDFSPArray_RealField::write_data_to_cache(int32 sdsid,
const string& cache_fpath,
short dtype_size,
const vector<char> &buf,
int nelms) {
627 vector<int32>woffset32(rank,0);
628 vector<int32>wstep32(rank,1);
629 vector<int32>wcount32;
630 wcount32.resize(rank);
632 for(
int i = 0; i <rank; i++){
633 wcount32[i] = (int32)dimsizes[i];
634 total_nelem = total_nelem*dimsizes[i];
637 if(DFNT_INT8 == dtype) {
639 #ifndef SIGNED_BYTE_TO_INT32
640 if(total_nelem == nelms)
641 llcache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&buf[0]);
643 val.resize(dtype_size*total_nelem);
644 if(SDreaddata (sdsid, &woffset32[0], &wstep32[0], &wcount32[0], &val[0])<0)
645 throw InternalErr (__FILE__, __LINE__,
"Cannot read the whole SDS for cache.");
646 llcache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&val[0]);
651 newval.resize(total_nelem);
652 if(total_nelem == nelms) {
653 for (
int i = 0; i < total_nelem;i++)
654 newval[i] = (
int)buf[i];
655 llcache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&newval[0]);
659 val2.resize(total_nelem);
660 if(SDreaddata (sdsid, &woffset32[0], &wstep32[0], &wcount32[0], &val2[0])<0)
661 throw InternalErr (__FILE__, __LINE__,
"Cannot read the whole SDS for cache.");
662 for (
int i = 0; i < total_nelem;i++)
663 newval[i] = (
int)val2[i];
664 llcache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&newval[0]);
669 if(total_nelem == nelms) {
670 llcache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&buf[0]);
673 val.resize(dtype_size*total_nelem);
674 if(SDreaddata (sdsid, &woffset32[0], &wstep32[0], &wcount32[0], &val[0])<0)
675 throw InternalErr (__FILE__, __LINE__,
"Cannot read the whole SDS for cache.");
677 llcache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&val[0]);
684 HDFSPArray_RealField::format_constraint (
int *offset,
int *step,
int *count)
689 Dim_iter p = dim_begin ();
690 while (p != dim_end ()) {
692 int start = dimension_start (p,
true);
693 int stride = dimension_stride (p,
true);
694 int stop = dimension_stop (p,
true);
699 oss <<
"Array/Grid hyperslab start point "<< start <<
700 " is greater than stop point " << stop <<
".";
701 throw Error(malformed_expr, oss.str());
706 count[id] = ((stop - start) / stride) + 1;
710 "=format_constraint():"
711 <<
"id=" <<
id <<
" offset=" << offset[
id]
712 <<
" step=" << step[
id]
713 <<
" count=" << count[
id]
735 template<
typename T>
int
736 HDFSPArray_RealField::subset(
741 vector<int> & stride,
743 std::vector<T> *poutput,
747 for(
int k=0; k<edge[index]; k++)
749 pos[index] = start[index] + k*stride[index];
751 subset(input, sf_rank, dim, start, stride, edge, poutput,pos,index+1);
754 poutput->push_back(input[INDEX_nD_TO_1D( dim, pos)]);