bes  Updated for version 3.20.6
HDF5DiskCache.cc
1 // This file includes cache handling routines for the HDF5 handler.
3 // The skeleton of the code is adapted from BESDapResponseCache.cc under bes/dap
4 // Authors: MuQun Yang <myang6@hdfgroup.org>
5 // Copyright (c) 2014 The HDF Group
7 
8 #include <sys/stat.h>
9 #include <iostream>
10 #include <sstream>
11 
12 #include "HDF5DiskCache.h"
13 #include "BESUtil.h"
14 
15 #include "BESInternalError.h"
16 #include "TheBESKeys.h"
17 #include "BESDebug.h"
18 #include "HDF5RequestHandler.h"
19 
20 using namespace std;
21 
22 HDF5DiskCache *HDF5DiskCache::d_instance = 0;
23 const string HDF5DiskCache::PATH_KEY = "H5.DiskCacheDataPath";
24 const string HDF5DiskCache::PREFIX_KEY = "H5.DiskCacheFilePrefix";
25 const string HDF5DiskCache::SIZE_KEY = "H5.DiskCacheSize";
26 
27 long HDF5DiskCache::getCacheSizeFromConfig(const long cache_size)
28 {
29  if (cache_size >0) {
30  BESDEBUG("cache",
31  "In HDF5DiskCache::getCacheSizeFromConfig(): Located BES key " << SIZE_KEY<< "=" << cache_size << endl);
32  return cache_size;
33  }
34  else {
35  string msg = "[ERROR] HDF5DiskCache::getCacheSize() - The BES Key " + SIZE_KEY
36  + " is either not set or the size is not a positive integer! It MUST be set and the size must be greater than 0 to use the HDF5 Disk cache. ";
37  BESDEBUG("cache", msg);
38  throw BESInternalError(msg, __FILE__, __LINE__);
39  }
40 }
41 
42 string HDF5DiskCache::getCachePrefixFromConfig(const string& cache_prefix)
43 {
44  if (cache_prefix!="") {
45  BESDEBUG("cache",
46  "In HDF5DiskCache::getCachePrefixFromConfig(): Located BES key " << PATH_KEY<< "=" << cache_prefix << endl);
47  return cache_prefix;
48  }
49  else {
50  string msg = "[ERROR] HDF5DiskCache::getCachePrefixFromConfig() - The BES Key " + PREFIX_KEY
51  + " is either not set or the value is an empty string! It MUST be set to be a valid string to utilize the HDF5 Disk cache. ";
52  BESDEBUG("cache", msg);
53  throw BESInternalError(msg, __FILE__, __LINE__);
54  }
55 }
56 
57 string HDF5DiskCache::getCacheDirFromConfig(const string& cache_dir)
58 {
59  if (cache_dir!="") {
60  BESDEBUG("cache",
61  "In HDF5DiskCache::getCacheDirFromConfig(): Located BES key " << PATH_KEY<< "=" << cache_dir << endl);
62  return cache_dir;
63  }
64  else {
65  string msg = "[ERROR] HDF5DiskCache::getCacheDirFromConfig() - The BES Key " + PREFIX_KEY
66  + " is either not set or the value is an empty string! It MUST be set to be a valid path to utilize the HDF5 Disk cache. ";
67  BESDEBUG("cache", msg);
68  throw BESInternalError(msg, __FILE__, __LINE__);
69  }
70 }
71 
72 
73 HDF5DiskCache::HDF5DiskCache(const unsigned long long _cache_size, const string &_cache_dir, const string &_cache_prefix)
74 {
75  BESDEBUG("cache", "In HDF5DiskCache::HDF5DiskCache()" << endl);
76 
77  string cacheDir = getCacheDirFromConfig(_cache_dir);
78  string prefix = getCachePrefixFromConfig(_cache_prefix);
79  unsigned long long size_in_megabytes = getCacheSizeFromConfig(_cache_size);
80 
81  BESDEBUG("cache",
82  "HDF5DiskCache() - Cache config params: " << cacheDir << ", " << prefix << ", " << size_in_megabytes << endl);
83 
84  // The required params must be present. If initialize() is not called,
85  // then d_cache will stay null and is_available() will return false.
86  // Also, the directory 'path' must exist, or d_cache will be null.
87  if (!cacheDir.empty() && size_in_megabytes > 0) {
88  BESDEBUG("cache", "Before calling initialize function." << endl);
89  initialize(cacheDir, prefix, size_in_megabytes);
90  }
91 
92  BESDEBUG("cache", "Leaving HDF5DiskCache::HDF5DiskCache()" << endl);
93 }
94 
99 HDF5DiskCache::get_instance(const long _cache_size, const string &_cache_dir, const string &_cache_prefix)
100 {
101  if (d_instance == 0) {
102  struct stat buf;
103  string cache_dir = getCacheDirFromConfig(_cache_dir);
104  if ((stat(cache_dir.c_str(), &buf) == 0) && (buf.st_mode & S_IFDIR)) {
105  try {
106  d_instance = new HDF5DiskCache(_cache_size,_cache_dir,_cache_prefix);
107 #ifdef HAVE_ATEXIT
108  atexit(delete_instance);
109 #endif
110  }
111  catch (BESInternalError &bie) {
112  BESDEBUG("cache",
113  "HDF5DiskCache::get_instance(): Failed to obtain cache! msg: " << bie.get_message() << endl);
114  }
115  }
116  }
117 
118  return d_instance;
119 }
120 
121 // Check whether the real lat/lon file size is the same as the expected lat/lon size. If not, return false.
122 bool HDF5DiskCache::is_valid(const string & cache_file_name, const int expected_file_size)
123 {
124 
125  struct stat st;
126  int result = stat(cache_file_name.c_str(), &st);
127  if (result != 0) {
128  string msg = "Cannot check the cached file " + cache_file_name;
129  throw BESInternalError(msg, __FILE__, __LINE__);
130  }
131  if (expected_file_size == st.st_size)
132  return true;
133  else
134  return false;
135 }
136 
137 // This call will try to obtain the read lock.
138 bool HDF5DiskCache::get_data_from_cache(const string & cache_file_name, const int expected_file_size, int &fd)
139 {
140 #if 0
141  cerr<<"coming to get_data_from_cache "<<endl;
142  BESDEBUG("cache", "In HDF5DiskCache::get_data_from_cache()" << endl);
143  cerr<<"cache_file_name is "<<cache_file_name <<endl;
144  int fd1;
145  string cache_file_name1 = cache_file_name;
146  get_read_lock(cache_file_name1,fd1);
147 
148  cerr<<"After get_read_lock "<<endl;
149 #endif
150  if (false == get_read_lock(cache_file_name, fd))
151  return false;
152  else if (false == is_valid(cache_file_name, expected_file_size)) {
153  unlock_and_close(cache_file_name);
154  purge_file(cache_file_name);
155  return false;
156  }
157  else
158  return true;
159 }
160 
161 bool HDF5DiskCache::write_cached_data(const string & cache_file_name, const int expected_file_size,
162  const vector<double> &val)
163 {
164 
165  BESDEBUG("cache", "In HDF5DiskCache::write_cached_data()" << endl);
166  int fd = 0;
167  bool ret_value = false;
168 
169  // 1. create_and_lock.
170  if (create_and_lock(cache_file_name, fd)) {
171 
172  ssize_t ret_val = 0;
173 
174  // 2. write the file.
175  ret_val = write(fd, &val[0], expected_file_size);
176 
177  // 3. If the written size is not the same as the expected file size, purge the file.
178  if (ret_val != expected_file_size) {
179  if (unlink(cache_file_name.c_str()) != 0) {
180  string msg = "Cannot remove the corrupt cached file " + cache_file_name;
181  throw BESInternalError(msg, __FILE__, __LINE__);
182  }
183 
184  }
185  else {
186  unsigned long long size = update_cache_info(cache_file_name);
187  if (cache_too_big(size)) update_and_purge(cache_file_name);
188  ret_value = true;
189  }
190  // 4. release the lock.
191  unlock_and_close(cache_file_name);
192 
193  }
194 
195  return ret_value;
196 
197 }
198 
199 bool HDF5DiskCache::write_cached_data2(const string & cache_file_name, const int expected_file_size, const void *buf)
200 {
201 
202  BESDEBUG("cache", "In HDF5DiskCache::write_cached_data()" << endl);
203  int fd = 0;
204  bool ret_value = false;
205 
206  // 1. create_and_lock.
207  if (create_and_lock(cache_file_name, fd)) {
208 
209  ssize_t ret_val = 0;
210 
211  // 2. write the file.
212  ret_val = write(fd, buf, expected_file_size);
213 
214  // 3. If the written size is not the same as the expected file size, purge the file.
215  if (ret_val != expected_file_size) {
216  if (unlink(cache_file_name.c_str()) != 0) {
217  string msg = "Cannot remove the corrupt cached file " + cache_file_name;
218  throw BESInternalError(msg, __FILE__, __LINE__);
219  }
220 
221  }
222  else {
223  unsigned long long size = update_cache_info(cache_file_name);
224  if (cache_too_big(size)) update_and_purge(cache_file_name);
225  ret_value = true;
226  }
227  // 4. release the lock.
228  unlock_and_close(cache_file_name);
229 
230  }
231 
232  return ret_value;
233 
234 }
235 #if 0
236 void HDF5DiskCache::dummy_test_func() {
237 
238  cerr<<"HDF5DiskCache function is fine "<<endl;
239 
240 }
241 
242 string HDF5DiskCache::get_cache_file_name_h4(const string & src, bool mangle) {
243 
244  return src;
245 }
246 #endif
247 
BESError::get_message
virtual std::string get_message()
get the error message for this exception
Definition: BESError.h:99
HDF5DiskCache
Definition: HDF5DiskCache.h:16
HDF5RequestHandler.h
include the entry functions to execute the handlers
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
HDF5DiskCache::get_instance
static HDF5DiskCache * get_instance(const long, const std::string &, const std::string &)
Definition: HDF5DiskCache.cc:99