bes  Updated for version 3.20.6
AggMemberDatasetDimensionCache.cc
1 /*
2  * AggMemberDatasetDimensionCache.cc
3  *
4  * Created on: Sep 25, 2015
5  * Author: ndp
6  */
7 
8 #include "AggMemberDatasetDimensionCache.h"
9 #include "AggMemberDataset.h"
10 #include <string>
11 #include <fstream>
12 #include <sstream>
13 #include <sys/stat.h>
14 
15 #include "util.h"
16 #include "BESInternalError.h"
17 #include "BESUtil.h"
18 #include "BESDebug.h"
19 #include "TheBESKeys.h"
20 
21 
22 static const string BES_DATA_ROOT("BES.Data.RootDirectory");
23 static const string BES_CATALOG_ROOT("BES.Catalog.catalog.RootDirectory");
24 
25 
26 namespace agg_util
27 {
28 
29 AggMemberDatasetDimensionCache *AggMemberDatasetDimensionCache::d_instance = 0;
30 bool AggMemberDatasetDimensionCache::d_enabled = true;
31 
32 const string AggMemberDatasetDimensionCache::CACHE_DIR_KEY = "NCML.DimensionCache.directory";
33 const string AggMemberDatasetDimensionCache::PREFIX_KEY = "NCML.DimensionCache.prefix";
34 const string AggMemberDatasetDimensionCache::SIZE_KEY = "NCML.DimensionCache.size";
35 // const string AggMemberDatasetDimensionCache::CACHE_CONTROL_FILE = "ncmlAggDimensions.cache.info";
36 
41 unsigned long AggMemberDatasetDimensionCache::getCacheSizeFromConfig(){
42 
43  bool found;
44  string size;
45  unsigned long size_in_megabytes = 0;
46  TheBESKeys::TheKeys()->get_value( SIZE_KEY, size, found ) ;
47  if( found ) {
48  std::istringstream iss(size);
49  iss >> size_in_megabytes;
50  }
51  else {
52  string msg = "[ERROR] AggMemberDatasetDimensionCache::getCacheSize() - The BES Key " + SIZE_KEY + " is not set! It MUST be set to utilize the NcML Dimension Cache. ";
53  BESDEBUG("cache", msg << endl);
54  throw BESInternalError(msg , __FILE__, __LINE__);
55  }
56  return size_in_megabytes;
57 }
58 
63 string AggMemberDatasetDimensionCache::getCacheDirFromConfig(){
64  bool found;
65  string subdir = "";
66  TheBESKeys::TheKeys()->get_value( CACHE_DIR_KEY, subdir, found ) ;
67 
68  if( !found ) {
69  string msg = "[ERROR] AggMemberDatasetDimensionCache::getSubDirFromConfig() - The BES Key " + CACHE_DIR_KEY + " is not set! It MUST be set to utilize the NcML Dimension Cache. ";
70  BESDEBUG("cache", msg << endl);
71  throw BESInternalError(msg , __FILE__, __LINE__);
72  }
73 
74  return subdir;
75 }
76 
77 
82 string AggMemberDatasetDimensionCache::getDimCachePrefixFromConfig(){
83  bool found;
84  string prefix = "";
85  TheBESKeys::TheKeys()->get_value( PREFIX_KEY, prefix, found ) ;
86  if( found ) {
87  prefix = BESUtil::lowercase( prefix ) ;
88  }
89  else {
90  string msg = "[ERROR] AggMemberDatasetDimensionCache::getResultPrefix() - The BES Key " + PREFIX_KEY + " is not set! It MUST be set to utilize the NcML Dimension Cache. ";
91  BESDEBUG("cache", msg << endl);
92  throw BESInternalError(msg , __FILE__, __LINE__);
93  }
94 
95  return prefix;
96 }
97 
98 
104 string AggMemberDatasetDimensionCache::getBesDataRootDirFromConfig(){
105  bool found;
106  string cacheDir = "";
107  TheBESKeys::TheKeys()->get_value( BES_CATALOG_ROOT, cacheDir, found ) ;
108  if( !found ) {
109  TheBESKeys::TheKeys()->get_value( BES_DATA_ROOT, cacheDir, found ) ;
110  if( !found ) {
111  string msg = ((string)"[ERROR] AggMemberDatasetDimensionCache::getStoredResultsDir() - Neither the BES Key ") + BES_CATALOG_ROOT +
112  "or the BES key " + BES_DATA_ROOT + " have been set! One MUST be set to utilize the NcML Dimension Cache. ";
113  BESDEBUG("cache", msg << endl);
114  throw BESInternalError(msg , __FILE__, __LINE__);
115  }
116  }
117  return cacheDir;
118 
119 }
120 
124 AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache()
125 {
126  BESDEBUG("cache", "AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - BEGIN" << endl);
127 
128  d_dimCacheDir = getCacheDirFromConfig();
129  d_dataRootDir = getBesDataRootDirFromConfig();
130 
131  d_dimCacheFilePrefix = getDimCachePrefixFromConfig();
132  d_maxCacheSize = getCacheSizeFromConfig();
133 
134  BESDEBUG("cache", "AggMemberDatasetDimensionCache() - Stored results cache configuration params: " << d_dimCacheDir << ", " << d_dimCacheFilePrefix << ", " << d_maxCacheSize << endl);
135 
136  // initialize(d_dimCacheDir, CACHE_CONTROL_FILE, d_dimCacheFilePrefix, d_maxCacheSize);
137  initialize(d_dimCacheDir, d_dimCacheFilePrefix, d_maxCacheSize);
138 
139  BESDEBUG("cache", "AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - END" << endl);
140 
141 }
142 
146 AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache(const string &data_root_dir, const string &cache_dir, const string &prefix, unsigned long long size){
147 
148  BESDEBUG("cache", "AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - BEGIN" << endl);
149 
150  d_dataRootDir = data_root_dir;
151  d_dimCacheDir = cache_dir;
152  d_dimCacheFilePrefix = prefix;
153  d_maxCacheSize = size;
154 
155 // initialize(d_dimCacheDir, CACHE_CONTROL_FILE, d_dimCacheFilePrefix, d_maxCacheSize);
156  initialize(d_dimCacheDir, d_dimCacheFilePrefix, d_maxCacheSize);
157 
158  BESDEBUG("cache", "AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - END" << endl);
159 }
160 
161 
166 AggMemberDatasetDimensionCache *
167 AggMemberDatasetDimensionCache::get_instance(const string &data_root_dir, const string &cache_dir, const string &result_file_prefix, unsigned long long max_cache_size)
168 {
169  if (d_enabled && d_instance == 0){
170  if (libdap::dir_exists(cache_dir)) {
171  d_instance = new AggMemberDatasetDimensionCache(data_root_dir, cache_dir, result_file_prefix, max_cache_size);
172  d_enabled = d_instance->cache_enabled();
173  if(!d_enabled){
174  delete d_instance;
175  d_instance = NULL;
176  BESDEBUG("cache", "AggMemberDatasetDimensionCache::"<<__func__ << "() - " <<
177  "Cache is DISABLED"<< endl);
178  }
179  else {
180  #ifdef HAVE_ATEXIT
181  atexit(delete_instance);
182  #endif
183  BESDEBUG("cache", "AggMemberDatasetDimensionCache::"<<__func__ << "() - " <<
184  "Cache is ENABLED"<< endl);
185  }
186  }
187  }
188  return d_instance;
189 }
190 
196 AggMemberDatasetDimensionCache *
198 {
199  if (d_enabled && d_instance == 0) {
200  d_instance = new AggMemberDatasetDimensionCache();
201  d_enabled = d_instance->cache_enabled();
202  if(!d_enabled){
203  delete d_instance;
204  d_instance = NULL;
205  BESDEBUG("cache", "AggMemberDatasetDimensionCache::"<<__func__ << "() - " <<
206  "Cache is DISABLED"<< endl);
207  }
208  else {
209 #ifdef HAVE_ATEXIT
210  atexit(delete_instance);
211 #endif
212  BESDEBUG("cache", "AggMemberDatasetDimensionCache::"<<__func__ << "() - " <<
213  "Cache is ENABLED"<< endl);
214  }
215 }
216 
217  return d_instance;
218 }
219 
220 
224 void AggMemberDatasetDimensionCache::delete_instance() {
225  BESDEBUG("cache","AggMemberDatasetDimensionCache::delete_instance() - Deleting singleton BESStoredDapResultCache instance." << endl);
226  delete d_instance;
227  d_instance = 0;
228 }
229 
230 
231 
232 AggMemberDatasetDimensionCache::~AggMemberDatasetDimensionCache()
233 {
234  // Nothing to do here....
235 }
236 
247 bool AggMemberDatasetDimensionCache::is_valid(const string &cache_file_name, const string &local_id)
248 {
249  // If the cached response is zero bytes in size, it's not valid.
250  // (hmmm...)
251  string datasetFileName = BESUtil::assemblePath(d_dataRootDir,local_id, true);
252 
253  off_t entry_size = 0;
254  time_t entry_time = 0;
255  struct stat buf;
256  if (stat(cache_file_name.c_str(), &buf) == 0) {
257  entry_size = buf.st_size;
258  entry_time = buf.st_mtime;
259  }
260  else {
261  return false;
262  }
263 
264  if (entry_size == 0)
265  return false;
266 
267  time_t dataset_time = entry_time;
268  if (stat(datasetFileName.c_str(), &buf) == 0) {
269  dataset_time = buf.st_mtime;
270  }
271 
272  // Trick: if the d_dataset is not a file, stat() returns error and
273  // the times stay equal and the code uses the cache entry.
274 
275  // TODO Fix this so that the code can get a LMT from the correct handler.
276  // TODO Consider adding a getLastModified() method to the libdap::DDS object to support this
277  // TODO The DDS may be expensive to instantiate - I think the handler may be a better location for an LMT method, if we can access the handler when/where needed.
278  if (dataset_time > entry_time)
279  return false;
280 
281  return true;
282 }
283 
284 
285 
293  BESDEBUG("cache", "AggMemberDatasetDimensionCache::loadDimensionCache() - BEGIN" << endl );
294 
295  // Get the cache filename for this thing, mangle name.
296  string local_id = amd->getLocation();
297  BESDEBUG("cache", "AggMemberDatasetDimensionCache::loadDimensionCache() - local resource id: "<< local_id << endl );
298  string cache_file_name = get_cache_file_name(local_id, true);
299  BESDEBUG("cache", "AggMemberDatasetDimensionCache::loadDimensionCache() - cache_file_name: "<< cache_file_name << endl );
300 
301  int fd;
302  try {
303  // If the object in the cache is not valid, remove it. The read_lock will
304  // then fail and the code will drop down to the create_and_lock() call.
305  // is_valid() tests for a non-zero length cache file (cache_file_name) and
306  // for the source data file (local_id) with a newer LMT than the cache file.
307  if (!is_valid(cache_file_name, local_id)){
308  BESDEBUG("cache", "AggMemberDatasetDimensionCache::loadDimensionCache() - File is not valid. Purging file from cache. filename: " << cache_file_name << endl);
309  purge_file(cache_file_name);
310  }
311 
312  if (get_read_lock(cache_file_name, fd)) {
313  BESDEBUG("cache", "AggMemberDatasetDimensionCache::loadDimensionCache() - Dimension cache file exists. Loading dimension cache from file: " << cache_file_name << endl);
314 
315  ifstream istrm(cache_file_name.c_str());
316  if (!istrm)
317  throw libdap::InternalErr(__FILE__, __LINE__, "Could not open '" + cache_file_name + "' to read cached dimensions.");
318 
319  amd->loadDimensionCache(istrm);
320 
321  istrm.close();
322 
323 
324  }
325  else {
326  // If here, the cache_file_name could not be locked for read access, or it was out of date.
327  // So we are going to (re)build the cache file.
328 
329  // We need to build the DDS object and extract the dimensions.
330  // We do not lock before this operation because it may take a _long_ time and
331  // we don't want to monopolize the cache while we do it.
333 
334  // Now, we try to make an empty cache file and get an exclusive lock on it.
335  if (create_and_lock(cache_file_name, fd)) {
336  // Woohoo! We got the exclusive lock on the new cache file.
337  BESDEBUG("cache", "AggMemberDatasetDimensionCache::loadDimensionCache() - Created and locked cache file: " << cache_file_name << endl);
338 
339  // Now we open it (again) using the more friendly ostream API.
340  ofstream ostrm(cache_file_name.c_str());
341  if (!ostrm)
342  throw libdap::InternalErr(__FILE__, __LINE__, "Could not open '" + cache_file_name + "' to write cached response.");
343 
344  // Save the dimensions to the cache file.
345  amd->saveDimensionCache(ostrm);
346 
347  // And close the cache file;s ostream.
348  ostrm.close();
349 
350  // Change the exclusive lock on the new file to a shared lock. This keeps
351  // other processes from purging the new file and ensures that the reading
352  // process can use it.
354 
355  // Now update the total cache size info and purge if needed. The new file's
356  // name is passed into the purge method because this process cannot detect its
357  // own lock on the file.
358  unsigned long long size = update_cache_info(cache_file_name);
359  if (cache_too_big(size))
360  update_and_purge(cache_file_name);
361  }
362  // get_read_lock() returns immediately if the file does not exist,
363  // but blocks waiting to get a shared lock if the file does exist.
364  else if (get_read_lock(cache_file_name, fd)) {
365  // If we got here then someone else rebuilt the cache file before we could do it.
366  // That's OK, and since we already built the DDS we have all of the cache info in memory
367  // from directly accessing the source dataset(s), so we need to do nothing more,
368  // Except send a debug statement so we can see that this happened.
369  BESDEBUG("cache", "AggMemberDatasetDimensionCache::loadDimensionCache() - Couldn't create and lock cache file, But I got a read lock. "
370  "Cache file may have been rebuilt by another process. "
371  "Cache file: " << cache_file_name << endl);
372  }
373  else {
374  throw libdap::InternalErr(__FILE__, __LINE__, "AggMemberDatasetDimensionCache::loadDimensionCache() - Cache error during function invocation.");
375  }
376  }
377 
378  BESDEBUG("cache", "AggMemberDatasetDimensionCache::loadDimensionCache() - unlocking and closing cache file "<< cache_file_name << endl );
379  unlock_and_close(cache_file_name);
380  }
381  catch (...) {
382  BESDEBUG("cache", "AggMemberDatasetDimensionCache::loadDimensionCache() - caught exception, unlocking cache and re-throw." << endl );
383  unlock_cache();
384  throw;
385  }
386 
387  BESDEBUG("cache", "AggMemberDatasetDimensionCache::loadDimensionCache() - END (local_id=`"<< local_id << "')" << endl );
388 
389 }
390 
391 
392 
393 
394 
395 
396 } /* namespace agg_util */
BESFileLockingCache::get_read_lock
virtual bool get_read_lock(const std::string &target, int &fd)
Get a read-only lock on the file if it exists.
Definition: BESFileLockingCache.cc:544
agg_util::AggMemberDatasetDimensionCache::loadDimensionCache
void loadDimensionCache(AggMemberDataset *amd)
Definition: AggMemberDatasetDimensionCache.cc:292
agg_util::AggMemberDatasetDimensionCache::get_instance
static AggMemberDatasetDimensionCache * get_instance()
Definition: AggMemberDatasetDimensionCache.cc:197
agg_util::AggMemberDatasetDimensionCache
Definition: AggMemberDatasetDimensionCache.h:32
BESFileLockingCache::cache_enabled
bool cache_enabled() const
Definition: BESFileLockingCache.h:198
BESFileLockingCache::create_and_lock
virtual bool create_and_lock(const std::string &target, int &fd)
Create a file in the cache and lock it for write access.
Definition: BESFileLockingCache.cc:599
agg_util::AggMemberDataset::fillDimensionCacheByUsingDDS
virtual void fillDimensionCacheByUsingDDS()=0
agg_util::AggMemberDataset::loadDimensionCache
virtual void loadDimensionCache(std::istream &istr)=0
agg_util
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
Definition: AggMemberDataset.cc:38
BESFileLockingCache::purge_file
virtual void purge_file(const std::string &file)
Purge a single file from the cache.
Definition: BESFileLockingCache.cc:1085
BESUtil::assemblePath
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
Definition: BESUtil.cc:821
BESFileLockingCache::unlock_and_close
virtual void unlock_and_close(const std::string &target)
Definition: BESFileLockingCache.cc:713
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:62
BESFileLockingCache::initialize
void initialize(const std::string &cache_dir, const std::string &prefix, unsigned long long size)
Initialize an instance of FileLockingCache.
Definition: BESFileLockingCache.cc:116
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
TheBESKeys::get_value
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:272
BESFileLockingCache::update_cache_info
virtual unsigned long long update_cache_info(const std::string &target)
Update the cache info file to include 'target'.
Definition: BESFileLockingCache.cc:737
BESFileLockingCache::get_cache_file_name
virtual std::string get_cache_file_name(const std::string &src, bool mangle=true)
Definition: BESFileLockingCache.cc:451
agg_util::AggMemberDataset
Definition: AggMemberDataset.h:63
BESFileLockingCache::update_and_purge
virtual void update_and_purge(const std::string &new_file)
Purge files from the cache.
Definition: BESFileLockingCache.cc:940
BESFileLockingCache::unlock_cache
virtual void unlock_cache()
Definition: BESFileLockingCache.cc:686
BESUtil::lowercase
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:200
agg_util::AggMemberDataset::saveDimensionCache
virtual void saveDimensionCache(std::ostream &ostr)=0
agg_util::AggMemberDataset::getLocation
const std::string & getLocation() const
Definition: AggMemberDataset.cc:62
BESFileLockingCache::cache_too_big
virtual bool cache_too_big(unsigned long long current_size) const
look at the cache size; is it too large? Look at the cache size and see if it is too big.
Definition: BESFileLockingCache.cc:780
BESFileLockingCache::exclusive_to_shared_lock
virtual void exclusive_to_shared_lock(int fd)
Transfer from an exclusive lock to a shared lock.
Definition: BESFileLockingCache.cc:630