bes  Updated for version 3.17.0
BESFileLockingCache.cc
1 // This file was originally part of bes, A C++ back-end server
2 // implementation framework for the OPeNDAP Data Access Protocol.
3 // Copied to libdap. This is used to cache responses built from
4 // functional CE expressions.
5 
6 // Moved back to the BES. 6/11/13 jhrg
7 
8 // Copyright (c) 2012 OPeNDAP, Inc
9 // Author: James Gallagher <jgallagher@opendap.org>
10 // Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
11 //
12 // This library is free software; you can redistribute it and/or
13 // modify it under the terms of the GNU Lesser General Public
14 // License as published by the Free Software Foundation; either
15 // version 2.1 of the License, or (at your option) any later version.
16 //
17 // This library is distributed in the hope that it will be useful,
18 // but WITHOUT ANY WARRANTY; without even the implied warranty of
19 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 // Lesser General Public License for more details.
21 //
22 // You should have received a copy of the GNU Lesser General Public
23 // License along with this library; if not, write to the Free Software
24 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 //
26 // You can contact University Corporation for Atmospheric Research at
27 // 3080 Center Green Drive, Boulder, CO 80301
28 
29 #include "config.h"
30 
31 #include <sys/file.h>
32 #include <sys/stat.h>
33 #include <unistd.h>
34 #include <dirent.h>
35 #include <fcntl.h>
36 
37 #ifdef HAVE_STDLIB_H
38 #include <stdlib.h>
39 #endif
40 
41 #include <string>
42 #include <sstream>
43 #include <vector>
44 #include <cstring>
45 #include <cerrno>
46 
47 //#define DODS_DEBUG
48 
49 #include "BESInternalError.h"
50 
51 #include "BESUtil.h"
52 #include "BESDebug.h"
53 #include "BESLog.h"
54 
55 #include "BESFileLockingCache.h"
56 
57 using namespace std;
58 
59 // conversion factor
60 static const unsigned long long BYTES_PER_MEG = 1048576ULL;
61 
62 // Max cache size in megs, so we can check the user input and warn.
63 // 2^64 / 2^20 == 2^44
64 static const unsigned long long MAX_CACHE_SIZE_IN_MEGABYTES = (1ULL << 44);
65 
79 BESFileLockingCache::BESFileLockingCache(const string &cache_dir, const string &prefix, unsigned long long size) :
80  d_cache_dir(cache_dir), d_prefix(prefix), d_max_cache_size_in_bytes(size), d_target_size(0), d_cache_info(""), d_cache_info_fd(-1)
81 {
82  m_initialize_cache_info();
83 }
84 
85 void BESFileLockingCache::initialize(const string &cache_dir, const string &prefix, unsigned long long size)
86 {
87  d_cache_dir = cache_dir;
88  d_prefix = prefix;
90 
91  m_initialize_cache_info();
92 }
93 
94 static inline string get_errno() {
95  char *s_err = strerror(errno);
96  if (s_err)
97  return s_err;
98  else
99  return "Unknown error.";
100 }
101 
102 // Build a lock of a certain type.
103 static inline struct flock *lock(int type) {
104  static struct flock lock;
105  lock.l_type = type;
106  lock.l_whence = SEEK_SET;
107  lock.l_start = 0;
108  lock.l_len = 0;
109  lock.l_pid = getpid();
110 
111  return &lock;
112 }
113 
114 inline void BESFileLockingCache::m_record_descriptor(const string &file, int fd) {
115  BESDEBUG("cache", "BESFileLockingCache::m_record_descriptor() - Recording descriptor: " << file << ", " << fd << endl);
116  d_locks.insert(std::pair<string, int>(file, fd));
117 }
118 
119 inline int BESFileLockingCache::m_get_descriptor(const string &file) {
120  BESDEBUG("cache", "BESFileLockingCache::m_get_descriptor(): d_locks size: " << d_locks.size() << endl);
121  FilesAndLockDescriptors::iterator i = d_locks.find(file);
122  if (i == d_locks.end())
123  return -1;
124 
125  int fd = i->second;
126  BESDEBUG("cache", "BESFileLockingCache::m_get_descriptor(): Found file descriptor [" << fd << "] for file: " << file << endl);
127  d_locks.erase(i);
128  return fd;
129 }
130 
136 static void unlock(int fd)
137 {
138  if (fcntl(fd, F_SETLK, lock(F_UNLCK)) == -1) {
139  throw BESInternalError( "An error occurred trying to unlock the file: " + get_errno(), __FILE__, __LINE__);
140  }
141 
142  if (close(fd) == -1)
143  throw BESInternalError("Could not close the (just) unlocked file.", __FILE__, __LINE__);
144 }
145 
158 static bool getSharedLock(const string &file_name, int &ref_fd)
159 {
160  BESDEBUG("cache", "getSharedLock(): Acquiring cache read lock for " << file_name <<endl);
161 
162  int fd;
163  if ((fd = open(file_name.c_str(), O_RDONLY)) < 0) {
164  switch (errno) {
165  case ENOENT:
166  return false;
167 
168  default:
169  throw BESInternalError( get_errno(), __FILE__, __LINE__);
170  }
171  }
172 
173  struct flock *l = lock(F_RDLCK);
174  if (fcntl(fd, F_SETLKW, l) == -1) {
175  close(fd);
176  ostringstream oss;
177  oss << "cache process: " << l->l_pid << " triggered a locking error: " << get_errno();
178  throw BESInternalError( oss.str(), __FILE__, __LINE__);
179  }
180 
181  BESDEBUG("cache", "getSharedLock(): SUCCESS Read Lock Acquired For " << file_name <<endl);
182 
183  // Success
184  ref_fd = fd;
185  return true;
186 }
187 
200 bool BESFileLockingCache::getExclusiveLock(string file_name, int &ref_fd)
201 {
202  BESDEBUG("cache", "BESFileLockingCache::getExclusiveLock() - " << file_name <<endl);
203 
204  int fd;
205  if ((fd = open(file_name.c_str(), O_RDWR)) < 0) {
206  switch (errno) {
207  case ENOENT:
208  return false;
209 
210  default:
211  BESDEBUG("cache", "BESFileLockingCache::getExclusiveLock() - FAILED to open file. name: " << file_name << " name_length: " << file_name.length( )<<endl);
212  throw BESInternalError(get_errno(), __FILE__, __LINE__);
213  }
214  }
215 
216  struct flock *l = lock(F_WRLCK);
217  if (fcntl(fd, F_SETLKW, l) == -1) {
218  close(fd);
219  ostringstream oss;
220  oss << "cache process: " << l->l_pid << " triggered a locking error: " << get_errno();
221  throw BESInternalError(oss.str(), __FILE__, __LINE__);
222  }
223 
224  BESDEBUG("cache", "BESFileLockingCache::getExclusiveLock() - exit: " << file_name <<endl);
225 
226  // Success
227  ref_fd = fd;
228  return true;
229 }
230 
242 static bool getExclusiveLockNB(string file_name, int &ref_fd)
243 {
244  BESDEBUG("cache", "getExclusiveLock_nonblocking: " << file_name <<endl);
245 
246  int fd;
247  if ((fd = open(file_name.c_str(), O_RDWR)) < 0) {
248  switch (errno) {
249  case ENOENT:
250  return false;
251 
252  default:
253  throw BESInternalError(get_errno(), __FILE__, __LINE__);
254  }
255  }
256 
257  struct flock *l = lock(F_WRLCK);
258  if (fcntl(fd, F_SETLK, l) == -1) {
259  switch (errno) {
260  case EAGAIN:
261  BESDEBUG("cache", "getExclusiveLock_nonblocking exit (false): " << file_name << " by: " << l->l_pid << endl);
262  close(fd);
263  return false;
264 
265  default: {
266  close(fd);
267  ostringstream oss;
268  oss << "cache process: " << l->l_pid << " triggered a locking error: " << get_errno();
269  throw BESInternalError(oss.str(), __FILE__, __LINE__);
270  }
271  }
272  }
273 
274  BESDEBUG("cache", "getExclusiveLock_nonblocking exit (true): " << file_name <<endl);
275 
276  // Success
277  ref_fd = fd;
278  return true;
279 }
280 
294 static bool createLockedFile(string file_name, int &ref_fd)
295 {
296  BESDEBUG("cache", "createLockedFile() - filename: " << file_name <<endl);
297 
298  int fd;
299  if ((fd = open(file_name.c_str(), O_CREAT | O_EXCL | O_RDWR, 0666)) < 0) {
300  switch (errno) {
301  case EEXIST:
302  return false;
303 
304  default:
305  throw BESInternalError(get_errno(), __FILE__, __LINE__);
306  }
307  }
308 
309  struct flock *l = lock(F_WRLCK);
310  if (fcntl(fd, F_SETLKW, l) == -1) {
311  close(fd);
312  ostringstream oss;
313  oss << "cache process: " << l->l_pid << " triggered a locking error: " << get_errno();
314  throw BESInternalError(oss.str(), __FILE__, __LINE__);
315  }
316 
317  BESDEBUG("cache", "createLockedFile exit: " << file_name <<endl);
318 
319  // Success
320  ref_fd = fd;
321  return true;
322 }
323 
325 void BESFileLockingCache::m_check_ctor_params()
326 {
327  // TODO Should this really be a fatal error? What about just not
328  // using the cache in ths case or writing out a warning message
329  // to the log. jhrg 10/23/15
330  if (d_cache_dir.empty()) {
331  string err = "BESFileLockingCache::m_check_ctor_params() - The cache directory was not specified";
332  throw BESInternalError(err, __FILE__, __LINE__);
333  }
334 
335 #if 0
336  // This code has a Time of check, time of Use (TOCTOU) error.
337  // It could be that stat returns that the directory does not
338  // exist, then another process makes the directory and then this
339  // code tries and fails. I think it would be better to just try
340  // and if it fails, return an error only when the code indicates
341  // there really is an error.
342  //
343  // jhrg 10/23/15
344  struct stat buf;
345  int statret = stat(d_cache_dir.c_str(), &buf);
346  if (statret != 0 || !S_ISDIR(buf.st_mode)) {
347  // Try to make the directory
348  int status = mkdir(d_cache_dir.c_str(), 0775);
349  if (status != 0) {
350  string err = "BESFileLockingCache::m_check_ctor_params() - The cache directory " + d_cache_dir + " does not exist or could not be created.";
351  throw BESInternalError(err, __FILE__, __LINE__);
352  }
353  }
354 #endif
355 
356  // I changed these to BES_SYNTAX_USER_ERROR. jhrg 10/23/15
357 
358  int status = mkdir(d_cache_dir.c_str(), 0775);
359  // If there is an error and it's not that the dir already exists,
360  // throw an exception.
361  if (status == -1 && errno != EEXIST) {
362  string err = "The cache directory " + d_cache_dir + " could not be created: " + strerror(errno);
363  throw BESError(err, BES_SYNTAX_USER_ERROR, __FILE__, __LINE__);
364  }
365 
366  if (d_prefix.empty()) {
367  string err = "The cache file prefix was not specified, must not be empty";
368  throw BESError(err, BES_SYNTAX_USER_ERROR, __FILE__, __LINE__);
369  }
370 
371  if (d_max_cache_size_in_bytes <= 0) {
372  string err = "The cache size was not specified, must be greater than zero";
373  throw BESError(err, BES_SYNTAX_USER_ERROR, __FILE__, __LINE__);
374  }
375 
376  BESDEBUG("cache", "BESFileLockingCache::m_check_ctor_params() - directory " << d_cache_dir << ", prefix " << d_prefix
377  << ", max size " << d_max_cache_size_in_bytes << endl );
378 }
379 
381 void BESFileLockingCache::m_initialize_cache_info()
382 {
383  BESDEBUG("cache", "BESFileLockingCache::m_initialize_cache_info() - BEGIN" << endl);
384 
385  // The value set in configuration files, etc., is the size in megabytes. The private
386  // variable holds the size in bytes (converted below).
387  d_max_cache_size_in_bytes = min(d_max_cache_size_in_bytes, MAX_CACHE_SIZE_IN_MEGABYTES);
388  d_max_cache_size_in_bytes *= BYTES_PER_MEG;
389  d_target_size = d_max_cache_size_in_bytes * 0.8;
390  BESDEBUG("cache", "BESFileLockingCache::m_initialize_cache_info() - d_max_cache_size_in_bytes: " << d_max_cache_size_in_bytes
391  << " d_target_size: "<<d_target_size<< endl);
392 
393 
394  m_check_ctor_params(); // Throws BESInternalError on error.
395 
396  d_cache_info = BESUtil::assemblePath(d_cache_dir, d_prefix+".cache_control", true);
397 
398  BESDEBUG("cache", "BESFileLockingCache::m_initialize_cache_info() - d_cache_info: " << d_cache_info << endl);
399 
400  // See if we can create it. If so, that means it doesn't exist. So make it and
401  // set the cache initial size to zero.
402  if (createLockedFile(d_cache_info, d_cache_info_fd)) {
403  // initialize the cache size to zero
404  unsigned long long size = 0;
405  if (write(d_cache_info_fd, &size, sizeof(unsigned long long)) != sizeof(unsigned long long))
406  throw BESInternalError("Could not write size info to the cache info file `"+d_cache_info+"`", __FILE__, __LINE__);
407 
408  // This leaves the d_cache_info_fd file descriptor open
409  unlock_cache();
410  }
411  else {
412  if ((d_cache_info_fd = open(d_cache_info.c_str(), O_RDWR)) == -1) {
413  throw BESInternalError(get_errno(), __FILE__, __LINE__);
414  }
415  }
416 
417  // cerr << endl << "***** BESFileLockingCache::m_initialize_cache_info() completed. d_cache_info_fd: " << d_cache_info_fd << endl;
418 
419  BESDEBUG("cache", "BESFileLockingCache::m_initialize_cache_info() - d_cache_info_fd: " << d_cache_info_fd << endl);
420  BESDEBUG("cache", "BESFileLockingCache::m_initialize_cache_info() - END" << endl);
421 }
422 
423 
437 const string chars_excluded_from_filenames = "<>=,/()\"\':? []()$";
438 string BESFileLockingCache::get_cache_file_name(const string &src, bool mangle)
439 {
440  // Old way of building String, retied 10/02/2015 - ndp
441  // Return d_cache_dir + "/" + d_prefix + BESFileLockingCache::DAP_CACHE_CHAR + target;
442  BESDEBUG("cache", "BESFileLockingCache::get_cache_file_name() - src: '" << src << "' mangle: "<< mangle << endl);
443 
444  string target = src;
445 
446  target = BESUtil::assemblePath( getCacheFilePrefix(), src);
447  BESDEBUG("cache", "BESFileLockingCache::get_cache_file_name() - target: '" << target << "'" << endl);
448 
449  if (mangle) {
450  if (target.at(0) == '/') {
451  target = src.substr(1, target.length() - 1);
452  }
453 
454  string::size_type pos = target.find_first_of(chars_excluded_from_filenames);
455  while (pos != string::npos) {
456  target.replace(pos, 1, "#", 1);
457  pos = target.find_first_of(chars_excluded_from_filenames);
458  }
459 
460 
461  }
462  BESDEBUG("cache", "BESFileLockingCache::get_cache_file_name() - target: '" << target << "'" << endl);
463 
464  if(target.length()>254){
465  ostringstream msg;
466  msg << "[ERROR} OOPS! Cache filename is longer than 254 characters. The file system is going to balk at that. name.length: ";
467  msg << target.length() << " characters. name.value: " << target;
468  BESDEBUG("cache","BESFileLockingCache::get_cache_file_name() - " << msg.str() << endl);
469  throw BESInternalError(msg.str(), __FILE__, __LINE__);
470 
471  }
472 
473  target = BESUtil::assemblePath( getCacheDirectory(), target, true);
474 
475  BESDEBUG("cache", "BESFileLockingCache::get_cache_file_name() - d_cache_dir: '" << d_cache_dir << "'" << endl);
476  BESDEBUG("cache", "BESFileLockingCache::get_cache_file_name() - d_prefix: '" << d_prefix << "'" << endl);
477  BESDEBUG("cache", "BESFileLockingCache::get_cache_file_name() - target: '" << target << "'" << endl);
478 
479  return target;
480 }
481 
499 bool BESFileLockingCache::get_read_lock(const string &target, int &fd)
500 {
501  lock_cache_read();
502 
503  bool status = getSharedLock(target, fd);
504 
505  BESDEBUG("cache2", "BESFileLockingCache::get_read_lock() - " << target << " (status: " << status << ", fd: " << fd << ")" << endl);
506 
507  if (status)
508  m_record_descriptor(target, fd);
509 
510  unlock_cache();
511 
512  return status;
513 }
514 
527 bool BESFileLockingCache::create_and_lock(const string &target, int &fd)
528 {
530 
531  bool status = createLockedFile(target, fd);
532 
533  BESDEBUG("cache2", "BESFileLockingCache::create_and_lock() - " << target << " (status: " << status << ", fd: " << fd << ")" << endl);
534 
535  if (status)
536  m_record_descriptor(target, fd);
537 
538  unlock_cache();
539 
540  return status;
541 
542 }
543 
558 {
559  struct flock lock;
560  lock.l_type = F_RDLCK;
561  lock.l_whence = SEEK_SET;
562  lock.l_start = 0;
563  lock.l_len = 0;
564  lock.l_pid = getpid();
565 
566  if (fcntl(fd, F_SETLKW, &lock) == -1) {
567  throw BESInternalError(get_errno(), __FILE__, __LINE__);
568  }
569 }
570 
580 {
581  BESDEBUG("cache", "BESFileLockingCache::lock_cache_write() - d_cache_info_fd: " << d_cache_info_fd << endl);
582 
583  if (fcntl(d_cache_info_fd, F_SETLKW, lock(F_WRLCK)) == -1) {
584  throw BESInternalError("An error occurred trying to lock the cache-control file" + get_errno(), __FILE__, __LINE__);
585  }
586 }
587 
592 {
593  BESDEBUG("cache", "BESFileLockingCache::lock_cache_read() - d_cache_info_fd: " << d_cache_info_fd << endl);
594 
595  if (fcntl(d_cache_info_fd, F_SETLKW, lock(F_RDLCK)) == -1) {
596  throw BESInternalError("An error occurred trying to lock the cache-control file" + get_errno(), __FILE__, __LINE__);
597  }
598 }
599 
606 {
607  BESDEBUG("cache", "BESFileLockingCache::unlock_cache() - d_cache_info_fd: " << d_cache_info_fd << endl);
608 
609  if (fcntl(d_cache_info_fd, F_SETLK, lock(F_UNLCK)) == -1) {
610  throw BESInternalError("An error occurred trying to unlock the cache-control file" + get_errno(), __FILE__, __LINE__);
611  }
612 }
613 
629 void BESFileLockingCache::unlock_and_close(const string &file_name)
630 {
631  BESDEBUG("cache2", "BESFileLockingCache::unlock_and_close() - BEGIN file: " << file_name << endl);
632 
633  int fd = m_get_descriptor(file_name); // returns -1 when no more files desp. remain
634  while (fd != -1) {
635  unlock(fd);
636  fd = m_get_descriptor(file_name);
637  }
638  BESDEBUG("cache2", "BESFileLockingCache::unlock_and_close() - END"<< endl);
639 
640 }
641 
652 unsigned long long BESFileLockingCache::update_cache_info(const string &target)
653 {
654  unsigned long long current_size;
655  try {
657 
658  if (lseek(d_cache_info_fd, 0, SEEK_SET) == -1)
659  throw BESInternalError("Could not rewind to front of cache info file.", __FILE__, __LINE__);
660 
661  // read the size from the cache info file
662  if (read(d_cache_info_fd, &current_size, sizeof(unsigned long long)) != sizeof(unsigned long long))
663  throw BESInternalError("Could not get read size info from the cache info file!", __FILE__, __LINE__);
664 
665  struct stat buf;
666  int statret = stat(target.c_str(), &buf);
667  if (statret == 0)
668  current_size += buf.st_size;
669  else
670  throw BESInternalError("Could not read the size of the new file: " + target + " : " + get_errno(), __FILE__, __LINE__);
671 
672  BESDEBUG("cache", "BESFileLockingCache::update_cache_info() - cache size updated to: " << current_size << endl);
673 
674  if (lseek(d_cache_info_fd, 0, SEEK_SET) == -1)
675  throw BESInternalError("Could not rewind to front of cache info file.", __FILE__, __LINE__);
676 
677  if(write(d_cache_info_fd, &current_size, sizeof(unsigned long long)) != sizeof(unsigned long long))
678  throw BESInternalError("Could not write size info from the cache info file!", __FILE__, __LINE__);
679 
680  unlock_cache();
681  }
682  catch (...) {
683  unlock_cache();
684  throw;
685  }
686 
687  return current_size;
688 }
689 
694 bool BESFileLockingCache::cache_too_big(unsigned long long current_size) const
695 {
696  return current_size > d_max_cache_size_in_bytes;
697 }
698 
707 {
708  unsigned long long current_size;
709  try {
710  lock_cache_read();
711 
712  if (lseek(d_cache_info_fd, 0, SEEK_SET) == -1)
713  throw BESInternalError("Could not rewind to front of cache info file.", __FILE__, __LINE__);
714  // read the size from the cache info file
715  if(read(d_cache_info_fd, &current_size, sizeof(unsigned long long)) != sizeof(unsigned long long))
716  throw BESInternalError("Could not get read size info from the cache info file!", __FILE__, __LINE__);
717 
718  unlock_cache();
719  }
720  catch(...) {
721  unlock_cache();
722  throw;
723  }
724 
725  return current_size;
726 }
727 
728 
729 static bool entry_op(cache_entry &e1, cache_entry &e2)
730 {
731  return e1.time < e2.time;
732 }
733 
735 unsigned long long BESFileLockingCache::m_collect_cache_dir_info(CacheFiles &contents)
736 {
737  DIR *dip = opendir(d_cache_dir.c_str());
738  if (!dip)
739  throw BESInternalError("Unable to open cache directory " + d_cache_dir, __FILE__, __LINE__);
740 
741  struct dirent *dit;
742  vector<string> files;
743  // go through the cache directory and collect all of the files that
744  // start with the matching prefix
745  while ((dit = readdir(dip)) != NULL) {
746  string dirEntry = dit->d_name;
747  if (dirEntry.compare(0, d_prefix.length(), d_prefix) == 0 && dirEntry!=d_cache_info) {
748  files.push_back(d_cache_dir + "/" + dirEntry);
749  }
750  }
751 
752  closedir(dip);
753 
754  unsigned long long current_size = 0;
755  struct stat buf;
756  for (vector<string>::iterator file = files.begin(); file != files.end(); ++file) {
757  if (stat(file->c_str(), &buf) == 0) {
758  current_size += buf.st_size;
759  cache_entry entry;
760  entry.name = *file;
761  entry.size = buf.st_size;
762  entry.time = buf.st_atime;
763  // Sanity check; Removed after initial testing since some files might be zero bytes
764 #if 0
765  if (entry.size == 0)
766  throw BESInternalError("Zero-byte file found in cache. " + *file, __FILE__, __LINE__);
767 #endif
768  contents.push_back(entry);
769  }
770  }
771 
772  // Sort so smaller (older) times are first.
773  contents.sort(entry_op);
774 
775  return current_size;
776 }
777 
789 void BESFileLockingCache::update_and_purge(const string &new_file)
790 {
791  BESDEBUG("cache", "purge - starting the purge" << endl);
792 
793  try {
795 
796  CacheFiles contents;
797  unsigned long long computed_size = m_collect_cache_dir_info(contents);
798 #if 0
799  if (BESISDEBUG( "cache_contents" )) {
800  BESDEBUG("cache", "BEFORE Purge " << computed_size/BYTES_PER_MEG << endl );
801  CacheFiles::iterator ti = contents.begin();
802  CacheFiles::iterator te = contents.end();
803  for (; ti != te; ti++) {
804  BESDEBUG("cache", (*ti).time << ": " << (*ti).name << ": size " << (*ti).size/BYTES_PER_MEG << endl );
805  }
806  }
807 #endif
808  BESDEBUG("cache", "BESFileLockingCache::update_and_purge() - current and target size (in MB) " << computed_size/BYTES_PER_MEG << ", " << d_target_size/BYTES_PER_MEG << endl );
809 
810  // This deletes files and updates computed_size
811  if (cache_too_big(computed_size)) {
812 
813  // d_target_size is 80% of the maximum cache size.
814  // Grab the first which is the oldest in terms of access time.
815  CacheFiles::iterator i = contents.begin();
816  while (i != contents.end() && computed_size > d_target_size) {
817  // Grab an exclusive lock but do not block - if another process has the file locked
818  // just move on to the next file. Also test to see if the current file is the file
819  // this process just added to the cache - don't purge that!
820  int cfile_fd;
821  if (i->name != new_file && getExclusiveLockNB(i->name, cfile_fd)) {
822  BESDEBUG("cache", "purge: " << i->name << " removed." << endl );
823 
824  if (unlink(i->name.c_str()) != 0)
825  throw BESInternalError("Unable to purge the file " + i->name + " from the cache: " + get_errno(), __FILE__, __LINE__);
826 
827  unlock(cfile_fd);
828  computed_size -= i->size;
829  }
830  ++i;
831 
832  BESDEBUG("cache", "BESFileLockingCache::update_and_purge() - current and target size (in MB) " << computed_size/BYTES_PER_MEG << ", " << d_target_size/BYTES_PER_MEG << endl );
833  }
834 
835  }
836 
837  if (lseek(d_cache_info_fd, 0, SEEK_SET) == -1)
838  throw BESInternalError("Could not rewind to front of cache info file.", __FILE__, __LINE__);
839 
840  if(write(d_cache_info_fd, &computed_size, sizeof(unsigned long long)) != sizeof(unsigned long long))
841  throw BESInternalError("Could not write size info to the cache info file!", __FILE__, __LINE__);
842 #if 0
843  if (BESISDEBUG( "cache_contents" )) {
844  contents.clear();
845  computed_size = m_collect_cache_dir_info(contents);
846  BESDEBUG("cache", "AFTER Purge " << computed_size/BYTES_PER_MEG << endl );
847  CacheFiles::iterator ti = contents.begin();
848  CacheFiles::iterator te = contents.end();
849  for (; ti != te; ti++) {
850  BESDEBUG("cache", (*ti).time << ": " << (*ti).name << ": size " << (*ti).size/BYTES_PER_MEG << endl );
851  }
852  }
853 #endif
854  unlock_cache();
855  }
856  catch(...) {
857  unlock_cache();
858  throw;
859  }
860 }
861 
873 void BESFileLockingCache::purge_file(const string &file)
874 {
875  BESDEBUG("cache", "BESFileLockingCache::purge_file() - starting the purge" << endl);
876 
877  try {
879 
880  // Grab an exclusive lock on the file
881  int cfile_fd;
882  if (getExclusiveLock(file, cfile_fd)) {
883  // Get the file's size
884  unsigned long long size = 0;
885  struct stat buf;
886  if (stat(file.c_str(), &buf) == 0) {
887  size = buf.st_size;
888  }
889 
890  BESDEBUG("cache", "BESFileLockingCache::purge_file() - " << file << " removed." << endl );
891 
892  if (unlink(file.c_str()) != 0)
893  throw BESInternalError("Unable to purge the file " + file + " from the cache: " + get_errno(), __FILE__, __LINE__);
894 
895  unlock(cfile_fd);
896 
897  unsigned long long cache_size = get_cache_size() - size;
898 
899  if (lseek(d_cache_info_fd, 0, SEEK_SET) == -1)
900  throw BESInternalError("Could not rewind to front of cache info file.", __FILE__, __LINE__);
901 
902  if (write(d_cache_info_fd, &cache_size, sizeof(unsigned long long)) != sizeof(unsigned long long))
903  throw BESInternalError("Could not write size info to the cache info file!", __FILE__, __LINE__);
904  }
905 
906  unlock_cache();
907  }
908  catch (...) {
909  unlock_cache();
910  throw;
911  }
912 }
913 
914 
915 const string BESFileLockingCache::getCacheFilePrefix(){
916  return d_prefix;
917 }
918 
919 const string BESFileLockingCache::getCacheDirectory(){
920  return d_cache_dir;
921 }
922 
929 bool BESFileLockingCache::dir_exists(const string &dir)
930 {
931  struct stat buf;
932 
933  return (stat(dir.c_str(), &buf) == 0) && (buf.st_mode & S_IFDIR);
934 }
935 
943 void BESFileLockingCache::dump(ostream &strm) const
944 {
945  strm << BESIndent::LMarg << "BESFileLockingCache::dump - (" << (void *) this << ")" << endl;
946  BESIndent::Indent();
947  strm << BESIndent::LMarg << "cache dir: " << d_cache_dir << endl;
948  strm << BESIndent::LMarg << "prefix: " << d_prefix << endl;
949  strm << BESIndent::LMarg << "size (bytes): " << d_max_cache_size_in_bytes << endl;
950  BESIndent::UnIndent();
951 }
exception thrown if inernal error encountered
virtual bool create_and_lock(const string &target, int &fd)
Create a file in the cache and lock it for write access. If the file does not exist, make it, open it for read-write access and get an exclusive lock on it. The locking operation blocks, although that should never happen.
STL namespace.
virtual unsigned long long get_cache_size()
Get the cache size. Read the size information from the cache info file and return it...
static string assemblePath(const string &firstPart, const string &secondPart, bool addLeadingSlash=false)
Assemble path fragments making sure that they are separated by a single &#39;/&#39; character.
Definition: BESUtil.cc:747
static bool dir_exists(const string &dir)
virtual void dump(ostream &strm) const
dumps information about this object
Abstract exception class for the BES with basic string message.
Definition: BESError.h:56
virtual void purge_file(const string &file)
Purge a single file from the cache.
virtual bool cache_too_big(unsigned long long current_size) const
look at the cache size; is it too large? Look at the cache size and see if it is too big...
virtual void lock_cache_write()
virtual bool get_read_lock(const string &target, int &fd)
Get a read-only lock on the file if it exists.
virtual void update_and_purge(const string &new_file)
Purge files from the cache.
virtual unsigned long long update_cache_info(const string &target)
Update the cache info file to include &#39;target&#39;.
unsigned long long d_max_cache_size_in_bytes
How many bytes can the cache hold before we have to purge.
virtual void lock_cache_read()
virtual void exclusive_to_shared_lock(int fd)
Transfer from an exclusive lock to a shared lock. If the file has an exclusive write lock on it...
virtual bool getExclusiveLock(string file_name, int &ref_fd)
virtual void unlock_and_close(const string &target)