bes  Updated for version 3.20.6
BESUncompressManager3.cc
1 // BESUncompressManager3.cc
2 
3 // This file is part of bes, A C++ back-end server implementation framework
4 // for the OPeNDAP Data Access Protocol.
5 
6 // Copyright (c) 2012 OPeNDAP, Inc
7 // Author: James Gallagher <jgallagher@opendap.org>
8 // Patrick West <pwest@ucar.edu> and
9 // Jose Garcia <jgarcia@ucar.edu>
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // You can contact University Corporation for Atmospheric Research at
26 // 3080 Center Green Drive, Boulder, CO 80301
27 
28 #include <sstream>
29 
30 using std::istringstream;
31 using std::endl;
32 using std::ostream;
33 using std::string;
34 
35 #include "BESUncompressManager3.h"
36 #include "BESUncompress3GZ.h"
37 #include "BESUncompress3BZ2.h"
38 #include "BESUncompress3Z.h"
39 
40 #include "BESFileLockingCache.h"
41 
42 #include "BESInternalError.h"
43 #include "BESDebug.h"
44 
45 #include "TheBESKeys.h"
46 
47 BESUncompressManager3 *BESUncompressManager3::_instance = 0;
48 
58 BESUncompressManager3::BESUncompressManager3()
59 {
63 }
64 
74 bool BESUncompressManager3::add_method(const string &name, p_bes_uncompress method)
75 {
76  BESUncompressManager3::UCIter i;
77  i = _uncompress_list.find(name);
78  if (i == _uncompress_list.end()) {
79  _uncompress_list[name] = method;
80  return true;
81  }
82  return false;
83 }
84 
93 p_bes_uncompress BESUncompressManager3::find_method(const string &name)
94 {
95  BESUncompressManager3::UCIter i;
96  i = _uncompress_list.find(name);
97  if (i != _uncompress_list.end()) {
98  return (*i).second;
99  }
100  return 0;
101 }
102 
135 bool BESUncompressManager3::uncompress(const string &src, string &cache_file, BESFileLockingCache *cache)
136 {
137  BESDEBUG( "uncompress2", "BESUncompressManager3::uncompress() - src: " << src << endl );
138 
149  if (cache == NULL) {
150  std::ostringstream oss;
151  oss << "BESUncompressManager3::" << __func__ << "() - ";
152  oss << "The supplied Cache object is NULL. Decompression Requires An Operational Cache.";
153  throw BESInternalError(oss.str(), __FILE__, __LINE__);
154  }
155 
156  // All compressed files have a 'dot extension'.
157  string::size_type dot = src.rfind(".");
158  if (dot == string::npos) {
159  BESDEBUG( "uncompress2", "BESUncompressManager3::uncompress() - no file extension" << endl );
160  return false;
161  }
162 
163  string ext = src.substr(dot + 1, src.length() - dot);
164 
165  // If there's no match for the extension, the file is not compressed and we return false.
166  // Otherwise, 'p' points to a function that uncompresses the data.
167  p_bes_uncompress p = find_method(ext);
168  if (!p) {
169  BESDEBUG( "uncompress2", "BESUncompressManager3::uncompress() - not compressed " << endl );
170  return false;
171  }
172 
173  // Get the name of the file in the cache (either the code finds this file or
174  // or it makes it).
175  cache_file = cache->get_cache_file_name(src);
176 
177  try {
178  BESDEBUG( "uncompress2", "BESUncompressManager3::uncompress() - is cached? " << src << endl );
179 
180  int fd;
181  if (cache->get_read_lock(cache_file, fd)) {
182  BESDEBUG( "uncompress", "BESUncompressManager3::uncompress() - cached hit: " << cache_file << endl );
183  return true;
184  }
185 
186  // Now we actually try to uncompress the file, given that there's not a decomp'd version
187  // in the cache. First make an empty file and get an exclusive lock on it.
188  if (cache->create_and_lock(cache_file, fd)) {
189  BESDEBUG( "uncompress", "BESUncompressManager3::uncompress() - caching " << cache_file << endl );
190 
191  // uncompress. Make sure that the decompression function does not close
192  // the file descriptor.
193  p(src, fd);
194 
195  // Change the exclusive lock on the new file to a shared lock. This keeps
196  // other processes from purging the new file and ensures that the reading
197  // process can use it.
198  cache->exclusive_to_shared_lock(fd);
199 
200  // Now update the total cache size info and purge if needed. The new file's
201  // name is passed into the purge method because this process cannot detect its
202  // own lock on the file.
203  unsigned long long size = cache->update_cache_info(cache_file);
204  if (cache->cache_too_big(size))
205  cache->update_and_purge(cache_file);
206 
207  return true;
208  }
209  else {
210  if (cache->get_read_lock(cache_file, fd)) {
211  BESDEBUG( "uncompress", "BESUncompressManager3::uncompress() - cached hit: " << cache_file << endl );
212  return true;
213  }
214  }
215 
216  return false;
217  }
218  catch (...) {
219  BESDEBUG( "uncompress", "BESUncompressManager3::uncompress() - caught exception, unlocking cache and re-throw." << endl );
220  cache->unlock_cache();
221  throw;
222  }
223 
224  return false; // gcc warns without this
225 }
226 
234 void BESUncompressManager3::dump(ostream &strm) const
235 {
236  strm << BESIndent::LMarg << "BESUncompressManager3::dump - (" << (void *) this << ")" << endl;
237  BESIndent::Indent();
238  if (_uncompress_list.size()) {
239  strm << BESIndent::LMarg << "registered uncompression methods:" << endl;
240  BESIndent::Indent();
241  BESUncompressManager3::UCIter i = _uncompress_list.begin();
242  BESUncompressManager3::UCIter ie = _uncompress_list.end();
243  for (; i != ie; i++) {
244  strm << BESIndent::LMarg << (*i).first << endl;
245  }
246  BESIndent::UnIndent();
247  }
248  else {
249  strm << BESIndent::LMarg << "registered uncompress methods: none" << endl;
250  }
251  BESIndent::UnIndent();
252 }
253 
255 BESUncompressManager3::TheManager()
256 {
257  if (_instance == 0) {
258  _instance = new BESUncompressManager3;
259  }
260  return _instance;
261 }
BESFileLockingCache::get_read_lock
virtual bool get_read_lock(const std::string &target, int &fd)
Get a read-only lock on the file if it exists.
Definition: BESFileLockingCache.cc:544
BESUncompress3GZ::uncompress
static void uncompress(const std::string &src, int dest_fd)
uncompress a file with the .gz file extension
Definition: BESUncompress3GZ.cc:56
BESFileLockingCache::create_and_lock
virtual bool create_and_lock(const std::string &target, int &fd)
Create a file in the cache and lock it for write access.
Definition: BESFileLockingCache.cc:599
BESUncompressManager3::uncompress
virtual bool uncompress(const std::string &src, std::string &target, BESFileLockingCache *cache)
If the file 'src' should be uncompressed, do so and return a new file name on the value-result param ...
Definition: BESUncompressManager3.cc:135
BESUncompressManager3::add_method
virtual bool add_method(const std::string &name, p_bes_uncompress method)
create_and_lock a uncompress method to the list
Definition: BESUncompressManager3.cc:74
BESUncompressManager3::find_method
virtual p_bes_uncompress find_method(const std::string &name)
returns the uncompression method specified
Definition: BESUncompressManager3.cc:93
BESUncompress3Z::uncompress
static void uncompress(const std::string &src, int fd)
uncompress a file with the .gz file extension
Definition: BESUncompress3Z.cc:57
BESUncompressManager3
List of all registered decompression methods.
Definition: BESUncompressManager3.h:55
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
BESFileLockingCache
Implementation of a caching mechanism for compressed data.
Definition: BESFileLockingCache.h:85
BESUncompressManager3::dump
virtual void dump(std::ostream &strm) const
dumps information about this object
Definition: BESUncompressManager3.cc:234
BESFileLockingCache::update_cache_info
virtual unsigned long long update_cache_info(const std::string &target)
Update the cache info file to include 'target'.
Definition: BESFileLockingCache.cc:737
BESFileLockingCache::get_cache_file_name
virtual std::string get_cache_file_name(const std::string &src, bool mangle=true)
Definition: BESFileLockingCache.cc:451
BESFileLockingCache::update_and_purge
virtual void update_and_purge(const std::string &new_file)
Purge files from the cache.
Definition: BESFileLockingCache.cc:940
BESFileLockingCache::unlock_cache
virtual void unlock_cache()
Definition: BESFileLockingCache.cc:686
BESUncompress3BZ2::uncompress
static void uncompress(const std::string &src, int fd)
uncompress a file with the .bz2 file extension
Definition: BESUncompress3BZ2.cc:70
BESFileLockingCache::cache_too_big
virtual bool cache_too_big(unsigned long long current_size) const
look at the cache size; is it too large? Look at the cache size and see if it is too big.
Definition: BESFileLockingCache.cc:780
BESFileLockingCache::exclusive_to_shared_lock
virtual void exclusive_to_shared_lock(int fd)
Transfer from an exclusive lock to a shared lock.
Definition: BESFileLockingCache.cc:630