37 #include <ConstraintEvaluator.h>
38 #include <DDXParserSAX2.h>
39 #include <XDRStreamMarshaller.h>
40 #include <XDRStreamUnMarshaller.h>
43 #include <mime_util.h>
58 using namespace libdap;
70 unsigned long size_in_megabytes = 0;
73 BESDEBUG(
"cache",
"In BESDapResponseCache::getDefaultCacheSize(): Located BES key " <<
74 SIZE_KEY<<
"=" << size << endl);
75 istringstream iss(size);
76 iss >> size_in_megabytes;
79 string msg =
"[ERROR] BESDapResponseCache::getCacheSize() - The BES Key " + SIZE_KEY +
" is not set! It MUST be set to utilize the DAP response cache. ";
83 return size_in_megabytes;
91 BESDEBUG(
"cache",
"In BESDapResponseCache::getDefaultCachePrefix(): Located BES key " <<
92 PREFIX_KEY<<
"=" << prefix << endl);
96 string msg =
"[ERROR] BESDapResponseCache::getCachePrefix() - The BES Key " + PREFIX_KEY +
" is not set! It MUST be set to utilize the DAP response cache. ";
107 string cacheDir =
"";
110 BESDEBUG(
"cache",
"In BESDapResponseCache::getCachePrefix(): Located BES key " <<
111 PATH_KEY<<
"=" << cacheDir << endl);
115 string msg =
"[ERROR] BESDapResponseCache::getCacheDir() - The BES Key " + PATH_KEY +
" is not set! It MUST be set to utilize the DAP response cache. ";
123 BESDapResponseCache::BESDapResponseCache(){
124 BESDEBUG(
"cache",
"In BESDapResponseCache::BESDapResponseCache()" << endl);
126 string cacheDir = getCacheDirFromConfig();
127 string prefix = getCachePrefixFromConfig();
128 unsigned long size_in_megabytes = getCacheSizeFromConfig();
130 BESDEBUG(
"cache",
"BESDapResponseCache() - Cache config params: " << cacheDir <<
", " << prefix <<
", " << size_in_megabytes << endl);
137 if (!cacheDir.empty() && size_in_megabytes > 0)
138 initialize(cacheDir, prefix, size_in_megabytes);
140 BESDEBUG(
"cache",
"Leaving BESDapResponseCache::BESDapResponseCache()" << endl);
157 BESDapResponseCache::BESDapResponseCache(
const string &cache_dir,
const string &prefix,
unsigned long long size):
BESFileLockingCache(cache_dir,prefix,size) {
176 if (d_instance == 0){
177 if(dir_exists(cache_dir)){
182 BESDEBUG(
"cache",
"BESDapResponseCache::get_instance(): Failed to obtain cache! msg: " << bie.
get_message() << endl);
195 if (d_instance == 0) {
201 BESDEBUG(
"cache",
"BESDapResponseCache::get_instance(): Failed to obtain cache! msg: " << bie.
get_message() << endl);
211 void BESDapResponseCache::delete_instance() {
212 BESDEBUG(
"cache",
"BESDapResponseCache::delete_instance() - Deleting singleton BESDapResponseCache instance." << endl);
213 cerr <<
"BESDapResponseCache::delete_instance() - Deleting singleton BESDapResponseCache instance. d_instance="<< d_instance << endl;
228 build_cache_file_name(
const string &dataset,
const string &ce)
230 BESDEBUG(
"cache",
"build_cache_file_name: dataset: " << dataset <<
", ce: " << ce << endl);
232 string name = dataset +
"#" + ce;
234 while (pos != string::npos) {
235 name.replace(pos, 1,
"#", 1);
239 BESDEBUG(
"cache",
"build_cache_file_name: name: " << name << endl);
253 bool BESDapResponseCache::is_valid(
const string &cache_file_name,
const string &dataset)
258 off_t entry_size = 0;
259 time_t entry_time = 0;
261 if (stat(cache_file_name.c_str(), &buf) == 0) {
262 entry_size = buf.st_size;
263 entry_time = buf.st_mtime;
272 time_t dataset_time = entry_time;
273 if (stat(dataset.c_str(), &buf) == 0) {
274 dataset_time = buf.st_mtime;
282 if (dataset_time > entry_time)
299 void BESDapResponseCache::read_data_from_cache(
const string &cache_file_name, DDS *fdds)
301 BESDEBUG(
"cache",
"Opening cache file: " << cache_file_name << endl);
302 ifstream data(cache_file_name.c_str());
305 string mime = get_next_mime_header(data);
306 while (!mime.empty()) {
307 mime = get_next_mime_header(data);
311 DDXParser ddx_parser(fdds->get_factory());
314 string boundary = read_multipart_boundary(data);
315 BESDEBUG(
"cache",
"MPM Boundary: " << boundary << endl);
317 read_multipart_headers(data,
"text/xml", dap4_ddx);
319 BESDEBUG(
"cache",
"Read the multipart haeaders" << endl);
325 ddx_parser.intern_stream(data, fdds, data_cid, boundary);
326 BESDEBUG(
"cache",
"Dataset name: " << fdds->get_dataset_name() << endl);
329 BESDEBUG(
"cache",
"DDX Parser Error: " << e.get_error_message() << endl);
334 BESDEBUG(
"cache",
"Data CID (before): " << data_cid << endl);
335 data_cid = cid_to_header_value(data_cid);
336 BESDEBUG(
"cache",
"Data CID (after): " << data_cid << endl);
340 read_multipart_headers(data,
"application/octet-stream", dap4_data, data_cid);
345 XDRStreamUnMarshaller um(data);
346 for (DDS::Vars_iter i = fdds->var_begin(); i != fdds->var_end(); i++) {
347 (*i)->deserialize(um, fdds);
356 BESDapResponseCache::get_cached_data_ddx(
const string &cache_file_name, BaseTypeFactory *factory,
const string &filename)
358 BESDEBUG(
"cache",
"Reading cache for " << cache_file_name << endl);
360 DDS *fdds =
new DDS(factory);
362 fdds->filename(filename) ;
365 read_data_from_cache(cache_file_name, fdds);
367 BESDEBUG(
"cache",
"DDS Filename: " << fdds->filename() << endl);
368 BESDEBUG(
"cache",
"DDS Dataset name: " << fdds->get_dataset_name() << endl);
370 fdds->set_factory( 0 ) ;
374 DDS::Vars_iter i = fdds->var_begin();
375 while(i != fdds->var_end()) {
376 (*i)->set_read_p(
true );
377 (*i++)->set_send_p(
true);
392 DDS *BESDapResponseCache::read_dataset(
const string &filename,
const string &constraint,
string &cache_token)
395 BaseTypeFactory factory;
400 string cache_file_name =
get_cache_file_name(build_cache_file_name(filename, constraint),
false);
403 if (
get_read_lock(cache_file_name, fd) && is_valid(cache_file_name, filename)) {
404 BESDEBUG(
"cache",
"function ce (change)- cached hit: " << cache_file_name << endl);
405 fdds = get_cached_data_ddx(cache_file_name, &factory, filename);
409 BESDEBUG(
"cache",
"caught exception, unlocking cache and re-throw." << endl );
415 cache_token = cache_file_name;
453 BaseTypeFactory factory;
458 string cache_file_name =
get_cache_file_name(build_cache_file_name(dds.filename(), constraint),
false);
465 if (!is_valid(cache_file_name, dds.filename()))
469 BESDEBUG(
"cache",
"function ce (change)- cached hit: " << cache_file_name << endl);
470 fdds = get_cached_data_ddx(cache_file_name, &factory, dds.filename());
475 BESDEBUG(
"cache",
"function ce - caching " << cache_file_name <<
", constraint: " << constraint << endl);
478 eval->parse_constraint(constraint, *fdds);
480 if (eval->function_clauses()) {
481 DDS *temp_fdds = eval->eval_function_clauses(*fdds);
486 ofstream data_stream(cache_file_name.c_str());
488 throw InternalErr(__FILE__, __LINE__,
"Could not open '" + cache_file_name +
"' to write cached response.");
490 string start=
"dataddx_cache_start", boundary=
"dataddx_cache_boundary";
494 ConstraintEvaluator eval;
498 fdds->set_dap_version(
"3.2");
505 set_mime_multipart(data_stream, boundary, start, dap4_data_ddx, x_plain, last_modified_time(rb->
get_dataset_name()));
510 data_stream <<
CRLF <<
"--" << boundary <<
"--" <<
CRLF;
529 BESDEBUG(
"cache",
"function ce - cached hit: " << cache_file_name << endl);
530 fdds = get_cached_data_ddx(cache_file_name, &factory, dds.get_dataset_name());
533 throw InternalErr(__FILE__, __LINE__,
"Cache error during function invocation.");
537 BESDEBUG(
"cache",
"caught exception, unlocking cache and re-throw." << endl );
543 cache_token = cache_file_name;
virtual void unlock_cache()
Unlock the cache info file.
static unsigned long getCacheSizeFromConfig()
virtual libdap::DDS * cache_dataset(libdap::DDS &dds, const std::string &constraint, BESDapResponseBuilder *rb, libdap::ConstraintEvaluator *eval, std::string &cache_token)
Get the cached DDS object.
exception thrown if inernal error encountered
virtual bool create_and_lock(const string &target, int &fd)
Create a file in the cache and lock it for write access.
static string lowercase(const string &s)
Convert a string to all lower case.
const string chars_excluded_from_filenames
static BESDapResponseCache * get_instance()
Get the default instance of the BESDapResponseCache object.
This class is used to cache DAP2 response objects.
Implementation of a caching mechanism for compressed data.
virtual string get_message()
get the error message for this exception
virtual void purge_file(const string &file)
Purge a single file from the cache.
virtual bool cache_too_big(unsigned long long current_size) const
look at the cache size; is it too large? Look at the cache size and see if it is too big...
static const string SIZE_KEY
virtual string get_cache_file_name(const string &src, bool mangle=true)
Build the name of file that will holds the uncompressed data from 'src' in the cache.
static string getCacheDirFromConfig()
static string getCachePrefixFromConfig()
virtual bool get_read_lock(const string &target, int &fd)
Get a read-only lock on the file if it exists.
void get_value(const string &s, string &val, bool &found)
Retrieve the value of a given key, if set.
virtual void update_and_purge(const string &new_file)
Purge files from the cache.
virtual unsigned long long update_cache_info(const string &target)
Update the cache info file to include 'target'.
virtual void exclusive_to_shared_lock(int fd)
Transfer from an exclusive lock to a shared lock.
This class is used to build responses for/by the BES.
#define BESDEBUG(x, y)
macro used to send debug information to the debug stream
static const string PREFIX_KEY
virtual void dataset_constraint_ddx(std::ostream &out, libdap::DDS &dds, libdap::ConstraintEvaluator &eval, const std::string &boundary, const std::string &start, bool ce_eval=true)
Build/return the DDX and the BLOB part of the DAP3.x data response.
static BESKeys * TheKeys()
virtual std::string get_dataset_name() const
The ``dataset name'' is the filename or other string that the filter program will use to access the d...
static const string PATH_KEY