bes  Updated for version 3.20.6
BESStoredDapResultCache.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
4 // Access Protocol.
5 
6 // Copyright (c) 2011 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 #include "config.h"
26 
27 //#define DODS_DEBUG
28 
29 #include <sys/stat.h>
30 
31 #include <iostream>
32 #ifdef HAVE_TR1_FUNCTIONAL
33 #include <tr1/functional>
34 #endif
35 #include <string>
36 #include <fstream>
37 #include <sstream>
38 
39 #include <DDS.h>
40 #include <DMR.h>
41 #include <DapXmlNamespaces.h>
42 #include <ConstraintEvaluator.h>
43 #include <DDXParserSAX2.h>
44 
45 // These are needed because D4ParserSax2.h does not properly declare
46 // the classes. I think. Check on that... jhrg 3/28/14
47 #include <D4EnumDefs.h>
48 #include <D4Dimensions.h>
49 #include <D4Group.h>
50 
51 #include <D4ParserSax2.h>
52 
53 // DAP2 Stored results are not supported by default. If we do start using this.
54 // It would be better to use the CacheMarshaller and CacheUnMarshaller code
55 // since that does not translate data into network byte order. Also, there
56 // may be a bug in the XDRStreamUnMarshaller code - in/with get_opaque() - that
57 // breaks Sequence::deserialize(). jhrg 5/25/16
58 #ifdef DAP2_STORED_RESULTS
59 #include <XDRStreamMarshaller.h>
60 #include <XDRStreamUnMarshaller.h>
61 #endif
62 
63 #include <chunked_istream.h>
64 #include <D4StreamUnMarshaller.h>
65 
66 #include <debug.h>
67 #include <mime_util.h> // for last_modified_time() and rfc_822_date()
68 #include <util.h>
69 
70 #include "BESStoredDapResultCache.h"
71 #include "BESDapResponseBuilder.h"
72 #include "BESInternalError.h"
73 
74 #include "BESUtil.h"
75 #include "TheBESKeys.h"
76 #include "BESDebug.h"
77 
78 #ifdef HAVE_TR1_FUNCTIONAL
79 #define HASH_OBJ std::tr1::hash
80 #else
81 #define HASH_OBJ std::hash
82 #endif
83 
84 #define CRLF "\r\n"
85 #define BES_DATA_ROOT "BES.Data.RootDirectory"
86 #define BES_CATALOG_ROOT "BES.Catalog.catalog.RootDirectory"
87 
88 using namespace std;
89 using namespace libdap;
90 
91 BESStoredDapResultCache *BESStoredDapResultCache::d_instance = 0;
92 bool BESStoredDapResultCache::d_enabled = true;
93 const string BESStoredDapResultCache::SUBDIR_KEY = "DAP.StoredResultsCache.subdir";
94 const string BESStoredDapResultCache::PREFIX_KEY = "DAP.StoredResultsCache.prefix";
95 const string BESStoredDapResultCache::SIZE_KEY = "DAP.StoredResultsCache.size";
96 
97 unsigned long BESStoredDapResultCache::getCacheSizeFromConfig()
98 {
99  bool found;
100  string size;
101  unsigned long size_in_megabytes = 0;
102  TheBESKeys::TheKeys()->get_value(SIZE_KEY, size, found);
103  if (found) {
104  istringstream iss(size);
105  iss >> size_in_megabytes;
106  }
107  else {
108  string msg = "[ERROR] BESStoredDapResultCache::getCacheSize() - The BES Key " + SIZE_KEY
109  + " is not set! It MUST be set to utilize the Stored Result Caching system. ";
110  BESDEBUG("cache", msg << endl);
111  throw BESInternalError(msg, __FILE__, __LINE__);
112  }
113  return size_in_megabytes;
114 }
115 
116 string BESStoredDapResultCache::getSubDirFromConfig()
117 {
118  bool found;
119  string subdir = "";
120  TheBESKeys::TheKeys()->get_value(SUBDIR_KEY, subdir, found);
121 
122  if (!found) {
123  string msg = "[ERROR] BESStoredDapResultCache::getSubDirFromConfig() - The BES Key " + SUBDIR_KEY
124  + " is not set! It MUST be set to utilize the Stored Result Caching system. ";
125  BESDEBUG("cache", msg << endl);
126  throw BESInternalError(msg, __FILE__, __LINE__);
127  }
128  else {
129  while (*subdir.begin() == '/' && subdir.length() > 0) {
130  subdir = subdir.substr(1);
131  }
132  // So if it's value is "/" or the empty string then the subdir will default to the root
133  // directory of the BES data system.
134  }
135 
136  return subdir;
137 }
138 
139 string BESStoredDapResultCache::getResultPrefixFromConfig()
140 {
141  bool found;
142  string prefix = "";
143  TheBESKeys::TheKeys()->get_value(PREFIX_KEY, prefix, found);
144  if (found) {
145  prefix = BESUtil::lowercase(prefix);
146  }
147  else {
148  string msg = "[ERROR] BESStoredDapResultCache::getResultPrefix() - The BES Key " + PREFIX_KEY
149  + " is not set! It MUST be set to utilize the Stored Result Caching system. ";
150  BESDEBUG("cache", msg << endl);
151  throw BESInternalError(msg, __FILE__, __LINE__);
152  }
153 
154  return prefix;
155 }
156 
157 string BESStoredDapResultCache::getBesDataRootDirFromConfig()
158 {
159  bool found;
160  string cacheDir = "";
161  TheBESKeys::TheKeys()->get_value( BES_CATALOG_ROOT, cacheDir, found);
162  if (!found) {
163  TheBESKeys::TheKeys()->get_value( BES_DATA_ROOT, cacheDir, found);
164  if (!found) {
165  string msg = ((string) "[ERROR] BESStoredDapResultCache::getStoredResultsDir() - Neither the BES Key ")
166  + BES_CATALOG_ROOT + "or the BES key " + BES_DATA_ROOT
167  + " have been set! One MUST be set to utilize the Stored Result Caching system. ";
168  BESDEBUG("cache", msg << endl);
169  throw BESInternalError(msg, __FILE__, __LINE__);
170  }
171  }
172  return cacheDir;
173 
174 }
175 
176 BESStoredDapResultCache::BESStoredDapResultCache()
177 {
178  BESDEBUG("cache", "BESStoredDapResultCache::BESStoredDapResultCache() - BEGIN" << endl);
179 
180  d_storedResultsSubdir = getSubDirFromConfig();
181  d_dataRootDir = getBesDataRootDirFromConfig();
182  string resultsDir = BESUtil::assemblePath(d_dataRootDir, d_storedResultsSubdir);
183 
184  d_resultFilePrefix = getResultPrefixFromConfig();
185  d_maxCacheSize = getCacheSizeFromConfig();
186 
187  BESDEBUG("cache",
188  "BESStoredDapResultCache() - Stored results cache configuration params: " << resultsDir << ", " << d_resultFilePrefix << ", " << d_maxCacheSize << endl);
189 
190  initialize(resultsDir, d_resultFilePrefix, d_maxCacheSize);
191 
192  BESDEBUG("cache", "BESStoredDapResultCache::BESStoredDapResultCache() - END" << endl);
193 }
194 
198 BESStoredDapResultCache::BESStoredDapResultCache(const string &data_root_dir, const string &stored_results_subdir,
199  const string &result_file_prefix, unsigned long long max_cache_size)
200 {
201 
202  d_storedResultsSubdir = stored_results_subdir;
203  d_dataRootDir = data_root_dir;
204  d_resultFilePrefix = result_file_prefix;
205  d_maxCacheSize = max_cache_size;
206  initialize(BESUtil::assemblePath(d_dataRootDir, stored_results_subdir), d_resultFilePrefix, d_maxCacheSize);
207 }
208 
210 BESStoredDapResultCache::get_instance(const string &data_root_dir, const string &stored_results_subdir,
211  const string &result_file_prefix, unsigned long long max_cache_size)
212 {
213  if (d_enabled && d_instance == 0) {
214  if (dir_exists(data_root_dir)) {
215  d_instance = new BESStoredDapResultCache(data_root_dir, stored_results_subdir, result_file_prefix,
216  max_cache_size);
217  d_enabled = d_instance->cache_enabled();
218  if(!d_enabled){
219  delete d_instance;
220  d_instance = NULL;
221  BESDEBUG("cache", "BESStoredDapResultCache::"<<__func__ << "() - " <<
222  "Cache is DISABLED"<< endl);
223  }
224  else {
225 #ifdef HAVE_ATEXIT
226  atexit(delete_instance);
227 #endif
228  BESDEBUG("cache", "BESStoredDapResultCache::"<<__func__ << "() - " <<
229  "Cache is ENABLED"<< endl);
230  }
231  }
232  }
233  return d_instance;
234 }
235 
241 {
242  if (d_enabled && d_instance == 0) {
243  d_instance = new BESStoredDapResultCache();
244  d_enabled = d_instance->cache_enabled();
245  if(!d_enabled){
246  delete d_instance;
247  d_instance = NULL;
248  BESDEBUG("cache", "BESStoredDapResultCache::"<<__func__ << "() - " <<
249  "Cache is DISABLED"<< endl);
250  }
251  else {
252 #ifdef HAVE_ATEXIT
253  atexit(delete_instance);
254 #endif
255  BESDEBUG("cache", "BESStoredDapResultCache::"<<__func__ << "() - " <<
256  "Cache is ENABLED"<< endl);
257  }
258  }
259 
260  return d_instance;
261 }
262 
272 bool BESStoredDapResultCache::is_valid(const string &cache_file_name, const string &dataset)
273 {
274  // If the cached response is zero bytes in size, it's not valid.
275  // (hmmm...)
276 
277  off_t entry_size = 0;
278  time_t entry_time = 0;
279  struct stat buf;
280  if (stat(cache_file_name.c_str(), &buf) == 0) {
281  entry_size = buf.st_size;
282  entry_time = buf.st_mtime;
283  }
284  else {
285  return false;
286  }
287 
288  if (entry_size == 0) return false;
289 
290  time_t dataset_time = entry_time;
291  if (stat(dataset.c_str(), &buf) == 0) {
292  dataset_time = buf.st_mtime;
293  }
294 
295  // Trick: if the d_dataset is not a file, stat() returns error and
296  // the times stay equal and the code uses the cache entry.
297 
298  // TODO Fix this so that the code can get a LMT from the correct
299  // handler.
300  if (dataset_time > entry_time) return false;
301 
302  return true;
303 }
304 
305 #ifdef DAP2_STORED_RESULTS
306 
317 bool BESStoredDapResultCache::read_dap2_data_from_cache(const string &cache_file_name, DDS *fdds)
318 {
319  BESDEBUG("cache",
320  "BESStoredDapResultCache::read_dap2_data_from_cache() - Opening cache file: " << cache_file_name << endl);
321 
322  int fd = 1;
323 
324  try {
325  if (get_read_lock(cache_file_name, fd)) {
326 
327  ifstream data(cache_file_name.c_str());
328 
329  // Rip off the MIME headers from the response if they are present
330  string mime = get_next_mime_header(data);
331  while (!mime.empty()) {
332  mime = get_next_mime_header(data);
333  }
334 
335  // Parse the DDX; throw an exception on error.
336  DDXParser ddx_parser(fdds->get_factory());
337 
338  // Read the MPM boundary and then read the subsequent headers
339  string boundary = read_multipart_boundary(data);
340  BESDEBUG("cache",
341  "BESStoredDapResultCache::read_dap2_data_from_cache() - MPM Boundary: " << boundary << endl);
342 
343  read_multipart_headers(data, "text/xml", dods_ddx);
344 
345  BESDEBUG("cache",
346  "BESStoredDapResultCache::read_dap2_data_from_cache() - Read the multipart haeaders" << endl);
347 
348  // Parse the DDX, reading up to and including the next boundary.
349  // Return the CID for the matching data part
350  string data_cid;
351  try {
352  ddx_parser.intern_stream(data, fdds, data_cid, boundary);
353  BESDEBUG("cache",
354  "BESStoredDapResultCache::read_dap2_data_from_cache() - Dataset name: " << fdds->get_dataset_name() << endl);
355  }
356  catch (Error &e) {
357  BESDEBUG("cache",
358  "BESStoredDapResultCache::read_dap2_data_from_cache() - DDX Parser Error: " << e.get_error_message() << endl);
359  throw;
360  }
361 
362  // Munge the CID into something we can work with
363  BESDEBUG("cache",
364  "BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (before): " << data_cid << endl);
365  data_cid = cid_to_header_value(data_cid);
366  BESDEBUG("cache",
367  "BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (after): " << data_cid << endl);
368 
369  // Read the data part's MPM part headers (boundary was read by
370  // DDXParse::intern)
371  read_multipart_headers(data, "application/octet-stream", dods_data_ddx, data_cid);
372 
373  // Now read the data
374 
375  // XDRFileUnMarshaller um(data);
376  XDRStreamUnMarshaller um(data);
377  for (DDS::Vars_iter i = fdds->var_begin(); i != fdds->var_end(); i++) {
378  (*i)->deserialize(um, fdds);
379  }
380 
381  data.close();
382  unlock_and_close(cache_file_name /* was fd */);
383  return true;
384  }
385  else {
386  BESDEBUG("cache", "BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
387 
388  return false;
389  }
390  }
391  catch (...) {
392  BESDEBUG("cache",
393  "BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl);
394  // I think this call is not needed. jhrg 10/23/12
395  if (fd != -1) unlock_and_close(cache_file_name /* was fd */);
396  throw;
397  }
398 }
399 #endif
400 
412 bool BESStoredDapResultCache::read_dap4_data_from_cache(const string &cache_file_name, libdap::DMR *dmr)
413 {
414  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - BEGIN" << endl);
415 
416  int fd = 1;
417 
418  try {
419  if (get_read_lock(cache_file_name, fd)) {
420  BESDEBUG("cache",
421  "BESStoredDapResultCache::read_dap4_data_from_cache() - Opening cache file: " << cache_file_name << endl);
422  fstream in(cache_file_name.c_str(), ios::in | ios::binary);
423 
424  // Gobble up the response's initial set of MIME headers. Normally
425  // a client would extract information from these headers.
426  // NOTE - I am dumping this call because it basically just
427  // slurps up lines until it finds a blank line, regardless of what the
428  // lines actually have in the. So basically if the stream DOESN't have
429  // a mime header then this call will read (and ignore) the entire
430  // XML encoding of the DMR. doh.
431  // remove_mime_header(in);
432 
433  chunked_istream cis(in, CHUNK_SIZE);
434 
435  bool debug = BESDebug::IsSet("parser");
436 
437  // parse the DMR, stopping when the boundary is found.
438  // force chunk read
439  // get chunk size
440  int chunk_size = cis.read_next_chunk();
441 
442  BESDEBUG("cache",
443  "BESStoredDapResultCache::read_dap4_data_from_cache() - First chunk_size: " << chunk_size << endl);
444 
445  if (chunk_size == EOF) {
446  throw InternalErr(__FILE__, __LINE__,
447  "BESStoredDapResultCache::read_dap4_data_from_cache() - Failed to read first chunk from file. Chunk size = EOF (aka "
448  + libdap::long_to_string(EOF) + ")");
449  }
450 
451  // get chunk
452  char chunk[chunk_size];
453  cis.read(chunk, chunk_size);
454  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Read first chunk." << endl);
455 
456  // parse char * with given size
457  D4ParserSax2 parser;
458  // '-2' to discard the CRLF pair
459  parser.intern(chunk, chunk_size - 2, dmr, debug);
460  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Parsed first chunk." << endl);
461 
462  D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
463 
464  dmr->root()->deserialize(um, *dmr);
465  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Deserialized data." << endl);
466 
467  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - END" << endl);
468 
469  in.close();
470  unlock_and_close(cache_file_name /* was fd */);
471 
472  return true;
473 
474  }
475  else {
476  BESDEBUG("cache", "BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
477 
478  return false;
479 
480  }
481  }
482  catch (...) {
483  BESDEBUG("cache",
484  "BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl);
485  // I think this call is not needed. jhrg 10/23/12
486  if (fd != -1) unlock_and_close(cache_file_name /* was fd */);
487  throw;
488  }
489 }
490 
491 #ifdef DAP2_STORED_RESULTS
492 
496 DDS *
497 BESStoredDapResultCache::get_cached_dap2_data_ddx(const string &cache_file_name, BaseTypeFactory *factory,
498  const string &filename)
499 {
500  BESDEBUG("cache",
501  "BESStoredDapResultCache::get_cached_dap2_data_ddx() - Reading cache for " << cache_file_name << endl);
502 
503  DDS *fdds = new DDS(factory);
504 
505  if (read_dap2_data_from_cache(cache_file_name, fdds)) {
506 
507  fdds->filename(filename);
508  //fdds->set_dataset_name( "function_result_" + name_path(filename) ) ;
509 
510  BESDEBUG("cache", "DDS Filename: " << fdds->filename() << endl);
511  BESDEBUG("cache", "DDS Dataset name: " << fdds->get_dataset_name() << endl);
512 
513  fdds->set_factory(0);
514 
515  // mark everything as read. and send. That is, make sure that when a response
516  // is retrieved from the cache, all of the variables are marked as to be sent
517  DDS::Vars_iter i = fdds->var_begin();
518  while (i != fdds->var_end()) {
519  (*i)->set_read_p(true);
520  (*i++)->set_send_p(true);
521  }
522 
523  return fdds;
524  }
525  else {
526  delete fdds;
527  return 0;
528  }
529 
530 }
531 #endif
532 
537 DMR *
538 BESStoredDapResultCache::get_cached_dap4_data(const string &cache_file_name, libdap::D4BaseTypeFactory *factory,
539  const string &filename)
540 {
541  BESDEBUG("cache",
542  "BESStoredDapResultCache::get_cached_dap4_data() - Reading cache for " << cache_file_name << endl);
543 
544  DMR *fdmr = new DMR(factory);
545 
546  BESDEBUG("cache", "BESStoredDapResultCache::get_cached_dap4_data() - DMR Filename: " << fdmr->filename() << endl);
547  fdmr->set_filename(filename);
548 
549  if (read_dap4_data_from_cache(cache_file_name, fdmr)) {
550  BESDEBUG("cache",
551  "BESStoredDapResultCache::get_cached_dap4_data() - DMR Dataset name: " << fdmr->name() << endl);
552 
553  fdmr->set_factory(0);
554 
555  // mark everything as read. and send. That is, make sure that when a response
556  // is retrieved from the cache, all of the variables are marked as to be sent
557  fdmr->root()->set_send_p(true);
558  fdmr->root()->set_read_p(true);
559 
560  return fdmr;
561  }
562 
563  return 0;
564 }
565 
566 #ifdef DAP2_STORED_RESULTS
567 
571 string BESStoredDapResultCache::store_dap2_result(DDS &dds, const string &constraint, BESDapResponseBuilder *rb,
572  ConstraintEvaluator *eval)
573 {
574  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - BEGIN" << endl);
575  // These are used for the cached or newly created DDS object
576  BaseTypeFactory factory;
577 
578  // Get the cache filename for this thing. Do not use the default
579  // name mangling; instead use what build_cache_file_name() does.
580  string local_id = get_stored_result_local_id(dds.filename(), constraint, DAP_3_2);
581  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - local_id: "<< local_id << endl);
582  string cache_file_name = get_cache_file_name(local_id, /*mangle*/false);
583  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - cache_file_name: "<< cache_file_name << endl);
584  int fd;
585  try {
586  // If the object in the cache is not valid, remove it. The read_lock will
587  // then fail and the code will drop down to the create_and_lock() call.
588  // is_valid() tests for a non-zero object and for d_dateset newer than
589  // the cached object.
590  if (!is_valid(cache_file_name, dds.filename())) purge_file(cache_file_name);
591 
592  if (get_read_lock(cache_file_name, fd)) {
593  BESDEBUG("cache",
594  "BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
595  }
596  else if (create_and_lock(cache_file_name, fd)) {
597  // If here, the cache_file_name could not be locked for read access;
598  // try to build it. First make an empty file and get an exclusive lock on it.
599  BESDEBUG("cache",
600  "BESStoredDapResultCache::store_dap2_result() - cache_file_name " << cache_file_name << ", constraint: " << constraint << endl);
601 
602 #if 0 // I shut this off because we know that the constraint and functions have already been evaluated - ndp
603  DDS *fdds;
604 
605  fdds = new DDS(dds);
606  eval->parse_constraint(constraint, *fdds);
607 
608  if (eval->function_clauses()) {
609  DDS *temp_fdds = eval->eval_function_clauses(*fdds);
610  delete fdds;
611  fdds = temp_fdds;
612  }
613 #endif
614 
615  ofstream data_stream(cache_file_name.c_str());
616  if (!data_stream)
617  throw InternalErr(__FILE__, __LINE__,
618  "Could not open '" + cache_file_name + "' to write cached response.");
619 
620  string start = "dataddx_cache_start", boundary = "dataddx_cache_boundary";
621 
622  // Use a ConstraintEvaluator that has not parsed a CE so the code can use
623  // the send method(s)
624  ConstraintEvaluator eval;
625 
626  // Setting the version to 3.2 causes send_data_ddx to write the MIME headers that
627  // the cache expects.
628  dds.set_dap_version("3.2");
629 
630  // This is a bit of a hack, but it effectively uses ResponseBuilder to write the
631  // cached object/response without calling the machinery in one of the send_*()
632  // methods. Those methods assume they need to evaluate the BESDapResponseBuilder's
633  // CE, which is not necessary and will alter the values of the send_p property
634  // of the DDS's variables.
635  set_mime_multipart(data_stream, boundary, start, dods_data_ddx, x_plain,
636  last_modified_time(rb->get_dataset_name()));
637  //data_stream << flush;
638  rb->serialize_dap2_data_ddx(data_stream, (DDS**) &dds, eval, boundary, start);
639  //data_stream << flush;
640 
641  data_stream << CRLF << "--" << boundary << "--" << CRLF;
642 
643  data_stream.close();
644 
645  // Change the exclusive lock on the new file to a shared lock. This keeps
646  // other processes from purging the new file and ensures that the reading
647  // process can use it.
648  exclusive_to_shared_lock(fd);
649 
650  // Now update the total cache size info and purge if needed. The new file's
651  // name is passed into the purge method because this process cannot detect its
652  // own lock on the file.
653  unsigned long long size = update_cache_info(cache_file_name);
654  if (cache_too_big(size)) update_and_purge(cache_file_name);
655 
656  }
657  // get_read_lock() returns immediately if the file does not exist,
658  // but blocks waiting to get a shared lock if the file does exist.
659  else if (get_read_lock(cache_file_name, fd)) {
660  BESDEBUG("cache",
661  "BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
662  }
663  else {
664  throw InternalErr(__FILE__, __LINE__,
665  "BESStoredDapResultCache::store_dap2_result() - Cache error during function invocation.");
666  }
667 
668  BESDEBUG("cache",
669  "BESStoredDapResultCache::store_dap2_result() - unlocking and closing cache file "<< cache_file_name << endl);
670  unlock_and_close(cache_file_name);
671  }
672  catch (...) {
673  BESDEBUG("cache",
674  "BESStoredDapResultCache::store_dap2_result() - caught exception, unlocking cache and re-throw." << endl);
675  // I think this call is not needed. jhrg 10/23/12
676  unlock_cache();
677  throw;
678  }
679 
680  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - END (local_id=`"<< local_id << "')" << endl);
681  return local_id;
682 }
683 #endif
684 
692 string BESStoredDapResultCache::get_stored_result_local_id(const string &dataset, const string &ce,
693  libdap::DAPVersion version)
694 {
695  BESDEBUG("cache", "get_stored_result_local_id() - BEGIN. dataset: " << dataset << ", ce: " << ce << endl);
696  std::ostringstream ostr;
697  HASH_OBJ<std::string> str_hash;
698  string name = dataset + "#" + ce;
699  ostr << str_hash(name);
700  string hashed_name = ostr.str();
701  BESDEBUG("cache", "get_stored_result_local_id() - hashed_name: " << hashed_name << endl);
702 
703  string suffix = "";
704  switch (version) {
705 #ifdef DAP2_STORED_RESULTS
706  case DAP_2_0:
707  suffix = ".dods";
708  break;
709 
710  case DAP_3_2:
711  suffix = ".data_ddx";
712  break;
713 #endif
714  case DAP_4_0:
715  suffix = ".dap";
716  break;
717 
718  default:
719  throw BESInternalError("BESStoredDapResultCache::get_stored_result_local_id() - Unrecognized DAP version!!",
720  __FILE__, __LINE__);
721  break;
722  }
723 
724  BESDEBUG("cache", "get_stored_result_local_id() - Data file suffix: " << suffix << endl);
725 
726  string local_id = d_resultFilePrefix + hashed_name + suffix;
727  BESDEBUG("cache", "get_stored_result_local_id() - file: " << local_id << endl);
728 
729  local_id = BESUtil::assemblePath(d_storedResultsSubdir, local_id);
730 
731  BESDEBUG("cache", "get_stored_result_local_id() - END. local_id: " << local_id << endl);
732  return local_id;
733 }
734 
739 string BESStoredDapResultCache::store_dap4_result(DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
740 {
741  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - BEGIN" << endl);
742  // These are used for the cached or newly created DDS object
743  BaseTypeFactory factory;
744 
745  // Get the cache filename for this thing. Do not use the default
746  // name mangling; instead use what build_cache_file_name() does.
747  string local_id = get_stored_result_local_id(dmr.filename(), constraint, DAP_4_0);
748  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - local_id: "<< local_id << endl);
749  string cache_file_name = get_cache_file_name(local_id, /*mangle*/false);
750  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - cache_file_name: "<< cache_file_name << endl);
751  int fd;
752  try {
753  // If the object in the cache is not valid, remove it. The read_lock will
754  // then fail and the code will drop down to the create_and_lock() call.
755  // is_valid() tests for a non-zero object and for d_dateset newer than
756  // the cached object.
757  if (!is_valid(cache_file_name, dmr.filename())) {
758  BESDEBUG("cache",
759  "BESStoredDapResultCache::store_dap4_result() - File is not valid. Purging file from cache. filename: " << cache_file_name << endl);
760  purge_file(cache_file_name);
761  }
762 
763  if (get_read_lock(cache_file_name, fd)) {
764  BESDEBUG("cache",
765  "BESStoredDapResultCache::store_dap4_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
766  }
767  else if (create_and_lock(cache_file_name, fd)) {
768  // If here, the cache_file_name could not be locked for read access;
769  // try to build it. First make an empty file and get an exclusive lock on it.
770  BESDEBUG("cache",
771  "BESStoredDapResultCache::store_dap4_result() - cache_file_name: " << cache_file_name << ", constraint: " << constraint << endl);
772 
773  ofstream data_stream(cache_file_name.c_str());
774  if (!data_stream)
775  throw InternalErr(__FILE__, __LINE__,
776  "Could not open '" + cache_file_name + "' to write cached response.");
777 
778  //data_stream << flush;
779  rb->serialize_dap4_data(data_stream, dmr, false);
780  //data_stream << flush;
781 
782  data_stream.close();
783 
784  // Change the exclusive lock on the new file to a shared lock. This keeps
785  // other processes from purging the new file and ensures that the reading
786  // process can use it.
787  exclusive_to_shared_lock(fd);
788 
789  // Now update the total cache size info and purge if needed. The new file's
790  // name is passed into the purge method because this process cannot detect its
791  // own lock on the file.
792  unsigned long long size = update_cache_info(cache_file_name);
793  if (cache_too_big(size)) update_and_purge(cache_file_name);
794  }
795  // get_read_lock() returns immediately if the file does not exist,
796  // but blocks waiting to get a shared lock if the file does exist.
797  else if (get_read_lock(cache_file_name, fd)) {
798  BESDEBUG("cache",
799  "BESStoredDapResultCache::store_dap4_result() - Couldn't create and lock file, But I got a read lock. " "Result may have been created by another process. " "Not rewriting file: " << cache_file_name << endl);
800  }
801  else {
802  throw InternalErr(__FILE__, __LINE__,
803  "BESStoredDapResultCache::store_dap4_result() - Cache error during function invocation.");
804  }
805 
806  BESDEBUG("cache",
807  "BESStoredDapResultCache::store_dap4_result() - unlocking and closing cache file "<< cache_file_name << endl);
808  unlock_and_close(cache_file_name);
809  }
810  catch (...) {
811  BESDEBUG("cache",
812  "BESStoredDapResultCache::store_dap4_result() - caught exception, unlocking cache and re-throw." << endl);
813  // I think this call is not needed. jhrg 10/23/12
814  unlock_cache();
815  throw;
816  }
817 
818  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - END (local_id=`"<< local_id << "')" << endl);
819  return local_id;
820 
821 }
822 
BESDapResponseBuilder::get_dataset_name
virtual std::string get_dataset_name() const
Get the dataset name.
Definition: BESDapResponseBuilder.cc:251
BESStoredDapResultCache::store_dap4_result
virtual string store_dap4_result(libdap::DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
Definition: BESStoredDapResultCache.cc:739
BESStoredDapResultCache::get_cached_dap4_data
libdap::DMR * get_cached_dap4_data(const string &cache_file_name, libdap::D4BaseTypeFactory *factory, const string &filename)
Definition: BESStoredDapResultCache.cc:538
BESUtil::assemblePath
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
Definition: BESUtil.cc:821
libdap
Definition: BESDapFunctionResponseCache.h:35
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:62
BESDebug::IsSet
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:157
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
BESStoredDapResultCache::get_instance
static BESStoredDapResultCache * get_instance()
Definition: BESStoredDapResultCache.cc:240
TheBESKeys::get_value
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:272
BESStoredDapResultCache
Definition: BESStoredDapResultCache.h:52
BESDapResponseBuilder::serialize_dap4_data
virtual void serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers=true)
Definition: BESDapResponseBuilder.cc:1444
BESDapResponseBuilder
Definition: BESDapResponseBuilder.h:53
Error
BESUtil::lowercase
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:200