bes  Updated for version 3.17.0
BESStoredDapResultCache.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
4 // Access Protocol.
5 
6 // Copyright (c) 2011 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 #include "config.h"
26 
27 //#define DODS_DEBUG
28 
29 #include <sys/stat.h>
30 
31 #include <iostream>
32 #ifdef HAVE_TR1_FUNCTIONAL
33 #include <tr1/functional>
34 #endif
35 #include <string>
36 #include <fstream>
37 #include <sstream>
38 
39 #include <DDS.h>
40 #include <DMR.h>
41 #include <DapXmlNamespaces.h>
42 #include <ConstraintEvaluator.h>
43 #include <DDXParserSAX2.h>
44 
45 // These are needed because D4ParserSax2.h does not properly declare
46 // the classes. I think. Check on that... jhrg 3/28/14
47 #include <D4EnumDefs.h>
48 #include <D4Dimensions.h>
49 #include <D4Group.h>
50 
51 #include <D4ParserSax2.h>
52 #include <XDRStreamMarshaller.h>
53 #include <XDRStreamUnMarshaller.h>
54 #include <chunked_istream.h>
55 #include <D4StreamUnMarshaller.h>
56 //<XDRFileUnMarshaller.h>
57 #include <debug.h>
58 #include <mime_util.h> // for last_modified_time() and rfc_822_date()
59 #include <util.h>
60 
61 #include "BESStoredDapResultCache.h"
62 #include "BESDapResponseBuilder.h"
63 #include "BESInternalError.h"
64 
65 #include "BESUtil.h"
66 #include "TheBESKeys.h"
67 #include "BESDebug.h"
68 
69 #ifdef HAVE_TR1_FUNCTIONAL
70 #define HASH_OBJ std::tr1::hash
71 #else
72 #define HASH_OBJ std::hash
73 #endif
74 
75 #define CRLF "\r\n"
76 #define BES_DATA_ROOT "BES.Data.RootDirectory"
77 #define BES_CATALOG_ROOT "BES.Catalog.catalog.RootDirectory"
78 
79 using namespace std;
80 using namespace libdap;
81 
82 BESStoredDapResultCache *BESStoredDapResultCache::d_instance = 0;
83 const string BESStoredDapResultCache::SUBDIR_KEY = "DAP.StoredResultsCache.subdir";
84 const string BESStoredDapResultCache::PREFIX_KEY = "DAP.StoredResultsCache.prefix";
85 const string BESStoredDapResultCache::SIZE_KEY = "DAP.StoredResultsCache.size";
86 
87 unsigned long BESStoredDapResultCache::getCacheSizeFromConfig(){
88 
89  bool found;
90  string size;
91  unsigned long size_in_megabytes = 0;
92  TheBESKeys::TheKeys()->get_value( SIZE_KEY, size, found ) ;
93  if( found ) {
94  istringstream iss(size);
95  iss >> size_in_megabytes;
96  }
97  else {
98  string msg = "[ERROR] BESStoredDapResultCache::getCacheSize() - The BES Key " + SIZE_KEY + " is not set! It MUST be set to utilize the Stored Result Caching system. ";
99  BESDEBUG("cache", msg << endl);
100  throw BESInternalError(msg , __FILE__, __LINE__);
101  }
102  return size_in_megabytes;
103 }
104 
105 string BESStoredDapResultCache::getSubDirFromConfig(){
106  bool found;
107  string subdir = "";
108  TheBESKeys::TheKeys()->get_value( SUBDIR_KEY, subdir, found ) ;
109 
110  if( !found ) {
111  string msg = "[ERROR] BESStoredDapResultCache::getSubDirFromConfig() - The BES Key " + SUBDIR_KEY + " is not set! It MUST be set to utilize the Stored Result Caching system. ";
112  BESDEBUG("cache", msg << endl);
113  throw BESInternalError(msg , __FILE__, __LINE__);
114  }
115  else {
116  while(*subdir.begin() == '/' && subdir.length()>0){
117  subdir = subdir.substr(1);
118  }
119  // So if it's value is "/" or the empty string then the subdir will default to the root
120  // directory of the BES data system.
121  }
122 
123  return subdir;
124 }
125 
126 
127 string BESStoredDapResultCache::getResultPrefixFromConfig(){
128  bool found;
129  string prefix = "";
130  TheBESKeys::TheKeys()->get_value( PREFIX_KEY, prefix, found ) ;
131  if( found ) {
132  prefix = BESUtil::lowercase( prefix ) ;
133  }
134  else {
135  string msg = "[ERROR] BESStoredDapResultCache::getResultPrefix() - The BES Key " + PREFIX_KEY + " is not set! It MUST be set to utilize the Stored Result Caching system. ";
136  BESDEBUG("cache", msg << endl);
137  throw BESInternalError(msg , __FILE__, __LINE__);
138  }
139 
140  return prefix;
141 }
142 
143 
144 string BESStoredDapResultCache::getBesDataRootDirFromConfig(){
145  bool found;
146  string cacheDir = "";
147  TheBESKeys::TheKeys()->get_value( BES_CATALOG_ROOT, cacheDir, found ) ;
148  if( !found ) {
149  TheBESKeys::TheKeys()->get_value( BES_DATA_ROOT, cacheDir, found ) ;
150  if( !found ) {
151  string msg = ((string)"[ERROR] BESStoredDapResultCache::getStoredResultsDir() - Neither the BES Key ") + BES_CATALOG_ROOT +
152  "or the BES key " + BES_DATA_ROOT + " have been set! One MUST be set to utilize the Stored Result Caching system. ";
153  BESDEBUG("cache", msg << endl);
154  throw BESInternalError(msg , __FILE__, __LINE__);
155  }
156  }
157  return cacheDir;
158 
159 }
160 
161 #if 0
162 string BESStoredDapResultCache::assemblePath(const string &firstPart, const string &secondPart, bool addLeadingSlash){
163 
164  //BESDEBUG("cache", "BESStoredDapResultCache::assemblePath() - BEGIN" << endl);
165  //BESDEBUG("cache", "BESStoredDapResultCache::assemblePath() - firstPart: "<< firstPart << endl);
166  //BESDEBUG("cache", "BESStoredDapResultCache::assemblePath() - secondPart: "<< secondPart << endl);
167 
168  string firstPathFragment = firstPart;
169  string secondPathFragment = secondPart;
170 
171 
172  if(addLeadingSlash){
173  if(*firstPathFragment.begin() != '/')
174  firstPathFragment = "/" + firstPathFragment;
175  }
176 
177  // make sure there are not multiple slashes at the end of the first part...
178  while(*firstPathFragment.rbegin() == '/' && firstPathFragment.length()>0){
179  firstPathFragment = firstPathFragment.substr(0,firstPathFragment.length()-1);
180  // BESDEBUG("cache", "BESStoredDapResultCache::assemblePath() - firstPathFragment: "<< firstPathFragment << endl);
181  }
182 
183  // make sure first part ends with a "/"
184  if(*firstPathFragment.rbegin() != '/'){
185  firstPathFragment += "/";
186  }
187  //BESDEBUG("cache", "BESStoredDapResultCache::assemblePath() - firstPathFragment: "<< firstPathFragment << endl);
188 
189  // make sure second part does not BEGIN with a slash
190  while(*secondPathFragment.begin() == '/' && secondPathFragment.length()>0){
191  secondPathFragment = secondPathFragment.substr(1);
192  }
193 
194  //BESDEBUG("cache", "BESStoredDapResultCache::assemblePath() - secondPathFragment: "<< secondPathFragment << endl);
195 
196  string newPath = firstPathFragment + secondPathFragment;
197 
198  //BESDEBUG("cache", "BESStoredDapResultCache::assemblePath() - newPath: "<< newPath << endl);
199  //BESDEBUG("cache", "BESStoredDapResultCache::assemblePath() - END" << endl);
200 
201  return newPath;
202 }
203 #endif
204 
205 BESStoredDapResultCache::BESStoredDapResultCache(){
206  BESDEBUG("cache", "BESStoredDapResultCache::BESStoredDapResultCache() - BEGIN" << endl);
207 
208  d_storedResultsSubdir = getSubDirFromConfig();
209  d_dataRootDir = getBesDataRootDirFromConfig();
210  string resultsDir = BESUtil::assemblePath(d_dataRootDir,d_storedResultsSubdir);
211 
212  d_resultFilePrefix = getResultPrefixFromConfig();
213  d_maxCacheSize = getCacheSizeFromConfig();
214 
215  BESDEBUG("cache", "BESStoredDapResultCache() - Stored results cache configuration params: " << resultsDir << ", " << d_resultFilePrefix << ", " << d_maxCacheSize << endl);
216 
217  initialize(resultsDir, d_resultFilePrefix, d_maxCacheSize);
218 
219  BESDEBUG("cache", "BESStoredDapResultCache::BESStoredDapResultCache() - END" << endl);
220 }
221 
222 
226 BESStoredDapResultCache::BESStoredDapResultCache( const string &data_root_dir, const string &stored_results_subdir, const string &result_file_prefix, unsigned long long max_cache_size){
227 
228  d_storedResultsSubdir = stored_results_subdir;
229  d_dataRootDir = data_root_dir;
230  d_resultFilePrefix = result_file_prefix;
231  d_maxCacheSize = max_cache_size;
232  initialize(BESUtil::assemblePath(d_dataRootDir,stored_results_subdir), d_resultFilePrefix, d_maxCacheSize);
233 }
234 
235 
237 BESStoredDapResultCache::get_instance(const string &data_root_dir, const string &stored_results_subdir, const string &result_file_prefix, unsigned long long max_cache_size)
238 {
239  if (d_instance == 0){
240  if(dir_exists(data_root_dir)){
241  try {
242  d_instance = new BESStoredDapResultCache(data_root_dir, stored_results_subdir, result_file_prefix, max_cache_size);
243 #ifdef HAVE_ATEXIT
244  atexit(delete_instance);
245 #endif
246  }
247  catch(BESInternalError &bie){
248  BESDEBUG("cache", "[ERROR] BESStoredDapResultCache::get_instance(): Failed to obtain cache! msg: " << bie.get_message() << endl);
249  }
250  }
251  }
252  return d_instance;
253 }
254 
260 {
261  if (d_instance == 0) {
262  try {
263  d_instance = new BESStoredDapResultCache();
264 #ifdef HAVE_ATEXIT
265  atexit(delete_instance);
266 #endif
267  }
268  catch(BESInternalError &bie){
269  BESDEBUG("cache", "[ERROR] BESStoredDapResultCache::get_instance(): Failed to obtain cache! msg: " << bie.get_message() << endl);
270  }
271  }
272 
273  return d_instance;
274 }
275 
276 
286 bool BESStoredDapResultCache::is_valid(const string &cache_file_name, const string &dataset)
287 {
288  // If the cached response is zero bytes in size, it's not valid.
289  // (hmmm...)
290 
291  off_t entry_size = 0;
292  time_t entry_time = 0;
293  struct stat buf;
294  if (stat(cache_file_name.c_str(), &buf) == 0) {
295  entry_size = buf.st_size;
296  entry_time = buf.st_mtime;
297  }
298  else {
299  return false;
300  }
301 
302  if (entry_size == 0)
303  return false;
304 
305  time_t dataset_time = entry_time;
306  if (stat(dataset.c_str(), &buf) == 0) {
307  dataset_time = buf.st_mtime;
308  }
309 
310  // Trick: if the d_dataset is not a file, stat() returns error and
311  // the times stay equal and the code uses the cache entry.
312 
313  // TODO Fix this so that the code can get a LMT from the correct
314  // handler.
315  if (dataset_time > entry_time)
316  return false;
317 
318  return true;
319 }
320 
332 bool BESStoredDapResultCache::read_dap2_data_from_cache(const string &cache_file_name, DDS *fdds)
333 {
334  BESDEBUG("cache", "BESStoredDapResultCache::read_dap2_data_from_cache() - Opening cache file: " << cache_file_name << endl);
335 
336  int fd = 1;
337 
338  try {
339  if (get_read_lock(cache_file_name, fd)) {
340 
341  ifstream data(cache_file_name.c_str());
342 
343  // Rip off the MIME headers from the response if they are present
344  string mime = get_next_mime_header(data);
345  while (!mime.empty()) {
346  mime = get_next_mime_header(data);
347  }
348 
349  // Parse the DDX; throw an exception on error.
350  DDXParser ddx_parser(fdds->get_factory());
351 
352  // Read the MPM boundary and then read the subsequent headers
353  string boundary = read_multipart_boundary(data);
354  BESDEBUG("cache", "BESStoredDapResultCache::read_dap2_data_from_cache() - MPM Boundary: " << boundary << endl);
355 
356  read_multipart_headers(data, "text/xml", dods_ddx);
357 
358  BESDEBUG("cache", "BESStoredDapResultCache::read_dap2_data_from_cache() - Read the multipart haeaders" << endl);
359 
360  // Parse the DDX, reading up to and including the next boundary.
361  // Return the CID for the matching data part
362  string data_cid;
363  try {
364  ddx_parser.intern_stream(data, fdds, data_cid, boundary);
365  BESDEBUG("cache", "BESStoredDapResultCache::read_dap2_data_from_cache() - Dataset name: " << fdds->get_dataset_name() << endl);
366  }
367  catch(Error &e) {
368  BESDEBUG("cache", "BESStoredDapResultCache::read_dap2_data_from_cache() - DDX Parser Error: " << e.get_error_message() << endl);
369  throw;
370  }
371 
372  // Munge the CID into something we can work with
373  BESDEBUG("cache", "BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (before): " << data_cid << endl);
374  data_cid = cid_to_header_value(data_cid);
375  BESDEBUG("cache", "BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (after): " << data_cid << endl);
376 
377  // Read the data part's MPM part headers (boundary was read by
378  // DDXParse::intern)
379  read_multipart_headers(data, "application/octet-stream", dods_data_ddx, data_cid);
380 
381  // Now read the data
382 
383  // XDRFileUnMarshaller um(data);
384  XDRStreamUnMarshaller um(data);
385  for (DDS::Vars_iter i = fdds->var_begin(); i != fdds->var_end(); i++) {
386  (*i)->deserialize(um, fdds);
387  }
388 
389  data.close();
390  unlock_and_close(cache_file_name /* was fd */);
391  return true;
392  }
393  else {
394  BESDEBUG( "cache", "BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
395 
396  return false;
397  }
398  }
399  catch (...) {
400  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl );
401  // I think this call is not needed. jhrg 10/23/12
402  if (fd != -1)
403  unlock_and_close(cache_file_name /* was fd */);
404  throw;
405  }
406 }
407 
419 bool BESStoredDapResultCache::read_dap4_data_from_cache(const string &cache_file_name, libdap::DMR *dmr)
420 {
421  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - BEGIN" << endl);
422 
423 
424  int fd = 1;
425 
426  try {
427  if (get_read_lock(cache_file_name, fd)) {
428  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Opening cache file: " << cache_file_name << endl);
429  fstream in(cache_file_name.c_str(), ios::in|ios::binary);
430 
431  // Gobble up the response's initial set of MIME headers. Normally
432  // a client would extract information from these headers.
433  // NOTE - I am dumping this call because it basically just
434  // slurps up lines until it finds a blank line, regardless of what the
435  // lines actually have in the. So basically if the stream DOESN't have
436  // a mime header then this call will read (and ignore) the entire
437  // XML encoding of the DMR. doh.
438  // remove_mime_header(in);
439 
440  chunked_istream cis(in, CHUNK_SIZE);
441 
442  bool debug = BESDebug::IsSet( "parser" );
443 
444  // parse the DMR, stopping when the boundary is found.
445  // force chunk read
446  // get chunk size
447  int chunk_size = cis.read_next_chunk();
448 
449  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - First chunk_size: " << chunk_size << endl);
450 
451 
452  if(chunk_size == EOF){
453  throw InternalErr(__FILE__, __LINE__, "BESStoredDapResultCache::read_dap4_data_from_cache() - Failed to read first chunk from file. Chunk size = EOF (aka " + libdap::long_to_string(EOF) + ")");
454  }
455 
456  // get chunk
457  char chunk[chunk_size];
458  cis.read(chunk, chunk_size);
459  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Read first chunk." << endl);
460 
461  // parse char * with given size
462  D4ParserSax2 parser;
463  // '-2' to discard the CRLF pair
464  parser.intern(chunk, chunk_size-2, dmr, debug);
465  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Parsed first chunk." << endl);
466 
467  D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
468 
469  dmr->root()->deserialize(um, *dmr);
470  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Deserialized data." << endl);
471 
472  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - END" << endl);
473 
474  in.close();
475  unlock_and_close(cache_file_name /* was fd */);
476 
477  return true;
478 
479  }
480  else {
481  BESDEBUG( "cache", "BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
482 
483  return false;
484 
485  }
486  }
487  catch (...) {
488  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl );
489  // I think this call is not needed. jhrg 10/23/12
490  if (fd != -1)
491  unlock_and_close(cache_file_name /* was fd */);
492  throw;
493  }
494 }
495 
500 DDS *
501 BESStoredDapResultCache::get_cached_dap2_data_ddx(const string &cache_file_name, BaseTypeFactory *factory, const string &filename)
502 {
503  BESDEBUG("cache", "BESStoredDapResultCache::get_cached_dap2_data_ddx() - Reading cache for " << cache_file_name << endl);
504 
505  DDS *fdds = new DDS(factory);
506 
507 
508  if(read_dap2_data_from_cache(cache_file_name, fdds)){
509 
510 
511  fdds->filename(filename) ;
512  //fdds->set_dataset_name( "function_result_" + name_path(filename) ) ;
513 
514  BESDEBUG("cache", "DDS Filename: " << fdds->filename() << endl);
515  BESDEBUG("cache", "DDS Dataset name: " << fdds->get_dataset_name() << endl);
516 
517  fdds->set_factory( 0 ) ;
518 
519  // mark everything as read. and send. That is, make sure that when a response
520  // is retrieved from the cache, all of the variables are marked as to be sent
521  DDS::Vars_iter i = fdds->var_begin();
522  while(i != fdds->var_end()) {
523  (*i)->set_read_p( true );
524  (*i++)->set_send_p(true);
525  }
526 
527  return fdds;
528  }
529  else {
530  delete fdds;
531  return 0;
532  }
533 
534 
535 }
536 
537 
542 DMR *
543 BESStoredDapResultCache::get_cached_dap4_data(const string &cache_file_name, libdap::D4BaseTypeFactory *factory, const string &filename)
544 {
545  BESDEBUG("cache", "BESStoredDapResultCache::get_cached_dap4_data() - Reading cache for " << cache_file_name << endl);
546 
547  DMR *fdmr = new DMR(factory);
548 
549  BESDEBUG("cache", "BESStoredDapResultCache::get_cached_dap4_data() - DMR Filename: " << fdmr->filename() << endl);
550  fdmr->set_filename(filename) ;
551 
552  if(read_dap4_data_from_cache(cache_file_name, fdmr)){
553  BESDEBUG("cache", "BESStoredDapResultCache::get_cached_dap4_data() - DMR Dataset name: " << fdmr->name() << endl);
554 
555  fdmr->set_factory( 0 ) ;
556 
557  // mark everything as read. and send. That is, make sure that when a response
558  // is retrieved from the cache, all of the variables are marked as to be sent
559  fdmr->root()->set_send_p(true);
560  fdmr->root()->set_read_p(true);
561 
562  return fdmr;
563  }
564 
565  return 0;
566 }
567 
572 string BESStoredDapResultCache::store_dap2_result(DDS &dds, const string &constraint, BESDapResponseBuilder *rb, ConstraintEvaluator *eval)
573 {
574  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - BEGIN" << endl );
575  // These are used for the cached or newly created DDS object
576  BaseTypeFactory factory;
577 
578  // Get the cache filename for this thing. Do not use the default
579  // name mangling; instead use what build_cache_file_name() does.
580  string local_id = get_stored_result_local_id(dds.filename(), constraint, DAP_3_2);
581  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - local_id: "<< local_id << endl );
582  string cache_file_name = get_cache_file_name(local_id, /*mangle*/false);
583  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - cache_file_name: "<< cache_file_name << endl );
584  int fd;
585  try {
586  // If the object in the cache is not valid, remove it. The read_lock will
587  // then fail and the code will drop down to the create_and_lock() call.
588  // is_valid() tests for a non-zero object and for d_dateset newer than
589  // the cached object.
590  if (!is_valid(cache_file_name, dds.filename()))
591  purge_file(cache_file_name);
592 
593  if (get_read_lock(cache_file_name, fd)) {
594  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
595  }
596  else if (create_and_lock(cache_file_name, fd)) {
597  // If here, the cache_file_name could not be locked for read access;
598  // try to build it. First make an empty file and get an exclusive lock on it.
599  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - cache_file_name " << cache_file_name << ", constraint: " << constraint << endl);
600  DDS *fdds;
601 
602  fdds = new DDS(dds);
603  eval->parse_constraint(constraint, *fdds);
604 
605  if (eval->function_clauses()) {
606  DDS *temp_fdds = eval->eval_function_clauses(*fdds);
607  delete fdds;
608  fdds = temp_fdds;
609  }
610 
611  ofstream data_stream(cache_file_name.c_str());
612  if (!data_stream)
613  throw InternalErr(__FILE__, __LINE__, "Could not open '" + cache_file_name + "' to write cached response.");
614 
615  string start="dataddx_cache_start", boundary="dataddx_cache_boundary";
616 
617  // Use a ConstraintEvaluator that has not parsed a CE so the code can use
618  // the send method(s)
619  ConstraintEvaluator eval;
620 
621  // Setting the version to 3.2 causes send_data_ddx to write the MIME headers that
622  // the cache expects.
623  fdds->set_dap_version("3.2");
624 
625  // This is a bit of a hack, but it effectively uses ResponseBuilder to write the
626  // cached object/response without calling the machinery in one of the send_*()
627  // methods. Those methods assume they need to evaluate the BESDapResponseBuilder's
628  // CE, which is not necessary and will alter the values of the send_p property
629  // of the DDS's variables.
630  set_mime_multipart(data_stream, boundary, start, dods_data_ddx, x_plain, last_modified_time(rb->get_dataset_name()));
631  //data_stream << flush;
632  rb->serialize_dap2_data_ddx(data_stream, *fdds, eval, boundary, start);
633  //data_stream << flush;
634 
635  data_stream << CRLF << "--" << boundary << "--" << CRLF;
636 
637  data_stream.close();
638 
639  // Change the exclusive lock on the new file to a shared lock. This keeps
640  // other processes from purging the new file and ensures that the reading
641  // process can use it.
642  exclusive_to_shared_lock(fd);
643 
644  // Now update the total cache size info and purge if needed. The new file's
645  // name is passed into the purge method because this process cannot detect its
646  // own lock on the file.
647  unsigned long long size = update_cache_info(cache_file_name);
648  if (cache_too_big(size))
649  update_and_purge(cache_file_name);
653  delete fdds;
654  }
655  // get_read_lock() returns immediately if the file does not exist,
656  // but blocks waiting to get a shared lock if the file does exist.
657  else if (get_read_lock(cache_file_name, fd)) {
658  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
659  }
660  else {
661  throw InternalErr(__FILE__, __LINE__, "BESStoredDapResultCache::store_dap2_result() - Cache error during function invocation.");
662  }
663 
664  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - unlocking and closing cache file "<< cache_file_name << endl );
665  unlock_and_close(cache_file_name);
666  }
667  catch (...) {
668  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - caught exception, unlocking cache and re-throw." << endl );
669  // I think this call is not needed. jhrg 10/23/12
670  unlock_cache();
671  throw;
672  }
673 
674  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - END (local_id=`"<< local_id << "')" << endl );
675  return local_id;
676 }
677 
685 string
686 BESStoredDapResultCache::get_stored_result_local_id(const string &dataset, const string &ce, libdap::DAPVersion version )
687 {
688  BESDEBUG("cache", "get_stored_result_local_id() - BEGIN. dataset: " << dataset << ", ce: " << ce << endl);
689  std::ostringstream ostr;
690  HASH_OBJ<std::string> str_hash;
691  string name = dataset + "#" + ce;
692  ostr << str_hash(name);
693  string hashed_name = ostr.str();
694  BESDEBUG("cache", "get_stored_result_local_id() - hashed_name: " << hashed_name << endl);
695 
696  string suffix="";
697  switch(version){
698  case DAP_2_0:
699  suffix = ".dods";
700  break;
701 
702  case DAP_3_2:
703  suffix = ".data_ddx";
704  break;
705 
706  case DAP_4_0:
707  suffix = ".dap";
708  break;
709  default:
710  throw BESInternalError("BESStoredDapResultCache::get_stored_result_local_id() - Unrecognized DAP version!!", __FILE__, __LINE__);
711  break;
712  }
713 
714  BESDEBUG("cache", "get_stored_result_local_id() - Data file suffix: " << suffix << endl);
715 
716 
717  string local_id = d_resultFilePrefix + hashed_name + suffix;
718  BESDEBUG("cache", "get_stored_result_local_id() - file: " << local_id << endl);
719 
720  local_id = BESUtil::assemblePath(d_storedResultsSubdir,local_id);
721 
722  BESDEBUG("cache", "get_stored_result_local_id() - END. local_id: " << local_id << endl);
723  return local_id;
724 }
725 
726 
727 
728 #if 0
729 
742 string BESStoredDapResultCache::get_cache_file_name(const string &local_id, bool mangle)
743 {
744  if(local_id.empty()){
745  throw BESInternalError("BESStoredDapResultCache: The target cache file name must not be an empty string. Srsly.", __FILE__, __LINE__);
746  }
747 
748  string cacheFile = assemblePath(d_dataRootDir,local_id);
749 
750  BESDEBUG("cache", "BESStoredDapResultCache::get_cache_file_name() - local_id: '" << local_id << "'" << endl);
751  BESDEBUG("cache", "BESStoredDapResultCache::get_cache_file_name() - cacheDir: '" << cacheFile << "'" << endl);
752 
753  if(mangle){
754  BESDEBUG("cache", "[WARNING] BESStoredDapResultCache::get_cache_file_name() - The parameter 'mangle' is ignored!" << endl);
755  }
756 
757 
758  return cacheFile;
759 }
760 #endif
761 
766 string BESStoredDapResultCache::store_dap4_result(DMR &dmr, const string &constraint, BESDapResponseBuilder *rb){
767  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - BEGIN" << endl );
768  // These are used for the cached or newly created DDS object
769  BaseTypeFactory factory;
770 
771  // Get the cache filename for this thing. Do not use the default
772  // name mangling; instead use what build_cache_file_name() does.
773  string local_id = get_stored_result_local_id(dmr.filename(), constraint, DAP_4_0);
774  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - local_id: "<< local_id << endl );
775  string cache_file_name = get_cache_file_name(local_id, /*mangle*/false);
776  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - cache_file_name: "<< cache_file_name << endl );
777  int fd;
778  try {
779  // If the object in the cache is not valid, remove it. The read_lock will
780  // then fail and the code will drop down to the create_and_lock() call.
781  // is_valid() tests for a non-zero object and for d_dateset newer than
782  // the cached object.
783  if (!is_valid(cache_file_name, dmr.filename())){
784  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - File is not valid. Purging file from cache. filename: " << cache_file_name << endl);
785  purge_file(cache_file_name);
786  }
787 
788  if (get_read_lock(cache_file_name, fd)) {
789  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
790  }
791  else if (create_and_lock(cache_file_name, fd)) {
792  // If here, the cache_file_name could not be locked for read access;
793  // try to build it. First make an empty file and get an exclusive lock on it.
794  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - cache_file_name: " << cache_file_name << ", constraint: " << constraint << endl);
795 
796  ofstream data_stream(cache_file_name.c_str());
797  if (!data_stream)
798  throw InternalErr(__FILE__, __LINE__, "Could not open '" + cache_file_name + "' to write cached response.");
799 
800  //data_stream << flush;
801  rb->serialize_dap4_data(data_stream, dmr, false);
802  //data_stream << flush;
803 
804  data_stream.close();
805 
806  // Change the exclusive lock on the new file to a shared lock. This keeps
807  // other processes from purging the new file and ensures that the reading
808  // process can use it.
809  exclusive_to_shared_lock(fd);
810 
811  // Now update the total cache size info and purge if needed. The new file's
812  // name is passed into the purge method because this process cannot detect its
813  // own lock on the file.
814  unsigned long long size = update_cache_info(cache_file_name);
815  if (cache_too_big(size))
816  update_and_purge(cache_file_name);
817  }
818  // get_read_lock() returns immediately if the file does not exist,
819  // but blocks waiting to get a shared lock if the file does exist.
820  else if (get_read_lock(cache_file_name, fd)) {
821  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - Couldn't create and lock file, But I got a read lock. "
822  "Result may have been created by another process. "
823  "Not rewriting file: " << cache_file_name << endl);
824  }
825  else {
826  throw InternalErr(__FILE__, __LINE__, "BESStoredDapResultCache::store_dap4_result() - Cache error during function invocation.");
827  }
828 
829  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - unlocking and closing cache file "<< cache_file_name << endl );
830  unlock_and_close(cache_file_name);
831  }
832  catch (...) {
833  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - caught exception, unlocking cache and re-throw." << endl );
834  // I think this call is not needed. jhrg 10/23/12
835  unlock_cache();
836  throw;
837  }
838 
839  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - END (local_id=`"<< local_id << "')" << endl );
840  return local_id;
841 
842 }
843 
exception thrown if inernal error encountered
static string lowercase(const string &s)
Definition: BESUtil.cc:184
virtual std::string get_message()
get the error message for this exception
Definition: BESError.h:97
STL namespace.
virtual string store_dap4_result(libdap::DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
libdap::DMR * get_cached_dap4_data(const string &cache_file_name, libdap::D4BaseTypeFactory *factory, const string &filename)
static string assemblePath(const string &firstPart, const string &secondPart, bool addLeadingSlash=false)
Assemble path fragments making sure that they are separated by a single &#39;/&#39; character.
Definition: BESUtil.cc:747
static BESStoredDapResultCache * get_instance()
libdap::DDS * get_cached_dap2_data_ddx(const std::string &cache_file_name, libdap::BaseTypeFactory *factory, const std::string &dataset)
static bool IsSet(const string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:167
void get_value(const string &s, string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: BESKeys.cc:481
virtual void serialize_dap2_data_ddx(std::ostream &out, libdap::DDS &dds, libdap::ConstraintEvaluator &eval, const std::string &boundary, const std::string &start, bool ce_eval=true)
static BESKeys * TheKeys()
Definition: TheBESKeys.cc:43
virtual std::string get_dataset_name() const
Get the dataset name.
virtual string store_dap2_result(libdap::DDS &dds, const std::string &constraint, BESDapResponseBuilder *rb, libdap::ConstraintEvaluator *eval)
virtual void serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers=true)