bes  Updated for version 3.20.6
GlobalMetadataStore.h
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of Hyrax, A C++ implementation of the OPeNDAP Data
4 // Access Protocol.
5 
6 // Copyright (c) 2018 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 #ifndef _global_metadata_cache_h
26 #define _global_metadata_cache_h
27 
28 #include <string>
29 #include <functional>
30 #include <fstream>
31 
32 #include "BESFileLockingCache.h"
33 #include "BESInternalFatalError.h"
34 #include "BESContainer.h"
35 
39 #define XML_BASE_MISSING_MEANS_OMIT_ATTRIBUTE 1
40 
41 namespace libdap {
42 class DapObj;
43 class DAS;
44 class DDS;
45 class DMR;
46 }
47 
48 namespace bes {
49 
90 private:
91  bool d_use_local_time; // Base on BES.LogTimeLocal
92  std::string d_ledger_name; // Name of the ledger file
93  std::string d_xml_base; // The value of the context xml:basse
94 
95  static bool d_enabled;
96  static GlobalMetadataStore *d_instance;
97 
98  std::ofstream of;
99 
100  // Called by atexit()
101  static void delete_instance() {
102  delete d_instance;
103  d_instance = 0;
104  }
105 
106  friend class DmrppMetadataStore;
107  friend class DmrppMetadataStoreTest;
108  friend class GlobalMetadataStoreTest;
109 
110 protected:
111  std::string d_ledger_entry; // Built up as info is added, written on success
112  void write_ledger();
113 
114  std::string get_hash(const std::string &name);
115 
132  struct StreamDAP : public std::unary_function<libdap::DapObj*, void> {
133  libdap::DDS *d_dds;
134  libdap::DMR *d_dmr;
135 
136  StreamDAP() : d_dds(0), d_dmr(0) {
137  throw BESInternalFatalError("Unknown DAP object type.", __FILE__, __LINE__);
138  }
139  StreamDAP(libdap::DDS *dds) : d_dds(dds), d_dmr(0) { }
140  StreamDAP(libdap::DMR *dmr) : d_dds(0), d_dmr(dmr) { }
141 
142  virtual void operator()(std::ostream &os) = 0;
143  };
144 
146  struct StreamDDS : public StreamDAP {
147  StreamDDS(libdap::DDS *dds) : StreamDAP(dds) { }
148  StreamDDS(libdap::DMR *dmr) : StreamDAP(dmr) { }
149 
150  virtual void operator()(std::ostream &os);
151  };
152 
154  struct StreamDAS : public StreamDAP {
155  StreamDAS(libdap::DDS *dds) : StreamDAP(dds) { }
156  StreamDAS(libdap::DMR *dmr) : StreamDAP(dmr) { }
157 
158  virtual void operator()(std::ostream &os);
159  };
160 
162  struct StreamDMR : public StreamDAP {
163  StreamDMR(libdap::DDS *dds) : StreamDAP(dds) { }
164  StreamDMR(libdap::DMR *dmr) : StreamDAP(dmr) { }
165 
166  virtual void operator()(std::ostream &os);
167  };
168 
169  bool store_dap_response(StreamDAP &writer, const std::string &key, const std::string &name, const std::string &response_name);
170 
171  void write_response_helper(const std::string &name, std::ostream &os, const std::string &suffix,
172  const std::string &object_name);
173 
174  // This version adds xml:base to the DMR/DMR++
175  void write_response_helper(const std::string &name, std::ostream &os, const std::string &suffix,
176  const std::string &xml_base, const std::string &object_name);
177 
178  bool remove_response_helper(const std::string& name, const std::string &suffix, const std::string &object_name);
179 
180  static void transfer_bytes(int fd, std::ostream &os);
181  static void insert_xml_base(int fd, std::ostream &os, const std::string &xml_base);
182 
183 public:
193  struct MDSReadLock : public std::unary_function<std::string, bool> {
194  std::string name;
195  bool locked;
196  GlobalMetadataStore *mds;
197  MDSReadLock() : name(""), locked(false), mds(0) { }
198  MDSReadLock(const std::string n, bool l, GlobalMetadataStore *store): name(n), locked(l), mds(store) { }
199  ~MDSReadLock() {
200  if (locked) mds->unlock_and_close(name);
201  locked = false;
202  }
203 
204  virtual bool operator()() { return locked; }
205 
206  //used to set 'locked' to false to force reload of file in cache. SBL 6/7/19
207  virtual void clearLock() {
208  if (locked) mds->unlock_and_close(name);
209  locked = false;
210  }//end clearLock()
211  };
212 
213  typedef struct MDSReadLock MDSReadLock;
214 
215 protected:
216  MDSReadLock get_read_lock_helper(const std::string &name, const std::string &suffix, const std::string &object_name);
217 
218  // Suppress the automatic generation of these ctors
220 
221  void initialize();
222 
223  // Only get_instance() should be used to instantiate this class
225  GlobalMetadataStore(const std::string &cache_dir, const std::string &prefix, unsigned long long size);
226 
227  // these are static because they are called by the static method get_instance()
228  static std::string get_cache_dir_from_config();
229  static std::string get_cache_prefix_from_config();
230  static unsigned long get_cache_size_from_config();
231 
232 public:
233  static GlobalMetadataStore *get_instance(const std::string &cache_dir, const std::string &prefix,
234  unsigned long long size);
236 
237  virtual ~GlobalMetadataStore()
238  {
239  }
240 
241  virtual bool add_responses(libdap::DDS *dds, const std::string &name);
242  virtual bool add_responses(libdap::DMR *dmr, const std::string &name);
243 
244  virtual MDSReadLock is_dmr_available(const std::string &name);
245  virtual MDSReadLock is_dmr_available(const BESContainer &container);
246  //added a third method here to handle case in build_dmrpp.cc - SBL 6.19.19
247  virtual MDSReadLock is_dmr_available(const std::string &realName, const std::string &relativeName, const std::string &fileType);
248 
249  virtual MDSReadLock is_dds_available(const std::string &name);
250  virtual MDSReadLock is_dds_available(const BESContainer &container);
251 
252  virtual MDSReadLock is_das_available(const std::string &name);
253  virtual MDSReadLock is_das_available(const BESContainer &container);
254 
255  virtual MDSReadLock is_dmrpp_available(const std::string &name);
256  virtual MDSReadLock is_dmrpp_available(const BESContainer &container);
257 
258  virtual bool is_available_helper(const std::string &realName, const std::string &relativeName, const std::string &fileType, const std::string &suffix);
259 
260  virtual time_t get_cache_lmt(const std::string &fileName, const std::string &suffix);
261 
262  virtual void write_dds_response(const std::string &name, std::ostream &os);
263  virtual void write_das_response(const std::string &name, std::ostream &os);
264 
265  // Add a third parameter to enable changing the value of xmlbase in this response.
266  // jhrg 2.28.18
267  virtual void write_dmr_response(const std::string &name, std::ostream &os);
268  virtual void write_dmrpp_response(const std::string &name, std::ostream &os);
269 
270  virtual bool remove_responses(const std::string &name);
271 
272  virtual libdap::DDS *get_dds_object(const std::string &name);
273  virtual libdap::DMR *get_dmr_object(const std::string &name);
274 
275  virtual void parse_das_from_mds(libdap::DAS*das, const std::string &name);
276 };
277 
278 } // namespace bes
279 
280 #endif // _global_metadata_cache_h
bes::GlobalMetadataStore::MDSReadLock
Unlock and close the MDS item when the ReadLock goes out of scope.
Definition: GlobalMetadataStore.h:193
BESInternalFatalError
exception thrown if an internal error is found and is fatal to the BES
Definition: BESInternalFatalError.h:43
bes::GlobalMetadataStore::StreamDDS::operator()
virtual void operator()(std::ostream &os)
Definition: GlobalMetadataStore.cc:524
bes::GlobalMetadataStore::is_das_available
virtual MDSReadLock is_das_available(const std::string &name)
Is the DAS response for.
Definition: GlobalMetadataStore.cc:879
bes::GlobalMetadataStore::get_dmr_object
virtual libdap::DMR * get_dmr_object(const std::string &name)
Build a DMR object from the cached Response.
Definition: GlobalMetadataStore.cc:1210
bes::GlobalMetadataStore::StreamDMR
Instantiate with a DDS or DMR and use to write the DMR response.
Definition: GlobalMetadataStore.h:162
bes::GlobalMetadataStore::insert_xml_base
static void insert_xml_base(int fd, std::ostream &os, const std::string &xml_base)
like transfer_bytes(), but adds the xml:base attribute to the DMR/++
Definition: GlobalMetadataStore.cc:155
bes::GlobalMetadataStore::write_dmr_response
virtual void write_dmr_response(const std::string &name, std::ostream &os)
Write the stored DMR response to a stream.
Definition: GlobalMetadataStore.cc:1106
bes::GlobalMetadataStore::remove_response_helper
bool remove_response_helper(const std::string &name, const std::string &suffix, const std::string &object_name)
Definition: GlobalMetadataStore.cc:1153
bes::GlobalMetadataStore::get_cache_lmt
virtual time_t get_cache_lmt(const std::string &fileName, const std::string &suffix)
Get the last modified time for the cached object file.
Definition: GlobalMetadataStore.cc:997
bes::GlobalMetadataStore::write_dmrpp_response
virtual void write_dmrpp_response(const std::string &name, std::ostream &os)
Write the stored DMR++ response to a stream.
Definition: GlobalMetadataStore.cc:1129
bes::GlobalMetadataStore::get_instance
static GlobalMetadataStore * get_instance()
Definition: GlobalMetadataStore.cc:336
bes::GlobalMetadataStore::transfer_bytes
static void transfer_bytes(int fd, std::ostream &os)
Definition: GlobalMetadataStore.cc:119
bes::GlobalMetadataStore::StreamDMR::operator()
virtual void operator()(std::ostream &os)
Use an object (DDS or DMR) to write data to the MDS.
Definition: GlobalMetadataStore.cc:504
bes::GlobalMetadataStore::write_das_response
virtual void write_das_response(const std::string &name, std::ostream &os)
Write the stored DAS response to a stream.
Definition: GlobalMetadataStore.cc:1094
bes::GlobalMetadataStore::StreamDAS::operator()
virtual void operator()(std::ostream &os)
Definition: GlobalMetadataStore.cc:534
libdap
Definition: BESDapFunctionResponseCache.h:35
BESFileLockingCache::unlock_and_close
virtual void unlock_and_close(const std::string &target)
Definition: BESFileLockingCache.cc:713
bes::GlobalMetadataStore
Store the DAP metadata responses.
Definition: GlobalMetadataStore.h:89
bes::GlobalMetadataStore::GlobalMetadataStore
GlobalMetadataStore()
Definition: GlobalMetadataStore.cc:399
bes::GlobalMetadataStore::remove_responses
virtual bool remove_responses(const std::string &name)
Remove all cached responses and objects for a granule.
Definition: GlobalMetadataStore.cc:1175
bes::GlobalMetadataStore::add_responses
virtual bool add_responses(libdap::DDS *dds, const std::string &name)
Add the DAP2 metadata responses using a DDS.
Definition: GlobalMetadataStore.cc:645
bes::GlobalMetadataStore::StreamDDS
Instantiate with a DDS or DMR and use to write the DDS response.
Definition: GlobalMetadataStore.h:146
bes::DmrppMetadataStore
Store the DAP DMR++ metadata responses.
Definition: DmrppMetadataStore.h:83
bes::GlobalMetadataStore::write_dds_response
virtual void write_dds_response(const std::string &name, std::ostream &os)
Write the stored DDS response to a stream.
Definition: GlobalMetadataStore.cc:1082
BESFileLockingCache
Implementation of a caching mechanism for compressed data.
Definition: BESFileLockingCache.h:85
bes::GlobalMetadataStore::get_read_lock_helper
MDSReadLock get_read_lock_helper(const std::string &name, const std::string &suffix, const std::string &object_name)
Definition: GlobalMetadataStore.cc:731
bes::GlobalMetadataStore::get_dds_object
virtual libdap::DDS * get_dds_object(const std::string &name)
Build a DDS object from the cached Response.
Definition: GlobalMetadataStore.cc:1248
bes::GlobalMetadataStore::initialize
void initialize()
Configure the ledger using LEDGER_KEY and LOCAL_TIME_KEY.
Definition: GlobalMetadataStore.cc:364
bes::GlobalMetadataStore::is_dds_available
virtual MDSReadLock is_dds_available(const std::string &name)
Is the DDS response for.
Definition: GlobalMetadataStore.cc:832
bes::GlobalMetadataStore::StreamDAS
Instantiate with a DDS or DMR and use to write the DAS response.
Definition: GlobalMetadataStore.h:154
BESContainer
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
bes::GlobalMetadataStore::get_hash
std::string get_hash(const std::string &name)
Definition: GlobalMetadataStore.cc:480
bes::GlobalMetadataStore::StreamDAP
Definition: GlobalMetadataStore.h:132
bes::GlobalMetadataStore::is_dmrpp_available
virtual MDSReadLock is_dmrpp_available(const std::string &name)
Is the DMR++ response for.
Definition: GlobalMetadataStore.cc:928
bes::GlobalMetadataStore::is_available_helper
virtual bool is_available_helper(const std::string &realName, const std::string &relativeName, const std::string &fileType, const std::string &suffix)
helper function that checks if last modified time is greater than cached file
Definition: GlobalMetadataStore.cc:969
bes::GlobalMetadataStore::write_ledger
void write_ledger()
Definition: GlobalMetadataStore.cc:444
bes::GlobalMetadataStore::is_dmr_available
virtual MDSReadLock is_dmr_available(const std::string &name)
Is the DMR response for.
Definition: GlobalMetadataStore.cc:771
bes::GlobalMetadataStore::store_dap_response
bool store_dap_response(StreamDAP &writer, const std::string &key, const std::string &name, const std::string &response_name)
Definition: GlobalMetadataStore.cc:558
bes::GlobalMetadataStore::write_response_helper
void write_response_helper(const std::string &name, std::ostream &os, const std::string &suffix, const std::string &object_name)
Definition: GlobalMetadataStore.cc:1020