bes  Updated for version 3.20.6
FONcTransmitter.cc
1 // FONcTransmitter.cc
2 
3 // This file is part of BES Netcdf File Out Module
4 
5 // Copyright (c) 2004,2005 University Corporation for Atmospheric Research
6 // Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact University Corporation for Atmospheric Research at
23 // 3080 Center Green Drive, Boulder, CO 80301
24 
25 // (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
26 // Please read the full copyright statement in the file COPYRIGHT_UCAR.
27 //
28 // Authors:
29 // pwest Patrick West <pwest@ucar.edu>
30 // jgarcia Jose Garcia <jgarcia@ucar.edu>
31 
32 #include "config.h"
33 
34 #include <stdio.h>
35 #include <stdlib.h>
36 #include <string.h>
37 
38 #ifdef HAVE_UNISTD_H
39 #include <unistd.h>
40 #endif
41 
42 #include <sys/types.h> // For umask
43 #include <sys/stat.h>
44 
45 #include <iostream>
46 #include <fstream>
47 #include <exception>
48 #include <sstream> // std::stringstream
49 #include <libgen.h>
50 
51 #include <DataDDS.h>
52 #include <BaseType.h>
53 #include <escaping.h>
54 #include <ConstraintEvaluator.h>
55 
56 #include <TheBESKeys.h>
57 #include <BESContextManager.h>
58 #include <BESDataDDSResponse.h>
59 #include <BESDapNames.h>
60 #include <BESDataNames.h>
61 #include <BESDebug.h>
62 #include <BESUtil.h>
63 #include <TempFile.h>
64 
65 #include <BESDapResponseBuilder.h>
66 
67 #include <BESError.h>
68 #include <BESDapError.h>
69 #include <BESForbiddenError.h>
70 #include <BESInternalFatalError.h>
71 #include <DapFunctionUtils.h>
72 
73 #include "FONcBaseType.h"
74 #include "FONcRequestHandler.h"
75 #include "FONcTransmitter.h"
76 #include "FONcTransform.h"
77 
78 using namespace ::libdap;
79 using namespace std;
80 
81 // size of the buffer used to read from the temporary file built on disk and
82 // send data to the client over the network connection (socket/stream)
83 #define OUTPUT_FILE_BLOCK_SIZE 4096
84 
98 {
99  add_method(DATA_SERVICE, FONcTransmitter::send_data);
100 }
101 
105 struct wrap_temp_descriptor {
106  int d_fd;
107  wrap_temp_descriptor(int fd) : d_fd(fd) {}
108  ~wrap_temp_descriptor() { close(d_fd); }
109 };
110 
111 #if 0
112 // Replaced by code in BESHandlerUtil. jhrg 8/25/17
113 
118 struct wrap_temp_name {
119  vector<char> d_name;
120  wrap_temp_name(vector<char> &name) : d_name(name) {}
121  ~wrap_temp_name() { unlink(&d_name[0]); }
122 };
123 #endif
124 
135 void updateHistoryAttribute(DDS *dds, const string ce)
136 {
137  bool foundIt = false;
138  string cf_history_entry = BESContextManager::TheManager()->get_context("cf_history_entry", foundIt);
139  if (!foundIt) {
140  // This code will be used only when the 'cf_histroy_context' is not set,
141  // which should be never in an operating server. However, when we are
142  // testing, often only the besstandalone code is running and the existing
143  // baselines don't set the context, so we have this. It must do something
144  // so the tests are not hopelessly obscure and filter out junk that varies
145  // by host (e.g., the names of cached files that have been decompressed).
146  // jhrg 6/3/16
147 
148  string request_url = dds->filename();
149  // remove path info
150  request_url = request_url.substr(request_url.find_last_of('/')+1);
151  // remove 'uncompress' cache mangling
152  request_url = request_url.substr(request_url.find_last_of('#')+1);
153  request_url += "?" + ce;
154 
155  std::stringstream ss;
156 
157  time_t raw_now;
158  struct tm * timeinfo;
159  time(&raw_now); /* get current time; same as: timer = time(NULL) */
160  timeinfo = localtime(&raw_now);
161 
162  char time_str[100];
163  // 2000-6-1 6:00:00
164  strftime(time_str, 100, "%Y-%m-%d %H:%M:%S", timeinfo);
165 
166  ss << time_str << " " << "Hyrax" << " " << request_url;
167  cf_history_entry = ss.str();
168  }
169 
170  BESDEBUG("fonc",
171  "FONcTransmitter::updateHistoryAttribute() - Adding cf_history_entry context. '" << cf_history_entry << "'" << endl);
172 
173  vector<string> hist_entry_vec;
174  hist_entry_vec.push_back(cf_history_entry);
175  BESDEBUG("fonc",
176  "FONcTransmitter::updateHistoryAttribute() - hist_entry_vec.size(): " << hist_entry_vec.size() << endl);
177 
178  // Add the new entry to the "history" attribute
179  // Get the top level Attribute table.
180  AttrTable &globals = dds->get_attr_table();
181 
182  // Since many files support "CF" conventions the history tag may already exist in the source data
183  // and we should add an entry to it if possible.
184  bool done = false; // Used to indicate that we located a toplevel ATtrTable whose name ends in "_GLOBAL" and that has an existing "history" attribute.
185  unsigned int num_attrs = globals.get_size();
186  if (num_attrs) {
187  // Here we look for a top level AttrTable whose name ends with "_GLOBAL" which is where, by convention,
188  // data ingest handlers place global level attributes found in the source dataset.
189  AttrTable::Attr_iter i = globals.attr_begin();
190  AttrTable::Attr_iter e = globals.attr_end();
191  for (; i != e && !done; i++) {
192  AttrType attrType = globals.get_attr_type(i);
193  string attr_name = globals.get_name(i);
194  // Test the entry...
195  if (attrType == Attr_container && BESUtil::endsWith(attr_name, "_GLOBAL")) {
196  // Look promising, but does it have an existing "history" Attribute?
197  AttrTable *source_file_globals = globals.get_attr_table(i);
198  AttrTable::Attr_iter history_attrItr = source_file_globals->simple_find("history");
199  if (history_attrItr != source_file_globals->attr_end()) {
200  // Yup! Add our entry...
201  BESDEBUG("fonc",
202  "FONcTransmitter::updateHistoryAttribute() - Adding history entry to " << attr_name << endl);
203  source_file_globals->append_attr("history", "string", &hist_entry_vec);
204  done = true;
205  }
206  }
207  }
208  }
209 
210  if (!done) {
211  // We never found an existing location to place the "history" entry, so we'll just stuff it into the top level AttrTable.
212  BESDEBUG("fonc",
213  "FONcTransmitter::updateHistoryAttribute() - Adding history entry to top level AttrTable" << endl);
214  globals.append_attr("history", "string", &hist_entry_vec);
215 
216  }
217 }
218 
236 {
237  BESDEBUG("fonc", "FONcTransmitter::send_data() - BEGIN" << endl);
238 
239  try { // Expanded try block so all DAP errors are caught. ndp 12/23/2015
240  BESDapResponseBuilder responseBuilder;
241  // Use the DDS from the ResponseObject along with the parameters
242  // from the DataHandlerInterface to load the DDS with values.
243  // Note that the BESResponseObject will manage the loaded_dds object's
244  // memory. Make this a shared_ptr<>. jhrg 9/6/16
245 
246  // Now that we are ready to start reading the response data we
247  // cancel any pending timeout alarm according to the configuration.
249 
250  BESDEBUG("fonc", "FONcTransmitter::send_data() - Reading data into DataDDS" << endl);
251  DDS *loaded_dds = responseBuilder.intern_dap2_data(obj, dhi);
252 
253  // ResponseBuilder splits the CE, so use the DHI or make two calls and
254  // glue the result together: responseBuilder.get_btp_func_ce() + " " + responseBuilder.get_ce()
255  // jhrg 9/6/16
256  updateHistoryAttribute(loaded_dds, dhi.data[POST_CONSTRAINT]);
257 
258 #if 0
259  // TODO Make this code and the two struct classes that wrap the name a fd part of
260  // a utility class or file. jhrg 9/7/16
261 
262  string temp_file_name = FONcRequestHandler::temp_dir + "/ncXXXXXX";
263  vector<char> temp_file(temp_file_name.length() + 1);
264  string::size_type len = temp_file_name.copy(&temp_file[0], temp_file_name.length());
265  temp_file[len] = '\0';
266  // cover the case where older versions of mkstemp() create the file using
267  // a mode of 666.
268  mode_t original_mode = umask(077);
269  int fd = mkstemp(&temp_file[0]);
270  umask(original_mode);
271 
272  // Hack: Wrap the name and file descriptors so that the descriptor is closed
273  // and temp file in unlinked no matter how we exit. jhrg 9/7/16
274  // Except if there is a hard crash.. jhrg 3/30/17
275  wrap_temp_name w_temp_file(temp_file);
276  wrap_temp_descriptor w_fd(fd);
277 
278  if (fd == -1) throw BESInternalError("Failed to open the temporary file.", __FILE__, __LINE__);
279 #endif
280  // This object closes the file when it goes out of scope.
281  bes::TempFile temp_file(FONcRequestHandler::temp_dir + "/ncXXXXXX");
282 
283  BESDEBUG("fonc", "FONcTransmitter::send_data - Building response file " << temp_file.get_name() << endl);
284  // Note that 'RETURN_CMD' is the same as the string that determines the file type:
285  // netcdf 3 or netcdf 4. Hack. jhrg 9/7/16
286  FONcTransform ft(loaded_dds, dhi, temp_file.get_name(), dhi.data[RETURN_CMD]);
287  ft.transform();
288 
289  ostream &strm = dhi.get_output_stream();
290  if (!strm) throw BESInternalError("Output stream is not set, can not return as", __FILE__, __LINE__);
291 
292  BESDEBUG("fonc", "FONcTransmitter::send_data - Transmitting temp file " << temp_file.get_name() << endl);
293 
294  FONcTransmitter::write_temp_file_to_stream(temp_file.get_fd(), strm); //, loaded_dds->filename(), ncVersion);
295  }
296  catch (Error &e) {
297  throw BESDapError("Failed to read data: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
298  }
299  catch (BESError &e) {
300  throw;
301  }
302  catch (std::exception &e) {
303  throw BESInternalError("Failed to read data: STL Error: " + string(e.what()), __FILE__, __LINE__);
304  }
305  catch (...) {
306  throw BESInternalError("Failed to get read data: Unknown exception caught", __FILE__, __LINE__);
307  }
308 
309  BESDEBUG("fonc", "FONcTransmitter::send_data - done transmitting to netcdf" << endl);
310 }
311 
321 void FONcTransmitter::write_temp_file_to_stream(int fd, ostream &strm) //, const string &filename, const string &ncVersion)
322 {
323  char block[OUTPUT_FILE_BLOCK_SIZE];
324 
325  int nbytes = read(fd, block, sizeof block);
326  while (nbytes > 0) {
327  strm.write(block, nbytes /*os.gcount()*/);
328  nbytes = read(fd, block, sizeof block);
329  }
330 }
331 
FONcTransmitter::send_data
static void send_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
The static method registered to transmit OPeNDAP data objects as a netcdf file.
Definition: FONcTransmitter.cc:235
BESUtil::conditional_timeout_cancel
static void conditional_timeout_cancel()
Definition: BESUtil.cc:967
FONcTransform::transform
virtual void transform()
Transforms each of the variables of the DataDDS to the NetCDF file.
Definition: FONcTransform.cc:127
libdap
Definition: BESDapFunctionResponseCache.h:35
bes::TempFile::get_name
std::string get_name() const
Definition: TempFile.h:70
BESDataHandlerInterface::data
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
Definition: BESDataHandlerInterface.h:90
BESUtil::endsWith
static bool endsWith(std::string const &fullString, std::string const &ending)
Definition: BESUtil.cc:942
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
BESTransmitter
Definition: BESTransmitter.h:47
BESDapResponseBuilder::intern_dap2_data
virtual libdap::DDS * intern_dap2_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
Definition: BESDapResponseBuilder.cc:980
BESDapResponseBuilder
Definition: BESDapResponseBuilder.h:53
Error
FONcTransmitter::FONcTransmitter
FONcTransmitter()
Construct the FONcTransmitter, adding it with name netcdf to be able to transmit a data response.
Definition: FONcTransmitter.cc:96
BESContextManager::get_context
virtual std::string get_context(const std::string &name, bool &found)
retrieve the value of the specified context from the BES
Definition: BESContextManager.cc:77
BESDapError
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
FONcTransform
Transformation object that converts an OPeNDAP DataDDS to a netcdf file.
Definition: FONcTransform.h:58
BESDataHandlerInterface
Structure storing information used by the BES to handle the request.
Definition: BESDataHandlerInterface.h:56
BESError
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
bes::TempFile::get_fd
int get_fd() const
Definition: TempFile.h:67
bes::TempFile
Get a new temporary file.
Definition: TempFile.h:46
BESResponseObject
Abstract base class representing a specific set of information in response to a request to the BES.
Definition: BESResponseObject.h:45