bes  Updated for version 3.20.6
DmrppCommon.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of the BES
4 
5 // Copyright (c) 2016 OPeNDAP, Inc.
6 // Author: James Gallagher <jgallagher@opendap.org>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23 
24 #include <string>
25 #include <sstream>
26 #include <vector>
27 #include <iterator>
28 #include <cstdlib>
29 #include <cstring>
30 
31 #include <curl/curl.h>
32 
33 #include <BaseType.h>
34 #include <D4Attributes.h>
35 #include <XMLWriter.h>
36 
37 #include <BESIndent.h>
38 #include <BESDebug.h>
39 #include <BESLog.h>
40 #include <BESInternalError.h>
41 
42 #include "DmrppRequestHandler.h"
43 #include "DmrppCommon.h"
44 #include "Chunk.h"
45 
46 using namespace std;
47 using namespace libdap;
48 
49 namespace dmrpp {
50 
51 // Used with BESDEBUG
52 static const string dmrpp_3 = "dmrpp:3";
53 static const string dmrpp_4 = "dmrpp:4";
54 
55 bool DmrppCommon::d_print_chunks = false;
56 string DmrppCommon::d_dmrpp_ns = "http://xml.opendap.org/dap/dmrpp/1.0.0#";
57 string DmrppCommon::d_ns_prefix = "dmrpp";
58 
69 void join_threads(pthread_t threads[], unsigned int num_threads)
70 {
71  int status;
72  for (unsigned int i = 0; i < num_threads; ++i) {
73  if (threads[i]) {
74  BESDEBUG(dmrpp_3, "Join thread " << i << " after an exception was caught.");
75  if ((status = pthread_join(threads[i], NULL)) < 0)
76  LOG("Failed to join thread " << i << "during clean up from an exception: " << strerror(status) << endl);
77  }
78  }
79 }
80 
90 void DmrppCommon::parse_chunk_dimension_sizes(string chunk_dims)
91 {
92  d_chunk_dimension_sizes.clear();
93 
94  if (chunk_dims.empty()) return;
95 
96  // If the input is anything other than integers and spaces, throw
97  if (chunk_dims.find_first_not_of("1234567890 ") != string::npos)
98  throw BESInternalError("while processing chunk dimension information, illegal character(s)", __FILE__, __LINE__);
99 
100  // istringstream can parse this kind of input more easily. jhrg 4/10/18
101 
102  string space(" ");
103  size_t strPos = 0;
104  string strVal;
105 
106  // Are there spaces or multiple values?
107  if (chunk_dims.find(space) != string::npos) {
108  // Process space delimited content
109  while ((strPos = chunk_dims.find(space)) != string::npos) {
110  strVal = chunk_dims.substr(0, strPos);
111 
112  d_chunk_dimension_sizes.push_back(strtol(strVal.c_str(), NULL, 10));
113  chunk_dims.erase(0, strPos + space.length());
114  }
115  }
116 
117  // If it's multi valued there's still one more value left to process
118  // If it's single valued the same is true, so let's ingest that.
119  d_chunk_dimension_sizes.push_back(strtol(chunk_dims.c_str(), NULL, 10));
120 }
121 
128 void DmrppCommon::ingest_compression_type(string compression_type_string)
129 {
130  if (compression_type_string.empty()) return;
131 
132  // Clear previous state
133  d_deflate = false;
134  d_shuffle = false;
135 
136  string deflate("deflate");
137  string shuffle("shuffle");
138 
139  // Process content
140  if (compression_type_string.find(deflate) != string::npos) {
141  d_deflate = true;
142  }
143 
144  if (compression_type_string.find(shuffle) != string::npos) {
145  d_shuffle = true;
146  }
147 }
148 
153 unsigned long DmrppCommon::add_chunk(const string &data_url, unsigned long long size, unsigned long long offset,
154  string position_in_array)
155 {
156  d_chunks.push_back(Chunk(data_url, size, offset, position_in_array));
157 
158  return d_chunks.size();
159 }
160 
161 unsigned long DmrppCommon::add_chunk(const string &data_url, unsigned long long size, unsigned long long offset,
162  const vector<unsigned int> &position_in_array)
163 {
164  d_chunks.push_back(Chunk(data_url, size, offset, position_in_array));
165 
166  return d_chunks.size();
167 }
168 
186 char *
187 DmrppCommon::read_atomic(const string &name)
188 {
189  vector<Chunk> &chunk_refs = get_chunk_vec();
190 
191  if (chunk_refs.size() != 1)
192  throw BESInternalError(string("Expected only a single chunk for variable ") + name, __FILE__, __LINE__);
193 
194  Chunk &chunk = chunk_refs[0];
195 
196  chunk.read_chunk();
197 
198  return chunk.get_rbuf();
199 }
200 
204 void
205 DmrppCommon::print_chunks_element(XMLWriter &xml, const string &name_space)
206 {
207  // Start element "chunks" with dmrpp namespace and attributes:
208  if (xmlTextWriterStartElementNS(xml.get_writer(), (const xmlChar*)name_space.c_str(), (const xmlChar*) "chunks", NULL) < 0)
209  throw BESInternalError("Could not start chunks element.", __FILE__, __LINE__);
210 
211  string compression = "";
212  if (is_shuffle_compression() && is_deflate_compression())
213  compression = "deflate shuffle";
214  else if (is_shuffle_compression())
215  compression.append("shuffle");
216  else if (is_deflate_compression())
217  compression.append("deflate");
218 
219  if (!compression.empty())
220  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "compressionType", (const xmlChar*) compression.c_str()) < 0)
221  throw BESInternalError("Could not write compression attribute.", __FILE__, __LINE__);
222 
223  if (d_chunk_dimension_sizes.size() > 0) {
224  // Write element "chunkDimensionSizes" with dmrpp namespace:
225  ostringstream oss;
226  copy(d_chunk_dimension_sizes.begin(), d_chunk_dimension_sizes.end(), ostream_iterator<unsigned int>(oss, " "));
227  string sizes = oss.str();
228  sizes.erase(sizes.size() - 1, 1); // trim the trailing space
229 
230  if (xmlTextWriterWriteElementNS(xml.get_writer(), (const xmlChar*) name_space.c_str(), (const xmlChar*) "chunkDimensionSizes", NULL,
231  (const xmlChar*) sizes.c_str()) < 0) throw BESInternalError("Could not write chunkDimensionSizes attribute.", __FILE__, __LINE__);
232  }
233 
234  // Start elements "chunk" with dmrpp namespace and attributes:
235  for (vector<Chunk>::iterator i = get_chunk_vec().begin(), e = get_chunk_vec().end(); i != e; ++i) {
236  Chunk &chunk = *i;
237 
238  if (xmlTextWriterStartElementNS(xml.get_writer(), (const xmlChar*)name_space.c_str(), (const xmlChar*) "chunk", NULL) < 0)
239  throw BESInternalError("Could not start element chunk", __FILE__, __LINE__);
240 
241  // Get offset string:
242  ostringstream offset;
243  offset << chunk.get_offset();
244  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "offset", (const xmlChar*) offset.str().c_str()) < 0)
245  throw BESInternalError("Could not write attribute offset", __FILE__, __LINE__);
246 
247  // Get nBytes string:
248  ostringstream nBytes;
249  nBytes << chunk.get_size();
250  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "nBytes", (const xmlChar*) nBytes.str().c_str()) < 0)
251  throw BESInternalError("Could not write attribute nBytes", __FILE__, __LINE__);
252 
253  if (chunk.get_position_in_array().size() > 0) {
254  // Get position in array string:
255  vector<unsigned int> pia = chunk.get_position_in_array();
256  ostringstream oss;
257  oss << "[";
258  copy(pia.begin(), pia.end(), ostream_iterator<unsigned int>(oss, ","));
259  string pia_str = oss.str();
260  if (pia.size() > 0) pia_str.replace(pia_str.size() - 1, 1, "]"); // replace the trailing ',' with ']'
261  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "chunkPositionInArray", (const xmlChar*) pia_str.c_str()) < 0)
262  throw BESInternalError("Could not write attribute position in array", __FILE__, __LINE__);
263  }
264 
265  // End element "chunk":
266  if (xmlTextWriterEndElement(xml.get_writer()) < 0) throw BESInternalError("Could not end chunk element", __FILE__, __LINE__);
267  }
268 
269  if (xmlTextWriterEndElement(xml.get_writer()) < 0) throw BESInternalError("Could not end chunks element", __FILE__, __LINE__);
270 }
271 
282 void DmrppCommon::print_dmrpp(XMLWriter &xml, bool constrained /*false*/)
283 {
284  BaseType &bt = dynamic_cast<BaseType&>(*this);
285  if (constrained && !bt.send_p())
286  return;
287 
288  if (xmlTextWriterStartElement(xml.get_writer(), (const xmlChar*)bt.type_name().c_str()) < 0)
289  throw InternalErr(__FILE__, __LINE__, "Could not write " + bt.type_name() + " element");
290 
291  if (!bt.name().empty())
292  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "name", (const xmlChar*)bt.name().c_str()) < 0)
293  throw InternalErr(__FILE__, __LINE__, "Could not write attribute for name");
294 
295  if (bt.is_dap4())
296  bt.attributes()->print_dap4(xml);
297 
298  if (!bt.is_dap4() && bt.get_attr_table().get_size() > 0)
299  bt.get_attr_table().print_xml_writer(xml);
300 
301  // This is the code added to libdap::BaseType::print_dap4(). jhrg 5/10/18
302  if (DmrppCommon::d_print_chunks && get_immutable_chunks().size() > 0)
303  print_chunks_element(xml, DmrppCommon::d_ns_prefix);
304 
305  if (xmlTextWriterEndElement(xml.get_writer()) < 0)
306  throw InternalErr(__FILE__, __LINE__, "Could not end " + bt.type_name() + " element");
307 }
308 
309 void DmrppCommon::dump(ostream & strm) const
310 {
311  strm << BESIndent::LMarg << "is_deflate: " << (is_deflate_compression() ? "true" : "false") << endl;
312  strm << BESIndent::LMarg << "is_shuffle_compression: " << (is_shuffle_compression() ? "true" : "false") << endl;
313 
314  const vector<unsigned int> &chunk_dim_sizes = get_chunk_dimension_sizes();
315 
316  strm << BESIndent::LMarg << "chunk dimension sizes: [";
317  for (unsigned int i = 0; i < chunk_dim_sizes.size(); i++) {
318  strm << (i ? "][" : "") << chunk_dim_sizes[i];
319  }
320  strm << "]" << endl;
321 
322  const vector<Chunk> &chunk_refs = get_immutable_chunks();
323  strm << BESIndent::LMarg << "Chunks (aka chunks):" << (chunk_refs.size() ? "" : "None Found.") << endl;
324  BESIndent::Indent();
325  for (unsigned int i = 0; i < chunk_refs.size(); i++) {
326  strm << BESIndent::LMarg;
327  chunk_refs[i].dump(strm);
328  strm << endl;
329  }
330 
331  BESIndent::UnIndent();
332 }
333 
334 } // namepsace dmrpp
335 
dmrpp::Chunk
Definition: Chunk.h:43
libdap
Definition: BESDapFunctionResponseCache.h:35
dmrpp::Chunk::get_offset
virtual unsigned long long get_offset() const
Get the offset to this Chunk's data block.
Definition: Chunk.h:169
dmrpp::Chunk::get_rbuf
virtual char * get_rbuf()
Definition: Chunk.h:235
dmrpp::Chunk::get_size
virtual unsigned long long get_size() const
Get the size of this Chunk's data block on disk.
Definition: Chunk.h:161
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
dmrpp::Chunk::read_chunk
virtual void read_chunk()
Definition: Chunk.cc:541
dmrpp::Chunk::get_position_in_array
virtual const std::vector< unsigned int > & get_position_in_array() const
Definition: Chunk.h:272