bes  Updated for version 3.20.6
Chunk.h
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of the BES
4 
5 // Copyright (c) 2016 OPeNDAP, Inc.
6 // Author: Nathan Potter <ndp@opendap.org>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23 
24 #ifndef _Chunk_h
25 #define _Chunk_h 1
26 
27 #include <string>
28 #include <vector>
29 
30 #define USE_PTHREADS 1
31 
32 namespace dmrpp {
33 
34 // Callback function used by chunk readers
35 size_t chunk_write_data(void *buffer, size_t size, size_t nmemb, void *data);
36 
43 class Chunk {
44 private:
45  std::string d_data_url;
46  std::string d_query_marker;
47  unsigned long long d_size;
48  unsigned long long d_offset;
49 
50  std::vector<unsigned int> d_chunk_position_in_array;
51 
52  // These are used only during the libcurl callback;
53  // they are not duplicated by the copy ctor or assignment
54  // operator.
55  unsigned long long d_bytes_read;
56  char *d_read_buffer;
57  unsigned long long d_read_buffer_size;
58  bool d_is_read;
59  bool d_is_inflated;
60 
61  static const std::string tracking_context;
62 
63  friend class ChunkTest;
64  friend class DmrppCommonTest;
65 
66 protected:
67 
68  void _duplicate(const Chunk &bs)
69  {
70  // See above
71  d_bytes_read = 0;
72  d_read_buffer = 0;
73  d_read_buffer_size = 0;
74  d_is_read = false;
75  d_is_inflated = false;
76 
77  d_size = bs.d_size;
78  d_offset = bs.d_offset;
79  d_data_url = bs.d_data_url;
80  d_query_marker = bs.d_query_marker;
81  d_chunk_position_in_array = bs.d_chunk_position_in_array;
82  }
83 
84 public:
85 
95  Chunk() :
96  d_data_url(""), d_query_marker(""), d_size(0), d_offset(0), d_bytes_read(0), d_read_buffer(0),
97  d_read_buffer_size(0), d_is_read(false), d_is_inflated(false)
98  {
99  }
100 
110  Chunk(const std::string &data_url, unsigned long long size, unsigned long long offset, std::string pia_str = "") :
111  d_data_url(data_url), d_query_marker(""), d_size(size), d_offset(offset), d_bytes_read(0), d_read_buffer(0),
112  d_read_buffer_size(0), d_is_read(false), d_is_inflated(false)
113  {
115  set_position_in_array(pia_str);
116  }
117 
127  Chunk(const std::string &data_url, unsigned long long size, unsigned long long offset, const std::vector<unsigned int> &pia_vec) :
128  d_data_url(data_url), d_query_marker(""), d_size(size), d_offset(offset), d_bytes_read(0), d_read_buffer(0),
129  d_read_buffer_size(0), d_is_read(false), d_is_inflated(false)
130  {
132  set_position_in_array(pia_vec);
133  }
134 
135  Chunk(const Chunk &h4bs)
136  {
137  _duplicate(h4bs);
138  }
139 
140  virtual ~Chunk()
141  {
142  delete[] d_read_buffer;
143  }
144 
149  Chunk &operator=(const Chunk &rhs)
150  {
151  if (this == &rhs) return *this;
152 
153  _duplicate(rhs);
154 
155  return *this;
156  }
157 
161  virtual unsigned long long get_size() const
162  {
163  return d_size;
164  }
165 
169  virtual unsigned long long get_offset() const
170  {
171  return d_offset;
172  }
173 
177  virtual std::string get_data_url() const
178  {
179  // A conditional call to void Chunk::add_tracking_query_param()
180  // here for the NASA cost model work THG's doing. jhrg 8/7/18
181 
182  if (!d_query_marker.empty()) {
183  return d_data_url + d_query_marker;
184  }
185 
186  return d_data_url;
187  }
188 
192  virtual void set_data_url(const std::string &data_url)
193  {
194  d_data_url = data_url;
195  }
196 
200  virtual unsigned long long get_bytes_read() const
201  {
202  return d_bytes_read;
203  }
204 
209  virtual void set_bytes_read(unsigned long long bytes_read)
210  {
211  d_bytes_read = bytes_read;
212  }
213 
222  virtual void set_rbuf_to_size()
223  {
224  delete[] d_read_buffer;
225 
226  d_read_buffer = new char[d_size];
227  d_read_buffer_size = d_size;
228  set_bytes_read(0);
229  }
230 
235  virtual char *get_rbuf()
236  {
237  return d_read_buffer;
238  }
239 
251  virtual void set_rbuf(char *buf, unsigned int size)
252  {
253  delete[] d_read_buffer;
254 
255  d_read_buffer = buf;
256  d_read_buffer_size = size;
257 
258  set_bytes_read(size);
259  }
260 
264  virtual unsigned long long get_rbuf_size() const
265  {
266  return d_read_buffer_size;
267  }
268 
272  virtual const std::vector<unsigned int> &get_position_in_array() const
273  {
274  return d_chunk_position_in_array;
275  }
276 
277  virtual void add_tracking_query_param();
278 
279  virtual void set_position_in_array(const std::string &pia);
280  virtual void set_position_in_array(const std::vector<unsigned int> &pia);
281 
282  virtual void read_chunk();
283 
284  virtual void inflate_chunk(bool deflate, bool shuffle, unsigned int chunk_size, unsigned int elem_width);
285 
286  virtual void set_is_read(bool state) { d_is_read = state; }
287 
288  virtual bool get_is_inflated() const { return d_is_inflated; }
289  virtual void set_is_inflated(bool state) { d_is_inflated = state; }
290 
291  virtual std::string get_curl_range_arg_string();
292 
293  virtual void dump(std::ostream & strm) const;
294 
295  virtual std::string to_string() const;
296 };
297 
298 #if 0
299 struct inflate_chunk_args {
301  Chunk *chunk;
302  bool deflate;
303  bool shuffle;
304  unsigned int chunk_size;
305  unsigned int elem_width;
306 
307  inflate_chunk_args(Chunk *c, bool d, bool s, unsigned int c_size, unsigned int e_size):
308  chunk(c), deflate(d), shuffle(s), chunk_size(c_size), elem_width(e_size) {}
309 };
310 
311 void *inflate_chunk(void *args);
312 #endif
313 
314 
315 } // namespace dmrpp
316 
317 #endif // _Chunk_h
dmrpp::Chunk::operator=
Chunk & operator=(const Chunk &rhs)
Definition: Chunk.h:149
dmrpp::Chunk
Definition: Chunk.h:43
dmrpp::Chunk::set_rbuf_to_size
virtual void set_rbuf_to_size()
Allocates the internal read buffer to be d_size bytes.
Definition: Chunk.h:222
dmrpp::Chunk::add_tracking_query_param
virtual void add_tracking_query_param()
Modify this chunk's data URL so that it includes tracking info.
Definition: Chunk.cc:401
dmrpp::Chunk::set_bytes_read
virtual void set_bytes_read(unsigned long long bytes_read)
Set the size of this Chunk's data block.
Definition: Chunk.h:209
dmrpp::Chunk::set_data_url
virtual void set_data_url(const std::string &data_url)
Get the data url string for this Chunk's data block.
Definition: Chunk.h:192
dmrpp::Chunk::get_offset
virtual unsigned long long get_offset() const
Get the offset to this Chunk's data block.
Definition: Chunk.h:169
dmrpp::Chunk::get_rbuf
virtual char * get_rbuf()
Definition: Chunk.h:235
dmrpp::Chunk::get_size
virtual unsigned long long get_size() const
Get the size of this Chunk's data block on disk.
Definition: Chunk.h:161
dmrpp::Chunk::Chunk
Chunk(const std::string &data_url, unsigned long long size, unsigned long long offset, const std::vector< unsigned int > &pia_vec)
Get a chunk initialized with values.
Definition: Chunk.h:127
dmrpp::Chunk::set_rbuf
virtual void set_rbuf(char *buf, unsigned int size)
Set the read buffer.
Definition: Chunk.h:251
dmrpp::Chunk::read_chunk
virtual void read_chunk()
Definition: Chunk.cc:541
dmrpp::Chunk::set_position_in_array
virtual void set_position_in_array(const std::string &pia)
parse the chunk position string
Definition: Chunk.cc:331
dmrpp::Chunk::inflate_chunk
virtual void inflate_chunk(bool deflate, bool shuffle, unsigned int chunk_size, unsigned int elem_width)
Decompress data in the chunk, managing the Chunk's data buffers.
Definition: Chunk.cc:472
dmrpp::Chunk::get_rbuf_size
virtual unsigned long long get_rbuf_size() const
Definition: Chunk.h:264
dmrpp::Chunk::get_curl_range_arg_string
virtual std::string get_curl_range_arg_string()
Returns a curl range argument. The libcurl requires a string argument for range-ge activitys,...
Definition: Chunk.cc:383
dmrpp::Chunk::get_data_url
virtual std::string get_data_url() const
Get the data url string for this Chunk's data block.
Definition: Chunk.h:177
dmrpp::Chunk::Chunk
Chunk(const std::string &data_url, unsigned long long size, unsigned long long offset, std::string pia_str="")
Get a chunk initialized with values.
Definition: Chunk.h:110
dmrpp::Chunk::Chunk
Chunk()
Get an empty chunk.
Definition: Chunk.h:95
dmrpp::Chunk::get_bytes_read
virtual unsigned long long get_bytes_read() const
Get the number of bytes read so far for this Chunk.
Definition: Chunk.h:200
dmrpp::Chunk::get_position_in_array
virtual const std::vector< unsigned int > & get_position_in_array() const
Definition: Chunk.h:272
dmrpp::Chunk::dump
virtual void dump(std::ostream &strm) const
Definition: Chunk.cc:578