Intel® RealSense™ Cross Platform API
Intel Realsense Cross-platform API
archive.h
Go to the documentation of this file.
1 // License: Apache 2.0. See LICENSE file in root directory.
2 // Copyright(c) 2015 Intel Corporation. All Rights Reserved.
3 
4 #pragma once
5 
6 #include "types.h"
7 #include "core/streaming.h"
8 
9 #include <atomic>
10 #include <array>
11 #include <math.h>
12 
13 namespace librealsense
14 {
15  class archive_interface;
16  class md_attribute_parser_base;
17  class frame;
18 }
19 
21 {
23  unsigned long long frame_number = 0;
27  uint32_t metadata_size = 0;
28  bool fisheye_ae_mode = false;
29  std::array<uint8_t,MAX_META_DATA_SIZE> metadata_blob;
32  unsigned long long last_frame_number = 0;
33 
35 
36  frame_additional_data(double in_timestamp,
37  unsigned long long in_frame_number,
38  double in_system_time,
39  uint8_t md_size,
40  const uint8_t* md_buf,
41  double backend_time,
43  unsigned long long last_frame_number)
44  : timestamp(in_timestamp),
45  frame_number(in_frame_number),
46  system_time(in_system_time),
47  metadata_size(md_size),
48  backend_timestamp(backend_time),
51  {
52  // Copy up to 255 bytes to preserve metadata as raw data
53  if (metadata_size)
54  std::copy(md_buf,md_buf+ std::min(md_size,MAX_META_DATA_SIZE),metadata_blob.begin());
55  }
56 };
57 
58 namespace librealsense
59 {
60  typedef std::map<rs2_frame_metadata_value, std::shared_ptr<md_attribute_parser_base>> metadata_parser_map;
61 
62  // Define a movable but explicitly noncopyable buffer type to hold our frame data
63  class frame : public frame_interface
64  {
65  public:
66  std::vector<byte> data;
68 
69  explicit frame() : ref_count(0), _kept(false), owner(nullptr), on_release() {}
70  frame(const frame& r) = delete;
71  frame(frame&& r)
72  : ref_count(r.ref_count.exchange(0)), _kept(r._kept.exchange(false)),
73  owner(r.owner), on_release()
74  {
75  *this = std::move(r);
76  }
77 
78  frame& operator=(const frame& r) = delete;
80  {
81  data = move(r.data);
82  owner = r.owner;
83  ref_count = r.ref_count.exchange(0);
84  _kept = r._kept.exchange(false);
85  on_release = std::move(r.on_release);
86  additional_data = std::move(r.additional_data);
87  r.owner.reset();
88  return *this;
89  }
90 
91  virtual ~frame() { on_release.reset(); }
92  rs2_metadata_type get_frame_metadata(const rs2_frame_metadata_value& frame_metadata) const override;
93  bool supports_frame_metadata(const rs2_frame_metadata_value& frame_metadata) const override;
94  const byte* get_frame_data() const override;
95  rs2_time_t get_frame_timestamp() const override;
97  void set_timestamp(double new_ts) override { additional_data.timestamp = new_ts; }
98  unsigned long long get_frame_number() const override;
99 
100  void set_timestamp_domain(rs2_timestamp_domain timestamp_domain) override
101  {
102  additional_data.timestamp_domain = timestamp_domain;
103  }
104 
105  rs2_time_t get_frame_system_time() const override;
106 
107  std::shared_ptr<stream_profile_interface> get_stream() const override { return stream; }
108  void set_stream(std::shared_ptr<stream_profile_interface> sp) override { stream = std::move(sp); }
109 
111  void update_frame_callback_start_ts(rs2_time_t ts) override;
112 
113  void acquire() override { ref_count.fetch_add(1); }
114  void release() override;
115  void keep() override;
116 
117  frame_interface* publish(std::shared_ptr<archive_interface> new_owner) override;
118  void attach_continuation(frame_continuation&& continuation) override { on_release = std::move(continuation); }
119  void disable_continuation() override { on_release.reset(); }
120 
121  archive_interface* get_owner() const override { return owner.get(); }
122 
123  std::shared_ptr<sensor_interface> get_sensor() const override;
124  void set_sensor(std::shared_ptr<sensor_interface> s) override;
125 
126 
127  void log_callback_start(rs2_time_t timestamp) override;
128  void log_callback_end(rs2_time_t timestamp) const override;
129 
130  void mark_fixed() override { _fixed = true; }
131  bool is_fixed() const override { return _fixed; }
132 
133  private:
134  // TODO: check boost::intrusive_ptr or an alternative
135  std::atomic<int> ref_count; // the reference count is on how many times this placeholder has been observed (not lifetime, not content)
136  std::shared_ptr<archive_interface> owner; // pointer to the owner to be returned to by last observe
137  std::weak_ptr<sensor_interface> sensor;
138  frame_continuation on_release;
139  bool _fixed = false;
140  std::atomic_bool _kept;
141  std::shared_ptr<stream_profile_interface> stream;
142  };
143 
144  class points : public frame
145  {
146  public:
147  float3* get_vertices();
148  void export_to_ply(const std::string& fname, const frame_holder& texture);
149  size_t get_vertex_count() const;
151  };
152 
154 
155  class composite_frame : public frame
156  {
157  public:
159 
161  {
162  auto frames = get_frames();
163  return frames[i];
164  }
165 
166  frame_interface** get_frames() const { return (frame_interface**)data.data(); }
167 
168  const frame_interface* first() const
169  {
170  return get_frame(0);
171  }
173  {
174  return get_frame(0);
175  }
176 
177  void keep() override
178  {
179  auto frames = get_frames();
180  for (int i = 0; i < get_embedded_frames_count(); i++)
181  if (frames[i]) frames[i]->keep();
182  frame::keep();
183  }
184 
185  size_t get_embedded_frames_count() const { return data.size() / sizeof(rs2_frame*); }
186 
187  // In the next section we make the composite frame "look and feel" like the first of its children
188  rs2_metadata_type get_frame_metadata(const rs2_frame_metadata_value& frame_metadata) const override
189  {
190  return first()->get_frame_metadata(frame_metadata);
191  }
192  bool supports_frame_metadata(const rs2_frame_metadata_value& frame_metadata) const override
193  {
194  return first()->supports_frame_metadata(frame_metadata);
195  }
196  const byte* get_frame_data() const override
197  {
198  return first()->get_frame_data();
199  }
201  {
202  return first()->get_frame_timestamp();
203  }
205  {
206  return first()->get_frame_timestamp_domain();
207  }
208  unsigned long long get_frame_number() const override
209  {
210  if (first())
211  return first()->get_frame_number();
212  else
213  return frame::get_frame_number();
214  }
216  {
217  return first()->get_frame_system_time();
218  }
219  std::shared_ptr<sensor_interface> get_sensor() const override
220  {
221  return first()->get_sensor();
222  }
223  };
224 
226 
227  class video_frame : public frame
228  {
229  public:
231  : frame(), _width(0), _height(0), _bpp(0), _stride(0)
232  {}
233 
234  int get_width() const { return _width; }
235  int get_height() const { return _height; }
236  int get_stride() const { return _stride; }
237  int get_bpp() const { return _bpp; }
238 
239  void assign(int width, int height, int stride, int bpp)
240  {
241  _width = width;
242  _height = height;
243  _stride = stride;
244  _bpp = bpp;
245  }
246 
247  private:
248  int _width, _height, _bpp, _stride;
249  };
250 
252 
253  class depth_frame : public video_frame
254  {
255  public:
257  {
258  }
259 
260  frame_interface* publish(std::shared_ptr<archive_interface> new_owner) override
261  {
263  return video_frame::publish(new_owner);
264  }
265 
266  void keep() override
267  {
268  if (_original) _original->keep();
270  }
271 
272  float get_distance(int x, int y) const
273  {
274  // If this frame does not itself contain Z16 depth data,
275  // fall back to the original frame it was created from
276  if (_original && get_stream()->get_format() != RS2_FORMAT_Z16)
277  return((depth_frame*)_original.frame)->get_distance(x, y);
278 
279  uint64_t pixel = 0;
280  switch (get_bpp()/8) // bits per pixel
281  {
282  case 1: pixel = get_frame_data()[y*get_width() + x]; break;
283  case 2: pixel = reinterpret_cast<const uint16_t*>(get_frame_data())[y*get_width() + x]; break;
284  case 4: pixel = reinterpret_cast<const uint32_t*>(get_frame_data())[y*get_width() + x]; break;
285  case 8: pixel = reinterpret_cast<const uint64_t*>(get_frame_data())[y*get_width() + x]; break;
286  default: throw std::runtime_error(to_string() << "Unrecognized depth format " << int(get_bpp() / 8) << " bytes per pixel");
287  }
288 
289  return pixel * get_units();
290  }
291 
292  float get_units() const
293  {
294  if (!_depth_units)
296  return _depth_units.value();
297  }
298 
300  {
301  auto res = _original.frame;
302  auto df = dynamic_cast<depth_frame*>(res);
303  if (df)
304  {
305  auto prev = df->get_original_depth();
306  if (prev) return prev;
307  }
308  return res;
309  }
310 
312  {
313  _original = std::move(h);
315  if (_original)
316  {
317  _original = {};
318  }
319  }, get_frame_data()));
320  }
321 
322  protected:
323  static float query_units(const std::shared_ptr<sensor_interface>& sensor)
324  {
325  if (sensor != nullptr)
326  {
327  try
328  {
329  auto depth_sensor = As<librealsense::depth_sensor>(sensor);
330  if(depth_sensor != nullptr)
331  {
332  return depth_sensor->get_depth_scale();
333  }
334  else
335  {
336  //For playback sensors
337  auto extendable = As<librealsense::extendable_interface>(sensor);
338  if (extendable && extendable->extend_to(TypeToExtension<librealsense::depth_sensor>::value, (void**)(&depth_sensor)))
339  {
340  return depth_sensor->get_depth_scale();
341  }
342  }
343  }
344  catch (const std::exception& e)
345  {
346  LOG_ERROR("Failed to query depth units from sensor. " << e.what());
347  }
348  catch (...)
349  {
350  LOG_ERROR("Failed to query depth units from sensor");
351  }
352  }
353  else
354  {
355  LOG_WARNING("sensor was nullptr");
356  }
357 
358  return 0;
359  }
360 
363  };
364 
366 
367  // Disparity frame provides an alternative representation of the depth data for stereo-based depth sensors
368  // In addition for the depth frame API it allows to query the stereoscopic baseline required to transform depth to disparity and vice versa
370  {
371  public:
373  {
374  }
375 
376  // TODO Refactor to framemetadata
377  float get_stereo_baseline(void) const { return query_stereo_baseline(this->get_sensor()); }
378 
379  protected:
380 
381  static float query_stereo_baseline(const std::shared_ptr<sensor_interface>& sensor)
382  {
383  if (sensor != nullptr)
384  {
385  try
386  {
387  auto stereo_sensor = As<librealsense::depth_stereo_sensor>(sensor);
388  if (stereo_sensor != nullptr)
389  {
390  return stereo_sensor->get_stereo_baseline_mm();
391  }
392  else
393  {
394  //For playback sensors
395  auto extendable = As<librealsense::extendable_interface>(sensor);
396  if (extendable && extendable->extend_to(TypeToExtension<librealsense::depth_stereo_sensor>::value, (void**)(&stereo_sensor)))
397  {
398  return stereo_sensor->get_stereo_baseline_mm();
399  }
400  }
401  }
402  catch (const std::exception& e)
403  {
404  LOG_ERROR("Failed to query stereo baseline from sensor. " << e.what());
405  }
406  catch (...)
407  {
408  LOG_ERROR("Failed to query stereo baseline from sensor");
409  }
410  }
411  else
412  {
413  LOG_WARNING("sensor was nullptr");
414  }
415 
416  return 0;
417  }
418  };
419 
421 
422  class motion_frame : public frame
423  {
424  public:
426  {}
427  };
428 
430 
431  class pose_frame : public frame
432  {
433  public:
434  // pose frame data buffer is pose info struct
435  struct pose_info
436  {
444  uint32_t mapper_confidence;
445  };
446 
447  pose_frame() : frame() {}
448 
449  float3 get_translation() const { return reinterpret_cast<const pose_info*>(data.data())->translation; }
450  float3 get_velocity() const { return reinterpret_cast<const pose_info*>(data.data())->velocity; }
451  float3 get_acceleration() const { return reinterpret_cast<const pose_info*>(data.data())->acceleration; }
452  float4 get_rotation() const { return reinterpret_cast<const pose_info*>(data.data())->rotation; }
453  float3 get_angular_velocity() const { return reinterpret_cast<const pose_info*>(data.data())->angular_velocity; }
454  float3 get_angular_acceleration() const { return reinterpret_cast<const pose_info*>(data.data())->angular_acceleration; }
455  uint32_t get_tracker_confidence() const { return reinterpret_cast<const pose_info*>(data.data())->tracker_confidence; }
456  uint32_t get_mapper_confidence() const { return reinterpret_cast<const pose_info*>(data.data())->mapper_confidence; }
457  };
458 
460 
461 
463  {
464  public:
466 
467  virtual frame_interface* alloc_and_track(const size_t size, const frame_additional_data& additional_data, bool requires_memory) = 0;
468 
469  virtual std::shared_ptr<metadata_parser_map> get_md_parsers() const = 0;
470 
471  virtual void flush() = 0;
472 
474  virtual void unpublish_frame(frame_interface* frame) = 0;
475  virtual void keep_frame(frame_interface* frame) = 0;
476 
477  virtual ~archive_interface() = default;
478 
479  };
480 
481  std::shared_ptr<archive_interface> make_archive(rs2_extension type,
482  std::atomic<uint32_t>* in_max_frame_queue_size,
483  std::shared_ptr<platform::time_service> ts,
484  std::shared_ptr<metadata_parser_map> parsers);
485 }
virtual rs2_metadata_type get_frame_metadata(const rs2_frame_metadata_value &frame_metadata) const =0
std::shared_ptr< stream_profile_interface > get_stream() const override
Definition: archive.h:107
bool supports_frame_metadata(const rs2_frame_metadata_value &frame_metadata) const override
float3 get_acceleration() const
Definition: archive.h:451
virtual const byte * get_frame_data() const =0
bool fisheye_ae_mode
Definition: archive.h:28
virtual rs2_timestamp_domain get_frame_timestamp_domain() const =0
T & value() &
Definition: types.h:1478
void disable_continuation() override
Definition: archive.h:119
uint32_t get_mapper_confidence() const
Definition: archive.h:456
uint32_t mapper_confidence
Definition: archive.h:444
frame_additional_data(double in_timestamp, unsigned long long in_frame_number, double in_system_time, uint8_t md_size, const uint8_t *md_buf, double backend_time, rs2_time_t last_timestamp, unsigned long long last_frame_number)
Definition: archive.h:36
int get_height() const
Definition: archive.h:235
float3 get_angular_acceleration() const
Definition: archive.h:454
unsigned char byte
Definition: types.h:33
frame_interface * publish(std::shared_ptr< archive_interface > new_owner) override
Definition: archive.h:260
Definition: types.h:413
frame(frame &&r)
Definition: archive.h:71
rs2_time_t frame_callback_started
Definition: archive.h:26
Definition: streaming.h:63
int get_width() const
Definition: archive.h:234
uint32_t metadata_size
Definition: archive.h:27
void keep() override
Definition: archive.h:177
Definition: rs_frame.h:21
Definition: archive.h:369
Definition: archive.h:435
Definition: rs_types.h:104
void keep() override
Definition: archive.h:266
#define LOG_WARNING(...)
Definition: types.h:109
float3 * get_vertices()
float2 * get_texture_coordinates()
Definition: archive.h:253
rs2_time_t get_frame_system_time() const override
Definition: archive.h:215
motion_frame()
Definition: archive.h:425
Definition: rs_sensor.h:55
const uint8_t MAX_META_DATA_SIZE
Definition: backend.h:30
frame()
Definition: archive.h:69
Definition: rs_types.h:106
Definition: archive.h:20
float3 velocity
Definition: archive.h:438
unsigned long long get_frame_number() const override
float get_units() const
Definition: archive.h:292
void mark_fixed() override
Definition: archive.h:130
optional_value< float > _depth_units
Definition: archive.h:362
void set_timestamp_domain(rs2_timestamp_domain timestamp_domain) override
Definition: archive.h:100
void export_to_ply(const std::string &fname, const frame_holder &texture)
void set_original(frame_holder h)
Definition: archive.h:311
archive_interface * get_owner() const override
Definition: archive.h:121
void log_callback_end(rs2_time_t timestamp) const override
int get_bpp() const
Definition: archive.h:237
uint32_t tracker_confidence
Definition: archive.h:443
frame_interface ** get_frames() const
Definition: archive.h:166
Definition: archive.h:155
float3 translation
Definition: archive.h:437
Definition: extension.h:112
frame & operator=(frame &&r)
Definition: archive.h:79
void release() override
frame_additional_data additional_data
Definition: archive.h:67
int get_stride() const
Definition: archive.h:236
std::shared_ptr< sensor_interface > get_sensor() const override
virtual float get_depth_scale() const =0
frame_interface * publish(std::shared_ptr< archive_interface > new_owner) override
Definition: archive.h:63
virtual std::shared_ptr< metadata_parser_map > get_md_parsers() const =0
Definition: algo.h:16
Definition: streaming.h:21
float3 get_velocity() const
Definition: archive.h:450
const frame_interface * first() const
Definition: archive.h:168
Definition: archive.h:422
void set_timestamp(double new_ts) override
Definition: archive.h:97
virtual void keep_frame(frame_interface *frame)=0
frame_additional_data()
Definition: archive.h:34
float4 rotation
Definition: archive.h:440
Definition: archive.h:431
Definition: rs_types.h:107
std::shared_ptr< sensor_interface > get_sensor() const override
Definition: archive.h:219
Definition: types.h:414
const frame_interface * get_original_depth() const
Definition: archive.h:299
rs2_time_t backend_timestamp
Definition: archive.h:30
virtual frame_interface * alloc_and_track(const size_t size, const frame_additional_data &additional_data, bool requires_memory)=0
void set_stream(std::shared_ptr< stream_profile_interface > sp) override
Definition: archive.h:108
Definition: rs_types.h:105
rs2_time_t system_time
Definition: archive.h:25
rs2_time_t get_frame_timestamp() const override
Definition: archive.h:200
Definition: rs_types.h:113
rs2_timestamp_domain get_frame_timestamp_domain() const override
float3 get_translation() const
Definition: archive.h:449
float get_stereo_baseline(void) const
Definition: archive.h:377
void attach_continuation(frame_continuation &&continuation) override
Definition: archive.h:118
virtual void unpublish_frame(frame_interface *frame)=0
rs2_time_t get_frame_callback_start_time_point() const override
std::shared_ptr< archive_interface > make_archive(rs2_extension type, std::atomic< uint32_t > *in_max_frame_queue_size, std::shared_ptr< platform::time_service > ts, std::shared_ptr< metadata_parser_map > parsers)
virtual ~archive_interface()=default
static float query_units(const std::shared_ptr< sensor_interface > &sensor)
Definition: archive.h:323
virtual rs2_time_t get_frame_timestamp() const =0
float3 get_angular_velocity() const
Definition: archive.h:453
bool supports_frame_metadata(const rs2_frame_metadata_value &frame_metadata) const override
Definition: archive.h:192
uint32_t get_tracker_confidence() const
Definition: archive.h:455
frame & operator=(const frame &r)=delete
depth_frame()
Definition: archive.h:256
virtual frame_interface * publish_frame(frame_interface *frame)=0
virtual std::shared_ptr< sensor_interface > get_sensor() const =0
const byte * get_frame_data() const override
Definition: archive.h:196
Definition: archive.h:227
void set_sensor(std::shared_ptr< sensor_interface > s) override
rs2_time_t timestamp
Definition: archive.h:22
Definition: types.h:54
Definition: types.h:415
Definition: rs_types.h:103
unsigned long long last_frame_number
Definition: archive.h:32
float4 get_rotation() const
Definition: archive.h:452
unsigned long long get_frame_number() const override
Definition: archive.h:208
video_frame()
Definition: archive.h:230
void keep() override
void assign(int width, int height, int stride, int bpp)
Definition: archive.h:239
pose_frame()
Definition: archive.h:447
void log_callback_start(rs2_time_t timestamp) override
frame_interface * frame
Definition: types.h:589
const byte * get_frame_data() const override
virtual rs2_time_t get_frame_system_time() const =0
rs2_extension
Specifies advanced interfaces (capabilities) objects may implement.
Definition: rs_types.h:93
Definition: types.h:1190
rs2_metadata_type get_frame_metadata(const rs2_frame_metadata_value &frame_metadata) const override
Definition: archive.h:188
float3 angular_velocity
Definition: archive.h:441
void update_frame_callback_start_ts(rs2_time_t ts) override
long long rs2_metadata_type
Definition: rs_types.h:180
Definition: stream.h:14
Definition: backend.h:348
Definition: types.h:587
composite_frame()
Definition: archive.h:158
virtual ~frame()
Definition: archive.h:91
float get_distance(int x, int y) const
Definition: archive.h:272
rs2_timestamp_domain timestamp_domain
Definition: archive.h:24
bool is_fixed() const override
Definition: archive.h:131
Definition: archive.h:144
unsigned long long frame_number
Definition: archive.h:23
float3 angular_acceleration
Definition: archive.h:442
frame_holder _original
Definition: archive.h:361
Definition: archive.h:462
frame_interface * get_frame(int i) const
Definition: archive.h:160
disparity_frame()
Definition: archive.h:372
float3 acceleration
Definition: archive.h:439
virtual callback_invocation_holder begin_callback()=0
std::array< uint8_t, MAX_META_DATA_SIZE > metadata_blob
Definition: archive.h:29
Definition: streaming.h:158
Definition: rs_types.h:115
size_t get_vertex_count() const
frame_interface * first()
Definition: archive.h:172
rs2_time_t get_frame_system_time() const override
static float query_stereo_baseline(const std::shared_ptr< sensor_interface > &sensor)
Definition: archive.h:381
double rs2_time_t
Definition: rs_types.h:179
rs2_timestamp_domain get_frame_timestamp_domain() const override
Definition: archive.h:204
void reset()
Definition: types.h:1216
virtual unsigned long long get_frame_number() const =0
std::map< rs2_frame_metadata_value, std::shared_ptr< md_attribute_parser_base > > metadata_parser_map
Definition: archive.h:60
void acquire() override
Definition: archive.h:113
rs2_time_t last_timestamp
Definition: archive.h:31
rs2_time_t get_frame_timestamp() const override
rs2_frame_metadata_value
Per-Frame-Metadata are set of read-only properties that might be exposed for each individual frame...
Definition: rs_frame.h:28
rs2_metadata_type get_frame_metadata(const rs2_frame_metadata_value &frame_metadata) const override
MAP_EXTENSION(RS2_EXTENSION_POINTS, librealsense::points)
struct rs2_frame rs2_frame
Definition: rs_types.h:150
#define LOG_ERROR(...)
Definition: types.h:110
void copy(void *dst, void const *src, size_t size)
virtual bool supports_frame_metadata(const rs2_frame_metadata_value &frame_metadata) const =0
std::vector< byte > data
Definition: archive.h:66
size_t get_embedded_frames_count() const
Definition: archive.h:185
rs2_timestamp_domain
Specifies the clock in relation to which the frame timestamp was measured.
Definition: rs_frame.h:19