protozero
Minimalistic protocol buffer decoder and encoder in C++.
pbf_writer.hpp
Go to the documentation of this file.
1 #ifndef PROTOZERO_PBF_WRITER_HPP
2 #define PROTOZERO_PBF_WRITER_HPP
3 
4 /*****************************************************************************
5 
6 protozero - Minimalistic protocol buffer decoder and encoder in C++.
7 
8 This file is from https://github.com/mapbox/protozero where you can find more
9 documentation.
10 
11 *****************************************************************************/
12 
19 #include <cstddef>
20 #include <cstdint>
21 #include <cstring>
22 #include <iterator>
23 #include <limits>
24 #include <string>
25 
26 #include <protozero/config.hpp>
27 #include <protozero/types.hpp>
28 #include <protozero/varint.hpp>
29 
30 #if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
31 # include <protozero/byteswap.hpp>
32 #endif
33 
34 namespace protozero {
35 
36 namespace detail {
37 
38  template <typename T> class packed_field_varint;
39  template <typename T> class packed_field_svarint;
40  template <typename T> class packed_field_fixed;
41 
42 } // end namespace detail
43 
50 class pbf_writer {
51 
52  // A pointer to a string buffer holding the data already written to the
53  // PBF message. For default constructed writers or writers that have been
54  // rolled back, this is a nullptr.
55  std::string* m_data;
56 
57  // A pointer to a parent writer object if this is a submessage. If this
58  // is a top-level writer, it is a nullptr.
59  pbf_writer* m_parent_writer;
60 
61  // This is usually 0. If there is an open submessage, this is set in the
62  // parent to the rollback position, ie. the last position before the
63  // submessage was started. This is the position where the header of the
64  // submessage starts.
65  std::size_t m_rollback_pos = 0;
66 
67  // This is usually 0. If there is an open submessage, this is set in the
68  // parent to the position where the data of the submessage is written to.
69  std::size_t m_pos = 0;
70 
71  inline void add_varint(uint64_t value) {
72  protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage");
73  protozero_assert(m_data);
74  write_varint(std::back_inserter(*m_data), value);
75  }
76 
77  inline void add_field(pbf_tag_type tag, pbf_wire_type type) {
78  protozero_assert(((tag > 0 && tag < 19000) || (tag > 19999 && tag <= ((1 << 29) - 1))) && "tag out of range");
79  uint32_t b = (tag << 3) | uint32_t(type);
80  add_varint(b);
81  }
82 
83  inline void add_tagged_varint(pbf_tag_type tag, uint64_t value) {
84  add_field(tag, pbf_wire_type::varint);
85  add_varint(value);
86  }
87 
88  template <typename T>
89  inline void add_fixed(T value) {
90  protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage");
91  protozero_assert(m_data);
92 #if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN
93  m_data->append(reinterpret_cast<const char*>(&value), sizeof(T));
94 #else
95  auto size = m_data->size();
96  m_data->resize(size + sizeof(T));
97  byteswap<sizeof(T)>(reinterpret_cast<const char*>(&value), const_cast<char*>(m_data->data() + size));
98 #endif
99  }
100 
101  template <typename T, typename It>
102  inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::input_iterator_tag) {
103  if (first == last) {
104  return;
105  }
106 
107  pbf_writer sw(*this, tag);
108 
109  while (first != last) {
110  sw.add_fixed<T>(*first++);
111  }
112  }
113 
114  template <typename T, typename It>
115  inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::forward_iterator_tag) {
116  if (first == last) {
117  return;
118  }
119 
120  auto length = std::distance(first, last);
121  add_length_varint(tag, sizeof(T) * pbf_length_type(length));
122  reserve(sizeof(T) * std::size_t(length));
123 
124  while (first != last) {
125  add_fixed<T>(*first++);
126  }
127  }
128 
129  template <typename It>
130  inline void add_packed_varint(pbf_tag_type tag, It first, It last) {
131  if (first == last) {
132  return;
133  }
134 
135  pbf_writer sw(*this, tag);
136 
137  while (first != last) {
138  sw.add_varint(uint64_t(*first++));
139  }
140  }
141 
142  template <typename It>
143  inline void add_packed_svarint(pbf_tag_type tag, It first, It last) {
144  if (first == last) {
145  return;
146  }
147 
148  pbf_writer sw(*this, tag);
149 
150  while (first != last) {
151  sw.add_varint(encode_zigzag64(*first++));
152  }
153  }
154 
155  // The number of bytes to reserve for the varint holding the length of
156  // a length-delimited field. The length has to fit into pbf_length_type,
157  // and a varint needs 8 bit for every 7 bit.
158  static const int reserve_bytes = sizeof(pbf_length_type) * 8 / 7 + 1;
159 
160  // If m_rollpack_pos is set to this special value, it means that when
161  // the submessage is closed, nothing needs to be done, because the length
162  // of the submessage has already been written correctly.
163  static const std::size_t size_is_known = std::numeric_limits<std::size_t>::max();
164 
165  inline void open_submessage(pbf_tag_type tag, std::size_t size) {
166  protozero_assert(m_pos == 0);
167  protozero_assert(m_data);
168  if (size == 0) {
169  m_rollback_pos = m_data->size();
170  add_field(tag, pbf_wire_type::length_delimited);
171  m_data->append(std::size_t(reserve_bytes), '\0');
172  } else {
173  m_rollback_pos = size_is_known;
174  add_length_varint(tag, pbf_length_type(size));
175  reserve(size);
176  }
177  m_pos = m_data->size();
178  }
179 
180  inline void rollback_submessage() {
181  protozero_assert(m_pos != 0);
182  protozero_assert(m_rollback_pos != size_is_known);
183  protozero_assert(m_data);
184  m_data->resize(m_rollback_pos);
185  m_pos = 0;
186  }
187 
188  inline void commit_submessage() {
189  protozero_assert(m_pos != 0);
190  protozero_assert(m_rollback_pos != size_is_known);
191  protozero_assert(m_data);
192  auto length = pbf_length_type(m_data->size() - m_pos);
193 
194  protozero_assert(m_data->size() >= m_pos - reserve_bytes);
195  auto n = write_varint(m_data->begin() + long(m_pos) - reserve_bytes, length);
196 
197  m_data->erase(m_data->begin() + long(m_pos) - reserve_bytes + n, m_data->begin() + long(m_pos));
198  m_pos = 0;
199  }
200 
201  inline void close_submessage() {
202  protozero_assert(m_data);
203  if (m_pos == 0 || m_rollback_pos == size_is_known) {
204  return;
205  }
206  if (m_data->size() - m_pos == 0) {
207  rollback_submessage();
208  } else {
209  commit_submessage();
210  }
211  }
212 
213  inline void add_length_varint(pbf_tag_type tag, pbf_length_type length) {
214  add_field(tag, pbf_wire_type::length_delimited);
215  add_varint(length);
216  }
217 
218 public:
219 
225  inline explicit pbf_writer(std::string& data) noexcept :
226  m_data(&data),
227  m_parent_writer(nullptr),
228  m_pos(0) {
229  }
230 
235  inline pbf_writer() noexcept :
236  m_data(nullptr),
237  m_parent_writer(nullptr),
238  m_pos(0) {
239  }
240 
251  inline pbf_writer(pbf_writer& parent_writer, pbf_tag_type tag, std::size_t size=0) :
252  m_data(parent_writer.m_data),
253  m_parent_writer(&parent_writer),
254  m_pos(0) {
255  m_parent_writer->open_submessage(tag, size);
256  }
257 
259  pbf_writer(const pbf_writer&) noexcept = default;
260 
262  pbf_writer& operator=(const pbf_writer&) noexcept = default;
263 
265  inline pbf_writer(pbf_writer&&) noexcept = default;
266 
268  inline pbf_writer& operator=(pbf_writer&&) noexcept = default;
269 
270  inline ~pbf_writer() {
271  if (m_parent_writer) {
272  m_parent_writer->close_submessage();
273  }
274  }
275 
284  void reserve(std::size_t size) {
285  protozero_assert(m_data);
286  m_data->reserve(m_data->size() + size);
287  }
288 
289  inline void rollback() {
290  protozero_assert(m_parent_writer && "you can't call rollback() on a pbf_writer without a parent");
291  protozero_assert(m_pos == 0 && "you can't call rollback() on a pbf_writer that has an open nested submessage");
292  m_parent_writer->rollback_submessage();
293  m_data = nullptr;
294  }
295 
297 
307  inline void add_bool(pbf_tag_type tag, bool value) {
308  add_field(tag, pbf_wire_type::varint);
309  protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage");
310  protozero_assert(m_data);
311  m_data->append(1, value);
312  }
313 
320  inline void add_enum(pbf_tag_type tag, int32_t value) {
321  add_tagged_varint(tag, uint64_t(value));
322  }
323 
330  inline void add_int32(pbf_tag_type tag, int32_t value) {
331  add_tagged_varint(tag, uint64_t(value));
332  }
333 
340  inline void add_sint32(pbf_tag_type tag, int32_t value) {
341  add_tagged_varint(tag, encode_zigzag32(value));
342  }
343 
350  inline void add_uint32(pbf_tag_type tag, uint32_t value) {
351  add_tagged_varint(tag, value);
352  }
353 
360  inline void add_int64(pbf_tag_type tag, int64_t value) {
361  add_tagged_varint(tag, uint64_t(value));
362  }
363 
370  inline void add_sint64(pbf_tag_type tag, int64_t value) {
371  add_tagged_varint(tag, encode_zigzag64(value));
372  }
373 
380  inline void add_uint64(pbf_tag_type tag, uint64_t value) {
381  add_tagged_varint(tag, value);
382  }
383 
390  inline void add_fixed32(pbf_tag_type tag, uint32_t value) {
391  add_field(tag, pbf_wire_type::fixed32);
392  add_fixed<uint32_t>(value);
393  }
394 
401  inline void add_sfixed32(pbf_tag_type tag, int32_t value) {
402  add_field(tag, pbf_wire_type::fixed32);
403  add_fixed<int32_t>(value);
404  }
405 
412  inline void add_fixed64(pbf_tag_type tag, uint64_t value) {
413  add_field(tag, pbf_wire_type::fixed64);
414  add_fixed<uint64_t>(value);
415  }
416 
423  inline void add_sfixed64(pbf_tag_type tag, int64_t value) {
424  add_field(tag, pbf_wire_type::fixed64);
425  add_fixed<int64_t>(value);
426  }
427 
434  inline void add_float(pbf_tag_type tag, float value) {
435  add_field(tag, pbf_wire_type::fixed32);
436  add_fixed<float>(value);
437  }
438 
445  inline void add_double(pbf_tag_type tag, double value) {
446  add_field(tag, pbf_wire_type::fixed64);
447  add_fixed<double>(value);
448  }
449 
457  inline void add_bytes(pbf_tag_type tag, const char* value, std::size_t size) {
458  protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage");
459  protozero_assert(m_data);
460  protozero_assert(size <= std::numeric_limits<pbf_length_type>::max());
461  add_length_varint(tag, pbf_length_type(size));
462  m_data->append(value, size);
463  }
464 
471  inline void add_bytes(pbf_tag_type tag, const std::string& value) {
472  add_bytes(tag, value.data(), value.size());
473  }
474 
482  inline void add_string(pbf_tag_type tag, const char* value, std::size_t size) {
483  add_bytes(tag, value, size);
484  }
485 
492  inline void add_string(pbf_tag_type tag, const std::string& value) {
493  add_bytes(tag, value.data(), value.size());
494  }
495 
503  inline void add_string(pbf_tag_type tag, const char* value) {
504  add_bytes(tag, value, std::strlen(value));
505  }
506 
514  inline void add_message(pbf_tag_type tag, const char* value, std::size_t size) {
515  add_bytes(tag, value, size);
516  }
517 
524  inline void add_message(pbf_tag_type tag, const std::string& value) {
525  add_bytes(tag, value.data(), value.size());
526  }
527 
529 
531 
544  template <typename InputIterator>
545  inline void add_packed_bool(pbf_tag_type tag, InputIterator first, InputIterator last) {
546  add_packed_varint(tag, first, last);
547  }
548 
558  template <typename InputIterator>
559  inline void add_packed_enum(pbf_tag_type tag, InputIterator first, InputIterator last) {
560  add_packed_varint(tag, first, last);
561  }
562 
572  template <typename InputIterator>
573  inline void add_packed_int32(pbf_tag_type tag, InputIterator first, InputIterator last) {
574  add_packed_varint(tag, first, last);
575  }
576 
586  template <typename InputIterator>
587  inline void add_packed_sint32(pbf_tag_type tag, InputIterator first, InputIterator last) {
588  add_packed_svarint(tag, first, last);
589  }
590 
600  template <typename InputIterator>
601  inline void add_packed_uint32(pbf_tag_type tag, InputIterator first, InputIterator last) {
602  add_packed_varint(tag, first, last);
603  }
604 
614  template <typename InputIterator>
615  inline void add_packed_int64(pbf_tag_type tag, InputIterator first, InputIterator last) {
616  add_packed_varint(tag, first, last);
617  }
618 
628  template <typename InputIterator>
629  inline void add_packed_sint64(pbf_tag_type tag, InputIterator first, InputIterator last) {
630  add_packed_svarint(tag, first, last);
631  }
632 
642  template <typename InputIterator>
643  inline void add_packed_uint64(pbf_tag_type tag, InputIterator first, InputIterator last) {
644  add_packed_varint(tag, first, last);
645  }
646 
656  template <typename InputIterator>
657  inline void add_packed_fixed32(pbf_tag_type tag, InputIterator first, InputIterator last) {
658  add_packed_fixed<uint32_t, InputIterator>(tag, first, last,
659  typename std::iterator_traits<InputIterator>::iterator_category());
660  }
661 
671  template <typename InputIterator>
672  inline void add_packed_sfixed32(pbf_tag_type tag, InputIterator first, InputIterator last) {
673  add_packed_fixed<int32_t, InputIterator>(tag, first, last,
674  typename std::iterator_traits<InputIterator>::iterator_category());
675  }
676 
686  template <typename InputIterator>
687  inline void add_packed_fixed64(pbf_tag_type tag, InputIterator first, InputIterator last) {
688  add_packed_fixed<uint64_t, InputIterator>(tag, first, last,
689  typename std::iterator_traits<InputIterator>::iterator_category());
690  }
691 
701  template <typename InputIterator>
702  inline void add_packed_sfixed64(pbf_tag_type tag, InputIterator first, InputIterator last) {
703  add_packed_fixed<int64_t, InputIterator>(tag, first, last,
704  typename std::iterator_traits<InputIterator>::iterator_category());
705  }
706 
716  template <typename InputIterator>
717  inline void add_packed_float(pbf_tag_type tag, InputIterator first, InputIterator last) {
718  add_packed_fixed<float, InputIterator>(tag, first, last,
719  typename std::iterator_traits<InputIterator>::iterator_category());
720  }
721 
731  template <typename InputIterator>
732  inline void add_packed_double(pbf_tag_type tag, InputIterator first, InputIterator last) {
733  add_packed_fixed<double, InputIterator>(tag, first, last,
734  typename std::iterator_traits<InputIterator>::iterator_category());
735  }
736 
738 
739  template <typename T> friend class detail::packed_field_varint;
740  template <typename T> friend class detail::packed_field_svarint;
741  template <typename T> friend class detail::packed_field_fixed;
742 
743 }; // class pbf_writer
744 
745 namespace detail {
746 
747  class packed_field {
748 
749  protected:
750 
751  pbf_writer m_writer;
752 
753  public:
754 
755  packed_field(pbf_writer& parent_writer, pbf_tag_type tag) :
756  m_writer(parent_writer, tag) {
757  }
758 
759  packed_field(pbf_writer& parent_writer, pbf_tag_type tag, std::size_t size) :
760  m_writer(parent_writer, tag, size) {
761  }
762 
763  void rollback() {
764  m_writer.rollback();
765  }
766 
767  }; // class packed_field
768 
769  template <typename T>
770  class packed_field_fixed : public packed_field {
771 
772  public:
773 
774  packed_field_fixed(pbf_writer& parent_writer, pbf_tag_type tag) :
775  packed_field(parent_writer, tag) {
776  }
777 
778  packed_field_fixed(pbf_writer& parent_writer, pbf_tag_type tag, std::size_t size) :
779  packed_field(parent_writer, tag, size * sizeof(T)) {
780  }
781 
782  void add_element(T value) {
783  m_writer.add_fixed<T>(value);
784  }
785 
786  }; // class packed_field_fixed
787 
788  template <typename T>
789  class packed_field_varint : public packed_field {
790 
791  public:
792 
793  packed_field_varint(pbf_writer& parent_writer, pbf_tag_type tag) :
794  packed_field(parent_writer, tag) {
795  }
796 
797  void add_element(T value) {
798  m_writer.add_varint(uint64_t(value));
799  }
800 
801  }; // class packed_field_varint
802 
803  template <typename T>
804  class packed_field_svarint : public packed_field {
805 
806  public:
807 
808  packed_field_svarint(pbf_writer& parent_writer, pbf_tag_type tag) :
809  packed_field(parent_writer, tag) {
810  }
811 
812  void add_element(T value) {
813  m_writer.add_varint(encode_zigzag64(value));
814  }
815 
816  }; // class packed_field_svarint
817 
818 } // end namespace detail
819 
834 
835 } // end namespace protozero
836 
837 #endif // PROTOZERO_PBF_WRITER_HPP
void add_packed_fixed64(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:687
void add_packed_sint32(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:587
void add_packed_sint64(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:629
void add_string(pbf_tag_type tag, const char *value)
Definition: pbf_writer.hpp:503
void add_packed_sfixed64(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:702
void add_packed_sfixed32(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:672
uint32_t pbf_length_type
Definition: types.hpp:45
void reserve(std::size_t size)
Definition: pbf_writer.hpp:284
uint64_t encode_zigzag64(int64_t value) noexcept
Definition: varint.hpp:112
void add_sint64(pbf_tag_type tag, int64_t value)
Definition: pbf_writer.hpp:370
void add_message(pbf_tag_type tag, const char *value, std::size_t size)
Definition: pbf_writer.hpp:514
void add_sfixed64(pbf_tag_type tag, int64_t value)
Definition: pbf_writer.hpp:423
void add_uint32(pbf_tag_type tag, uint32_t value)
Definition: pbf_writer.hpp:350
void add_bytes(pbf_tag_type tag, const std::string &value)
Definition: pbf_writer.hpp:471
void add_string(pbf_tag_type tag, const char *value, std::size_t size)
Definition: pbf_writer.hpp:482
void add_packed_enum(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:559
Definition: pbf_writer.hpp:50
void add_int64(pbf_tag_type tag, int64_t value)
Definition: pbf_writer.hpp:360
Contains macro checks for different configurations.
Contains the declaration of low-level types used in the pbf format.
void add_int32(pbf_tag_type tag, int32_t value)
Definition: pbf_writer.hpp:330
Definition: pbf_writer.hpp:38
void add_string(pbf_tag_type tag, const std::string &value)
Definition: pbf_writer.hpp:492
void add_uint64(pbf_tag_type tag, uint64_t value)
Definition: pbf_writer.hpp:380
pbf_wire_type
Definition: types.hpp:33
void add_message(pbf_tag_type tag, const std::string &value)
Definition: pbf_writer.hpp:524
void add_float(pbf_tag_type tag, float value)
Definition: pbf_writer.hpp:434
uint32_t pbf_tag_type
Definition: types.hpp:26
void add_enum(pbf_tag_type tag, int32_t value)
Definition: pbf_writer.hpp:320
void add_packed_uint64(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:643
pbf_writer() noexcept
Definition: pbf_writer.hpp:235
Definition: pbf_writer.hpp:39
Contains functions to swap bytes in values (for different endianness).
void add_packed_uint32(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:601
uint32_t encode_zigzag32(int32_t value) noexcept
Definition: varint.hpp:105
void add_packed_int64(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:615
void add_fixed64(pbf_tag_type tag, uint64_t value)
Definition: pbf_writer.hpp:412
void add_bool(pbf_tag_type tag, bool value)
Definition: pbf_writer.hpp:307
void add_packed_bool(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:545
void add_packed_double(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:732
int write_varint(OutputIterator data, uint64_t value)
Definition: varint.hpp:89
void add_packed_float(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:717
void add_packed_int32(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:573
pbf_writer(pbf_writer &parent_writer, pbf_tag_type tag, std::size_t size=0)
Definition: pbf_writer.hpp:251
void add_sfixed32(pbf_tag_type tag, int32_t value)
Definition: pbf_writer.hpp:401
pbf_writer(std::string &data) noexcept
Definition: pbf_writer.hpp:225
Contains low-level varint and zigzag encoding and decoding functions.
Definition: pbf_writer.hpp:40
void add_sint32(pbf_tag_type tag, int32_t value)
Definition: pbf_writer.hpp:340
void add_fixed32(pbf_tag_type tag, uint32_t value)
Definition: pbf_writer.hpp:390
Definition: pbf_writer.hpp:747
void add_packed_fixed32(pbf_tag_type tag, InputIterator first, InputIterator last)
Definition: pbf_writer.hpp:657
void add_double(pbf_tag_type tag, double value)
Definition: pbf_writer.hpp:445
void add_bytes(pbf_tag_type tag, const char *value, std::size_t size)
Definition: pbf_writer.hpp:457
All parts of the protozero header-only library are in this namespace.
Definition: byteswap.hpp:24