Libosmium  2.15.4
Fast and flexible C++ library for working with OpenStreetMap data
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2019 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/handler.hpp>
38 #include <osmium/memory/buffer.hpp>
39 #include <osmium/osm/item_type.hpp>
40 #include <osmium/osm/object.hpp>
41 #include <osmium/osm/relation.hpp>
42 #include <osmium/osm/types.hpp>
43 #include <osmium/relations/detail/member_meta.hpp>
44 #include <osmium/relations/detail/relation_meta.hpp>
45 #include <osmium/util/iterator.hpp>
46 #include <osmium/visitor.hpp>
47 
48 #include <algorithm>
49 #include <array>
50 #include <cassert>
51 #include <cstddef>
52 #include <cstdint>
53 #include <functional>
54 #include <iomanip>
55 #include <iostream>
56 #include <utility>
57 #include <vector>
58 
59 namespace osmium {
60 
61  class Node;
62  class Way;
63 
67  namespace relations {
68 
97  template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
98  class Collector {
99 
104 
105  TCollector& m_collector;
106 
107  public:
108 
109  explicit HandlerPass1(TCollector& collector) noexcept :
110  m_collector(collector) {
111  }
112 
114  if (m_collector.keep_relation(relation)) {
115  m_collector.add_relation(relation);
116  }
117  }
118 
119  }; // class HandlerPass1
120 
121  public:
122 
127 
129  TCollector& m_collector;
130 
131  public:
132 
133  explicit HandlerPass2(TCollector& collector) noexcept :
134  m_collector(collector) {
135  }
136 
137  void node(const osmium::Node& node) {
138  if (TNodes) {
140  if (!m_collector.find_and_add_object(node)) {
141  m_collector.node_not_in_any_relation(node);
142  }
143  }
144  }
145 
146  void way(const osmium::Way& way) {
147  if (TWays) {
149  if (!m_collector.find_and_add_object(way)) {
150  m_collector.way_not_in_any_relation(way);
151  }
152  }
153  }
154 
156  if (TRelations) {
158  if (!m_collector.find_and_add_object(relation)) {
159  m_collector.relation_not_in_any_relation(relation);
160  }
161  }
162  }
163 
164  void flush() {
165  m_collector.flush();
166  }
167 
168  }; // class HandlerPass2
169 
170  private:
171 
172  HandlerPass2 m_handler_pass2;
173 
174  // All relations we are interested in will be kept in this buffer
176 
177  // All members we are interested in will be kept in this buffer
179 
181  std::vector<RelationMeta> m_relations;
182 
187  using mm_vector_type = std::vector<MemberMeta>;
188  using mm_iterator = mm_vector_type::iterator;
189  std::array<mm_vector_type, 3> m_member_meta;
190 
192 
193  using callback_func_type = std::function<void(osmium::memory::Buffer&&)>;
195 
196  enum {
197  initial_buffer_size = 1024UL * 1024UL
198  };
199 
201  auto& mmv = member_meta(type);
202  return make_range(std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id)));
203  }
204 
205  public:
206 
211  m_handler_pass2(*static_cast<TCollector*>(this)),
212  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
213  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes) {
214  }
215 
216  protected:
217 
218  std::vector<MemberMeta>& member_meta(const item_type type) {
219  return m_member_meta[static_cast<uint16_t>(type) - 1];
220  }
221 
223  return m_callback;
224  }
225 
226  const std::vector<RelationMeta>& relations() const {
227  return m_relations;
228  }
229 
239  bool keep_relation(const osmium::Relation& /*relation*/) const {
240  return true;
241  }
242 
253  bool keep_member(const RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
254  return true;
255  }
256 
264  void node_not_in_any_relation(const osmium::Node& /*node*/) {
265  }
266 
274  void way_not_in_any_relation(const osmium::Way& /*way*/) {
275  }
276 
284  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
285  }
286 
298  void flush() {
299  }
300 
301  const osmium::Relation& get_relation(size_t offset) const {
302  assert(m_relations_buffer.committed() > offset);
303  return m_relations_buffer.get<osmium::Relation>(offset);
304  }
305 
309  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
310  return get_relation(relation_meta.relation_offset());
311  }
312 
316  const osmium::Relation& get_relation(const MemberMeta& member_meta) const {
317  return get_relation(m_relations[member_meta.relation_pos()]);
318  }
319 
320  osmium::OSMObject& get_member(size_t offset) const {
321  assert(m_members_buffer.committed() > offset);
322  return m_members_buffer.get<osmium::OSMObject>(offset);
323  }
324 
325  private:
326 
335  void add_relation(const osmium::Relation& relation) {
336  const size_t offset = m_relations_buffer.committed();
337  m_relations_buffer.add_item(relation);
338 
339  RelationMeta relation_meta{offset};
340 
341  int n = 0;
342  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
343  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
344  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
345  relation_meta.increment_need_members();
346  } else {
347  member.set_ref(0); // set member id to zero to indicate we are not interested
348  }
349  ++n;
350  }
351 
352  assert(offset == m_relations_buffer.committed());
353  if (relation_meta.has_all_members()) {
355  } else {
357  m_relations.push_back(relation_meta);
358  }
359  }
360 
366  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
367  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
368  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
369  }
370 
372  return std::count_if(range.begin(), range.end(), [](MemberMeta& mm) {
373  return !mm.removed();
374  });
375  }
376 
385  auto range = find_member_meta(object.type(), object.id());
386 
387  if (count_not_removed(range) == 0) {
388  // nothing found
389  return false;
390  }
391 
392  {
393  members_buffer().add_item(object);
394  const size_t member_offset = members_buffer().commit();
395 
396  for (auto& member_meta : range) {
397  member_meta.set_buffer_offset(member_offset);
398  }
399  }
400 
401  for (auto& member_meta : range) {
402  if (member_meta.removed()) {
403  break;
404  }
405  assert(member_meta.member_id() == object.id());
406  assert(member_meta.relation_pos() < m_relations.size());
407  RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
408  assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
409  relation_meta.got_one_member();
410  if (relation_meta.has_all_members()) {
411  const size_t relation_offset = member_meta.relation_pos();
412  static_cast<TCollector*>(this)->complete_relation(relation_meta);
413  clear_member_metas(relation_meta);
414  m_relations[relation_offset] = RelationMeta{};
416  }
417  }
418 
419  return true;
420  }
421 
422  void clear_member_metas(const RelationMeta& relation_meta) {
423  const osmium::Relation& relation = get_relation(relation_meta);
424  for (const auto& member : relation.members()) {
425  if (member.ref() != 0) {
426  const auto range = find_member_meta(member.type(), member.ref());
427  assert(!range.empty());
428 
429  // if this is the last time this object was needed
430  // then mark it as removed
431  if (count_not_removed(range) == 1) {
432  get_member(range.begin()->buffer_offset()).set_removed(true);
433  }
434 
435  for (auto& member_meta : range) {
436  if (!member_meta.removed() && relation.id() == get_relation(member_meta).id()) {
437  member_meta.remove();
438  break;
439  }
440  }
441  }
442  }
443  }
444 
445  public:
446 
447  uint64_t used_memory() const {
448  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
449  const uint64_t members = nmembers * sizeof(MemberMeta);
450  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
451  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
452  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
453 
454  std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
455  std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
456  std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
457  std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
458  std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
459 
460  std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
461  std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
462 
463  std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
464  std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
465  std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
466  std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
467 
468  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
469 
470  std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
471  std::cerr << " =======================================================\n";
472 
473  return relations_buffer_capacity + members_buffer_capacity + relations + members;
474  }
475 
479  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
481  return m_handler_pass2;
482  }
483 
485  return m_members_buffer;
486  }
487 
500  const auto range = find_member_meta(type, id);
501  assert(!range.empty());
502  return range.begin()->is_available();
503  }
504 
515  const auto range = find_member_meta(type, id);
516  assert(!range.empty());
517  assert(range.begin()->is_available());
518  return range.begin()->buffer_offset();
519  }
520 
538  const auto range = find_member_meta(type, id);
539  assert(!range.empty());
540  if (range.begin()->is_available()) {
541  return std::make_pair(true, range.begin()->buffer_offset());
542  }
543  return std::make_pair(false, 0);
544  }
545 
546  template <typename TIter>
547  void read_relations(TIter begin, TIter end) {
548  HandlerPass1 handler(*static_cast<TCollector*>(this));
551  }
552 
553  template <typename TSource>
554  void read_relations(TSource& source) {
555  using std::begin;
556  using std::end;
557  read_relations(begin(source), end(source));
558  source.close();
559  }
560 
561  void moving_in_buffer(size_t old_offset, size_t new_offset) {
562  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
563  auto range = find_member_meta(object.type(), object.id());
564  for (auto& member_meta : range) {
565  assert(member_meta.buffer_offset() == old_offset);
566  member_meta.set_buffer_offset(new_offset);
567  }
568  }
569 
578  if (m_count_complete > 10000) { // XXX
579 // const size_t size_before = m_members_buffer.committed();
581 /*
582  const size_t size_after = m_members_buffer.committed();
583  double percent = static_cast<double>(size_before - size_after);
584  percent /= size_before;
585  percent *= 100;
586  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
587 */
588  m_count_complete = 0;
589  }
590  }
591 
600  std::vector<const osmium::Relation*> get_incomplete_relations() const {
601  std::vector<const osmium::Relation*> relations;
602  for (const auto& relation_meta : m_relations) {
603  if (!relation_meta.has_all_members()) {
604  relations.push_back(&get_relation(relation_meta));
605  }
606  }
607  return relations;
608  }
609 
610  }; // class Collector
611 
612  } // namespace relations
613 
614 } // namespace osmium
615 
616 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
osmium::relations::Collector::get_relation
const osmium::Relation & get_relation(const MemberMeta &member_meta) const
Definition: collector.hpp:316
osmium::relations::Collector::get_availability_and_offset
std::pair< bool, size_t > get_availability_and_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:537
osmium::make_range
iterator_range< It > make_range(P &&p) noexcept
Definition: iterator.hpp:68
osmium::relations::Collector::possibly_purge_removed_members
void possibly_purge_removed_members()
Definition: collector.hpp:576
osmium::relations::Collector::keep_relation
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:239
osmium::apply
void apply(TIterator it, TIterator end, THandlers &&... handlers)
Definition: visitor.hpp:323
relation.hpp
osmium::relations::Collector::moving_in_buffer
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:561
osmium::relations::Collector::find_and_add_object
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:384
osmium::OSMObject
Definition: object.hpp:64
iterator.hpp
osmium::relations::Collector::m_members_buffer
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:178
osmium::relations::Collector::find_member_meta
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:200
osmium::relations::Collector::way_not_in_any_relation
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:274
osmium::handler::Handler
Definition: handler.hpp:71
types.hpp
osmium::memory::Buffer::rollback
void rollback()
Definition: buffer.hpp:484
osmium::relations::Collector::m_callback
callback_func_type m_callback
Definition: collector.hpp:194
osmium::relations::Collector::sort_member_meta
void sort_member_meta()
Definition: collector.hpp:365
osmium::object_id_type
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
osmium::handler::CheckOrder::relation
void relation(const osmium::Relation &relation)
Definition: check_order.hpp:126
osmium::relations::Collector::HandlerPass2::way
void way(const osmium::Way &way)
Definition: collector.hpp:146
osmium::memory::Buffer::purge_removed
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:854
osmium::memory::Buffer::committed
std::size_t committed() const noexcept
Definition: buffer.hpp:356
osmium::memory::Buffer::add_item
T & add_item(const T &item)
Definition: buffer.hpp:601
osmium::relations::Collector::m_count_complete
int m_count_complete
Definition: collector.hpp:191
osmium::Node
Definition: node.hpp:48
visitor.hpp
osmium::relations::Collector::HandlerPass1::relation
void relation(const osmium::Relation &relation)
Definition: collector.hpp:113
osmium::handler::CheckOrder
Definition: check_order.hpp:87
osmium::relations::Collector< MultipolygonCollector< TAssembler >, false, true, false >::mm_iterator
mm_vector_type::iterator mm_iterator
Definition: collector.hpp:188
osmium::relations::Collector< MultipolygonCollector< TAssembler >, false, true, false >::mm_vector_type
std::vector< MemberMeta > mm_vector_type
Definition: collector.hpp:187
osmium::relations::Collector::HandlerPass1::m_collector
TCollector & m_collector
Definition: collector.hpp:105
osmium::relations::Collector
Definition: collector.hpp:98
osmium::relations::Collector::count_not_removed
static iterator_range< mm_iterator >::iterator::difference_type count_not_removed(const iterator_range< mm_iterator > &range)
Definition: collector.hpp:371
osmium::handler::CheckOrder::node
void node(const osmium::Node &node)
Definition: check_order.hpp:95
osmium::memory::Item::set_removed
void set_removed(const bool removed) noexcept
Definition: item.hpp:179
osmium::relations::Collector::member_meta
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:218
osmium::relations::Collector::HandlerPass2::m_check_order
osmium::handler::CheckOrder m_check_order
Definition: collector.hpp:128
osmium::memory::Buffer
Definition: buffer.hpp:97
osmium::relations::Collector::get_relation
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:301
osmium::io::begin
InputIterator< Reader > begin(Reader &reader)
Definition: reader_iterator.hpp:43
osmium
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
osmium::relations::Collector::keep_member
bool keep_member(const RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:253
check_order.hpp
osmium::relations::Collector::clear_member_metas
void clear_member_metas(const RelationMeta &relation_meta)
Definition: collector.hpp:422
osmium::relations::Collector::HandlerPass1
Definition: collector.hpp:103
osmium::relations::Collector::members_buffer
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:484
osmium::iterator_range
Definition: iterator.hpp:42
osmium::relations::Collector::get_offset
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:514
osmium::Way
Definition: way.hpp:72
osmium::memory::Buffer::get
T & get(const std::size_t offset) const
Definition: buffer.hpp:518
osmium::relations::Collector::flush
void flush()
Definition: collector.hpp:298
osmium::Relation
Definition: relation.hpp:168
osmium::iterator_range::begin
It begin() const noexcept
Definition: iterator.hpp:50
osmium::relations::Collector::get_relation
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:309
osmium::relations::Collector::m_relations
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:181
osmium::RelationMember
Definition: relation.hpp:57
osmium::relations::Collector::get_incomplete_relations
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:600
osmium::relations::Collector::node_not_in_any_relation
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:264
osmium::relations::Collector::read_relations
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:547
osmium::relations::Collector::is_available
bool is_available(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:499
osmium::memory::Buffer::commit
std::size_t commit()
Definition: buffer.hpp:468
osmium::relations::Collector< MultipolygonCollector< TAssembler >, false, true, false >::callback_func_type
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:193
osmium::relations::Collector::m_handler_pass2
HandlerPass2 m_handler_pass2
Definition: collector.hpp:172
osmium::relations::Collector::callback
callback_func_type callback()
Definition: collector.hpp:222
osmium::relations::Collector::Collector
Collector()
Definition: collector.hpp:210
osmium::relations::Collector::m_member_meta
std::array< mm_vector_type, 3 > m_member_meta
Definition: collector.hpp:189
osmium::relations::Collector::HandlerPass2::relation
void relation(const osmium::Relation &relation)
Definition: collector.hpp:155
handler.hpp
osmium::relations::Collector::get_member
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:320
osmium::relations::Collector::handler
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:479
osmium::memory::Buffer::capacity
std::size_t capacity() const noexcept
Definition: buffer.hpp:348
osmium::relations::Collector::HandlerPass2::flush
void flush()
Definition: collector.hpp:164
osmium::relations::Collector::HandlerPass2
Definition: collector.hpp:126
osmium::relations::Collector::add_relation
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:335
osmium::relations::Collector::initial_buffer_size
@ initial_buffer_size
Definition: collector.hpp:197
buffer.hpp
osmium::relations::Collector::relations
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:226
osmium::handler::CheckOrder::way
void way(const osmium::Way &way)
Definition: check_order.hpp:112
osmium::relations::Collector::HandlerPass2::HandlerPass2
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:133
osmium::Relation::members
RelationMemberList & members()
Get a reference to the member list.
Definition: relation.hpp:186
osmium::iterator_range::end
It end() const noexcept
Definition: iterator.hpp:54
object.hpp
osmium::relations::Collector::used_memory
uint64_t used_memory() const
Definition: collector.hpp:447
item_type.hpp
osmium::io::end
InputIterator< Reader > end(Reader &)
Definition: reader_iterator.hpp:47
osmium::relations::Collector::HandlerPass2::node
void node(const osmium::Node &node)
Definition: collector.hpp:137
osmium::relations::Collector::m_relations_buffer
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:175
osmium::relations::Collector::relation_not_in_any_relation
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:284
osmium::relations::Collector::HandlerPass2::m_collector
TCollector & m_collector
Definition: collector.hpp:129
osmium::relations::Collector::read_relations
void read_relations(TSource &source)
Definition: collector.hpp:554
osmium::osm_entity_bits::type
type
Definition: entity_bits.hpp:63
osmium::item_type
item_type
Definition: item_type.hpp:43
osmium::relations::Collector::HandlerPass1::HandlerPass1
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:109