Libosmium  2.6.1
Fast and flexible C++ library for working with OpenStreetMap data
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstdint>
40 #include <functional>
41 #include <iomanip>
42 //#include <iostream>
43 #include <vector>
44 
45 #include <osmium/fwd.hpp>
46 #include <osmium/osm/item_type.hpp>
47 #include <osmium/osm/object.hpp>
48 #include <osmium/osm/relation.hpp> // IWYU pragma: keep
49 #include <osmium/osm/types.hpp>
50 #include <osmium/handler.hpp>
51 #include <osmium/memory/buffer.hpp>
52 #include <osmium/util/iterator.hpp>
53 #include <osmium/visitor.hpp>
54 
55 #include <osmium/relations/detail/relation_meta.hpp>
56 #include <osmium/relations/detail/member_meta.hpp>
57 
58 namespace osmium {
59 
63  namespace relations {
64 
65  namespace detail {
66 
67  template <typename R>
68  inline typename std::iterator_traits<typename R::iterator>::difference_type count_not_removed(const R& range) {
69  return std::count_if(range.begin(), range.end(), [](MemberMeta& mm) {
70  return !mm.removed();
71  });
72  }
73 
74  } // namespace detail
75 
104  template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
105  class Collector {
106 
111 
112  TCollector& m_collector;
113 
114  public:
115 
116  HandlerPass1(TCollector& collector) noexcept :
117  m_collector(collector) {
118  }
119 
120  void relation(const osmium::Relation& relation) {
121  if (m_collector.keep_relation(relation)) {
122  m_collector.add_relation(relation);
123  }
124  }
125 
126  }; // class HandlerPass1
127 
128  public:
129 
134 
135  TCollector& m_collector;
136 
137  public:
138 
139  HandlerPass2(TCollector& collector) noexcept :
140  m_collector(collector) {
141  }
142 
143  void node(const osmium::Node& node) {
144  if (TNodes) {
145  if (! m_collector.find_and_add_object(node)) {
146  m_collector.node_not_in_any_relation(node);
147  }
148  }
149  }
150 
151  void way(const osmium::Way& way) {
152  if (TWays) {
153  if (! m_collector.find_and_add_object(way)) {
154  m_collector.way_not_in_any_relation(way);
155  }
156  }
157  }
158 
159  void relation(const osmium::Relation& relation) {
160  if (TRelations) {
161  if (! m_collector.find_and_add_object(relation)) {
162  m_collector.relation_not_in_any_relation(relation);
163  }
164  }
165  }
166 
167  void flush() {
168  m_collector.flush();
169  }
170 
171  }; // class HandlerPass2
172 
173  private:
174 
175  HandlerPass2 m_handler_pass2;
176 
177  // All relations we are interested in will be kept in this buffer
179 
180  // All members we are interested in will be kept in this buffer
182 
184  std::vector<RelationMeta> m_relations;
185 
190  using mm_vector_type = std::vector<MemberMeta>;
191  using mm_iterator = mm_vector_type::iterator;
192  mm_vector_type m_member_meta[3];
193 
194  int m_count_complete = 0;
195 
196  typedef std::function<void(osmium::memory::Buffer&&)> callback_func_type;
197  callback_func_type m_callback;
198 
199  static constexpr size_t initial_buffer_size = 1024 * 1024;
200 
202  auto& mmv = member_meta(type);
203  return iterator_range<mm_iterator>{std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id))};
204  }
205 
206  public:
207 
212  m_handler_pass2(*static_cast<TCollector*>(this)),
213  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
214  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
215  m_relations(),
216  m_member_meta() {
217  }
218 
219  protected:
220 
221  std::vector<MemberMeta>& member_meta(const item_type type) {
222  return m_member_meta[static_cast<uint16_t>(type) - 1];
223  }
224 
225  callback_func_type callback() {
226  return m_callback;
227  }
228 
229  const std::vector<RelationMeta>& relations() const {
230  return m_relations;
231  }
232 
242  bool keep_relation(const osmium::Relation& /*relation*/) const {
243  return true;
244  }
245 
256  bool keep_member(const osmium::relations::RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
257  return true;
258  }
259 
267  void node_not_in_any_relation(const osmium::Node& /*node*/) {
268  }
269 
277  void way_not_in_any_relation(const osmium::Way& /*way*/) {
278  }
279 
287  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
288  }
289 
301  void flush() {
302  }
303 
309  m_relations.erase(
310  std::remove_if(m_relations.begin(), m_relations.end(), has_all_members()),
311  m_relations.end()
312  );
313  }
314 
315  const osmium::Relation& get_relation(size_t offset) const {
316  return m_relations_buffer.get<osmium::Relation>(offset);
317  }
318 
322  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
323  return get_relation(relation_meta.relation_offset());
324  }
325 
326  osmium::OSMObject& get_member(size_t offset) const {
327  return m_members_buffer.get<osmium::OSMObject>(offset);
328  }
329 
330  private:
331 
340  void add_relation(const osmium::Relation& relation) {
341  const size_t offset = m_relations_buffer.committed();
342  m_relations_buffer.add_item(relation);
343 
344  RelationMeta relation_meta(offset);
345 
346  int n = 0;
347  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
348  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
349  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
350  relation_meta.increment_need_members();
351  } else {
352  member.ref(0); // set member id to zero to indicate we are not interested
353  }
354  ++n;
355  }
356 
357  assert(offset == m_relations_buffer.committed());
358  if (relation_meta.has_all_members()) {
359  m_relations_buffer.rollback();
360  } else {
361  m_relations_buffer.commit();
362  m_relations.push_back(std::move(relation_meta));
363 // std::cerr << "added relation id=" << relation.id() << "\n";
364  }
365  }
366 
372 /* std::cerr << "relations: " << m_relations.size() << "\n";
373  std::cerr << "node members: " << m_member_meta[0].size() << "\n";
374  std::cerr << "way members: " << m_member_meta[1].size() << "\n";
375  std::cerr << "relation members: " << m_member_meta[2].size() << "\n";*/
376  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
377  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
378  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
379  }
380 
389  auto range = find_member_meta(object.type(), object.id());
390 
391  if (detail::count_not_removed(range) == 0) {
392  // nothing found
393  return false;
394  }
395 
396  {
397  members_buffer().add_item(object);
398  const size_t member_offset = members_buffer().commit();
399 
400  for (auto& member_meta : range) {
401  member_meta.set_buffer_offset(member_offset);
402  }
403  }
404 
405  for (auto& member_meta : range) {
406  if (member_meta.removed()) {
407  break;
408  }
409  assert(member_meta.member_id() == object.id());
410  assert(member_meta.relation_pos() < m_relations.size());
411  RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
412 // std::cerr << " => " << member_meta.member_pos() << " < " << get_relation(relation_meta).members().size() << " (id=" << get_relation(relation_meta).id() << ")\n";
413  assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
414 // std::cerr << " add way " << member_meta.member_id() << " to rel " << get_relation(relation_meta).id() << " at pos " << member_meta.member_pos() << "\n";
415  relation_meta.got_one_member();
416  if (relation_meta.has_all_members()) {
417  const size_t relation_offset = member_meta.relation_pos();
418  static_cast<TCollector*>(this)->complete_relation(relation_meta);
419  clear_member_metas(relation_meta);
420  m_relations[relation_offset] = RelationMeta();
421  possibly_purge_removed_members();
422  }
423  }
424 
425  return true;
426  }
427 
428  void clear_member_metas(const osmium::relations::RelationMeta& relation_meta) {
429  const osmium::Relation& relation = get_relation(relation_meta);
430  for (const auto& member : relation.members()) {
431  if (member.ref() != 0) {
432  auto range = find_member_meta(member.type(), member.ref());
433  assert(!range.empty());
434 
435  // if this is the last time this object was needed
436  // then mark it as removed
437  if (detail::count_not_removed(range) == 1) {
438  get_member(range.begin()->buffer_offset()).set_removed(true);
439  }
440 
441  for (auto& member_meta : range) {
442  if (!member_meta.removed() && relation.id() == get_relation(member_meta.relation_pos()).id()) {
443  member_meta.remove();
444  break;
445  }
446  }
447  }
448  }
449  }
450 
451  public:
452 
453  uint64_t used_memory() const {
454  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
455  const uint64_t members = nmembers * sizeof(MemberMeta);
456  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
457  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
458  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
459 
460  std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
461  std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
462  std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
463  std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
464  std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
465 
466  std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
467  std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
468 
469  std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
470  std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
471  std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
472  std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
473 
474  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
475 
476  std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
477  std::cerr << " =======================================================\n";
478 
479  return relations_buffer_capacity + members_buffer_capacity + relations + members;
480  }
481 
485  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
486  m_callback = callback;
487  return m_handler_pass2;
488  }
489 
491  return m_members_buffer;
492  }
493 
495  const auto range = find_member_meta(type, id);
496  assert(!range.empty());
497  return range.begin()->buffer_offset();
498  }
499 
500  template <typename TIter>
501  void read_relations(TIter begin, TIter end) {
502  HandlerPass1 handler(*static_cast<TCollector*>(this));
503  osmium::apply(begin, end, handler);
504  sort_member_meta();
505  }
506 
507  template <typename TSource>
508  void read_relations(TSource& source) {
509  read_relations(std::begin(source), std::end(source));
510  source.close();
511  }
512 
513  void moving_in_buffer(size_t old_offset, size_t new_offset) {
514  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
515  auto range = find_member_meta(object.type(), object.id());
516  for (auto& member_meta : range) {
517  assert(member_meta.buffer_offset() == old_offset);
518  member_meta.set_buffer_offset(new_offset);
519  }
520  }
521 
529  ++m_count_complete;
530  if (m_count_complete > 10000) { // XXX
531 // const size_t size_before = m_members_buffer.committed();
532  m_members_buffer.purge_removed(this);
533 /*
534  const size_t size_after = m_members_buffer.committed();
535  double percent = static_cast<double>(size_before - size_after);
536  percent /= size_before;
537  percent *= 100;
538  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
539 */
540  m_count_complete = 0;
541  }
542  }
543 
552  std::vector<const osmium::Relation*> get_incomplete_relations() const {
553  std::vector<const osmium::Relation*> relations;
554  for (const auto& relation_meta : m_relations) {
555  if (!relation_meta.has_all_members()) {
556  relations.push_back(&get_relation(relation_meta));
557  }
558  }
559  return relations;
560  }
561 
562  }; // class Collector
563 
564  } // namespace relations
565 
566 } // namespace osmium
567 
568 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:552
void relation(const osmium::Relation &relation)
Definition: collector.hpp:159
void clear_member_metas(const osmium::relations::RelationMeta &relation_meta)
Definition: collector.hpp:428
Definition: iterator.hpp:42
callback_func_type m_callback
Definition: collector.hpp:197
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:490
type
Definition: entity_bits.hpp:60
RelationMemberList & members()
Definition: relation.hpp:177
void way(const osmium::Way &way)
Definition: collector.hpp:151
item_type
Definition: item_type.hpp:43
void clean_assembled_relations()
Definition: collector.hpp:308
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:513
Definition: relation.hpp:165
size_t capacity() const noexcept
Definition: buffer.hpp:233
Definition: handler.hpp:45
bool keep_member(const osmium::relations::RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:256
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:181
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:229
void read_relations(TSource &source)
Definition: collector.hpp:508
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:494
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:287
Definition: way.hpp:65
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:221
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:388
Definition: collector.hpp:133
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:139
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:184
void sort_member_meta()
Definition: collector.hpp:371
void apply(TIterator it, TIterator end, THandlers &...handlers)
Definition: visitor.hpp:234
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:267
Definition: relation.hpp:54
TCollector & m_collector
Definition: collector.hpp:112
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
T & add_item(const T &item)
Definition: buffer.hpp:457
Definition: collector.hpp:105
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:201
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:315
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:690
Definition: attr.hpp:298
HandlerPass2 m_handler_pass2
Definition: collector.hpp:175
Collector()
Definition: collector.hpp:211
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:196
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:326
void relation(const osmium::Relation &relation)
Definition: collector.hpp:120
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
Definition: collector.hpp:110
void flush()
Definition: collector.hpp:167
size_t committed() const noexcept
Definition: buffer.hpp:241
std::vector< MemberMeta > mm_vector_type
Definition: collector.hpp:190
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:178
uint64_t used_memory() const
Definition: collector.hpp:453
callback_func_type callback()
Definition: collector.hpp:225
Definition: buffer.hpp:97
T & get(const size_t offset) const
Definition: buffer.hpp:379
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:340
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:485
object_id_type id() const noexcept
Get ID of this object.
Definition: object.hpp:110
void flush()
Definition: collector.hpp:301
Definition: node.hpp:47
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:277
void node(const osmium::Node &node)
Definition: collector.hpp:143
void possibly_purge_removed_members()
Definition: collector.hpp:528
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:242
TCollector & m_collector
Definition: collector.hpp:135
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:501
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:116
void rollback()
Definition: buffer.hpp:349
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:322
Definition: object.hpp:58
size_t commit()
Definition: buffer.hpp:335