Libosmium  2.7.2
Fast and flexible C++ library for working with OpenStreetMap data
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstdint>
40 #include <functional>
41 #include <iomanip>
42 //#include <iostream>
43 #include <vector>
44 
45 #include <osmium/fwd.hpp>
46 #include <osmium/osm/item_type.hpp>
47 #include <osmium/osm/object.hpp>
48 #include <osmium/osm/relation.hpp> // IWYU pragma: keep
49 #include <osmium/osm/types.hpp>
50 #include <osmium/handler.hpp>
51 #include <osmium/memory/buffer.hpp>
52 #include <osmium/util/iterator.hpp>
53 #include <osmium/visitor.hpp>
54 
55 #include <osmium/relations/detail/relation_meta.hpp>
56 #include <osmium/relations/detail/member_meta.hpp>
57 
58 namespace osmium {
59 
63  namespace relations {
64 
65  namespace detail {
66 
67  } // namespace detail
68 
97  template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
98  class Collector {
99 
104 
105  TCollector& m_collector;
106 
107  public:
108 
109  HandlerPass1(TCollector& collector) noexcept :
110  m_collector(collector) {
111  }
112 
113  void relation(const osmium::Relation& relation) {
114  if (m_collector.keep_relation(relation)) {
115  m_collector.add_relation(relation);
116  }
117  }
118 
119  }; // class HandlerPass1
120 
121  public:
122 
127 
128  TCollector& m_collector;
129 
130  public:
131 
132  HandlerPass2(TCollector& collector) noexcept :
133  m_collector(collector) {
134  }
135 
136  void node(const osmium::Node& node) {
137  if (TNodes) {
138  if (! m_collector.find_and_add_object(node)) {
139  m_collector.node_not_in_any_relation(node);
140  }
141  }
142  }
143 
144  void way(const osmium::Way& way) {
145  if (TWays) {
146  if (! m_collector.find_and_add_object(way)) {
147  m_collector.way_not_in_any_relation(way);
148  }
149  }
150  }
151 
152  void relation(const osmium::Relation& relation) {
153  if (TRelations) {
154  if (! m_collector.find_and_add_object(relation)) {
155  m_collector.relation_not_in_any_relation(relation);
156  }
157  }
158  }
159 
160  void flush() {
161  m_collector.flush();
162  }
163 
164  }; // class HandlerPass2
165 
166  private:
167 
168  HandlerPass2 m_handler_pass2;
169 
170  // All relations we are interested in will be kept in this buffer
172 
173  // All members we are interested in will be kept in this buffer
175 
177  std::vector<RelationMeta> m_relations;
178 
183  using mm_vector_type = std::vector<MemberMeta>;
184  using mm_iterator = mm_vector_type::iterator;
185  mm_vector_type m_member_meta[3];
186 
187  int m_count_complete = 0;
188 
189  using callback_func_type = std::function<void(osmium::memory::Buffer&&)>;
191 
192  static constexpr size_t initial_buffer_size = 1024 * 1024;
193 
195  auto& mmv = member_meta(type);
196  return make_range(std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id)));
197  }
198 
199  public:
200 
205  m_handler_pass2(*static_cast<TCollector*>(this)),
206  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
207  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
208  m_relations(),
209  m_member_meta() {
210  }
211 
212  protected:
213 
214  std::vector<MemberMeta>& member_meta(const item_type type) {
215  return m_member_meta[static_cast<uint16_t>(type) - 1];
216  }
217 
219  return m_callback;
220  }
221 
222  const std::vector<RelationMeta>& relations() const {
223  return m_relations;
224  }
225 
235  bool keep_relation(const osmium::Relation& /*relation*/) const {
236  return true;
237  }
238 
249  bool keep_member(const osmium::relations::RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
250  return true;
251  }
252 
260  void node_not_in_any_relation(const osmium::Node& /*node*/) {
261  }
262 
270  void way_not_in_any_relation(const osmium::Way& /*way*/) {
271  }
272 
280  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
281  }
282 
294  void flush() {
295  }
296 
302  m_relations.erase(
303  std::remove_if(m_relations.begin(), m_relations.end(), has_all_members()),
304  m_relations.end()
305  );
306  }
307 
308  const osmium::Relation& get_relation(size_t offset) const {
309  assert(m_relations_buffer.committed() > offset);
310  return m_relations_buffer.get<osmium::Relation>(offset);
311  }
312 
316  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
317  return get_relation(relation_meta.relation_offset());
318  }
319 
323  const osmium::Relation& get_relation(const MemberMeta& member_meta) const {
324  return get_relation(m_relations[member_meta.relation_pos()]);
325  }
326 
327  osmium::OSMObject& get_member(size_t offset) const {
328  assert(m_members_buffer.committed() > offset);
329  return m_members_buffer.get<osmium::OSMObject>(offset);
330  }
331 
332  private:
333 
342  void add_relation(const osmium::Relation& relation) {
343  const size_t offset = m_relations_buffer.committed();
344  m_relations_buffer.add_item(relation);
345 
346  RelationMeta relation_meta(offset);
347 
348  int n = 0;
349  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
350  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
351  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
352  relation_meta.increment_need_members();
353  } else {
354  member.ref(0); // set member id to zero to indicate we are not interested
355  }
356  ++n;
357  }
358 
359  assert(offset == m_relations_buffer.committed());
360  if (relation_meta.has_all_members()) {
361  m_relations_buffer.rollback();
362  } else {
363  m_relations_buffer.commit();
364  m_relations.push_back(std::move(relation_meta));
365 // std::cerr << "added relation id=" << relation.id() << "\n";
366  }
367  }
368 
374 /* std::cerr << "relations: " << m_relations.size() << "\n";
375  std::cerr << "node members: " << m_member_meta[0].size() << "\n";
376  std::cerr << "way members: " << m_member_meta[1].size() << "\n";
377  std::cerr << "relation members: " << m_member_meta[2].size() << "\n";*/
378  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
379  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
380  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
381  }
382 
384  return std::count_if(range.begin(), range.end(), [](MemberMeta& mm) {
385  return !mm.removed();
386  });
387  }
388 
397  auto range = find_member_meta(object.type(), object.id());
398 
399  if (count_not_removed(range) == 0) {
400  // nothing found
401  return false;
402  }
403 
404  {
405  members_buffer().add_item(object);
406  const size_t member_offset = members_buffer().commit();
407 
408  for (auto& member_meta : range) {
409  member_meta.set_buffer_offset(member_offset);
410  }
411  }
412 
413  for (auto& member_meta : range) {
414  if (member_meta.removed()) {
415  break;
416  }
417  assert(member_meta.member_id() == object.id());
418  assert(member_meta.relation_pos() < m_relations.size());
419  RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
420 // std::cerr << " => " << member_meta.member_pos() << " < " << get_relation(relation_meta).members().size() << " (id=" << get_relation(relation_meta).id() << ")\n";
421  assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
422 // std::cerr << " add way " << member_meta.member_id() << " to rel " << get_relation(relation_meta).id() << " at pos " << member_meta.member_pos() << "\n";
423  relation_meta.got_one_member();
424  if (relation_meta.has_all_members()) {
425  const size_t relation_offset = member_meta.relation_pos();
426  static_cast<TCollector*>(this)->complete_relation(relation_meta);
427  clear_member_metas(relation_meta);
428  m_relations[relation_offset] = RelationMeta();
429  possibly_purge_removed_members();
430  }
431  }
432 
433  return true;
434  }
435 
436  void clear_member_metas(const osmium::relations::RelationMeta& relation_meta) {
437  const osmium::Relation& relation = get_relation(relation_meta);
438  for (const auto& member : relation.members()) {
439  if (member.ref() != 0) {
440  auto range = find_member_meta(member.type(), member.ref());
441  assert(!range.empty());
442 
443  // if this is the last time this object was needed
444  // then mark it as removed
445  if (count_not_removed(range) == 1) {
446  get_member(range.begin()->buffer_offset()).set_removed(true);
447  }
448 
449  for (auto& member_meta : range) {
450  if (!member_meta.removed() && relation.id() == get_relation(member_meta).id()) {
451  member_meta.remove();
452  break;
453  }
454  }
455  }
456  }
457  }
458 
459  public:
460 
461  uint64_t used_memory() const {
462  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
463  const uint64_t members = nmembers * sizeof(MemberMeta);
464  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
465  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
466  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
467 
468  std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
469  std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
470  std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
471  std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
472  std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
473 
474  std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
475  std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
476 
477  std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
478  std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
479  std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
480  std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
481 
482  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
483 
484  std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
485  std::cerr << " =======================================================\n";
486 
487  return relations_buffer_capacity + members_buffer_capacity + relations + members;
488  }
489 
493  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
494  m_callback = callback;
495  return m_handler_pass2;
496  }
497 
499  return m_members_buffer;
500  }
501 
503  const auto range = find_member_meta(type, id);
504  assert(!range.empty());
505  return range.begin()->buffer_offset();
506  }
507 
508  template <typename TIter>
509  void read_relations(TIter begin, TIter end) {
510  HandlerPass1 handler(*static_cast<TCollector*>(this));
511  osmium::apply(begin, end, handler);
512  sort_member_meta();
513  }
514 
515  template <typename TSource>
516  void read_relations(TSource& source) {
517  read_relations(std::begin(source), std::end(source));
518  source.close();
519  }
520 
521  void moving_in_buffer(size_t old_offset, size_t new_offset) {
522  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
523  auto range = find_member_meta(object.type(), object.id());
524  for (auto& member_meta : range) {
525  assert(member_meta.buffer_offset() == old_offset);
526  member_meta.set_buffer_offset(new_offset);
527  }
528  }
529 
537  ++m_count_complete;
538  if (m_count_complete > 10000) { // XXX
539 // const size_t size_before = m_members_buffer.committed();
540  m_members_buffer.purge_removed(this);
541 /*
542  const size_t size_after = m_members_buffer.committed();
543  double percent = static_cast<double>(size_before - size_after);
544  percent /= size_before;
545  percent *= 100;
546  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
547 */
548  m_count_complete = 0;
549  }
550  }
551 
560  std::vector<const osmium::Relation*> get_incomplete_relations() const {
561  std::vector<const osmium::Relation*> relations;
562  for (const auto& relation_meta : m_relations) {
563  if (!relation_meta.has_all_members()) {
564  relations.push_back(&get_relation(relation_meta));
565  }
566  }
567  return relations;
568  }
569 
570  }; // class Collector
571 
572  } // namespace relations
573 
574 } // namespace osmium
575 
576 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:560
void relation(const osmium::Relation &relation)
Definition: collector.hpp:152
void clear_member_metas(const osmium::relations::RelationMeta &relation_meta)
Definition: collector.hpp:436
Definition: iterator.hpp:42
callback_func_type m_callback
Definition: collector.hpp:190
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:498
type
Definition: entity_bits.hpp:63
iterator_range< It > make_range(P &&p)
Definition: iterator.hpp:76
RelationMemberList & members()
Definition: relation.hpp:177
void way(const osmium::Way &way)
Definition: collector.hpp:144
item_type
Definition: item_type.hpp:43
void clean_assembled_relations()
Definition: collector.hpp:301
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:521
Definition: relation.hpp:165
Definition: handler.hpp:45
bool keep_member(const osmium::relations::RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:249
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:174
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:222
void read_relations(TSource &source)
Definition: collector.hpp:516
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:502
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:280
Definition: way.hpp:65
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:214
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:396
Definition: collector.hpp:126
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:132
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:177
void sort_member_meta()
Definition: collector.hpp:373
void apply(TIterator it, TIterator end, THandlers &...handlers)
Definition: visitor.hpp:234
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:260
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:189
Definition: relation.hpp:54
TCollector & m_collector
Definition: collector.hpp:105
Namespace for everything in the Osmium library.
Definition: assembler.hpp:66
T & add_item(const T &item)
Definition: buffer.hpp:457
Definition: collector.hpp:98
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:194
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:308
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:700
Definition: attr.hpp:298
HandlerPass2 m_handler_pass2
Definition: collector.hpp:168
Collector()
Definition: collector.hpp:204
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:327
size_t capacity() const noexcept
Definition: buffer.hpp:233
void relation(const osmium::Relation &relation)
Definition: collector.hpp:113
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
Definition: collector.hpp:103
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
void flush()
Definition: collector.hpp:160
std::vector< MemberMeta > mm_vector_type
Definition: collector.hpp:183
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:171
object_id_type id() const noexcept
Get ID of this object.
Definition: object.hpp:112
uint64_t used_memory() const
Definition: collector.hpp:461
size_t committed() const noexcept
Definition: buffer.hpp:241
callback_func_type callback()
Definition: collector.hpp:218
Definition: buffer.hpp:97
T & get(const size_t offset) const
Definition: buffer.hpp:379
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:342
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:493
void flush()
Definition: collector.hpp:294
Definition: node.hpp:47
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:270
void node(const osmium::Node &node)
Definition: collector.hpp:136
void possibly_purge_removed_members()
Definition: collector.hpp:536
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:235
const osmium::Relation & get_relation(const MemberMeta &member_meta) const
Definition: collector.hpp:323
It begin() const
Definition: iterator.hpp:58
TCollector & m_collector
Definition: collector.hpp:128
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:509
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:109
void rollback()
Definition: buffer.hpp:349
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:316
static iterator_range< mm_iterator >::iterator::difference_type count_not_removed(const iterator_range< mm_iterator > &range)
Definition: collector.hpp:383
It end() const
Definition: iterator.hpp:62
Definition: object.hpp:60
size_t commit()
Definition: buffer.hpp:335