Libosmium  2.10.3
Fast and flexible C++ library for working with OpenStreetMap data
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstring>
40 #include <functional>
41 #include <iterator>
42 #include <memory>
43 #include <stdexcept>
44 #include <utility>
45 
46 #include <osmium/memory/item.hpp>
48 #include <osmium/osm/entity.hpp>
50 
51 namespace osmium {
52 
58  struct buffer_is_full : public std::runtime_error {
59 
61  std::runtime_error("Osmium buffer is full") {
62  }
63 
64  }; // struct buffer_is_full
65 
69  namespace memory {
70 
97  class Buffer {
98 
99  public:
100 
101  // This is needed so we can call std::back_inserter() on a Buffer.
102  using value_type = Item;
103 
104  enum class auto_grow : bool {
105  yes = true,
106  no = false
107  }; // enum class auto_grow
108 
109  private:
110 
111  std::unique_ptr<unsigned char[]> m_memory;
112  unsigned char* m_data;
113  size_t m_capacity;
114  size_t m_written;
115  size_t m_committed;
116 #ifndef NDEBUG
117  uint8_t m_builder_count{0};
118 #endif
119  auto_grow m_auto_grow {auto_grow::no};
120  std::function<void(Buffer&)> m_full;
121 
122  public:
123 
132  Buffer() noexcept :
133  m_memory(),
134  m_data(nullptr),
135  m_capacity(0),
136  m_written(0),
137  m_committed(0) {
138  }
139 
150  explicit Buffer(unsigned char* data, size_t size) :
151  m_memory(),
152  m_data(data),
153  m_capacity(size),
154  m_written(size),
155  m_committed(size) {
156  if (size % align_bytes != 0) {
157  throw std::invalid_argument("buffer size needs to be multiple of alignment");
158  }
159  }
160 
172  explicit Buffer(unsigned char* data, size_t capacity, size_t committed) :
173  m_memory(),
174  m_data(data),
175  m_capacity(capacity),
176  m_written(committed),
177  m_committed(committed) {
178  if (capacity % align_bytes != 0) {
179  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
180  }
181  if (committed % align_bytes != 0) {
182  throw std::invalid_argument("buffer parameter 'committed' needs to be multiple of alignment");
183  }
184  }
185 
200  explicit Buffer(size_t capacity, auto_grow auto_grow = auto_grow::yes) :
201  m_memory(new unsigned char[capacity]),
202  m_data(m_memory.get()),
203  m_capacity(capacity),
204  m_written(0),
205  m_committed(0),
206  m_auto_grow(auto_grow) {
207  if (capacity % align_bytes != 0) {
208  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
209  }
210  }
211 
212  // buffers can not be copied
213  Buffer(const Buffer&) = delete;
214  Buffer& operator=(const Buffer&) = delete;
215 
216  // buffers can be moved
217  Buffer(Buffer&&) = default;
218  Buffer& operator=(Buffer&&) = default;
219 
220  ~Buffer() = default;
221 
222 #ifndef NDEBUG
223  void increment_builder_count() noexcept {
224  ++m_builder_count;
225  }
226 
227  void decrement_builder_count() noexcept {
228  assert(m_builder_count > 0);
229  --m_builder_count;
230  }
231 
232  uint8_t builder_count() const noexcept {
233  return m_builder_count;
234  }
235 #endif
236 
242  unsigned char* data() const noexcept {
243  assert(m_data && "This must be a valid buffer");
244  return m_data;
245  }
246 
251  size_t capacity() const noexcept {
252  return m_capacity;
253  }
254 
259  size_t committed() const noexcept {
260  return m_committed;
261  }
262 
268  size_t written() const noexcept {
269  return m_written;
270  }
271 
278  bool is_aligned() const noexcept {
279  assert(m_data && "This must be a valid buffer");
280  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
281  }
282 
303  OSMIUM_DEPRECATED void set_full_callback(std::function<void(Buffer&)> full) {
304  assert(m_data && "This must be a valid buffer");
305  m_full = full;
306  }
307 
324  void grow(size_t size) {
325  assert(m_data && "This must be a valid buffer");
326  if (!m_memory) {
327  throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
328  }
329  if (m_capacity < size) {
330  if (size % align_bytes != 0) {
331  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
332  }
333  std::unique_ptr<unsigned char[]> memory(new unsigned char[size]);
334  std::copy_n(m_memory.get(), m_capacity, memory.get());
335  using std::swap;
336  swap(m_memory, memory);
337  m_data = m_memory.get();
338  m_capacity = size;
339  }
340  }
341 
354  size_t commit() {
355  assert(m_data && "This must be a valid buffer");
356  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
357  assert(is_aligned());
358 
359  const size_t offset = m_committed;
360  m_committed = m_written;
361  return offset;
362  }
363 
370  void rollback() {
371  assert(m_data && "This must be a valid buffer");
372  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
373  m_written = m_committed;
374  }
375 
385  size_t clear() {
386  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
387  const size_t committed = m_committed;
388  m_written = 0;
389  m_committed = 0;
390  return committed;
391  }
392 
403  template <typename T>
404  T& get(const size_t offset) const {
405  assert(m_data && "This must be a valid buffer");
406  return *reinterpret_cast<T*>(&m_data[offset]);
407  }
408 
442  unsigned char* reserve_space(const size_t size) {
443  assert(m_data && "This must be a valid buffer");
444  // try to flush the buffer empty first.
445  if (m_written + size > m_capacity && m_full) {
446  m_full(*this);
447  }
448  // if there's still not enough space, then try growing the buffer.
449  if (m_written + size > m_capacity) {
450  if (m_memory && (m_auto_grow == auto_grow::yes)) {
451  // double buffer size until there is enough space
452  size_t new_capacity = m_capacity * 2;
453  while (m_written + size > new_capacity) {
454  new_capacity *= 2;
455  }
456  grow(new_capacity);
457  } else {
458  throw osmium::buffer_is_full();
459  }
460  }
461  unsigned char* data = &m_data[m_written];
462  m_written += size;
463  return data;
464  }
465 
481  template <typename T>
482  T& add_item(const T& item) {
483  assert(m_data && "This must be a valid buffer");
484  unsigned char* target = reserve_space(item.padded_size());
485  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
486  return *reinterpret_cast<T*>(target);
487  }
488 
500  void add_buffer(const Buffer& buffer) {
501  assert(m_data && "This must be a valid buffer");
502  assert(buffer && "Buffer parameter must be a valid buffer");
503  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
504  unsigned char* target = reserve_space(buffer.committed());
505  std::copy_n(buffer.data(), buffer.committed(), target);
506  }
507 
517  void push_back(const osmium::memory::Item& item) {
518  assert(m_data && "This must be a valid buffer");
519  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
520  add_item(item);
521  commit();
522  }
523 
528  template <typename T>
530 
535  template <typename T>
537 
543 
549 
550  template <typename T>
552  return ItemIteratorRange<T>{m_data, m_data + m_committed};
553  }
554 
555  template <typename T>
557  return ItemIteratorRange<const T>{m_data, m_data + m_committed};
558  }
559 
568  template <typename T>
570  assert(m_data && "This must be a valid buffer");
571  return t_iterator<T>(m_data, m_data + m_committed);
572  }
573 
583  assert(m_data && "This must be a valid buffer");
584  return iterator(m_data, m_data + m_committed);
585  }
586 
596  template <typename T>
597  t_iterator<T> get_iterator(size_t offset) {
598  assert(m_data && "This must be a valid buffer");
599  return t_iterator<T>(m_data + offset, m_data + m_committed);
600  }
601 
611  iterator get_iterator(size_t offset) {
612  assert(m_data && "This must be a valid buffer");
613  return iterator(m_data + offset, m_data + m_committed);
614  }
615 
624  template <typename T>
626  assert(m_data && "This must be a valid buffer");
627  return t_iterator<T>(m_data + m_committed, m_data + m_committed);
628  }
629 
639  assert(m_data && "This must be a valid buffer");
640  return iterator(m_data + m_committed, m_data + m_committed);
641  }
642 
643  template <typename T>
645  assert(m_data && "This must be a valid buffer");
646  return t_const_iterator<T>(m_data, m_data + m_committed);
647  }
648 
650  assert(m_data && "This must be a valid buffer");
651  return const_iterator(m_data, m_data + m_committed);
652  }
653 
654  template <typename T>
655  t_const_iterator<T> get_iterator(size_t offset) const {
656  assert(m_data && "This must be a valid buffer");
657  return t_const_iterator<T>(m_data + offset, m_data + m_committed);
658  }
659 
660  const_iterator get_iterator(size_t offset) const {
661  assert(m_data && "This must be a valid buffer");
662  return const_iterator(m_data + offset, m_data + m_committed);
663  }
664 
665  template <typename T>
667  assert(m_data && "This must be a valid buffer");
668  return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
669  }
670 
672  assert(m_data && "This must be a valid buffer");
673  return const_iterator(m_data + m_committed, m_data + m_committed);
674  }
675 
676  template <typename T>
678  return cbegin<T>();
679  }
680 
682  return cbegin();
683  }
684 
685  template <typename T>
687  return cend<T>();
688  }
689 
690  const_iterator end() const {
691  return cend();
692  }
693 
697  explicit operator bool() const noexcept {
698  return m_data != nullptr;
699  }
700 
701  void swap(Buffer& other) {
702  using std::swap;
703 
704  swap(m_memory, other.m_memory);
705  swap(m_data, other.m_data);
706  swap(m_capacity, other.m_capacity);
707  swap(m_written, other.m_written);
708  swap(m_committed, other.m_committed);
709  swap(m_auto_grow, other.m_auto_grow);
710  swap(m_full, other.m_full);
711  }
712 
729  template <typename TCallbackClass>
730  void purge_removed(TCallbackClass* callback) {
731  assert(m_data && "This must be a valid buffer");
732  if (begin() == end()) {
733  return;
734  }
735 
736  iterator it_write = begin();
737 
738  iterator next;
739  for (iterator it_read = begin(); it_read != end(); it_read = next) {
740  next = std::next(it_read);
741  if (!it_read->removed()) {
742  if (it_read != it_write) {
743  assert(it_read.data() >= data());
744  assert(it_write.data() >= data());
745  size_t old_offset = static_cast<size_t>(it_read.data() - data());
746  size_t new_offset = static_cast<size_t>(it_write.data() - data());
747  callback->moving_in_buffer(old_offset, new_offset);
748  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
749  }
750  it_write.advance_once();
751  }
752  }
753 
754  assert(it_write.data() >= data());
755  m_written = static_cast<size_t>(it_write.data() - data());
756  m_committed = m_written;
757  }
758 
759  }; // class Buffer
760 
761  inline void swap(Buffer& lhs, Buffer& rhs) {
762  lhs.swap(rhs);
763  }
764 
772  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
773  if (!lhs || !rhs) {
774  return !lhs && !rhs;
775  }
776  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
777  }
778 
779  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
780  return ! (lhs == rhs);
781  }
782 
783  } // namespace memory
784 
785 } // namespace osmium
786 
787 #endif // OSMIUM_MEMORY_BUFFER_HPP
size_t m_written
Definition: buffer.hpp:114
void swap(Buffer &other)
Definition: buffer.hpp:701
t_const_iterator< T > begin() const
Definition: buffer.hpp:677
size_t clear()
Definition: buffer.hpp:385
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
bool is_aligned() const noexcept
Definition: buffer.hpp:278
t_const_iterator< T > get_iterator(size_t offset) const
Definition: buffer.hpp:655
size_t written() const noexcept
Definition: buffer.hpp:268
OSMIUM_DEPRECATED void set_full_callback(std::function< void(Buffer &)> full)
Definition: buffer.hpp:303
Definition: item_iterator.hpp:248
iterator get_iterator(size_t offset)
Definition: buffer.hpp:611
void grow(size_t size)
Definition: buffer.hpp:324
t_const_iterator< T > cend() const
Definition: buffer.hpp:666
const_iterator cend() const
Definition: buffer.hpp:671
Definition: item_iterator.hpp:132
constexpr bool operator==(const Box &lhs, const Box &rhs) noexcept
Definition: box.hpp:221
unsigned char * m_data
Definition: buffer.hpp:112
const_iterator get_iterator(size_t offset) const
Definition: buffer.hpp:660
void increment_builder_count() noexcept
Definition: buffer.hpp:223
Buffer(unsigned char *data, size_t capacity, size_t committed)
Definition: buffer.hpp:172
Definition: reader_iterator.hpp:39
ItemIteratorRange< const T > select() const
Definition: buffer.hpp:556
void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:761
Buffer(size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:200
const_iterator begin() const
Definition: buffer.hpp:681
t_iterator< T > end()
Definition: buffer.hpp:625
Definition: item.hpp:105
Namespace for everything in the Osmium library.
Definition: assembler.hpp:73
T & add_item(const T &item)
Definition: buffer.hpp:482
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:730
t_iterator< T > begin()
Definition: buffer.hpp:569
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:500
uint8_t builder_count() const noexcept
Definition: buffer.hpp:232
size_t m_committed
Definition: buffer.hpp:115
ItemIterator< TMember > & advance_once() noexcept
Definition: item_iterator.hpp:187
unsigned char * data() const noexcept
Definition: buffer.hpp:242
Buffer() noexcept
Definition: buffer.hpp:132
size_t capacity() const noexcept
Definition: buffer.hpp:251
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
unsigned char * reserve_space(const size_t size)
Definition: buffer.hpp:442
auto_grow m_auto_grow
Definition: buffer.hpp:119
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:517
constexpr const item_size_type align_bytes
Definition: item.hpp:62
iterator end()
Definition: buffer.hpp:638
iterator begin()
Definition: buffer.hpp:582
size_t m_capacity
Definition: buffer.hpp:113
size_t committed() const noexcept
Definition: buffer.hpp:259
Buffer(unsigned char *data, size_t size)
Definition: buffer.hpp:150
Definition: buffer.hpp:97
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:644
Definition: buffer.hpp:58
void decrement_builder_count() noexcept
Definition: buffer.hpp:227
const_iterator end() const
Definition: buffer.hpp:690
auto_grow
Definition: buffer.hpp:104
t_const_iterator< T > end() const
Definition: buffer.hpp:686
data_type data() noexcept
Definition: item_iterator.hpp:208
buffer_is_full()
Definition: buffer.hpp:60
ItemIteratorRange< T > select()
Definition: buffer.hpp:551
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
void rollback()
Definition: buffer.hpp:370
std::unique_ptr< unsigned char[]> m_memory
Definition: buffer.hpp:111
bool operator!=(const Changeset &lhs, const Changeset &rhs)
Definition: changeset.hpp:434
t_iterator< T > get_iterator(size_t offset)
Definition: buffer.hpp:597
const_iterator cbegin() const
Definition: buffer.hpp:649
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:120
size_t commit()
Definition: buffer.hpp:354