Libosmium  2.10.3
Fast and flexible C++ library for working with OpenStreetMap data
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <atomic>
37 #include <cerrno>
38 #include <cstddef>
39 #include <functional>
40 #include <map>
41 #include <memory>
42 #include <string>
43 #include <system_error>
44 #include <tuple>
45 #include <utility>
46 
47 #ifndef _MSC_VER
48 # include <unistd.h>
49 #else
50 # include <io.h>
51 #endif
52 
53 #include <osmium/io/detail/read_write.hpp>
54 #include <osmium/io/error.hpp>
58 #include <osmium/util/file.hpp>
59 
60 namespace osmium {
61 
62  namespace io {
63 
64  class Compressor {
65 
67 
68  protected:
69 
70  bool do_fsync() const {
71  return m_fsync == fsync::yes;
72  }
73 
74  public:
75 
76  explicit Compressor(fsync sync) :
77  m_fsync(sync) {
78  }
79 
80  virtual ~Compressor() noexcept {
81  }
82 
83  virtual void write(const std::string& data) = 0;
84 
85  virtual void close() = 0;
86 
87  }; // class Compressor
88 
89  class Decompressor {
90 
91  std::atomic<size_t> m_file_size {0};
92  std::atomic<size_t> m_offset {0};
93 
94  public:
95 
96  static constexpr unsigned int input_buffer_size = 1024 * 1024;
97 
98  Decompressor() = default;
99 
100  Decompressor(const Decompressor&) = delete;
101  Decompressor& operator=(const Decompressor&) = delete;
102 
103  Decompressor(Decompressor&&) = delete;
104  Decompressor& operator=(Decompressor&&) = delete;
105 
106  virtual ~Decompressor() noexcept {
107  }
108 
109  virtual std::string read() = 0;
110 
111  virtual void close() = 0;
112 
113  size_t file_size() const noexcept {
114  return m_file_size;
115  }
116 
117  void set_file_size(size_t size) noexcept {
118  m_file_size = size;
119  }
120 
121  size_t offset() const noexcept {
122  return m_offset;
123  }
124 
125  void set_offset(size_t offset) noexcept {
126  m_offset = offset;
127  }
128 
129  }; // class Decompressor
130 
139 
140  public:
141 
142  using create_compressor_type = std::function<osmium::io::Compressor*(int, fsync)>;
143  using create_decompressor_type_fd = std::function<osmium::io::Decompressor*(int)>;
144  using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, size_t)>;
145 
146  private:
147 
148  using callbacks_type = std::tuple<create_compressor_type,
151 
152  using compression_map_type = std::map<const osmium::io::file_compression, callbacks_type>;
153 
155 
156  CompressionFactory() = default;
157 
158  CompressionFactory(const CompressionFactory&) = delete;
159  CompressionFactory& operator=(const CompressionFactory&) = delete;
160 
162  CompressionFactory& operator=(CompressionFactory&&) = delete;
163 
165  const auto it = m_callbacks.find(compression);
166 
167  if (it != m_callbacks.end()) {
168  return it->second;
169  }
170 
171  std::string error_message{"Support for compression '"};
172  error_message += as_string(compression);
173  error_message += "' not compiled into this binary";
174  throw unsupported_file_format_error{error_message};
175  }
176 
177  public:
178 
180  static CompressionFactory factory;
181  return factory;
182  }
183 
185  osmium::io::file_compression compression,
186  create_compressor_type create_compressor,
187  create_decompressor_type_fd create_decompressor_fd,
188  create_decompressor_type_buffer create_decompressor_buffer) {
189 
190  compression_map_type::value_type cc(compression,
191  std::make_tuple(create_compressor,
192  create_decompressor_fd,
193  create_decompressor_buffer));
194 
195  return m_callbacks.insert(cc).second;
196  }
197 
198  template <typename... TArgs>
199  std::unique_ptr<osmium::io::Compressor> create_compressor(osmium::io::file_compression compression, TArgs&&... args) const {
200  const auto callbacks = find_callbacks(compression);
201  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(callbacks)(std::forward<TArgs>(args)...));
202  }
203 
204  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, int fd) const {
205  const auto callbacks = find_callbacks(compression);
206  auto p = std::unique_ptr<osmium::io::Decompressor>(std::get<1>(callbacks)(fd));
207  p->set_file_size(osmium::util::file_size(fd));
208  return p;
209  }
210 
211  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, const char* buffer, size_t size) const {
212  const auto callbacks = find_callbacks(compression);
213  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(callbacks)(buffer, size));
214  }
215 
216  }; // class CompressionFactory
217 
218  class NoCompressor : public Compressor {
219 
220  int m_fd;
221 
222  public:
223 
224  NoCompressor(int fd, fsync sync) :
225  Compressor(sync),
226  m_fd(fd) {
227  }
228 
229  ~NoCompressor() noexcept final {
230  try {
231  close();
232  } catch (...) {
233  // Ignore any exceptions because destructor must not throw.
234  }
235  }
236 
237  void write(const std::string& data) final {
238  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
239  }
240 
241  void close() final {
242  if (m_fd >= 0) {
243  int fd = m_fd;
244  m_fd = -1;
245  if (do_fsync()) {
246  osmium::io::detail::reliable_fsync(fd);
247  }
248  osmium::io::detail::reliable_close(fd);
249  }
250  }
251 
252  }; // class NoCompressor
253 
254  class NoDecompressor : public Decompressor {
255 
256  int m_fd;
257  const char *m_buffer;
259  size_t m_offset = 0;
260 
261  public:
262 
263  explicit NoDecompressor(int fd) :
264  Decompressor(),
265  m_fd(fd),
266  m_buffer(nullptr),
267  m_buffer_size(0) {
268  }
269 
270  NoDecompressor(const char* buffer, size_t size) :
271  Decompressor(),
272  m_fd(-1),
273  m_buffer(buffer),
274  m_buffer_size(size) {
275  }
276 
277  ~NoDecompressor() noexcept final {
278  try {
279  close();
280  } catch (...) {
281  // Ignore any exceptions because destructor must not throw.
282  }
283  }
284 
285  std::string read() final {
286  std::string buffer;
287 
288  if (m_buffer) {
289  if (m_buffer_size != 0) {
290  size_t size = m_buffer_size;
291  m_buffer_size = 0;
292  buffer.append(m_buffer, size);
293  }
294  } else {
296  auto nread = ::read(m_fd, const_cast<char*>(buffer.data()), osmium::io::Decompressor::input_buffer_size);
297  if (nread < 0) {
298  throw std::system_error(errno, std::system_category(), "Read failed");
299  }
300  buffer.resize(std::string::size_type(nread));
301  }
302 
303  m_offset += buffer.size();
304  set_offset(m_offset);
305 
306  return buffer;
307  }
308 
309  void close() final {
310  if (m_fd >= 0) {
311  int fd = m_fd;
312  m_fd = -1;
313  osmium::io::detail::reliable_close(fd);
314  }
315  }
316 
317  }; // class NoDecompressor
318 
319  namespace detail {
320 
321  // we want the register_compression() function to run, setting
322  // the variable is only a side-effect, it will never be used
324  [](int fd, fsync sync) { return new osmium::io::NoCompressor(fd, sync); },
325  [](int fd) { return new osmium::io::NoDecompressor(fd); },
326  [](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); }
327  );
328 
329  // dummy function to silence the unused variable warning from above
330  inline bool get_registered_no_compression() noexcept {
331  return registered_no_compression;
332  }
333 
334  } // namespace detail
335 
336  } // namespace io
337 
338 } // namespace osmium
339 
340 #endif // OSMIUM_IO_COMPRESSION_HPP
void close() final
Definition: compression.hpp:309
NoDecompressor(int fd)
Definition: compression.hpp:263
size_t file_size(int fd)
Definition: file.hpp:67
std::string read() final
Definition: compression.hpp:285
const callbacks_type & find_callbacks(osmium::io::file_compression compression) const
Definition: compression.hpp:164
static CompressionFactory & instance()
Definition: compression.hpp:179
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:96
void close() final
Definition: compression.hpp:241
void write(const std::string &data) final
Definition: compression.hpp:237
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, const char *buffer, size_t size) const
Definition: compression.hpp:211
std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer > callbacks_type
Definition: compression.hpp:150
Definition: compression.hpp:89
fsync m_fsync
Definition: compression.hpp:66
virtual void write(const std::string &data)=0
Compressor(fsync sync)
Definition: compression.hpp:76
int m_fd
Definition: compression.hpp:256
const char * as_string(file_compression compression)
Definition: file_compression.hpp:51
size_t m_buffer_size
Definition: compression.hpp:258
Namespace for everything in the Osmium library.
Definition: assembler.hpp:73
Definition: attr.hpp:333
size_t offset() const noexcept
Definition: compression.hpp:121
fsync
Definition: writer_options.hpp:51
bool do_fsync() const
Definition: compression.hpp:70
virtual void close()=0
size_t file_size() const noexcept
Definition: compression.hpp:113
~NoCompressor() noexcept final
Definition: compression.hpp:229
void set_file_size(size_t size) noexcept
Definition: compression.hpp:117
std::map< const osmium::io::file_compression, callbacks_type > compression_map_type
Definition: compression.hpp:152
file_compression
Definition: file_compression.hpp:42
std::function< osmium::io::Compressor *(int, fsync)> create_compressor_type
Definition: compression.hpp:142
virtual ~Decompressor() noexcept
Definition: compression.hpp:106
Definition: compression.hpp:218
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, int fd) const
Definition: compression.hpp:204
Definition: compression.hpp:64
int m_fd
Definition: compression.hpp:220
compression_map_type m_callbacks
Definition: compression.hpp:154
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:143
std::function< osmium::io::Decompressor *(const char *, size_t)> create_decompressor_type_buffer
Definition: compression.hpp:144
virtual ~Compressor() noexcept
Definition: compression.hpp:80
Definition: compression.hpp:254
Definition: compression.hpp:138
NoDecompressor(const char *buffer, size_t size)
Definition: compression.hpp:270
std::unique_ptr< osmium::io::Compressor > create_compressor(osmium::io::file_compression compression, TArgs &&... args) const
Definition: compression.hpp:199
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:184
const char * m_buffer
Definition: compression.hpp:257
NoCompressor(int fd, fsync sync)
Definition: compression.hpp:224
~NoDecompressor() noexcept final
Definition: compression.hpp:277
void set_offset(size_t offset) noexcept
Definition: compression.hpp:125