Libosmium  2.14.2
Fast and flexible C++ library for working with OpenStreetMap data
bzip2_compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_BZIP2_COMPRESSION_HPP
2 #define OSMIUM_IO_BZIP2_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2018 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
46 #include <osmium/io/detail/read_write.hpp>
47 #include <osmium/io/error.hpp>
51 
52 #include <bzlib.h>
53 
54 #include <cassert>
55 #include <cerrno>
56 #include <cstdio>
57 #include <limits>
58 #include <string>
59 #include <system_error>
60 
61 #ifndef _MSC_VER
62 # include <unistd.h>
63 #endif
64 
65 namespace osmium {
66 
71  struct bzip2_error : public io_error {
72 
75 
76  bzip2_error(const std::string& what, int error_code) :
77  io_error(what),
78  bzip2_error_code(error_code),
79  system_errno(error_code == BZ_IO_ERROR ? errno : 0) {
80  }
81 
82  }; // struct bzip2_error
83 
84  namespace io {
85 
86  namespace detail {
87 
88  OSMIUM_NORETURN inline void throw_bzip2_error(BZFILE* bzfile, const char* msg, int bzlib_error = 0) {
89  std::string error{"bzip2 error: "};
90  error += msg;
91  error += ": ";
92  int errnum = bzlib_error;
93  if (bzlib_error) {
94  error += std::to_string(bzlib_error);
95  } else {
96  error += ::BZ2_bzerror(bzfile, &errnum);
97  }
98  throw osmium::bzip2_error{error, errnum};
99  }
100 
101  } // namespace detail
102 
103  class Bzip2Compressor : public Compressor {
104 
105  FILE* m_file;
107  BZFILE* m_bzfile;
108 
109  public:
110 
111  explicit Bzip2Compressor(int fd, fsync sync) :
112  Compressor(sync),
113  m_file(fdopen(::dup(fd), "wb")),
114  m_bzerror(BZ_OK),
115  m_bzfile(::BZ2_bzWriteOpen(&m_bzerror, m_file, 6, 0, 0)) {
116  if (!m_bzfile) {
117  detail::throw_bzip2_error(m_bzfile, "write open failed", m_bzerror);
118  }
119  }
120 
121  Bzip2Compressor(const Bzip2Compressor&) = delete;
122  Bzip2Compressor& operator=(const Bzip2Compressor&) = delete;
123 
124  Bzip2Compressor(Bzip2Compressor&&) = delete;
125  Bzip2Compressor& operator=(Bzip2Compressor&&) = delete;
126 
127  ~Bzip2Compressor() noexcept final {
128  try {
129  close();
130  } catch (...) {
131  // Ignore any exceptions because destructor must not throw.
132  }
133  }
134 
135  void write(const std::string& data) final {
136  int error;
137  assert(data.size() < std::numeric_limits<int>::max());
138  ::BZ2_bzWrite(&error, m_bzfile, const_cast<char*>(data.data()), static_cast<int>(data.size()));
139  if (error != BZ_OK && error != BZ_STREAM_END) {
140  detail::throw_bzip2_error(m_bzfile, "write failed", error);
141  }
142  }
143 
144  void close() final {
145  if (m_bzfile) {
146  int error;
147  ::BZ2_bzWriteClose(&error, m_bzfile, 0, nullptr, nullptr);
148  m_bzfile = nullptr;
149  if (m_file) {
150  if (do_fsync()) {
151  osmium::io::detail::reliable_fsync(fileno(m_file));
152  }
153  if (fclose(m_file) != 0) {
154  throw std::system_error{errno, std::system_category(), "Close failed"};
155  }
156  }
157  if (error != BZ_OK) {
158  detail::throw_bzip2_error(m_bzfile, "write close failed", error);
159  }
160  }
161  }
162 
163  }; // class Bzip2Compressor
164 
166 
167  FILE* m_file;
168  int m_bzerror = BZ_OK;
169  BZFILE* m_bzfile;
170  bool m_stream_end = false;
171 
172  public:
173 
174  explicit Bzip2Decompressor(int fd) :
175  m_file(fdopen(::dup(fd), "rb")),
176  m_bzfile(::BZ2_bzReadOpen(&m_bzerror, m_file, 0, 0, nullptr, 0)) {
177  if (!m_bzfile) {
178  detail::throw_bzip2_error(m_bzfile, "read open failed", m_bzerror);
179  }
180  }
181 
182  Bzip2Decompressor(const Bzip2Decompressor&) = delete;
183  Bzip2Decompressor& operator=(const Bzip2Decompressor&) = delete;
184 
186  Bzip2Decompressor& operator=(Bzip2Decompressor&&) = delete;
187 
188  ~Bzip2Decompressor() noexcept final {
189  try {
190  close();
191  } catch (...) {
192  // Ignore any exceptions because destructor must not throw.
193  }
194  }
195 
196  std::string read() final {
197  std::string buffer;
198 
199  if (!m_stream_end) {
201  int error;
202  assert(buffer.size() < std::numeric_limits<int>::max());
203  const int nread = ::BZ2_bzRead(&error, m_bzfile, const_cast<char*>(buffer.data()), static_cast<int>(buffer.size()));
204  if (error != BZ_OK && error != BZ_STREAM_END) {
205  detail::throw_bzip2_error(m_bzfile, "read failed", error);
206  }
207  if (error == BZ_STREAM_END) {
208  void* unused;
209  int nunused;
210  if (! feof(m_file)) {
211  ::BZ2_bzReadGetUnused(&error, m_bzfile, &unused, &nunused);
212  if (error != BZ_OK) {
213  detail::throw_bzip2_error(m_bzfile, "get unused failed", error);
214  }
215  std::string unused_data(static_cast<const char*>(unused), static_cast<std::string::size_type>(nunused));
216  ::BZ2_bzReadClose(&error, m_bzfile);
217  if (error != BZ_OK) {
218  detail::throw_bzip2_error(m_bzfile, "read close failed", error);
219  }
220  assert(unused_data.size() < std::numeric_limits<int>::max());
221  m_bzfile = ::BZ2_bzReadOpen(&error, m_file, 0, 0, const_cast<void*>(static_cast<const void*>(unused_data.data())), static_cast<int>(unused_data.size()));
222  if (error != BZ_OK) {
223  detail::throw_bzip2_error(m_bzfile, "read open failed", error);
224  }
225  } else {
226  m_stream_end = true;
227  }
228  }
229  buffer.resize(static_cast<std::string::size_type>(nread));
230  }
231 
232  set_offset(size_t(ftell(m_file)));
233 
234  return buffer;
235  }
236 
237  void close() final {
238  if (m_bzfile) {
239  int error;
240  ::BZ2_bzReadClose(&error, m_bzfile);
241  m_bzfile = nullptr;
242  if (m_file) {
243  if (fclose(m_file) != 0) {
244  throw std::system_error{errno, std::system_category(), "Close failed"};
245  }
246  }
247  if (error != BZ_OK) {
248  detail::throw_bzip2_error(m_bzfile, "read close failed", error);
249  }
250  }
251  }
252 
253  }; // class Bzip2Decompressor
254 
256 
257  const char* m_buffer;
259  bz_stream m_bzstream;
260 
261  public:
262 
263  Bzip2BufferDecompressor(const char* buffer, size_t size) :
264  m_buffer(buffer),
265  m_buffer_size(size),
266  m_bzstream() {
267  m_bzstream.next_in = const_cast<char*>(buffer);
268  assert(size < std::numeric_limits<unsigned int>::max());
269  m_bzstream.avail_in = static_cast<unsigned int>(size);
270  const int result = BZ2_bzDecompressInit(&m_bzstream, 0, 0);
271  if (result != BZ_OK) {
272  std::string message{"bzip2 error: decompression init failed: "};
273  throw bzip2_error{message, result};
274  }
275  }
276 
278  Bzip2BufferDecompressor& operator=(const Bzip2BufferDecompressor&) = delete;
279 
281  Bzip2BufferDecompressor& operator=(Bzip2BufferDecompressor&&) = delete;
282 
283  ~Bzip2BufferDecompressor() noexcept final {
284  try {
285  close();
286  } catch (...) {
287  // Ignore any exceptions because destructor must not throw.
288  }
289  }
290 
291  std::string read() final {
292  std::string output;
293 
294  if (m_buffer) {
295  const size_t buffer_size = 10240;
296  output.resize(buffer_size);
297  m_bzstream.next_out = const_cast<char*>(output.data());
298  m_bzstream.avail_out = buffer_size;
299  const int result = BZ2_bzDecompress(&m_bzstream);
300 
301  if (result != BZ_OK) {
302  m_buffer = nullptr;
303  m_buffer_size = 0;
304  }
305 
306  if (result != BZ_OK && result != BZ_STREAM_END) {
307  std::string message{"bzip2 error: decompress failed: "};
308  throw bzip2_error{message, result};
309  }
310 
311  output.resize(static_cast<std::size_t>(m_bzstream.next_out - output.data()));
312  }
313 
314  return output;
315  }
316 
317  void close() final {
318  BZ2_bzDecompressEnd(&m_bzstream);
319  }
320 
321  }; // class Bzip2BufferDecompressor
322 
323  namespace detail {
324 
325  // we want the register_compression() function to run, setting
326  // the variable is only a side-effect, it will never be used
328  [](int fd, fsync sync) { return new osmium::io::Bzip2Compressor{fd, sync}; },
329  [](int fd) { return new osmium::io::Bzip2Decompressor{fd}; },
330  [](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor{buffer, size}; }
331  );
332 
333  // dummy function to silence the unused variable warning from above
334  inline bool get_registered_bzip2_compression() noexcept {
335  return registered_bzip2_compression;
336  }
337 
338  } // namespace detail
339 
340  } // namespace io
341 
342 } // namespace osmium
343 
344 #endif // OSMIUM_IO_BZIP2_COMPRESSION_HPP
bz_stream m_bzstream
Definition: bzip2_compression.hpp:259
Definition: bzip2_compression.hpp:165
int bzip2_error_code
Definition: bzip2_compression.hpp:73
Bzip2Compressor(int fd, fsync sync)
Definition: bzip2_compression.hpp:111
#define OSMIUM_NORETURN
Definition: compatibility.hpp:41
~Bzip2Decompressor() noexcept final
Definition: bzip2_compression.hpp:188
void close() final
Definition: bzip2_compression.hpp:317
void write(const std::string &data) final
Definition: bzip2_compression.hpp:135
const char * m_buffer
Definition: bzip2_compression.hpp:257
int m_bzerror
Definition: bzip2_compression.hpp:106
size_t m_buffer_size
Definition: bzip2_compression.hpp:258
static CompressionFactory & instance()
Definition: compression.hpp:184
~Bzip2BufferDecompressor() noexcept final
Definition: bzip2_compression.hpp:283
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:100
bzip2_error(const std::string &what, int error_code)
Definition: bzip2_compression.hpp:76
std::string read() final
Definition: bzip2_compression.hpp:196
Definition: bzip2_compression.hpp:103
int system_errno
Definition: bzip2_compression.hpp:74
Definition: compression.hpp:93
FILE * m_file
Definition: bzip2_compression.hpp:105
Definition: bzip2_compression.hpp:255
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: attr.hpp:333
Bzip2BufferDecompressor(const char *buffer, size_t size)
Definition: bzip2_compression.hpp:263
fsync
Definition: writer_options.hpp:51
BZFILE * m_bzfile
Definition: bzip2_compression.hpp:169
BZFILE * m_bzfile
Definition: bzip2_compression.hpp:107
Definition: error.hpp:44
void close() final
Definition: bzip2_compression.hpp:144
void close() final
Definition: bzip2_compression.hpp:237
std::string read() final
Definition: bzip2_compression.hpp:291
Definition: compression.hpp:63
FILE * m_file
Definition: bzip2_compression.hpp:167
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:189
Bzip2Decompressor(int fd)
Definition: bzip2_compression.hpp:174
~Bzip2Compressor() noexcept final
Definition: bzip2_compression.hpp:127
Definition: bzip2_compression.hpp:71