OpenVDB  1.1.0
Compression.h
Go to the documentation of this file.
1 
2 //
3 // Copyright (c) 2012-2013 DreamWorks Animation LLC
4 //
5 // All rights reserved. This software is distributed under the
6 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )
7 //
8 // Redistributions of source code must retain the above copyright
9 // and license notice and the following restrictions and disclaimer.
10 //
11 // * Neither the name of DreamWorks Animation nor the names of
12 // its contributors may be used to endorse or promote products derived
13 // from this software without specific prior written permission.
14 //
15 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY INDIRECT, INCIDENTAL,
20 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 // IN NO EVENT SHALL THE COPYRIGHT HOLDERS' AND CONTRIBUTORS' AGGREGATE
27 // LIABILITY FOR ALL CLAIMS REGARDLESS OF THEIR BASIS EXCEED US$250.00.
28 //
30 
31 #ifndef OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
32 #define OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
33 
34 #include <openvdb/Types.h>
35 #include <openvdb/math/Math.h> // for negative()
36 #include <boost/scoped_array.hpp>
37 #include <algorithm>
38 #include <iostream>
39 #include <string>
40 #include <vector>
41 
42 
43 namespace openvdb {
45 namespace OPENVDB_VERSION_NAME {
46 namespace io {
47 
68 enum {
70  COMPRESS_ZIP = 0x1,
72 };
73 
75 OPENVDB_API std::string compressionToString(uint32_t flags);
76 
77 
79 
80 
83 enum {
84  /*0*/ NO_MASK_OR_INACTIVE_VALS, // no inactive vals, or all inactive vals are +background
85  /*1*/ NO_MASK_AND_MINUS_BG, // all inactive vals are -background
86  /*2*/ NO_MASK_AND_ONE_INACTIVE_VAL, // all inactive vals have the same non-background val
87  /*3*/ MASK_AND_NO_INACTIVE_VALS, // mask selects between -background and +background
88  /*4*/ MASK_AND_ONE_INACTIVE_VAL, // mask selects between backgd and one other inactive val
89  /*5*/ MASK_AND_TWO_INACTIVE_VALS // mask selects between two non-background inactive vals
90 };
91 
92 
94 
95 
98 template<typename T>
99 struct RealToHalf {
100  enum { isReal = false }; // unless otherwise specified, type T is not a floating-point type
101  typedef T HalfT; // type T's half float analogue is T itself
102 };
103 template<> struct RealToHalf<float> { enum { isReal = true }; typedef half HalfT; };
104 template<> struct RealToHalf<double> { enum { isReal = true }; typedef half HalfT; };
105 template<> struct RealToHalf<Vec2s> { enum { isReal = true }; typedef Vec2H HalfT; };
106 template<> struct RealToHalf<Vec2d> { enum { isReal = true }; typedef Vec2H HalfT; };
107 template<> struct RealToHalf<Vec3s> { enum { isReal = true }; typedef Vec3H HalfT; };
108 template<> struct RealToHalf<Vec3d> { enum { isReal = true }; typedef Vec3H HalfT; };
109 
110 
112 template<typename T>
113 inline T
114 truncateRealToHalf(const T& val)
115 {
116  return T(typename RealToHalf<T>::HalfT(val));
117 }
118 
119 
121 
122 
123 OPENVDB_API void zipToStream(std::ostream&, const char* data, size_t numBytes);
124 OPENVDB_API void unzipFromStream(std::istream&, char* data, size_t numBytes);
125 
133 template<typename T>
134 inline void
135 readData(std::istream& is, T* data, Index count, bool compressed)
136 {
137  if (compressed) {
138  unzipFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
139  } else {
140  is.read(reinterpret_cast<char*>(data), sizeof(T) * count);
141  }
142 }
143 
145 template<>
146 inline void
147 readData<std::string>(std::istream& is, std::string* data, Index count, bool /*compressed*/)
148 {
149  for (Index i = 0; i < count; ++i) {
150  size_t len = 0;
151  is >> len;
152  //data[i].resize(len);
153  //is.read(&(data[i][0]), len);
154 
155  std::string buffer(len+1, ' ');
156  is.read(&buffer[0], len+1 );
157  data[i].assign(buffer, 0, len);
158  }
159 }
160 
165 template<bool IsReal, typename T> struct HalfReader;
167 template<typename T>
168 struct HalfReader</*IsReal=*/false, T> {
169  static inline void read(std::istream& is, T* data, Index count, bool compressed) {
170  readData(is, data, count, compressed);
171  }
172 };
174 template<typename T>
175 struct HalfReader</*IsReal=*/true, T> {
176  typedef typename RealToHalf<T>::HalfT HalfT;
177  static inline void read(std::istream& is, T* data, Index count, bool compressed) {
178  if (count < 1) return;
179  std::vector<HalfT> halfData(count); // temp buffer into which to read half float values
180  readData<HalfT>(is, reinterpret_cast<HalfT*>(&halfData[0]), count, compressed);
181  // Copy half float values from the temporary buffer to the full float output array.
182  std::copy(halfData.begin(), halfData.end(), data);
183  }
184 };
185 
186 
194 template<typename T>
195 inline void
196 writeData(std::ostream &os, const T *data, Index count, bool compress)
197 {
198  if (compress) {
199  zipToStream(os, reinterpret_cast<const char*>(data), sizeof(T) * count);
200  } else {
201  os.write(reinterpret_cast<const char*>(data), sizeof(T) * count);
202  }
203 }
204 
207 template<>
208 inline void
209 writeData<std::string>(std::ostream& os, const std::string* data, Index count, bool /*compress*/)
210 {
211  for (Index i = 0; i < count; ++i) {
212  const size_t len = data[i].size();
213  os << len;
214  os.write(data[i].c_str(), len+1);
215  //os.write(&(data[i][0]), len );
216  }
217 }
218 
223 template<bool IsReal, typename T> struct HalfWriter;
225 template<typename T>
226 struct HalfWriter</*IsReal=*/false, T> {
227  static inline void write(std::ostream& os, const T* data, Index count, bool compress) {
228  writeData(os, data, count, compress);
229  }
230 };
232 template<typename T>
233 struct HalfWriter</*IsReal=*/true, T> {
234  typedef typename RealToHalf<T>::HalfT HalfT;
235  static inline void write(std::ostream& os, const T* data, Index count, bool compress) {
236  if (count < 1) return;
237  // Convert full float values to half float, then output the half float array.
238  std::vector<HalfT> halfData(count);
239  std::copy(data, data + count, halfData.begin());
240  writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compress);
241  }
242 };
243 #ifdef _MSC_VER
244 
245 template<>
246 struct HalfWriter</*IsReal=*/true, double> {
247  typedef RealToHalf<double>::HalfT HalfT;
248  static inline void write(std::ostream& os, const double* data, Index count, bool compress) {
249  if (count < 1) return;
250  // Convert full float values to half float, then output the half float array.
251  std::vector<HalfT> halfData(count);
252  for (Index i = 0; i < count; ++i) halfData[i] = float(data[i]);
253  writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compress);
254  }
255 };
256 #endif // _MSC_VER
257 
258 
260 
261 
274 template<typename ValueT, typename MaskT>
275 inline void
276 readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
277  const MaskT& valueMask, bool fromHalf)
278 {
279  // Get the stream's compression settings.
280  const uint32_t compression = getDataCompression(is);
281  const bool
282  zipped = compression & COMPRESS_ZIP,
283  maskCompressed = compression & COMPRESS_ACTIVE_MASK;
284 
285  int8_t metadata = NO_MASK_OR_INACTIVE_VALS;
287  // Read the flag that specifies what, if any, additional metadata
288  // (selection mask and/or inactive value(s)) is saved.
289  is.read(reinterpret_cast<char*>(&metadata), /*bytes=*/1);
290  }
291 
292  ValueT background = zeroVal<ValueT>();
293  if (const void* bgPtr = getGridBackgroundValuePtr(is)) {
294  background = *static_cast<const ValueT*>(bgPtr);
295  }
296  ValueT inactiveVal1 = background;
297  ValueT inactiveVal0 =
298  ((metadata == NO_MASK_OR_INACTIVE_VALS) ? background : negative(background));
299 
300  if (metadata != NO_MASK_OR_INACTIVE_VALS &&
301  metadata != NO_MASK_AND_MINUS_BG &&
302  metadata != MASK_AND_NO_INACTIVE_VALS)
303  {
304  // Read one of at most two distinct inactive values.
305  is.read(reinterpret_cast<char*>(&inactiveVal0), sizeof(ValueT));
306  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
307  // Read the second of two distinct inactive values.
308  is.read(reinterpret_cast<char*>(&inactiveVal1), sizeof(ValueT));
309  }
310  }
311 
312  MaskT selectionMask;
313  if (metadata != NO_MASK_OR_INACTIVE_VALS &&
314  metadata != NO_MASK_AND_MINUS_BG &&
315  metadata != NO_MASK_AND_ONE_INACTIVE_VAL)
316  {
317  // For use in mask compression (only), read the bitmask that selects
318  // between two distinct inactive values.
319  selectionMask.load(is);
320  }
321 
322  ValueT* tempBuf = destBuf;
323  boost::scoped_array<ValueT> scopedTempBuf;
324 
325  Index tempCount = destCount;
326  if (maskCompressed && getFormatVersion(is) >= OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) {
327  tempCount = valueMask.countOn();
328  if (tempCount != destCount) {
329  // If this node has inactive voxels, allocate a temporary buffer
330  // into which to read just the active values.
331  scopedTempBuf.reset(new ValueT[tempCount]);
332  tempBuf = scopedTempBuf.get();
333  }
334  }
335 
336  // Read in the buffer.
337  if (fromHalf) {
338  HalfReader<RealToHalf<ValueT>::isReal, ValueT>::read(is, tempBuf, tempCount, zipped);
339  } else {
340  readData<ValueT>(is, tempBuf, tempCount, zipped);
341  }
342 
343  // If mask compression is enabled and the number of active values read into
344  // the temp buffer is smaller than the size of the destination buffer,
345  // then there are missing (inactive) values.
346  if (maskCompressed && tempCount != destCount) {
347  // Restore inactive values, using the background value and, if available,
348  // the inside/outside mask. (For fog volumes, the destination buffer is assumed
349  // to be initialized to background value zero, so inactive values can be ignored.)
350  for (Index destIdx = 0, tempIdx = 0; destIdx < MaskT::SIZE; ++destIdx) {
351  if (valueMask.isOn(destIdx)) {
352  // Copy a saved active value into this node's buffer.
353  destBuf[destIdx] = tempBuf[tempIdx];
354  ++tempIdx;
355  } else {
356  // Reconstruct an unsaved inactive value and copy it into this node's buffer.
357  destBuf[destIdx] = (selectionMask.isOn(destIdx) ? inactiveVal1 : inactiveVal0);
358  }
359  }
360  }
361 }
362 
363 
376 template<typename ValueT, typename MaskT>
377 inline void
378 writeCompressedValues(std::ostream& os, ValueT* srcBuf, Index srcCount,
379  const MaskT& valueMask, const MaskT& childMask, bool toHalf)
380 {
381  struct Local {
382  // Comparison function for values
383  static inline bool eq(const ValueT& a, const ValueT& b) {
384  return math::isExactlyEqual(a, b);
385  }
386  };
387 
388  // Get the stream's compression settings.
389  const uint32_t compress = getDataCompression(os);
390  const bool
391  zip = compress & COMPRESS_ZIP,
392  maskCompress = compress & COMPRESS_ACTIVE_MASK;
393 
394  Index tempCount = srcCount;
395  ValueT* tempBuf = srcBuf;
396  boost::scoped_array<ValueT> scopedTempBuf;
397 
398  int8_t metadata = NO_MASK_OR_INACTIVE_VALS;
399 
400  if (!maskCompress) {
401  os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
402  } else {
403  // A valid level set's inactive values are either +background (outside)
404  // or -background (inside), and a fog volume's inactive values are all zero.
405  // Rather than write out all of these values, we can store just the active values
406  // (given that the value mask specifies their positions) and, if necessary,
407  // an inside/outside bitmask.
408 
409  const ValueT zero = zeroVal<ValueT>();
410  ValueT background = zero;
411  if (const void* bgPtr = getGridBackgroundValuePtr(os)) {
412  background = *static_cast<const ValueT*>(bgPtr);
413  }
414 
416  ValueT inactiveVal[2] = { background, background };
417  int numUniqueInactiveVals = 0;
418  for (typename MaskT::OffIterator it = valueMask.beginOff();
419  numUniqueInactiveVals < 3 && it; ++it)
420  {
421  const Index32 idx = it.pos();
422 
423  // Skip inactive values that are actually child node pointers.
424  if (childMask.isOn(idx)) continue;
425 
426  const ValueT& val = srcBuf[idx];
427  const bool unique = !(
428  (numUniqueInactiveVals > 0 && Local::eq(val, inactiveVal[0])) ||
429  (numUniqueInactiveVals > 1 && Local::eq(val, inactiveVal[1]))
430  );
431  if (unique) {
432  if (numUniqueInactiveVals < 2) inactiveVal[numUniqueInactiveVals] = val;
433  ++numUniqueInactiveVals;
434  }
435  }
436 
437  metadata = NO_MASK_OR_INACTIVE_VALS;
438 
439  if (numUniqueInactiveVals == 1) {
440  if (!Local::eq(inactiveVal[0], background)) {
441  if (Local::eq(inactiveVal[0], negative(background))) {
442  metadata = NO_MASK_AND_MINUS_BG;
443  } else {
444  metadata = NO_MASK_AND_ONE_INACTIVE_VAL;
445  }
446  }
447  } else if (numUniqueInactiveVals == 2) {
448  metadata = NO_MASK_OR_INACTIVE_VALS;
449  if (!Local::eq(inactiveVal[0], background) && !Local::eq(inactiveVal[1], background)) {
450  // If neither inactive value is equal to the background, both values
451  // need to be saved, along with a mask that selects between them.
452  metadata = MASK_AND_TWO_INACTIVE_VALS;
453 
454  } else if (Local::eq(inactiveVal[1], background)) {
455  if (Local::eq(inactiveVal[0], negative(background))) {
456  // If the second inactive value is equal to the background and
457  // the first is equal to -background, neither value needs to be saved,
458  // but save a mask that selects between -background and +background.
459  metadata = MASK_AND_NO_INACTIVE_VALS;
460  } else {
461  // If the second inactive value is equal to the background, only
462  // the first value needs to be saved, along with a mask that selects
463  // between it and the background.
464  metadata = MASK_AND_ONE_INACTIVE_VAL;
465  }
466  } else if (Local::eq(inactiveVal[0], background)) {
467  if (Local::eq(inactiveVal[1], negative(background))) {
468  // If the first inactive value is equal to the background and
469  // the second is equal to -background, neither value needs to be saved,
470  // but save a mask that selects between -background and +background.
471  metadata = MASK_AND_NO_INACTIVE_VALS;
472  std::swap(inactiveVal[0], inactiveVal[1]);
473  } else {
474  // If the first inactive value is equal to the background, swap it
475  // with the second value and save only that value, along with a mask
476  // that selects between it and the background.
477  std::swap(inactiveVal[0], inactiveVal[1]);
478  metadata = MASK_AND_ONE_INACTIVE_VAL;
479  }
480  }
481  }
482 
483  os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
484 
485  if (metadata != NO_MASK_OR_INACTIVE_VALS &&
486  metadata != NO_MASK_AND_MINUS_BG &&
487  metadata != MASK_AND_NO_INACTIVE_VALS)
488  {
489  if (!toHalf) {
490  // Write one of at most two distinct inactive values.
491  os.write(reinterpret_cast<const char*>(&inactiveVal[0]), sizeof(ValueT));
492  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
493  // Write the second of two distinct inactive values.
494  os.write(reinterpret_cast<const char*>(&inactiveVal[1]), sizeof(ValueT));
495  }
496  } else {
497  // Write one of at most two distinct inactive values.
498  ValueT truncatedVal = truncateRealToHalf(inactiveVal[0]);
499  os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
500  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
501  // Write the second of two distinct inactive values.
502  truncatedVal = truncateRealToHalf(inactiveVal[1]);
503  os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
504  }
505  }
506  }
507 
508  if (metadata == NO_MASK_OR_INACTIVE_VALS && numUniqueInactiveVals > 2) {
509  // If there are more than two unique inactive values, the entire input buffer
510  // needs to be saved (both active and inactive values).
513  } else {
514  // Create a new array to hold just the active values.
515  scopedTempBuf.reset(new ValueT[srcCount]);
516  tempBuf = scopedTempBuf.get();
517 
518  if (metadata == NO_MASK_OR_INACTIVE_VALS ||
519  metadata == NO_MASK_AND_MINUS_BG ||
520  metadata == NO_MASK_AND_ONE_INACTIVE_VAL)
521  {
522  // Copy active values to the contiguous array.
523  tempCount = 0;
524  for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
525  tempBuf[tempCount] = srcBuf[it.pos()];
526  }
527  } else {
528  // Copy active values to a new, contiguous array and populate a bitmask
529  // that selects between two distinct inactive values.
530  MaskT selectionMask;
531  tempCount = 0;
532  for (Index srcIdx = 0; srcIdx < srcCount; ++srcIdx) {
533  if (valueMask.isOn(srcIdx)) { // active value
534  tempBuf[tempCount] = srcBuf[srcIdx];
535  ++tempCount;
536  } else { // inactive value
537  if (Local::eq(srcBuf[srcIdx], inactiveVal[1])) {
538  selectionMask.setOn(srcIdx); // inactive value 1
539  } // else inactive value 0
540  }
541  }
542  assert(tempCount == valueMask.countOn());
543 
544  // Write out the mask that selects between two inactive values.
545  selectionMask.save(os);
546  }
547  }
548  }
549 
550  // Write out the buffer.
551  if (toHalf) {
552  HalfWriter<RealToHalf<ValueT>::isReal, ValueT>::write(os, tempBuf, tempCount, zip);
553  } else {
554  writeData(os, tempBuf, tempCount, zip);
555  }
556 }
557 
558 } // namespace io
559 } // namespace OPENVDB_VERSION_NAME
560 } // namespace openvdb
561 
562 #endif // OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
563 
564 // Copyright (c) 2012-2013 DreamWorks Animation LLC
565 // All rights reserved. This software is distributed under the
566 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )