Field3D
SparseFieldIO.h
Go to the documentation of this file.
1 //----------------------------------------------------------------------------//
2 
3 /*
4  * Copyright (c) 2009 Sony Pictures Imageworks Inc
5  *
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the
17  * distribution. Neither the name of Sony Pictures Imageworks nor the
18  * names of its contributors may be used to endorse or promote
19  * products derived from this software without specific prior written
20  * permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33  * OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 //----------------------------------------------------------------------------//
37 
44 //----------------------------------------------------------------------------//
45 
46 #ifndef _INCLUDED_Field3D_SparseFieldIO_H_
47 #define _INCLUDED_Field3D_SparseFieldIO_H_
48 
49 //----------------------------------------------------------------------------//
50 
51 #include <string>
52 #include <cmath>
53 
54 #include <hdf5.h>
55 
56 #include "SparseDataReader.h"
57 #include "SparseField.h"
58 #include "SparseFile.h"
59 #include "FieldIO.h"
60 #include "Field3DFile.h"
61 
62 //----------------------------------------------------------------------------//
63 
64 #include "ns.h"
65 
67 
68 //----------------------------------------------------------------------------//
69 // SparseFieldIO
70 //----------------------------------------------------------------------------//
71 
77 //----------------------------------------------------------------------------//
78 
79 class SparseFieldIO : public FieldIO
80 {
81 
82 public:
83 
84  // Typedefs ------------------------------------------------------------------
85 
86  typedef boost::intrusive_ptr<SparseFieldIO> Ptr;
87 
88  // RTTI replacement ----------------------------------------------------------
89 
92 
93  const char *staticClassType() const
94  {
95  return "SparseFieldIO";
96  }
97 
98  // Constructors --------------------------------------------------------------
99 
102  : FieldIO()
103  { }
104 
106  virtual ~SparseFieldIO()
107  { /* Empty */ }
108 
109 
111  { return Ptr(new SparseFieldIO); }
112 
113  // From FieldIO --------------------------------------------------------------
114 
118  virtual FieldBase::Ptr read(hid_t layerGroup, const std::string &filename,
119  const std::string &layerPath,
120  DataTypeEnum typeEnum);
121 
124  virtual bool write(hid_t layerGroup, FieldBase::Ptr field);
125 
127  virtual std::string className() const
128  { return "SparseField"; }
129 
130 private:
131 
132  // Internal methods ----------------------------------------------------------
133 
135  template <class Data_T>
136  bool writeInternal(hid_t layerGroup, typename SparseField<Data_T>::Ptr field);
137 
139  template <class Data_T>
140  bool readData(hid_t location,
141  int numBlocks,
142  const std::string &filename,
143  const std::string &layerPath,
144  typename SparseField<Data_T>::Ptr result);
145 
146  // Strings -------------------------------------------------------------------
147 
148  static const int k_versionNumber;
149  static const std::string k_versionAttrName;
150  static const std::string k_extentsStr;
151  static const std::string k_dataWindowStr;
152  static const std::string k_componentsStr;
153  static const std::string k_blockOrderStr;
154  static const std::string k_numBlocksStr;
155  static const std::string k_blockResStr;
156  static const std::string k_bitsPerComponentStr;
157  static const std::string k_numOccupiedBlocksStr;
158  static const std::string k_dataStr;
159 
160  // Typedefs ------------------------------------------------------------------
161 
163  typedef FieldIO base;
164 };
165 
166 //----------------------------------------------------------------------------//
167 // Template methods
168 //----------------------------------------------------------------------------//
169 
171 template <class Data_T>
172 bool SparseFieldIO::writeInternal(hid_t layerGroup,
173  typename SparseField<Data_T>::Ptr field)
174 {
175  using namespace std;
176  using namespace Exc;
177  using namespace Hdf5Util;
178  using namespace Sparse;
179 
180  GlobalLock lock(g_hdf5Mutex);
181 
182  Box3i ext(field->extents()), dw(field->dataWindow());
183 
184  int components = FieldTraits<Data_T>::dataDims();
185 
186  int valuesPerBlock = (1 << (field->m_blockOrder * 3)) * components;
187 
188  // Add extents attribute ---
189 
190  int extents[6] =
191  { ext.min.x, ext.min.y, ext.min.z, ext.max.x, ext.max.y, ext.max.z };
192 
193  if (!writeAttribute(layerGroup, k_extentsStr, 6, extents[0])) {
194  Msg::print(Msg::SevWarning, "Error adding size attribute.");
195  return false;
196  }
197 
198  // Add data window attribute ---
199 
200  int dataWindow[6] =
201  { dw.min.x, dw.min.y, dw.min.z, dw.max.x, dw.max.y, dw.max.z };
202 
203  if (!writeAttribute(layerGroup, k_dataWindowStr, 6, dataWindow[0])) {
204  Msg::print(Msg::SevWarning, "Error adding size attribute.");
205  return false;
206  }
207 
208  // Add components attribute ---
209 
210  if (!writeAttribute(layerGroup, k_componentsStr, 1, components)) {
211  Msg::print(Msg::SevWarning, "Error adding components attribute.");
212  return false;
213  }
214 
215  // Add block order attribute ---
216 
217  int blockOrder = field->m_blockOrder;
218 
219  if (!writeAttribute(layerGroup, k_blockOrderStr, 1, blockOrder)) {
220  Msg::print(Msg::SevWarning, "Error adding block order attribute.");
221  return false;
222  }
223 
224  // Add number of blocks attribute ---
225 
226  V3i &blockRes = field->m_blockRes;
227  int numBlocks = blockRes.x * blockRes.y * blockRes.z;
228 
229  if (!writeAttribute(layerGroup, k_numBlocksStr, 1, numBlocks)) {
230  Msg::print(Msg::SevWarning, "Error adding number of blocks attribute.");
231  return false;
232  }
233 
234  // Add block resolution in each dimension ---
235 
236  if (!writeAttribute(layerGroup, k_blockResStr, 3, blockRes.x)) {
237  Msg::print(Msg::SevWarning, "Error adding block res attribute.");
238  return false;
239  }
240 
241  // Add the bits per component attribute ---
242 
243  int bits = DataTypeTraits<Data_T>::h5bits();
244  if (!writeAttribute(layerGroup, k_bitsPerComponentStr, 1, bits)) {
245  Msg::print(Msg::SevWarning, "Error adding bits per component attribute.");
246  return false;
247  }
248 
249  // Write the block info data sets ---
250 
251  SparseBlock<Data_T> *blocks = field->m_blocks;
252 
253  // ... Write the isAllocated array
254  {
255  vector<char> isAllocated(numBlocks);
256  for (int i = 0; i < numBlocks; ++i) {
257  isAllocated[i] = static_cast<char>(blocks[i].isAllocated);
258  }
259  writeSimpleData<char>(layerGroup, "block_is_allocated_data", isAllocated);
260  }
261 
262  // ... Write the emptyValue array
263  {
264  vector<Data_T> emptyValue(numBlocks);
265  for (int i = 0; i < numBlocks; ++i) {
266  emptyValue[i] = static_cast<Data_T>(blocks[i].emptyValue);
267  }
268  writeSimpleData<Data_T>(layerGroup, "block_empty_value_data", emptyValue);
269  }
270 
271  // Count the number of occupied blocks ---
272  int occupiedBlocks = 0;
273  for (int i = 0; i < numBlocks; ++i) {
274  if (blocks[i].isAllocated) {
275  occupiedBlocks++;
276  }
277  }
278 
279  if (!writeAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks)) {
280  throw WriteAttributeException("Couldn't add attribute " +
282  }
283 
284  if (occupiedBlocks > 0) {
285 
286  // Make the memory data space
287  hsize_t memDims[1];
288  memDims[0] = valuesPerBlock;
289  H5ScopedScreate memDataSpace(H5S_SIMPLE);
290  H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL);
291 
292  // Make the file data space
293  hsize_t fileDims[2];
294  fileDims[0] = occupiedBlocks;
295  fileDims[1] = valuesPerBlock;
296  H5ScopedScreate fileDataSpace(H5S_SIMPLE);
297  H5Sset_extent_simple(fileDataSpace.id(), 2, fileDims, NULL);
298 
299  // Set up gzip property list
300  bool gzipAvailable = checkHdf5Gzip();
301  hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
302  hsize_t chunkSize[2];
303  chunkSize[0] = 1;
304  chunkSize[1] = valuesPerBlock;
305  if (gzipAvailable) {
306  herr_t status = H5Pset_deflate(dcpl, 9);
307  if (status < 0) {
308  return false;
309  }
310  status = H5Pset_chunk(dcpl, 2, chunkSize);
311  if (status < 0) {
312  return false;
313  }
314  }
315 
316  // Add the data set
317  H5ScopedDcreate dataSet(layerGroup, k_dataStr,
319  fileDataSpace.id(),
320  H5P_DEFAULT, dcpl, H5P_DEFAULT);
321  if (dataSet.id() < 0)
322  throw CreateDataSetException("Couldn't create data set in "
323  "SparseFieldIO::writeInternal");
324 
325  // For each allocated block ---
326 
327  int nextBlockIdx = 0;
328  hsize_t offset[2];
329  hsize_t count[2];
330  herr_t status;
331 
332  for (int i = 0; i < numBlocks; ++i) {
333  if (blocks[i].isAllocated) {
334  offset[0] = nextBlockIdx; // Index of next block
335  offset[1] = 0; // Index of first data in block. Always 0
336  count[0] = 1; // Number of columns to read. Always 1
337  count[1] = valuesPerBlock; // Number of values in one column
338  status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET,
339  offset, NULL, count, NULL);
340  if (status < 0) {
341  throw WriteHyperSlabException(
342  "Couldn't select slab " +
343  boost::lexical_cast<std::string>(nextBlockIdx));
344  }
345  Data_T *data = field->m_blocks[i].data;
346  status = H5Dwrite(dataSet.id(), DataTypeTraits<Data_T>::h5type(),
347  memDataSpace.id(),
348  fileDataSpace.id(), H5P_DEFAULT, data);
349  if (status < 0) {
350  throw WriteHyperSlabException(
351  "Couldn't write slab " +
352  boost::lexical_cast<std::string>(nextBlockIdx));
353  }
354  // Increment nextBlockIdx
355  nextBlockIdx++;
356  }
357  }
358 
359  } // if occupiedBlocks > 0
360 
361  return true;
362 
363 }
364 
365 //----------------------------------------------------------------------------//
366 
367 template <class Data_T>
368 bool SparseFieldIO::readData(hid_t location,
369  int numBlocks,
370  const std::string &filename,
371  const std::string &layerPath,
372  typename SparseField<Data_T>::Ptr result)
373 {
374  using namespace std;
375  using namespace Exc;
376  using namespace Hdf5Util;
377  using namespace Sparse;
378 
379  int occupiedBlocks;
380 
381  bool dynamicLoading = SparseFileManager::singleton().doLimitMemUse();
382 
383  int components = FieldTraits<Data_T>::dataDims();
384  int valuesPerBlock = (1 << (result->m_blockOrder * 3)) * components;
385 
386  // Read the number of occupied blocks ---
387 
388  if (!readAttribute(location, k_numOccupiedBlocksStr, 1, occupiedBlocks))
389  throw MissingAttributeException("Couldn't find attribute: " +
391 
392  // Set up the dynamic read info ---
393 
394  if (dynamicLoading) {
395  // Set up the field reference
396  result->addReference(filename, layerPath,
397  valuesPerBlock,
398  occupiedBlocks);
399  }
400 
401  // Read the block info data sets ---
402 
403  SparseBlock<Data_T> *blocks = result->m_blocks;
404 
405  // ... Read the isAllocated array
406 
407  {
408  vector<char> isAllocated(numBlocks);
409  readSimpleData<char>(location, "block_is_allocated_data", isAllocated);
410  for (int i = 0; i < numBlocks; ++i) {
411  blocks[i].isAllocated = isAllocated[i];
412  if (!dynamicLoading && isAllocated[i]) {
413  blocks[i].resize(valuesPerBlock);
414  }
415  }
416  }
417 
418  // ... Read the emptyValue array ---
419 
420  {
421  vector<Data_T> emptyValue(numBlocks);
422  readSimpleData<Data_T>(location, "block_empty_value_data", emptyValue);
423  for (int i = 0; i < numBlocks; ++i) {
424  blocks[i].emptyValue = emptyValue[i];
425  }
426  }
427 
428  // Read the data ---
429 
430  if (occupiedBlocks > 0) {
431 
432  if (dynamicLoading) {
433 
434  result->setupReferenceBlocks();
435 
436  } else {
437 
438  size_t b = 0, bend = b + numBlocks;
439 
440  SparseDataReader<Data_T> reader(location, valuesPerBlock, occupiedBlocks);
441 
442  // We'll read at most 50meg at a time
443  static const long maxMemPerPass = 50*1024*1024;
444 
445  for (int nextBlockIdx = 0;;) {
446 
447  long mem = 0;
448  std::vector<Data_T*> memoryList;
449 
450  for (; b != bend && mem < maxMemPerPass; ++b) {
451  if (blocks[b].isAllocated) {
452  mem += sizeof(Data_T)*valuesPerBlock;
453  memoryList.push_back(blocks[b].data);
454  }
455  }
456 
457  // all done.
458  if (!memoryList.size()) {
459  break;
460  }
461 
462  reader.readBlockList(nextBlockIdx, memoryList);
463  nextBlockIdx += memoryList.size();
464  }
465 
466  }
467 
468  } // if occupiedBlocks > 0
469 
470  return true;
471 
472 }
473 
474 //----------------------------------------------------------------------------//
475 
477 
478 //----------------------------------------------------------------------------//
479 
480 #endif
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition: ns.h:58
SparseFieldIO()
Ctor.
Imath::Box3i Box3i
Definition: SpiMathLib.h:77
void addReference(const std::string &filename, const std::string &layerPath, int valuesPerBlock, int occupiedBlocks)
Internal function to create a Reference for the current field, for use in dynamic reading...
Definition: SparseField.h:1365
V3i m_blockRes
Block array resolution.
Definition: SparseField.h:612
Contains utility functions and classes for Hdf5 files.
Definition: Hdf5Util.h:86
const Box3i & extents() const
Returns the extents of the data. This signifies the relevant area that the data exists over...
Definition: Field.h:254
FIELD3D_API bool writeAttribute(hid_t location, const std::string &attrName, const std::string &value)
Writes a string attribute.
Contains the Field3DFile classesOSS sanitized.
static int dataDims()
Dimensions of the given data type. i.e. 3 for V3f, 1 for float.
Namespace for Exception objects.
Definition: Exception.h:57
static const std::string k_dataStr
void readBlockList(int idx, const std::vector< Data_T * > &memoryList)
Reads a series of blocks, storing each block of data in memoryList, which is assumed to contain enoug...
virtual std::string className() const
Returns the class name.
static const std::string k_versionAttrName
Contains functions controlling the loading of sparse fields.
virtual bool write(hid_t layerGroup, FieldBase::Ptr field)
Writes the given field to disk.
static const std::string k_dataWindowStr
FIELD3D_API bool readAttribute(hid_t location, const std::string &attrName, std::string &value)
Reads a string attribute.
void setupReferenceBlocks()
Internal function to setup the Reference&#39;s block pointers, for use with dynamic reading.
Definition: SparseField.h:1397
boost::intrusive_ptr< FieldBase > Ptr
Definition: Field.h:97
static const std::string k_numOccupiedBlocksStr
static const std::string k_extentsStr
Data_T * data
Pointer to data. Null if block is unallocated.
Definition: SparseField.h:311
virtual ~SparseFieldIO()
Dtor.
boost::recursive_mutex::scoped_lock GlobalLock
Definition: Hdf5Util.h:78
DEFINE_FIELD_RTTI_CONCRETE_CLASS
Definition: SparseFieldIO.h:91
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
Definition: Log.cpp:66
int m_blockOrder
Block order (size = 2^blockOrder)
Definition: SparseField.h:610
boost::intrusive_ptr< SparseField > Ptr
Definition: SparseField.h:357
const char * staticClassType() const
Definition: SparseFieldIO.h:93
boost::intrusive_ptr< SparseFieldIO > Ptr
Definition: SparseFieldIO.h:86
static const int k_versionNumber
Data_T emptyValue
The value to use if the block isn&#39;t allocated. We allow setting this per block so that we for example...
Definition: SparseField.h:308
Imath::V3i V3i
Definition: SpiMathLib.h:71
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
FIELD3D_API bool checkHdf5Gzip()
Checks whether gzip is available in the current hdf5 library.
Definition: Hdf5Util.cpp:722
virtual FieldBase::Ptr read(hid_t layerGroup, const std::string &filename, const std::string &layerPath, DataTypeEnum typeEnum)
Reads the field at the given location and tries to create a SparseField object from it...
static const std::string k_blockOrderStr
static FieldIO::Ptr create()
static hid_t h5type()
Namespace for sparse field specifics.
Definition: SparseField.h:221
static const std::string k_numBlocksStr
static const std::string k_componentsStr
void resize(int n)
Alloc data.
Definition: SparseField.h:259
static const std::string k_blockResStr
Contains functions controlling the loading of sparse fields.
static int h5bits()
bool isAllocated
Whether the block is allocated or not.
Definition: SparseField.h:303
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
Definition: SparseFile.cpp:80
Storage for one individual block of a SparseField.
Definition: SparseField.h:227
SparseFieldIO class_type
Definition: SparseFieldIO.h:90
Scoped object - creates a dataspace on creation and closes it on destruction.
Definition: Hdf5Util.h:234
FIELD3D_NAMESPACE_OPEN FIELD3D_API boost::recursive_mutex g_hdf5Mutex
Definition: Hdf5Util.cpp:67
static const std::string k_bitsPerComponentStr
FieldIO base
Convenience typedef for referring to base class.
Contains the SparseField class.
Scoped object - creates a dataset on creation and closes it on destruction.
Definition: Hdf5Util.h:263
boost::intrusive_ptr< FieldIO > Ptr
Definition: FieldIO.h:90
DataTypeEnum
Definition: Traits.h:66
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
Definition: SparseFile.cpp:63
const Box3i & dataWindow() const
Returns the data window. Any coordinate inside this window is safe to pass to value() in the Field su...
Definition: Field.h:258
Contains FieldIO class.
bool writeInternal(hid_t layerGroup, typename SparseField< Data_T >::Ptr field)
This call writes all the attributes and sets up the data space.
bool readData(hid_t location, int numBlocks, const std::string &filename, const std::string &layerPath, typename SparseField< Data_T >::Ptr result)
Reads the data that is dependent on the data type on disk.
Block * m_blocks
Array of blocks. Not using std::vector since SparseBlock is noncopyable.
Definition: SparseField.h:616
hid_t id() const
Query the hid_t value.
Definition: Hdf5Util.h:100