46 #ifndef _INCLUDED_Field3D_SparseFieldIO_H_ 47 #define _INCLUDED_Field3D_SparseFieldIO_H_ 86 typedef boost::intrusive_ptr<SparseFieldIO>
Ptr;
95 return "SparseFieldIO";
119 const std::string &layerPath,
128 {
return "SparseField"; }
135 template <
class Data_T>
139 template <
class Data_T>
142 const std::string &filename,
143 const std::string &layerPath,
171 template <
class Data_T>
186 int valuesPerBlock = (1 << (field->
m_blockOrder * 3)) * components;
191 { ext.min.x, ext.min.y, ext.min.z, ext.max.x, ext.max.y, ext.max.z };
201 { dw.min.x, dw.min.y, dw.min.z, dw.max.x, dw.max.y, dw.max.z };
227 int numBlocks = blockRes.x * blockRes.y * blockRes.z;
255 vector<char> isAllocated(numBlocks);
256 for (
int i = 0; i < numBlocks; ++i) {
257 isAllocated[i] =
static_cast<char>(blocks[i].
isAllocated);
259 writeSimpleData<char>(layerGroup,
"block_is_allocated_data", isAllocated);
264 vector<Data_T> emptyValue(numBlocks);
265 for (
int i = 0; i < numBlocks; ++i) {
266 emptyValue[i] =
static_cast<Data_T
>(blocks[i].
emptyValue);
268 writeSimpleData<Data_T>(layerGroup,
"block_empty_value_data", emptyValue);
272 int occupiedBlocks = 0;
273 for (
int i = 0; i < numBlocks; ++i) {
274 if (blocks[i].isAllocated) {
280 throw WriteAttributeException(
"Couldn't add attribute " +
284 if (occupiedBlocks > 0) {
288 memDims[0] = valuesPerBlock;
290 H5Sset_extent_simple(memDataSpace.
id(), 1, memDims, NULL);
294 fileDims[0] = occupiedBlocks;
295 fileDims[1] = valuesPerBlock;
297 H5Sset_extent_simple(fileDataSpace.
id(), 2, fileDims, NULL);
301 hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
302 hsize_t chunkSize[2];
304 chunkSize[1] = valuesPerBlock;
306 herr_t status = H5Pset_deflate(dcpl, 9);
310 status = H5Pset_chunk(dcpl, 2, chunkSize);
320 H5P_DEFAULT, dcpl, H5P_DEFAULT);
321 if (dataSet.id() < 0)
322 throw CreateDataSetException(
"Couldn't create data set in " 323 "SparseFieldIO::writeInternal");
327 int nextBlockIdx = 0;
332 for (
int i = 0; i < numBlocks; ++i) {
333 if (blocks[i].isAllocated) {
334 offset[0] = nextBlockIdx;
337 count[1] = valuesPerBlock;
338 status = H5Sselect_hyperslab(fileDataSpace.
id(), H5S_SELECT_SET,
339 offset, NULL, count, NULL);
341 throw WriteHyperSlabException(
342 "Couldn't select slab " +
343 boost::lexical_cast<std::string>(nextBlockIdx));
348 fileDataSpace.
id(), H5P_DEFAULT, data);
350 throw WriteHyperSlabException(
351 "Couldn't write slab " +
352 boost::lexical_cast<std::string>(nextBlockIdx));
367 template <
class Data_T>
370 const std::string &filename,
371 const std::string &layerPath,
384 int valuesPerBlock = (1 << (result->
m_blockOrder * 3)) * components;
389 throw MissingAttributeException(
"Couldn't find attribute: " +
394 if (dynamicLoading) {
408 vector<char> isAllocated(numBlocks);
409 readSimpleData<char>(location,
"block_is_allocated_data", isAllocated);
410 for (
int i = 0; i < numBlocks; ++i) {
412 if (!dynamicLoading && isAllocated[i]) {
413 blocks[i].
resize(valuesPerBlock);
421 vector<Data_T> emptyValue(numBlocks);
422 readSimpleData<Data_T>(location,
"block_empty_value_data", emptyValue);
423 for (
int i = 0; i < numBlocks; ++i) {
430 if (occupiedBlocks > 0) {
432 if (dynamicLoading) {
438 size_t b = 0, bend = b + numBlocks;
443 static const long maxMemPerPass = 50*1024*1024;
445 for (
int nextBlockIdx = 0;;) {
448 std::vector<Data_T*> memoryList;
450 for (; b != bend && mem < maxMemPerPass; ++b) {
451 if (blocks[b].isAllocated) {
452 mem +=
sizeof(Data_T)*valuesPerBlock;
453 memoryList.push_back(blocks[b].data);
458 if (!memoryList.size()) {
463 nextBlockIdx += memoryList.size();
#define FIELD3D_NAMESPACE_HEADER_CLOSE
void addReference(const std::string &filename, const std::string &layerPath, int valuesPerBlock, int occupiedBlocks)
Internal function to create a Reference for the current field, for use in dynamic reading...
V3i m_blockRes
Block array resolution.
Contains utility functions and classes for Hdf5 files.
const Box3i & extents() const
Returns the extents of the data. This signifies the relevant area that the data exists over...
FIELD3D_API bool writeAttribute(hid_t location, const std::string &attrName, const std::string &value)
Writes a string attribute.
Contains the Field3DFile classesOSS sanitized.
static int dataDims()
Dimensions of the given data type. i.e. 3 for V3f, 1 for float.
Namespace for Exception objects.
static const std::string k_dataStr
void readBlockList(int idx, const std::vector< Data_T * > &memoryList)
Reads a series of blocks, storing each block of data in memoryList, which is assumed to contain enoug...
virtual std::string className() const
Returns the class name.
static const std::string k_versionAttrName
Contains functions controlling the loading of sparse fields.
virtual bool write(hid_t layerGroup, FieldBase::Ptr field)
Writes the given field to disk.
static const std::string k_dataWindowStr
FIELD3D_API bool readAttribute(hid_t location, const std::string &attrName, std::string &value)
Reads a string attribute.
void setupReferenceBlocks()
Internal function to setup the Reference's block pointers, for use with dynamic reading.
boost::intrusive_ptr< FieldBase > Ptr
static const std::string k_numOccupiedBlocksStr
static const std::string k_extentsStr
Data_T * data
Pointer to data. Null if block is unallocated.
virtual ~SparseFieldIO()
Dtor.
boost::recursive_mutex::scoped_lock GlobalLock
DEFINE_FIELD_RTTI_CONCRETE_CLASS
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
int m_blockOrder
Block order (size = 2^blockOrder)
boost::intrusive_ptr< SparseField > Ptr
const char * staticClassType() const
boost::intrusive_ptr< SparseFieldIO > Ptr
static const int k_versionNumber
Data_T emptyValue
The value to use if the block isn't allocated. We allow setting this per block so that we for example...
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
FIELD3D_API bool checkHdf5Gzip()
Checks whether gzip is available in the current hdf5 library.
virtual FieldBase::Ptr read(hid_t layerGroup, const std::string &filename, const std::string &layerPath, DataTypeEnum typeEnum)
Reads the field at the given location and tries to create a SparseField object from it...
static const std::string k_blockOrderStr
static FieldIO::Ptr create()
Namespace for sparse field specifics.
static const std::string k_numBlocksStr
static const std::string k_componentsStr
void resize(int n)
Alloc data.
static const std::string k_blockResStr
Contains functions controlling the loading of sparse fields.
bool isAllocated
Whether the block is allocated or not.
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
Storage for one individual block of a SparseField.
Scoped object - creates a dataspace on creation and closes it on destruction.
FIELD3D_NAMESPACE_OPEN FIELD3D_API boost::recursive_mutex g_hdf5Mutex
static const std::string k_bitsPerComponentStr
FieldIO base
Convenience typedef for referring to base class.
Contains the SparseField class.
Scoped object - creates a dataset on creation and closes it on destruction.
boost::intrusive_ptr< FieldIO > Ptr
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
const Box3i & dataWindow() const
Returns the data window. Any coordinate inside this window is safe to pass to value() in the Field su...
bool writeInternal(hid_t layerGroup, typename SparseField< Data_T >::Ptr field)
This call writes all the attributes and sets up the data space.
bool readData(hid_t location, int numBlocks, const std::string &filename, const std::string &layerPath, typename SparseField< Data_T >::Ptr result)
Reads the data that is dependent on the data type on disk.
Block * m_blocks
Array of blocks. Not using std::vector since SparseBlock is noncopyable.
hid_t id() const
Query the hid_t value.