44 #ifndef _INCLUDED_Field3D_SparseFile_H_ 45 #define _INCLUDED_Field3D_SparseFile_H_ 72 template <
typename Data_T>
77 template <
typename Data_T>
95 template <
class Data_T>
105 typedef boost::shared_ptr<Reference> Ptr;
109 typedef std::vector<Sparse::SparseBlock<Data_T>*>
BlockPtrs;
151 static Ptr create(
const std::string a_filename,
152 const std::string a_layerPath);
156 void setNumBlocks(
int numBlocks);
166 void loadBlock(
int blockIdx);
168 void unloadBlock(
int blockIdx);
171 void incBlockRef(
int blockIdx);
173 void decBlockRef(
int blockIdx);
175 int blockSize(
int blockIdx)
const;
178 int totalLoads()
const;
181 int numLoadedBlocks()
const;
184 int totalLoadedBlocks()
const;
187 float averageLoads()
const;
189 void resetCacheStatistics();
198 Reference(
const std::string filename,
const std::string layerPath);
253 template <
class Data_T>
258 template <
class Data_T>
262 template <
class Data_T>
263 size_t numRefs()
const;
292 blockType(blockTypeIn), refIdx(refIdxIn), blockIdx(blockIdxIn)
366 template <
class Data_T>
380 void setLimitMemUse(
bool enabled);
384 bool doLimitMemUse()
const;
387 void setMaxMemUse(
float maxMemUse);
394 long long totalLoads();
397 long long numLoadedBlocks();
400 long long totalLoadedBlocks();
404 float cacheFractionLoaded();
407 float cacheLoadsPerBlock();
411 float cacheEfficiency();
414 void resetCacheStatistics();
423 template <
class Data_T>
430 template <
class Data_T>
436 template <
class Data_T>
442 template <
class Data_T>
447 template <
class Data_T>
448 int getNextId(
const std::string filename,
const std::string layerPath);
450 template <
class Data_T>
451 void removeFieldFromCache(
int refIdx);
462 void addBlockToCache(
DataTypeEnum blockType,
int fileId,
int blockIdx);
466 void deallocateBlocks(int64_t bytesNeeded);
470 template <
class Data_T>
474 template <
class Data_T>
475 void deallocateBlock(CacheList::iterator &it);
519 template <
class Data_T>
521 const std::string a_layerPath)
522 : filename(a_filename), layerPath(a_layerPath),
523 valuesPerBlock(-1), occupiedBlocks(-1),
524 blockMutex(NULL), m_fileHandle(-1), m_reader(NULL),
532 template <
class Data_T>
546 template <
class Data_T>
556 template <
class Data_T>
602 template <
class Data_T>
605 const std::string a_layerPath)
612 template <
class Data_T>
620 template <
class Data_T>
623 boost::mutex::scoped_lock lock(
m_mutex);
627 blocks.resize(numBlocks, 0);
638 template <
class Data_T>
644 boost::mutex::scoped_lock lock_A(
m_mutex);
657 printf(
"openFile(): No file handle. Dying.\n");
658 throw NoSuchFileException(
filename);
664 "Couldn't find layer group " +
layerPath +
666 throw FileIntegrityException(
filename);
680 template <
class Data_T>
692 template <
class Data_T>
695 boost::mutex::scoped_lock lock(
m_mutex);
699 assert(
blocks[blockIdx]->data != NULL);
711 template <
class Data_T>
715 blocks[blockIdx]->clear();
730 template <
class Data_T>
733 boost::mutex::scoped_lock lock(
blockMutex[blockIdx]);
739 template <
class Data_T>
742 boost::mutex::scoped_lock lock(
blockMutex[blockIdx]);
748 template <
class Data_T>
756 template <
class Data_T>
759 std::vector<int>::const_iterator i =
loadCounts.begin();
760 std::vector<int>::const_iterator end =
loadCounts.end();
762 for (; i != end; ++i)
770 template <
class Data_T>
773 std::vector<int>::const_iterator i =
blockLoaded.begin();
774 std::vector<int>::const_iterator end =
blockLoaded.end();
776 for (; i != end; ++i)
785 template <
class Data_T>
788 std::vector<int>::const_iterator i =
loadCounts.begin();
789 std::vector<int>::const_iterator li =
blockLoaded.begin();
790 std::vector<int>::const_iterator end =
loadCounts.end();
794 for (; i != end; ++i)
800 for (; i != end; ++i, ++li)
810 template <
class Data_T>
813 std::vector<int>::const_iterator i =
loadCounts.begin();
814 std::vector<int>::const_iterator end =
loadCounts.end();
815 int numLoads = 0, numBlocks = 0;
816 for (; i != end; ++i) {
823 return (
float)numLoads / std::max(1, numBlocks);
828 template <
class Data_T>
831 std::vector<int>::iterator li =
loadCounts.begin();
832 std::vector<int>::iterator lend =
loadCounts.end();
833 for (; li != lend; ++li)
852 for (
size_t i = 0, end = m_hRefs.size(); i < end; ++i) {
855 for (
size_t i = 0, end = m_fRefs.size(); i < end; ++i) {
858 for (
size_t i = 0, end = m_dRefs.size(); i < end; ++i) {
861 for (
size_t i = 0, end = m_vhRefs.size(); i < end; ++i) {
864 for (
size_t i = 0, end = m_vfRefs.size(); i < end; ++i) {
867 for (
size_t i = 0, end = m_vdRefs.size(); i < end; ++i) {
880 return m_hRefs[idx].get();
893 return m_vhRefs[idx].get();
895 return m_vhRefs[idx];
906 return m_fRefs[idx].get();
919 return m_vfRefs[idx].get();
921 return m_vfRefs[idx];
932 return m_dRefs[idx].get();
945 return m_vdRefs[idx].get();
947 return m_vdRefs[idx];
956 Mutex::scoped_lock lock(
m_mutex);
958 m_hRefs.push_back(ref);
959 return m_hRefs.size() - 1;
967 Mutex::scoped_lock lock(
m_mutex);
969 m_vhRefs.push_back(ref);
970 return m_vhRefs.size() - 1;
978 Mutex::scoped_lock lock(
m_mutex);
980 m_fRefs.push_back(ref);
981 return m_fRefs.size() - 1;
989 Mutex::scoped_lock lock(
m_mutex);
991 m_vfRefs.push_back(ref);
992 return m_vfRefs.size() - 1;
1000 Mutex::scoped_lock lock(
m_mutex);
1002 m_dRefs.push_back(ref);
1003 return m_dRefs.size() - 1;
1011 Mutex::scoped_lock lock(
m_mutex);
1013 m_vdRefs.push_back(ref);
1014 return m_vdRefs.size() - 1;
1020 inline size_t FileReferences::numRefs<half>()
const 1022 Mutex::scoped_lock lock(
m_mutex);
1024 return m_hRefs.size();
1030 inline size_t FileReferences::numRefs<V3h>()
const 1032 Mutex::scoped_lock lock(
m_mutex);
1034 return m_vhRefs.size();
1040 inline size_t FileReferences::numRefs<float>()
const 1042 Mutex::scoped_lock lock(
m_mutex);
1044 return m_fRefs.size();
1050 inline size_t FileReferences::numRefs<V3f>()
const 1052 Mutex::scoped_lock lock(
m_mutex);
1054 return m_vfRefs.size();
1060 inline size_t FileReferences::numRefs<double>()
const 1062 Mutex::scoped_lock lock(
m_mutex);
1064 return m_dRefs.size();
1070 inline size_t FileReferences::numRefs<V3d>()
const 1072 Mutex::scoped_lock lock(
m_mutex);
1074 return m_vdRefs.size();
1083 template <
class Data_T>
1086 assert(
false &&
"Do not use memory limiting on sparse fields that aren't " 1087 "simple scalars or vectors!");
1089 "FileReferences::ref(): Do not use memory limiting on sparse " 1090 "fields that aren't simple scalars or vectors!");
1097 template <
class Data_T>
1100 assert(
false &&
"Do not use memory limiting on sparse fields that aren't " 1101 "simple scalars or vectors!");
1103 "FileReferences::append(): Do not use memory limiting on sparse " 1104 "fields that aren't simple scalars or vectors!");
1110 template <
class Data_T>
1113 assert(
false &&
"Do not use memory limiting on sparse fields that aren't " 1114 "simple scalars or vectors!");
1116 "FileReferences::numRefs(): " 1117 "Do not use memory limiting on sparse " 1118 "fields that aren't " 1119 "simple scalars or vectors!");
1133 template <
class Data_T>
1141 boost::mutex::scoped_lock lock(
m_mutex);
1150 template <
class Data_T>
1154 boost::mutex::scoped_lock lock(
m_mutex);
1159 CacheList::iterator it = m_blockCacheList.begin();
1160 CacheList::iterator end = m_blockCacheList.end();
1161 CacheList::iterator next;
1163 int64_t bytesFreed = 0;
1166 if (it->blockType == blockType && it->refIdx == refIdx) {
1167 if (it == m_nextBlock) {
1172 bytesFreed += reference->
blockSize(it->blockIdx);
1173 m_blockCacheList.erase(it);
1179 m_memUse -= bytesFreed;
1186 std::vector<bool>().swap(reference->
blockUsed);
1187 std::vector<int>().swap(reference->
loadCounts);
1188 std::vector<int>().swap(reference->
refCounts);
1195 template <
class Data_T>
1199 return m_fileData.ref<Data_T>(index);
1204 template <
class Data_T>
1213 if (m_limitMemUse) {
1216 deallocateBlocks(blockSize);
1223 boost::mutex::scoped_lock lock_A(
m_mutex);
1224 boost::mutex::scoped_lock lock_B(reference->
blockMutex[blockIdx]);
1240 template <
class Data_T>
1253 template <
class Data_T>
void removeFieldFromCache(int refIdx)
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Contains utility functions and classes for Hdf5 files.
size_t m_numActiveBlocks
Number of currently active blocks.
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full...
Namespace for Exception objects.
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
void activateBlock(int fileId, int blockIdx)
Called by SparseField when it's about to read from a block. This should not be called by the user...
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector<bool> is weird.
void incBlockRef(int fileId, int blockIdx)
Increments the usage reference count on the specified block, to prevent it from getting unloaded whil...
void incBlockRef(int blockIdx)
Increment reference count on a block, indicates the block is currently in use, so prevents it from be...
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn)
std::vector< Sparse::SparseBlock< Data_T > * > BlockPtrs
static Ptr create(const std::string a_filename, const std::string a_layerPath)
Returns a shared pointer to a reference. Preferred way of creating References.
boost::recursive_mutex::scoped_lock GlobalLock
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
std::vector< int > fileBlockIndices
Index in file for each block.
void activateBlock(const int blockId) const
Activates a given block.
std::deque< Reference< V3f >::Ptr > m_vfRefs
void setNumBlocks(int numBlocks)
Sets the number of blocks used by the SparseField we're supporting.
static DataTypeEnum typeEnum()
void decBlockRef(int blockIdx)
Decrement reference count on a block.
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zer...
Contains various utility functions for Hdf5.
Mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
std::deque< Reference< V3d >::Ptr > m_vdRefs
float m_maxMemUse
Max amount om memory to use in megabytes.
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
float averageLoads() const
Returns the average number of loads per accessed block in this file, for cache statistics.
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
static SparseFileManager * ms_singleton
Pointer to singleton.
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please. ...
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Namespace for sparse field specifics.
void decBlockRef(const int blockId) const
Decrements the block ref count for the given block.
std::list< SparseFile::CacheBlock > CacheList
std::deque< Reference< float >::Ptr > m_fRefs
int totalLoadedBlocks() const
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks)...
Contains functions controlling the loading of sparse fields.
int64_t m_memUse
Current amount of memory in use in bytes.
int totalLoads() const
Returns the total number of loads of the blocks of this file, for cache statistics.
void resetCacheStatistics()
Resets counts of total block loads.
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
int getNextId(const std::string filename, const std::string layerPath)
Returns the id of the next cache item. This is stored in the SparseField in order to reference its fi...
Reference< Data_T > * ref(size_t idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed...
void decBlockRef(int fileId, int blockIdx)
Decrements the usage reference count on the specified block, after its value is no longer being used ...
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
Reference & operator=(const Reference &o)
Assignment operator. Clears ref counts and rebuilds mutex array.
void incBlockRef(const int blockId) const
Increments the block ref count for the given block.
size_t numRefs() const
Returns the number of file references of the corresponding collection.
FIELD3D_NAMESPACE_OPEN FIELD3D_API boost::recursive_mutex g_hdf5Mutex
std::deque< Reference< double >::Ptr > m_dRefs
Reference(const std::string filename, const std::string layerPath)
Constructor. Requires the filename and layer path of the field to be known.
bool fileIsOpen()
Checks if the file used by this reference is open already.
std::vector< int > loadCounts
Per-block counts of the number of times each block has been loaded, for cache statistics.
This Field subclass stores voxel data in block-allocated arrays.
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
std::deque< Reference< V3h >::Ptr > m_vhRefs
void closeFile()
Closes the file. Will be re-opened as needed.
Scoped object - opens a group on creation and closes it on destruction.
Contains Exception base class.
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
size_t append(typename Reference< Data_T >::Ptr ref)
Appends a reference to the collection. This is specialized so that the correct data member is accesse...
void open(hid_t parentLocation, const std::string &name)
int numLoadedBlocks() const
Returns the total number of blocks that are currently loaded, for statistics.
std::deque< Reference< half >::Ptr > m_hRefs
hid_t id() const
Query the hid_t value.