Field3D
SparseFile.cpp
Go to the documentation of this file.
1 //----------------------------------------------------------------------------//
2 
3 /*
4  * Copyright (c) 2009 Sony Pictures Imageworks Inc
5  *
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the
17  * distribution. Neither the name of Sony Pictures Imageworks nor the
18  * names of its contributors may be used to endorse or promote
19  * products derived from this software without specific prior written
20  * permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33  * OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 //----------------------------------------------------------------------------//
37 
42 //----------------------------------------------------------------------------//
43 
44 // SparseField.h includes SparseFile.h, but we need the definition of
45 // SparseBlock from SparseField.h, so just include that to get both
46 // files
47 #include "SparseField.h"
48 
49 //----------------------------------------------------------------------------//
50 
52 
53 //----------------------------------------------------------------------------//
54 // Static instances
55 //----------------------------------------------------------------------------//
56 
58 
59 //----------------------------------------------------------------------------//
60 // SparseFileManager
61 //----------------------------------------------------------------------------//
62 
64 {
65  if (!ms_singleton) {
67  }
68  return *ms_singleton;
69 }
70 
71 //----------------------------------------------------------------------------//
72 
74 {
75  m_limitMemUse = enabled;
76 }
77 
78 //----------------------------------------------------------------------------//
79 
81 {
82  return m_limitMemUse;
83 }
84 
85 //----------------------------------------------------------------------------//
86 
87 void SparseFileManager::setMaxMemUse(float maxMemUse)
88 {
89  m_maxMemUse = maxMemUse;
90  m_maxMemUseInBytes = static_cast<int64_t>(m_maxMemUse * 1024 * 1024);
91 }
92 
93 //----------------------------------------------------------------------------//
94 
95 template <class Data_T>
97 {
98  int64_t bytesFreed = 0;
100 
101  // Note: we don't need to lock the block's mutex because
102  // deallocateBlock() is only called while the SparseFileManager's
103  // mutex is also locked (in flushCache() or deallocateBlocks()).
104  // Don't lock the block, to make sure we don't have a deadlock by
105  // holding two locks at the same time. (Because addBlockToCache()
106  // locks the manager but is also in a block-specific lock.)
107 
108  // lock the current block to make sure its blockUsed flag and ref
109  // counts don't change
110  // Note: this lock order is made consistent w/ allocate to prevent
111  // deadlocks and crashes.
112 
113  boost::mutex::scoped_lock lock_B(reference->blockMutex[cb.blockIdx]);
114 
115  // check whether the block is still in use
116  if (reference->refCounts[cb.blockIdx] > 0)
117  return bytesFreed;
118 
119  if (reference->blockUsed[cb.blockIdx]) {
120  // the block was recently used according to Second-chance paging
121  // algorithm, so skip it
122  reference->blockUsed[cb.blockIdx] = false;
123  }
124  else {
125 
126  // the block wasn't in use, so free it
127  reference->unloadBlock(cb.blockIdx);
128  bytesFreed = reference->blockSize(cb.blockIdx);
129  m_memUse -= bytesFreed;
130  CacheList::iterator toRemove = m_nextBlock;
131  ++m_nextBlock;
132  m_blockCacheList.erase(toRemove);
133  }
134  return bytesFreed;
135 }
136 
137 //----------------------------------------------------------------------------//
138 
139 template <class Data_T>
140 void SparseFileManager::deallocateBlock(CacheList::iterator &it)
141 {
142  SparseFile::CacheBlock &cb = *it;
144  int64_t bytesFreed = reference->blockSize(cb.blockIdx);
145  m_memUse -= bytesFreed;
146  reference->unloadBlock(cb.blockIdx);
147  it = m_blockCacheList.erase(it);
148 }
149 
150 //----------------------------------------------------------------------------//
151 
152 void SparseFileManager::deallocateBlocks(int64_t bytesNeeded)
153 {
154  boost::mutex::scoped_lock lock_A(m_mutex);
155 
156  while (m_blockCacheList.begin() != m_blockCacheList.end() &&
157  m_maxMemUseInBytes-m_memUse < bytesNeeded) {
158 
159  if (m_nextBlock == m_blockCacheList.end())
160  m_nextBlock = m_blockCacheList.begin();
161 
163 
164  // if bytesFreed is set to >0, then we've already freed a block
165  // and advanced the "clock hand" iterator
166  int64_t bytesFreed = 0;
167 
168  switch(cb.blockType) {
169  case DataTypeHalf:
170  bytesFreed = deallocateBlock<half>(cb);
171  if (bytesFreed > 0) {
172  continue;
173  }
174  break;
175  case DataTypeFloat:
176  bytesFreed = deallocateBlock<float>(cb);
177  if (bytesFreed > 0) {
178  continue;
179  }
180  break;
181  case DataTypeDouble:
182  bytesFreed = deallocateBlock<double>(cb);
183  if (bytesFreed > 0) {
184  continue;
185  }
186  break;
187  case DataTypeVecHalf:
188  bytesFreed = deallocateBlock<V3h>(cb);
189  if (bytesFreed > 0) {
190  continue;
191  }
192  break;
193  case DataTypeVecFloat:
194  bytesFreed = deallocateBlock<V3f>(cb);
195  if (bytesFreed > 0) {
196  continue;
197  }
198  break;
199  case DataTypeVecDouble:
200  bytesFreed = deallocateBlock<V3d>(cb);
201  if (bytesFreed > 0) {
202  continue;
203  }
204  break;
205  case DataTypeUnknown:
206  default:
207  break;
208  }
209  ++m_nextBlock;
210  }
211 }
212 
213 //----------------------------------------------------------------------------//
214 
216 {
217  boost::mutex::scoped_lock lock(m_mutex);
218 
219  CacheList::iterator it = m_blockCacheList.begin();
220  while (it != m_blockCacheList.end()) {
221  SparseFile::CacheBlock &cb = *it;
222 
223  switch(cb.blockType) {
224  case DataTypeHalf:
225  deallocateBlock<half>(it);
226  break;
227  case DataTypeFloat:
228  deallocateBlock<float>(it);
229  break;
230  case DataTypeDouble:
231  deallocateBlock<double>(it);
232  break;
233  case DataTypeVecHalf:
234  deallocateBlock<V3h>(it);
235  break;
236  case DataTypeVecFloat:
237  deallocateBlock<V3f>(it);
238  break;
239  case DataTypeVecDouble:
240  deallocateBlock<V3d>(it);
241  break;
242  case DataTypeUnknown:
243  default:
244  break;
245  }
246  }
247  m_nextBlock = m_blockCacheList.begin();
248 }
249 
250 //----------------------------------------------------------------------------//
251 
253  int fileId, int blockIdx)
254 {
255  // Note: this lock is obtained while we also have a lock on the
256  // specific block (in activateBlock()), so we should make sure we
257  // never lock the SparseFileManager and *then* a block, to ensure we
258  // don't have a deadlock.
259  //
260  // Note: this was changed so the order was consistent w/ dealloc
261  // again, see activateBlock()
262  // boost::mutex::scoped_lock lock(m_mutex);
263 
264  SparseFile::CacheBlock block(blockType, fileId, blockIdx);
265  if (m_nextBlock == m_blockCacheList.end()) {
266  m_blockCacheList.push_back(block);
267  } else {
268  m_blockCacheList.insert(m_nextBlock, block);
269  }
270 }
271 
272 //----------------------------------------------------------------------------//
273 
275  : m_memUse(0),
276  m_limitMemUse(false)
277 {
278  setMaxMemUse(1000.0);
279  m_nextBlock = m_blockCacheList.begin();
280 }
281 
282 //----------------------------------------------------------------------------//
283 
285 {
286 
287  long long int numLoads = 0;
288 
289  for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
290  numLoads += m_fileData.ref<half>(i)->totalLoads();
291  }
292 
293  for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
294  numLoads += m_fileData.ref<V3h>(i)->totalLoads();
295  }
296 
297  for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
298  numLoads += m_fileData.ref<float>(i)->totalLoads();
299  }
300 
301  for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
302  numLoads += m_fileData.ref<V3f>(i)->totalLoads();
303  }
304 
305  for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
306  numLoads += m_fileData.ref<double>(i)->totalLoads();
307  }
308 
309  for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
310  numLoads += m_fileData.ref<V3d>(i)->totalLoads();
311  }
312  return numLoads;
313 }
314 
315 //----------------------------------------------------------------------------//
316 
318 {
319 
320  long long int numBlocks = 0;
321 
322  for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
323  numBlocks += m_fileData.ref<half>(i)->numLoadedBlocks();
324  }
325 
326  for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
327  numBlocks += m_fileData.ref<V3h>(i)->numLoadedBlocks();
328  }
329 
330  for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
331  numBlocks += m_fileData.ref<float>(i)->numLoadedBlocks();
332  }
333 
334  for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
335  numBlocks += m_fileData.ref<V3f>(i)->numLoadedBlocks();
336  }
337 
338  for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
339  numBlocks += m_fileData.ref<double>(i)->numLoadedBlocks();
340  }
341 
342  for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
343  numBlocks += m_fileData.ref<V3d>(i)->numLoadedBlocks();
344  }
345  return numBlocks;
346 }
347 
348 //----------------------------------------------------------------------------//
349 
351 {
352 
353  long long int numBlocks = 0;
354 
355  for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
356  numBlocks += m_fileData.ref<half>(i)->totalLoadedBlocks();
357  }
358 
359  for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
360  numBlocks += m_fileData.ref<V3h>(i)->totalLoadedBlocks();
361  }
362 
363  for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
364  numBlocks += m_fileData.ref<float>(i)->totalLoadedBlocks();
365  }
366 
367  for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
368  numBlocks += m_fileData.ref<V3f>(i)->totalLoadedBlocks();
369  }
370 
371  for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
372  numBlocks += m_fileData.ref<double>(i)->totalLoadedBlocks();
373  }
374 
375  for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
376  numBlocks += m_fileData.ref<V3d>(i)->totalLoadedBlocks();
377  }
378  return numBlocks;
379 }
380 
381 //----------------------------------------------------------------------------//
382 
384 {
385  return ((double)numLoadedBlocks())/std::max(1.0, ((double)totalLoadedBlocks()));
386 }
387 
388 //----------------------------------------------------------------------------//
389 
391 {
392  return ((double)totalLoads())/std::max(1.0, ((double)totalLoadedBlocks()));
393 }
394 
395 //----------------------------------------------------------------------------//
396 
398 {
399  return ((double)totalLoadedBlocks())/std::max(1.0, ((double)totalLoads()));
400 }
401 
402 //----------------------------------------------------------------------------//
403 
405 {
406 
407  for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
409  }
410 
411  for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
413  }
414 
415  for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
416  m_fileData.ref<float>(i)->resetCacheStatistics();
417  }
418 
419  for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
421  }
422 
423  for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
424  m_fileData.ref<double>(i)->resetCacheStatistics();
425  }
426 
427  for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
429  }
430 }
431 
432 //----------------------------------------------------------------------------//
433 
435 
436 //----------------------------------------------------------------------------//
437 
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition: ns.h:58
long long totalLoads()
Returns the total number of block loads in the cache.
Definition: SparseFile.cpp:284
void resetCacheStatistics()
Resets block load.
Definition: SparseFile.cpp:404
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full...
Definition: SparseFile.h:499
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
Definition: SparseFile.h:749
float cacheEfficiency()
Computes the efficiency, the ratio of the number of blocks ever loaded to the number of loads...
Definition: SparseFile.cpp:397
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
Definition: SparseFile.h:129
long long totalLoadedBlocks()
Returns the total number of blocks loaded (max 1 per block) into cache.
Definition: SparseFile.cpp:350
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block&#39;s ref count is non-zer...
Definition: SparseFile.h:136
Imath::Vec3< half > V3h
Definition: SpiMathLib.h:72
float m_maxMemUse
Max amount om memory to use in megabytes.
Definition: SparseFile.h:478
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
Definition: SparseFile.h:481
FIELD3D_NAMESPACE_OPEN typedef::half half
Definition: SpiMathLib.h:64
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
Definition: SparseFile.h:507
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
Definition: SparseFile.h:488
static SparseFileManager * ms_singleton
Pointer to singleton.
Definition: SparseFile.h:459
void setLimitMemUse(bool enabled)
Sets whether to limit memory usage and do dynamic loading for sparse fields.
Definition: SparseFile.cpp:73
float cacheFractionLoaded()
Computes the ratio of blocks in the cache to the total number of blocks that have been loaded (includ...
Definition: SparseFile.cpp:383
long long numLoadedBlocks()
Returns the total number of blocks currently loaded into cache.
Definition: SparseFile.cpp:317
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
Definition: SparseFile.h:503
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
Definition: SparseFile.h:712
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Definition: SparseFile.h:492
Imath::V3d V3d
Definition: SpiMathLib.h:74
int64_t m_memUse
Current amount of memory in use in bytes.
Definition: SparseFile.h:484
Imath::V3f V3f
Definition: SpiMathLib.h:73
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
Definition: SparseFile.cpp:252
Reference< Data_T > * ref(size_t idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed...
SparseFileManager()
Private to prevent instantiation.
Definition: SparseFile.cpp:274
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
Definition: SparseFile.cpp:80
void flushCache()
Flushes the entire block cache for all files, should probably only be used for debugging.
Definition: SparseFile.cpp:215
float cacheLoadsPerBlock()
Computes the overall loaded-blocks-to-load ratio for cached files.
Definition: SparseFile.cpp:390
size_t numRefs() const
Returns the number of file references of the corresponding collection.
void deallocateBlocks(int64_t bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks...
Definition: SparseFile.cpp:152
Contains the SparseField class.
DataTypeEnum blockType
Definition: SparseFile.h:288
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
Definition: SparseFile.h:1197
DataTypeEnum
Definition: Traits.h:66
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
Definition: SparseFile.cpp:63
void setMaxMemUse(float maxMemUse)
Sets the maximum memory usage, in MB, by dynamically loaded sparse fields.
Definition: SparseFile.cpp:87
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
Definition: SparseFile.h:140
int64_t deallocateBlock(const SparseFile::CacheBlock &cb)
Utility function to attempt to deallocate a single block and advance the "hand".
Definition: SparseFile.cpp:96