Field3D
SparseFile.h
Go to the documentation of this file.
1 //----------------------------------------------------------------------------//
2 
3 /*
4  * Copyright (c) 2009 Sony Pictures Imageworks Inc
5  *
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the
17  * distribution. Neither the name of Sony Pictures Imageworks nor the
18  * names of its contributors may be used to endorse or promote
19  * products derived from this software without specific prior written
20  * permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33  * OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 //----------------------------------------------------------------------------//
37 
42 //----------------------------------------------------------------------------//
43 
44 #ifndef _INCLUDED_Field3D_SparseFile_H_
45 #define _INCLUDED_Field3D_SparseFile_H_
46 
47 //----------------------------------------------------------------------------//
48 
49 #include <deque>
50 #include <list>
51 #include <vector>
52 
53 #include <hdf5.h>
54 
55 #include "Exception.h"
56 #include "Hdf5Util.h"
57 #include "SparseDataReader.h"
58 #include "Traits.h"
59 
60 //----------------------------------------------------------------------------//
61 
62 #include "ns.h"
63 
65 
66 //----------------------------------------------------------------------------//
67 // Forward declarations
68 //----------------------------------------------------------------------------//
69 
70 namespace Sparse {
71 
72  template <typename Data_T>
73  struct SparseBlock;
74 
75 }
76 
77 template <typename Data_T>
78 class SparseField;
79 
80 //----------------------------------------------------------------------------//
81 
82 namespace SparseFile {
83 
84 //----------------------------------------------------------------------------//
85 // Reference
86 //----------------------------------------------------------------------------//
87 
93 //----------------------------------------------------------------------------//
94 
95 template <class Data_T>
96 class Reference
97 {
98 public:
99 
100  // Typedefs ------------------------------------------------------------------
101 
102 #define USE_SHPTR 0
103 
104 #if USE_SHPTR
105  typedef boost::shared_ptr<Reference> Ptr;
106 #else
107  typedef Reference* Ptr;
108 #endif
109  typedef std::vector<Sparse::SparseBlock<Data_T>*> BlockPtrs;
110 
111  // Public data members -------------------------------------------------------
112 
113  std::string filename;
114  std::string layerPath;
117 
119  std::vector<int> fileBlockIndices;
122  std::vector<int> blockLoaded;
125  BlockPtrs blocks;
129  std::vector<bool> blockUsed;
132  std::vector<int> loadCounts;
136  std::vector<int> refCounts;
140  boost::mutex *blockMutex;
141 
142  // Ctors, dtor ---------------------------------------------------------------
143 
145  ~Reference();
146 
147  // Main methods --------------------------------------------------------------
148 
151  static Ptr create(const std::string a_filename,
152  const std::string a_layerPath);
154  bool fileIsOpen();
156  void setNumBlocks(int numBlocks);
160  void openFile();
162  void closeFile();
166  void loadBlock(int blockIdx);
168  void unloadBlock(int blockIdx);
171  void incBlockRef(int blockIdx);
173  void decBlockRef(int blockIdx);
175  int blockSize(int blockIdx) const;
178  int totalLoads() const;
181  int numLoadedBlocks() const;
184  int totalLoadedBlocks() const;
187  float averageLoads() const;
189  void resetCacheStatistics();
190 
191 private:
192 
193  typedef boost::mutex Mutex;
194 
195  // Private constructors ---
196 
198  Reference(const std::string filename, const std::string layerPath);
199 
201  Reference(const Reference &o);
202 
204  Reference & operator=(const Reference &o);
205 
206  // Data members ---
207 
210 
214 
218 
220  // boost::mutex m_mutex;
221  Mutex m_mutex;
222 
225 
226 };
227 
228 //----------------------------------------------------------------------------//
229 // FileReferences
230 //----------------------------------------------------------------------------//
231 
233 {
234 public:
235 
236  /* A note on thread safety
237 
238  FileReferences uses a std::deque to store the References, which means
239  that already-added references 'stay put' in memory, even as more elements
240  are added. For threading purposes, we only need to lock during append()
241  and numRefs(), but ref() is safe, since we never remove references.
242 
243  */
244 
245  // Ctors, dtor ---------------------------------------------------------------
246 
247  ~FileReferences();
248 
249  // Main methods --------------------------------------------------------------
250 
253  template <class Data_T>
254  Reference<Data_T>* ref(size_t idx);
255 
258  template <class Data_T>
259  size_t append(typename Reference<Data_T>::Ptr ref);
260 
262  template <class Data_T>
263  size_t numRefs() const;
264 
265 private:
266 
267  // Data members --------------------------------------------------------------
268 
269  std::deque<Reference<half>::Ptr> m_hRefs;
270  std::deque<Reference<V3h>::Ptr> m_vhRefs;
271  std::deque<Reference<float>::Ptr> m_fRefs;
272  std::deque<Reference<V3f>::Ptr> m_vfRefs;
273  std::deque<Reference<double>::Ptr> m_dRefs;
274  std::deque<Reference<V3d>::Ptr> m_vdRefs;
275 
276  // Mutexes -------------------------------------------------------------------
277 
278  typedef boost::mutex Mutex;
279 
280  mutable Mutex m_mutex;
281 
282 };
283 
284 //----------------------------------------------------------------------------//
285 
286 class CacheBlock {
287 public:
289  int refIdx;
290  int blockIdx;
291  CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn) :
292  blockType(blockTypeIn), refIdx(refIdxIn), blockIdx(blockIdxIn)
293  { }
294 };
295 
296 //----------------------------------------------------------------------------//
297 
298 } // namespace SparseFile
299 
300 //----------------------------------------------------------------------------//
301 // SparseFileManager
302 //----------------------------------------------------------------------------//
303 
359 //----------------------------------------------------------------------------//
360 
362 {
363 
364 public:
365 
366  template <class Data_T>
367  friend class SparseField;
368 
369  // typedefs ------------------------------------------------------------------
370 
371  typedef std::list<SparseFile::CacheBlock> CacheList;
372 
373  // Main methods --------------------------------------------------------------
374 
376  static SparseFileManager &singleton();
377 
380  void setLimitMemUse(bool enabled);
381 
384  bool doLimitMemUse() const;
385 
387  void setMaxMemUse(float maxMemUse);
388 
391  void flushCache();
392 
394  long long totalLoads();
395 
397  long long numLoadedBlocks();
398 
400  long long totalLoadedBlocks();
401 
404  float cacheFractionLoaded();
405 
407  float cacheLoadsPerBlock();
408 
411  float cacheEfficiency();
412 
414  void resetCacheStatistics();
415 
416  //--------------------------------------------------------------------------//
417  // Utility functions
418 
423  template <class Data_T>
424  void incBlockRef(int fileId, int blockIdx);
425 
430  template <class Data_T>
431  void decBlockRef(int fileId, int blockIdx);
432 
436  template <class Data_T>
437  void activateBlock(int fileId, int blockIdx);
438 
439 protected:
440 
442  template <class Data_T>
443  SparseFile::Reference<Data_T>* reference(int index);
444 
447  template <class Data_T>
448  int getNextId(const std::string filename, const std::string layerPath);
449 
450  template <class Data_T>
451  void removeFieldFromCache(int refIdx);
452 
453 private:
454 
457 
460 
462  void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx);
463 
466  void deallocateBlocks(int64_t bytesNeeded);
467 
470  template <class Data_T>
471  int64_t deallocateBlock(const SparseFile::CacheBlock &cb);
472 
474  template <class Data_T>
475  void deallocateBlock(CacheList::iterator &it);
476 
478  float m_maxMemUse;
479 
482 
484  int64_t m_memUse;
485 
489 
493 
499  CacheList m_blockCacheList;
500 
503  CacheList::iterator m_nextBlock;
504 
507  boost::mutex m_mutex;
508 
509 };
510 
511 //----------------------------------------------------------------------------//
512 // Reference implementations
513 //----------------------------------------------------------------------------//
514 
515 namespace SparseFile {
516 
517 //----------------------------------------------------------------------------//
518 
519 template <class Data_T>
520 Reference<Data_T>::Reference(const std::string a_filename,
521  const std::string a_layerPath)
522  : filename(a_filename), layerPath(a_layerPath),
523  valuesPerBlock(-1), occupiedBlocks(-1),
524  blockMutex(NULL), m_fileHandle(-1), m_reader(NULL),
525  m_numActiveBlocks(0)
526 {
527  /* Empty */
528 }
529 
530 //----------------------------------------------------------------------------//
531 
532 template <class Data_T>
534 {
535  closeFile();
536 
537  if (m_reader)
538  delete m_reader;
539 
540  if (blockMutex)
541  delete [] blockMutex;
542 }
543 
544 //----------------------------------------------------------------------------//
545 
546 template <class Data_T>
548 {
549  m_reader = NULL;
550  blockMutex = NULL;
551  *this = o;
552 }
553 
554 //----------------------------------------------------------------------------//
555 
556 template <class Data_T>
559 {
560  if (this == &o) {
561  return *this;
562  }
563 
564  // Copy public member variables (where appropriate)
565  filename = o.filename;
566  layerPath = o.layerPath;
571  blocks = o.blocks;
572  blockUsed = o.blockUsed;
574  refCounts = o.refCounts;
575  if (blockMutex)
576  delete[] blockMutex;
577  blockMutex = new boost::mutex[blocks.size()];
578 
579 #if 0
580  // MW: Should this be copying the file handle? Never seems to happen,
581  // but it also seems bad. Ifdef'ing out for now.
582  // Copy private member variables (where appropriate)
584  // Don't copy id, let hdf5 generate a new one.
585  if (m_fileHandle >= 0) {
587  }
588 #else
589  m_fileHandle = -1;
590 #endif
591 
592  // Re-allocate reader
593  if (m_reader)
594  delete m_reader;
595  m_reader = NULL;
596 
597  return *this;
598 }
599 
600 //----------------------------------------------------------------------------//
601 
602 template <class Data_T>
603 typename Reference<Data_T>::Ptr
604 Reference<Data_T>::create(const std::string a_filename,
605  const std::string a_layerPath)
606 {
607  return Ptr(new Reference(a_filename, a_layerPath));
608 }
609 
610 //----------------------------------------------------------------------------//
611 
612 template <class Data_T>
614 {
615  return m_fileHandle >= 0;
616 }
617 
618 //----------------------------------------------------------------------------//
619 
620 template <class Data_T>
622 {
623  boost::mutex::scoped_lock lock(m_mutex);
624 
625  fileBlockIndices.resize(numBlocks);
626  blockLoaded.resize(numBlocks, 0);
627  blocks.resize(numBlocks, 0);
628  blockUsed.resize(numBlocks, false);
629  loadCounts.resize(numBlocks, 0);
630  refCounts.resize(numBlocks, 0);
631  if (blockMutex)
632  delete[] blockMutex;
633  blockMutex = new boost::mutex[numBlocks];
634 }
635 
636 //----------------------------------------------------------------------------//
637 
638 template <class Data_T>
640 {
641  using namespace Exc;
642  using namespace Hdf5Util;
643 
644  boost::mutex::scoped_lock lock_A(m_mutex);
645 
646  // check that the file wasn't already opened before obtaining the lock
647  if (fileIsOpen()) {
648  return;
649  }
650 
651  {
652  // Hold the global lock
653  GlobalLock lock(g_hdf5Mutex);
654  // Open the file
655  m_fileHandle = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
656  if (m_fileHandle < 0) {
657  printf("openFile(): No file handle. Dying.\n");
658  throw NoSuchFileException(filename);
659  }
660  // Open the layer group
662  if (m_layerGroup.id() < 0) {
663  Msg::print(Msg::SevWarning, "In SparseFile::Reference::openFile: "
664  "Couldn't find layer group " + layerPath +
665  " in .f3d file ");
666  throw FileIntegrityException(filename);
667  }
668  }
669 
670  // Re-allocate reader
671  if (m_reader) {
672  delete m_reader;
673  }
676 }
677 
678 //----------------------------------------------------------------------------//
679 
680 template <class Data_T>
682 {
683  if (m_fileHandle >= 0) {
684  if (H5Fclose(m_fileHandle) < 0) {
685  Msg::print("In ~Reference(): Error closing file " + filename);
686  }
687  }
688 }
689 
690 //----------------------------------------------------------------------------//
691 
692 template <class Data_T>
694 {
695  boost::mutex::scoped_lock lock(m_mutex);
696 
697  // Allocate the block
698  blocks[blockIdx]->resize(valuesPerBlock);
699  assert(blocks[blockIdx]->data != NULL);
700  // Read the data
701  assert(m_reader);
702  m_reader->readBlock(fileBlockIndices[blockIdx], *blocks[blockIdx]->data);
703  // Mark block as loaded
704  blockLoaded[blockIdx] = 1;
705  // Track count
707 }
708 
709 //----------------------------------------------------------------------------//
710 
711 template <class Data_T>
713 {
714  // Deallocate the block
715  blocks[blockIdx]->clear();
716  // Mark block as unloaded
717  blockLoaded[blockIdx] = 0;
718  // Track count
720 #if 0
721  // If no active blocks, close the file. De-activate for now.
722  if (m_numActiveBlocks == 0) {
723  closeFile();
724  }
725 #endif
726 }
727 
728 //----------------------------------------------------------------------------//
729 
730 template <class Data_T>
732 {
733  boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
734  ++refCounts[blockIdx];
735 }
736 
737 //----------------------------------------------------------------------------//
738 
739 template <class Data_T>
741 {
742  boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
743  --refCounts[blockIdx];
744 }
745 
746 //----------------------------------------------------------------------------//
747 
748 template <class Data_T>
749 int Reference<Data_T>::blockSize(int /* blockIdx */) const
750 {
751  return valuesPerBlock * sizeof(Data_T);
752 }
753 
754 //----------------------------------------------------------------------------//
755 
756 template <class Data_T>
758 {
759  std::vector<int>::const_iterator i = loadCounts.begin();
760  std::vector<int>::const_iterator end = loadCounts.end();
761  int numLoads = 0;
762  for (; i != end; ++i)
763  numLoads += *i;
764 
765  return numLoads;
766 }
767 
768 //----------------------------------------------------------------------------//
769 
770 template <class Data_T>
772 {
773  std::vector<int>::const_iterator i = blockLoaded.begin();
774  std::vector<int>::const_iterator end = blockLoaded.end();
775  int numBlocks = 0;
776  for (; i != end; ++i)
777  if (*i)
778  numBlocks++;
779 
780  return numBlocks;
781 }
782 
783 //----------------------------------------------------------------------------//
784 
785 template <class Data_T>
787 {
788  std::vector<int>::const_iterator i = loadCounts.begin();
789  std::vector<int>::const_iterator li = blockLoaded.begin();
790  std::vector<int>::const_iterator end = loadCounts.end();
791  int numBlocks = 0;
792 
793  if (blockLoaded.size() == 0) {
794  for (; i != end; ++i)
795  if (*i)
796  numBlocks++;
797  } else {
798  assert(loadCounts.size() == blockLoaded.size());
799 
800  for (; i != end; ++i, ++li)
801  if (*i || *li)
802  numBlocks++;
803  }
804 
805  return numBlocks;
806 }
807 
808 //----------------------------------------------------------------------------//
809 
810 template <class Data_T>
812 {
813  std::vector<int>::const_iterator i = loadCounts.begin();
814  std::vector<int>::const_iterator end = loadCounts.end();
815  int numLoads = 0, numBlocks = 0;
816  for (; i != end; ++i) {
817  if (*i) {
818  numLoads += *i;
819  numBlocks++;
820  }
821  }
822 
823  return (float)numLoads / std::max(1, numBlocks);
824 }
825 
826 //----------------------------------------------------------------------------//
827 
828 template <class Data_T>
830 {
831  std::vector<int>::iterator li = loadCounts.begin();
832  std::vector<int>::iterator lend = loadCounts.end();
833  for (; li != lend; ++li)
834  *li = 0;
835 }
836 
837 //----------------------------------------------------------------------------//
838 
839 } // namespace SparseFile
840 
841 //----------------------------------------------------------------------------//
842 // Specializations for FileReferences
843 //----------------------------------------------------------------------------//
844 
845 namespace SparseFile {
846 
847 //----------------------------------------------------------------------------//
848 
850 {
851 #if !USE_SHPTR
852  for (size_t i = 0, end = m_hRefs.size(); i < end; ++i) {
853  delete m_hRefs[i];
854  }
855  for (size_t i = 0, end = m_fRefs.size(); i < end; ++i) {
856  delete m_fRefs[i];
857  }
858  for (size_t i = 0, end = m_dRefs.size(); i < end; ++i) {
859  delete m_dRefs[i];
860  }
861  for (size_t i = 0, end = m_vhRefs.size(); i < end; ++i) {
862  delete m_vhRefs[i];
863  }
864  for (size_t i = 0, end = m_vfRefs.size(); i < end; ++i) {
865  delete m_vfRefs[i];
866  }
867  for (size_t i = 0, end = m_vdRefs.size(); i < end; ++i) {
868  delete m_vdRefs[i];
869  }
870 #endif
871 }
872 
873 //----------------------------------------------------------------------------//
874 
875 template <>
876 inline Reference<half>*
878 {
879 #if USE_SHPTR
880  return m_hRefs[idx].get();
881 #else
882  return m_hRefs[idx];
883 #endif
884 }
885 
886 //----------------------------------------------------------------------------//
887 
888 template <>
889 inline Reference<V3h>*
891 {
892 #if USE_SHPTR
893  return m_vhRefs[idx].get();
894 #else
895  return m_vhRefs[idx];
896 #endif
897 }
898 
899 //----------------------------------------------------------------------------//
900 
901 template <>
902 inline Reference<float>*
904 {
905 #if USE_SHPTR
906  return m_fRefs[idx].get();
907 #else
908  return m_fRefs[idx];
909 #endif
910 }
911 
912 //----------------------------------------------------------------------------//
913 
914 template <>
915 inline Reference<V3f>*
917 {
918 #if USE_SHPTR
919  return m_vfRefs[idx].get();
920 #else
921  return m_vfRefs[idx];
922 #endif
923 }
924 
925 //----------------------------------------------------------------------------//
926 
927 template <>
928 inline Reference<double>*
930 {
931 #if USE_SHPTR
932  return m_dRefs[idx].get();
933 #else
934  return m_dRefs[idx];
935 #endif
936 }
937 
938 //----------------------------------------------------------------------------//
939 
940 template <>
941 inline Reference<V3d>*
943 {
944 #if USE_SHPTR
945  return m_vdRefs[idx].get();
946 #else
947  return m_vdRefs[idx];
948 #endif
949 }
950 
951 //----------------------------------------------------------------------------//
952 
953 template <>
954 inline size_t FileReferences::append<half>(Reference<half>::Ptr ref)
955 {
956  Mutex::scoped_lock lock(m_mutex);
957 
958  m_hRefs.push_back(ref);
959  return m_hRefs.size() - 1;
960 }
961 
962 //----------------------------------------------------------------------------//
963 
964 template <>
965 inline size_t FileReferences::append<V3h>(Reference<V3h>::Ptr ref)
966 {
967  Mutex::scoped_lock lock(m_mutex);
968 
969  m_vhRefs.push_back(ref);
970  return m_vhRefs.size() - 1;
971 }
972 
973 //----------------------------------------------------------------------------//
974 
975 template <>
976 inline size_t FileReferences::append<float>(Reference<float>::Ptr ref)
977 {
978  Mutex::scoped_lock lock(m_mutex);
979 
980  m_fRefs.push_back(ref);
981  return m_fRefs.size() - 1;
982 }
983 
984 //----------------------------------------------------------------------------//
985 
986 template <>
987 inline size_t FileReferences::append<V3f>(Reference<V3f>::Ptr ref)
988 {
989  Mutex::scoped_lock lock(m_mutex);
990 
991  m_vfRefs.push_back(ref);
992  return m_vfRefs.size() - 1;
993 }
994 
995 //----------------------------------------------------------------------------//
996 
997 template <>
998 inline size_t FileReferences::append<double>(Reference<double>::Ptr ref)
999 {
1000  Mutex::scoped_lock lock(m_mutex);
1001 
1002  m_dRefs.push_back(ref);
1003  return m_dRefs.size() - 1;
1004 }
1005 
1006 //----------------------------------------------------------------------------//
1007 
1008 template <>
1009 inline size_t FileReferences::append<V3d>(Reference<V3d>::Ptr ref)
1010 {
1011  Mutex::scoped_lock lock(m_mutex);
1012 
1013  m_vdRefs.push_back(ref);
1014  return m_vdRefs.size() - 1;
1015 }
1016 
1017 //----------------------------------------------------------------------------//
1018 
1019 template <>
1020 inline size_t FileReferences::numRefs<half>() const
1021 {
1022  Mutex::scoped_lock lock(m_mutex);
1023 
1024  return m_hRefs.size();
1025 }
1026 
1027 //----------------------------------------------------------------------------//
1028 
1029 template <>
1030 inline size_t FileReferences::numRefs<V3h>() const
1031 {
1032  Mutex::scoped_lock lock(m_mutex);
1033 
1034  return m_vhRefs.size();
1035 }
1036 
1037 //----------------------------------------------------------------------------//
1038 
1039 template <>
1040 inline size_t FileReferences::numRefs<float>() const
1041 {
1042  Mutex::scoped_lock lock(m_mutex);
1043 
1044  return m_fRefs.size();
1045 }
1046 
1047 //----------------------------------------------------------------------------//
1048 
1049 template <>
1050 inline size_t FileReferences::numRefs<V3f>() const
1051 {
1052  Mutex::scoped_lock lock(m_mutex);
1053 
1054  return m_vfRefs.size();
1055 }
1056 
1057 //----------------------------------------------------------------------------//
1058 
1059 template <>
1060 inline size_t FileReferences::numRefs<double>() const
1061 {
1062  Mutex::scoped_lock lock(m_mutex);
1063 
1064  return m_dRefs.size();
1065 }
1066 
1067 //----------------------------------------------------------------------------//
1068 
1069 template <>
1070 inline size_t FileReferences::numRefs<V3d>() const
1071 {
1072  Mutex::scoped_lock lock(m_mutex);
1073 
1074  return m_vdRefs.size();
1075 }
1076 
1077 //----------------------------------------------------------------------------//
1078 // Implementations for FileReferences
1079 //----------------------------------------------------------------------------//
1080 
1081 #if 0
1082 
1083 template <class Data_T>
1085 {
1086  assert(false && "Do not use memory limiting on sparse fields that aren't "
1087  "simple scalars or vectors!");
1089  "FileReferences::ref(): Do not use memory limiting on sparse "
1090  "fields that aren't simple scalars or vectors!");
1091  static Reference<Data_T> dummy("", "");
1092  return dummy;
1093 }
1094 
1095 //----------------------------------------------------------------------------//
1096 
1097 template <class Data_T>
1098 size_t FileReferences::append(const Reference<Data_T>& ref)
1099 {
1100  assert(false && "Do not use memory limiting on sparse fields that aren't "
1101  "simple scalars or vectors!");
1103  "FileReferences::append(): Do not use memory limiting on sparse "
1104  "fields that aren't simple scalars or vectors!");
1105  return -1;
1106 }
1107 
1108 //----------------------------------------------------------------------------//
1109 
1110 template <class Data_T>
1111 size_t FileReferences::numRefs() const
1112 {
1113  assert(false && "Do not use memory limiting on sparse fields that aren't "
1114  "simple scalars or vectors!");
1116  "FileReferences::numRefs(): "
1117  "Do not use memory limiting on sparse "
1118  "fields that aren't "
1119  "simple scalars or vectors!");
1120  return -1;
1121 }
1122 
1123 #endif
1124 
1125 //----------------------------------------------------------------------------//
1126 
1127 } // namespace SparseFile
1128 
1129 //----------------------------------------------------------------------------//
1130 // SparseFileManager implementations
1131 //----------------------------------------------------------------------------//
1132 
1133 template <class Data_T>
1134 int
1136  const std::string layerPath)
1137 {
1138  using namespace SparseFile;
1139 
1140  // Must hold a mutex while appending to m_fileData
1141  boost::mutex::scoped_lock lock(m_mutex);
1142 
1143  int id = m_fileData.append<Data_T>(Reference<Data_T>::create(filename,
1144  layerPath));
1145  return id;
1146 }
1147 
1148 //----------------------------------------------------------------------------//
1149 
1150 template <class Data_T>
1151 void
1153 {
1154  boost::mutex::scoped_lock lock(m_mutex);
1155 
1157  SparseFile::Reference<Data_T> *reference = m_fileData.ref<Data_T>(refIdx);
1158 
1159  CacheList::iterator it = m_blockCacheList.begin();
1160  CacheList::iterator end = m_blockCacheList.end();
1161  CacheList::iterator next;
1162 
1163  int64_t bytesFreed = 0;
1164 
1165  while (it != end) {
1166  if (it->blockType == blockType && it->refIdx == refIdx) {
1167  if (it == m_nextBlock) {
1168  ++m_nextBlock;
1169  }
1170  next = it;
1171  ++next;
1172  bytesFreed += reference->blockSize(it->blockIdx);
1173  m_blockCacheList.erase(it);
1174  it = next;
1175  } else {
1176  ++it;
1177  }
1178  }
1179  m_memUse -= bytesFreed;
1180 
1181  std::vector<int>().swap(reference->fileBlockIndices);
1182  reference->fileBlockIndices.resize(reference->blocks.size(), -1);
1184  BlockPtrs().swap(reference->blocks);
1185  std::vector<int>().swap(reference->blockLoaded);
1186  std::vector<bool>().swap(reference->blockUsed);
1187  std::vector<int>().swap(reference->loadCounts);
1188  std::vector<int>().swap(reference->refCounts);
1189  delete[] reference->blockMutex;
1190  reference->blockMutex = NULL;
1191 }
1192 
1193 //----------------------------------------------------------------------------//
1194 
1195 template <class Data_T>
1198 {
1199  return m_fileData.ref<Data_T>(index);
1200 }
1201 
1202 //----------------------------------------------------------------------------//
1203 
1204 template <class Data_T>
1205 void
1206 SparseFileManager::activateBlock(int fileId, int blockIdx)
1207 {
1208  SparseFile::Reference<Data_T> *reference = m_fileData.ref<Data_T>(fileId);
1209 
1210  if (reference->fileBlockIndices[blockIdx] >= 0) {
1211  if (!reference->blockLoaded[blockIdx]) {
1212  int blockSize = reference->blockSize(blockIdx);
1213  if (m_limitMemUse) {
1214  // if we already have enough free memory, deallocateBlocks()
1215  // will just return
1216  deallocateBlocks(blockSize);
1217  }
1218 
1219  if (!reference->fileIsOpen()) {
1220  reference->openFile();
1221  }
1222 
1223  boost::mutex::scoped_lock lock_A(m_mutex);
1224  boost::mutex::scoped_lock lock_B(reference->blockMutex[blockIdx]);
1225  // check to see if it was loaded between when the function
1226  // started and we got the lock on the block
1227  if (!reference->blockLoaded[blockIdx]) {
1228  reference->loadBlock(blockIdx);
1229  reference->loadCounts[blockIdx]++;
1230  addBlockToCache(DataTypeTraits<Data_T>::typeEnum(), fileId, blockIdx);
1231  m_memUse += blockSize;
1232  }
1233  }
1234  }
1235  reference->blockUsed[blockIdx] = true;
1236 }
1237 
1238 //----------------------------------------------------------------------------//
1239 
1240 template <class Data_T>
1241 void
1242 SparseFileManager::incBlockRef(int fileId, int blockIdx)
1243 {
1244  SparseFile::Reference<Data_T> *reference = m_fileData.ref<Data_T>(fileId);
1245 
1246  if (reference->fileBlockIndices[blockIdx] >= 0) {
1247  reference->incBlockRef(blockIdx);
1248  }
1249 }
1250 
1251 //----------------------------------------------------------------------------//
1252 
1253 template <class Data_T>
1254 void
1255 SparseFileManager::decBlockRef(int fileId, int blockIdx)
1256 {
1257  SparseFile::Reference<Data_T> *reference = m_fileData.ref<Data_T>(fileId);
1258 
1259  if (reference->fileBlockIndices[blockIdx] >= 0) {
1260  reference->decBlockRef(blockIdx);
1261  }
1262 }
1263 
1264 //----------------------------------------------------------------------------//
1265 
1267 
1268 //----------------------------------------------------------------------------//
1269 
1270 #endif
void removeFieldFromCache(int refIdx)
Definition: SparseFile.h:1152
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition: ns.h:58
Contains utility functions and classes for Hdf5 files.
Definition: Hdf5Util.h:86
size_t m_numActiveBlocks
Number of currently active blocks.
Definition: SparseFile.h:224
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
Definition: SparseFile.h:209
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full...
Definition: SparseFile.h:499
Namespace for Exception objects.
Definition: Exception.h:57
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
Definition: SparseFile.h:749
void activateBlock(int fileId, int blockIdx)
Called by SparseField when it&#39;s about to read from a block. This should not be called by the user...
Definition: SparseFile.h:1206
std::vector< int > blockLoaded
Whether each block is loaded. We don&#39;t use bools since vector<bool> is weird.
Definition: SparseFile.h:122
void incBlockRef(int fileId, int blockIdx)
Increments the usage reference count on the specified block, to prevent it from getting unloaded whil...
Definition: SparseFile.h:1242
void incBlockRef(int blockIdx)
Increment reference count on a block, indicates the block is currently in use, so prevents it from be...
Definition: SparseFile.h:731
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
Definition: SparseFile.h:129
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn)
Definition: SparseFile.h:291
std::vector< Sparse::SparseBlock< Data_T > * > BlockPtrs
Definition: SparseFile.h:109
static Ptr create(const std::string a_filename, const std::string a_layerPath)
Returns a shared pointer to a reference. Preferred way of creating References.
Definition: SparseFile.h:604
boost::recursive_mutex::scoped_lock GlobalLock
Definition: Hdf5Util.h:78
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
Definition: Log.cpp:66
std::vector< int > fileBlockIndices
Index in file for each block.
Definition: SparseFile.h:119
void activateBlock(const int blockId) const
Activates a given block.
Definition: SparseField.h:1897
std::deque< Reference< V3f >::Ptr > m_vfRefs
Definition: SparseFile.h:272
void setNumBlocks(int numBlocks)
Sets the number of blocks used by the SparseField we&#39;re supporting.
Definition: SparseFile.h:621
std::string layerPath
Definition: SparseFile.h:114
static DataTypeEnum typeEnum()
void decBlockRef(int blockIdx)
Decrement reference count on a block.
Definition: SparseFile.h:740
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block&#39;s ref count is non-zer...
Definition: SparseFile.h:136
Contains various utility functions for Hdf5.
Mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
Definition: SparseFile.h:221
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
Definition: SparseFile.h:639
std::deque< Reference< V3d >::Ptr > m_vdRefs
Definition: SparseFile.h:274
float m_maxMemUse
Max amount om memory to use in megabytes.
Definition: SparseFile.h:478
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
Definition: SparseFile.h:217
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
Definition: SparseFile.h:481
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
Definition: SparseFile.h:507
float averageLoads() const
Returns the average number of loads per accessed block in this file, for cache statistics.
Definition: SparseFile.h:811
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
Definition: SparseFile.h:488
static SparseFileManager * ms_singleton
Pointer to singleton.
Definition: SparseFile.h:459
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please. ...
Definition: SparseFile.h:125
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
Definition: SparseFile.h:503
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
Definition: SparseFile.h:712
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Definition: SparseFile.h:492
Namespace for sparse field specifics.
Definition: SparseField.h:221
void decBlockRef(const int blockId) const
Decrements the block ref count for the given block.
Definition: SparseField.h:1905
std::list< SparseFile::CacheBlock > CacheList
Definition: SparseFile.h:371
std::deque< Reference< float >::Ptr > m_fRefs
Definition: SparseFile.h:271
boost::mutex Mutex
Definition: SparseFile.h:193
int totalLoadedBlocks() const
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks)...
Definition: SparseFile.h:786
~Reference()
Destructor.
Definition: SparseFile.h:533
#define FIELD3D_API
Definition: ns.h:77
Contains functions controlling the loading of sparse fields.
int64_t m_memUse
Current amount of memory in use in bytes.
Definition: SparseFile.h:484
int totalLoads() const
Returns the total number of loads of the blocks of this file, for cache statistics.
Definition: SparseFile.h:757
void resetCacheStatistics()
Resets counts of total block loads.
Definition: SparseFile.h:829
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference&#39;s existence.
Definition: SparseFile.h:213
int getNextId(const std::string filename, const std::string layerPath)
Returns the id of the next cache item. This is stored in the SparseField in order to reference its fi...
Definition: SparseFile.h:1135
Reference< Data_T > * ref(size_t idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed...
void decBlockRef(int fileId, int blockIdx)
Decrements the usage reference count on the specified block, after its value is no longer being used ...
Definition: SparseFile.h:1255
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don&#39;t pass in a reference to where the data shou...
Definition: SparseFile.h:693
Reference & operator=(const Reference &o)
Assignment operator. Clears ref counts and rebuilds mutex array.
Definition: SparseFile.h:558
std::string filename
Definition: SparseFile.h:113
void incBlockRef(const int blockId) const
Increments the block ref count for the given block.
Definition: SparseField.h:1889
size_t numRefs() const
Returns the number of file references of the corresponding collection.
FIELD3D_NAMESPACE_OPEN FIELD3D_API boost::recursive_mutex g_hdf5Mutex
Definition: Hdf5Util.cpp:67
std::deque< Reference< double >::Ptr > m_dRefs
Definition: SparseFile.h:273
Reference(const std::string filename, const std::string layerPath)
Constructor. Requires the filename and layer path of the field to be known.
Definition: SparseFile.h:520
bool fileIsOpen()
Checks if the file used by this reference is open already.
Definition: SparseFile.h:613
std::vector< int > loadCounts
Per-block counts of the number of times each block has been loaded, for cache statistics.
Definition: SparseFile.h:132
DataTypeEnum blockType
Definition: SparseFile.h:288
This Field subclass stores voxel data in block-allocated arrays.
Definition: SparseField.h:350
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
Definition: SparseFile.h:1197
DataTypeEnum
Definition: Traits.h:66
std::deque< Reference< V3h >::Ptr > m_vhRefs
Definition: SparseFile.h:270
void closeFile()
Closes the file. Will be re-opened as needed.
Definition: SparseFile.h:681
Scoped object - opens a group on creation and closes it on destruction.
Definition: Hdf5Util.h:194
Contains Exception base class.
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
Definition: SparseFile.h:140
size_t append(typename Reference< Data_T >::Ptr ref)
Appends a reference to the collection. This is specialized so that the correct data member is accesse...
void open(hid_t parentLocation, const std::string &name)
Definition: Hdf5Util.h:210
int numLoadedBlocks() const
Returns the total number of blocks that are currently loaded, for statistics.
Definition: SparseFile.h:771
std::deque< Reference< half >::Ptr > m_hRefs
Definition: SparseFile.h:269
hid_t id() const
Query the hid_t value.
Definition: Hdf5Util.h:100