casacore
TiledStMan.h
Go to the documentation of this file.
1 //# TiledStMan.h: Base class for Tiled Storage Managers
2 //# Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id$
27 
28 #ifndef TABLES_TILEDSTMAN_H
29 #define TABLES_TILEDSTMAN_H
30 
31 
32 //# Includes
33 #include <casacore/casa/aips.h>
34 #include <casacore/tables/DataMan/DataManager.h>
35 #include <casacore/casa/Containers/Block.h>
36 #include <casacore/casa/Arrays/IPosition.h>
37 #include <casacore/casa/OS/Conversion.h>
38 #include <casacore/casa/BasicSL/String.h>
39 
40 namespace casacore { //# NAMESPACE CASACORE - BEGIN
41 
42 //# Forward Declarations
43 class TSMColumn;
44 class TSMDataColumn;
45 class TSMCube;
46 class TSMFile;
47 class TableDesc;
48 class Record;
49 template<class T> class Vector;
50 
51 
52 // <summary>
53 // Base class for Tiled Storage Manager classes
54 // </summary>
55 
56 // <use visibility=export>
57 
58 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
59 // </reviewed>
60 
61 // <prerequisite>
62 //# Classes you should understand before using this one.
63 // <li> Description of Tiled Storage Manager in module file
64 // <linkto module=Tables:TiledStMan>Tables.h</linkto>
65 // <li> <linkto class=DataManager>DataManager</linkto>
66 // <li> <linkto class=TSMColumn>TSMColumn</linkto>
67 // <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
68 // for a discussion of the maximum cache size
69 // </prerequisite>
70 
71 // <synopsis>
72 // TiledStMan is the base class for Tiled Storage Managers.
73 // A tiled storage manager is capable of storing a hypercolumn
74 // (as defined by <linkto file="TableDesc.h#defineHypercolumn">
75 // TableDesc::defineHypercolumn</linkto>)
76 // in one or more hypercubes.
77 // <br>It is not necessary to define a hypercolumn. If not defined,
78 // it is assumed that all columns bound to this storage manager are
79 // data columns. At least one of the columns must have a fixed
80 // dimensionality and is used to determine the hypercube dimnensionality.
81 // <br>The general concept of these storage managers is explained in the
82 // <linkto module="Tables:TiledStMan">Tables module description</linkto>.
83 // <p>
84 // TiledStMan contains all common functions for the different tiled
85 // storage managers. In particular, it contains functions
86 // to check if the definitions of the shapes of hypercubes, coordinates, and
87 // data cells are consistent.
88 // It also contains various data members and functions to make them
89 // persistent by writing them into an AipsIO stream.
90 // </synopsis>
91 
92 // <motivation>
93 // This base class contains the common functionality of all
94 // tiled storage managers. The base class is still abstract.
95 // Only concrete tiled storage managers derived from it can
96 // be instantiated.
97 // <p>
98 // Tiled storage managers make access to array data possible with
99 // more or less the same efficiency for access along different axes.
100 // </motivation>
101 
102 //# <todo asof="$DATE:$">
103 //# A List of bugs, limitations, extensions or planned refinements.
104 //# </todo>
105 
106 
107 class TiledStMan : public DataManager
108 {
109 public:
110  // Create a TiledStMan.
112 
113  // Create a TiledStMan storage manager.
114  // The given maximum cache size (in MibiByte) is persistent,
115  // thus will be reused when the table is read back. Note that the class
116  // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
117  // allows one to overwrite the maximum cache size temporarily.
118  // Its description contains a discussion about the effects of
119  // setting a maximum cache.
120  TiledStMan (const String& hypercolumnName, uInt maximumCacheSizeMiB);
121 
122  virtual ~TiledStMan();
123 
124  // Get the name given to the storage manager.
125  // This is the name of the hypercolumn.
126  virtual String dataManagerName() const;
127 
128  void setDataManagerName (const String& newHypercolumnName);
129 
130  // Return a record containing data manager specifications.
131  virtual Record dataManagerSpec() const;
132 
133  // Get data manager properties that can be modified.
134  // It is only MaxCacheSize (the maximum cache size in MibiByte).
135  // It is a subset of the data manager specification.
136  virtual Record getProperties() const;
137 
138  // Modify data manager properties.
139  // Only MaxCacheSize can be used. It is similar to function setCacheSize
140  // with <src>canExceedNrBuckets=False</src>.
141  virtual void setProperties (const Record& spec);
142 
143  // Set the flag to "data has changed since last flush".
144  void setDataChanged();
145 
146  // Derive the tile shape from the hypercube shape for the given
147  // number of pixels per tile. It is tried to get the same number
148  // of tiles for each dimension.
149  // When a weight vector is given, the number of tiles for a dimension
150  // is proportional to the weight.
151  // <br>After the initial guess it tries to optimize it by trying
152  // to waste as little space as possible, while trying to keep as close
153  // to the initial guess. The given tolerance (possibly per axis)
154  // gives the minimum and maximum possible length of a tile axis
155  // (minimum = initial_guess*tolerance; maximum = initial_guess/tolerance).
156  // The heuristic is such that a tile axis length dividing the cube length
157  // exactly is always favoured.
158  // The test program <src>tTiledStMan</src> can be used to see how
159  // the algorithm works out for a given tile size and cube shape.
160  // <group>
162  Double tolerance = 0.5,
163  uInt maxNrPixelsPerTile = 32768);
165  const Vector<double>& weight,
166  const Vector<double>& tolerance,
167  uInt maxNrPixelsPerTile = 32768);
168  // </group>
169 
170  // Set the maximum cache size (in MiB) in a non-persistent way.
171  virtual void setMaximumCacheSize (uInt nMiB);
172 
173  // Get the current maximum cache size (in MiB (MibiByte)).
174  uInt maximumCacheSize() const;
175 
176  // Get the current cache size (in buckets) for the hypercube in
177  // the given row.
178  uInt cacheSize (uInt rownr) const;
179 
180  // Get the hypercube shape of the data in the given row.
181  const IPosition& hypercubeShape (uInt rownr) const;
182 
183  // Get the tile shape of the data in the given row.
184  const IPosition& tileShape (uInt rownr) const;
185 
186  // Get the bucket size (in bytes) of the hypercube in the given row.
187  uInt bucketSize (uInt rownr) const;
188 
189  // Can the tiled storage manager handle changing array shapes?
190  // The default is no (but TiledCellStMan can).
191  virtual Bool canChangeShape() const;
192 
193  // Can the tiled storage manager access an entire column.
194  // TiledColumnStMan can always do that.
195  // The others might be able to do it (for this time).
196  // The default implementation returns True if there is only 1 hypercube.
197  // reask is set to True (because next time things might be different).
198  virtual Bool canAccessColumn (Bool& reask) const;
199 
200  // The data manager supports use of MultiFile.
201  virtual Bool hasMultiFileSupport() const;
202 
203  // Calculate the cache size (in buckets) for accessing the hypercube
204  // containing the given row. It takes the maximum cache size into
205  // account (allowing an overdraft of 10%).
206  // It uses the given axisPath (i.e. traversal order) to determine
207  // the optimum size. A window can be specified to indicate that only
208  // the given subset of the hypercube will be accessed.
209  // <br>
210  // The length of the slice and window arguments and <src>axisPath</src>
211  // must be less or equal to the dimensionality of the hypercube.
212  // The non-specified <src>windowStart</src> parts default to 0.
213  // The non-specified <src>windowLength</src> parts default to
214  // the hypercube shape.
215  // The non-specified <src>sliceShape</src> parts default to 1.
216  // <br>
217  // Axispath = [2,0,1] indicates that the z-axis changes most rapidly,
218  // thereafter x and y. An axis can occur only once in the axisPath.
219  // The non-specified <src>axisPath</src> parts get the natural order.
220  // E.g. in the previous example axisPath=[2] defines the same path.
221  // <br>When forceSmaller is False, the cache is not resized when the
222  // new size is smaller.
223  // <br>A flag is set indicating that the TSMDataColumn
224  // access functions do not need to size the cache.
225  uInt calcCacheSize (uInt rownr, const IPosition& sliceShape,
226  const IPosition& windowStart,
227  const IPosition& windowLength,
228  const IPosition& axisPath) const;
229 
230  // Set the cache size using the <src>calcCacheSize</src>
231  // function mentioned above.
232  void setCacheSize (uInt rownr, const IPosition& sliceShape,
233  const IPosition& windowStart,
234  const IPosition& windowLength,
235  const IPosition& axisPath,
236  Bool forceSmaller);
237 
238  // Set the cache size for accessing the hypercube containing the given row.
239  // When the give cache size exceeds the maximum cache size with more
240  // than 10%, the maximum cache size is used instead.
241  // <br>When forceSmaller is False, the cache is not resized when the
242  // new size is smaller.
243  // <br>A flag is set indicating that the TSMDataColumn
244  // access functions do not need to size the cache.
245  void setCacheSize (uInt rownr, uInt nbuckets, Bool forceSmaller);
246 
247  // Sets the cache size using the hypercube instead of the row number.
248  // Useful for iterating over all hypercubes.
249  void setHypercubeCacheSize (uInt hypercube, uInt nbuckets, Bool forceSmaller);
250 
251  // Determine if the user set the cache size (using setCacheSize).
252  Bool userSetCache (uInt rownr) const;
253 
254  // Empty the caches used by the hypercubes in this storage manager.
255  // It will flush the caches as needed and remove all buckets from them
256  // resulting in a possibly large drop in memory used.
257  // It also clears the userSetCache flag.
258  void emptyCaches();
259 
260  // Show the statistics of all caches used.
261  void showCacheStatistics (ostream& os) const;
262 
263  // Get the length of the data for the given number of pixels.
264  // This can be used to calculate the length of a tile.
265  uInt getLengthOffset (uInt nrPixels, Block<uInt>& dataOffset,
266  Block<uInt>& localOffset,
267  uInt& localTileLength) const;
268 
269  // Get the number of coordinate vectors.
270  uInt nrCoordVector() const;
271 
272  // Get the nr of rows in this storage manager.
273  uInt nrow() const;
274 
275  // Does the storage manager allow to add rows? (yes)
276  Bool canAddRow() const;
277 
278  // Get the default tile shape.
279  // By default it returns a zero-length IPosition.
280  virtual IPosition defaultTileShape() const;
281 
282  // Return the number of hypercubes.
283  uInt nhypercubes() const;
284 
285  // Test if only one hypercube is used by this storage manager.
286  // If not, throw an exception. Otherwise return the hypercube.
288 
289  // Get the given hypercube.
290  // <group>
291  const TSMCube* getTSMCube (uInt hypercube) const;
292  TSMCube* getTSMCube (uInt hypercube);
293  // </group>
294 
295  // Get the hypercube in which the given row is stored.
296  // <group>
297  const TSMCube* getHypercube (uInt rownr) const;
298  virtual TSMCube* getHypercube (uInt rownr) = 0;
299  // </group>
300 
301  // Get the hypercube in which the given row is stored.
302  // It also returns the position of the row in that hypercube.
303  virtual TSMCube* getHypercube (uInt rownr, IPosition& position) = 0;
304 
305  // Make the correct TSMCube type (depending on tsmOption()).
306  TSMCube* makeTSMCube (TSMFile* file, const IPosition& cubeShape,
307  const IPosition& tileShape,
308  const Record& values, Int64 fileOffset=-1);
309 
310  // Read a tile and convert the data to local format.
311  void readTile (char* local, const Block<uInt>& localOffset,
312  const char* external, const Block<uInt>& externalOffset,
313  uInt nrpixels);
314 
315  // Write a tile after converting the data to external format.
316  void writeTile (char* external, const Block<uInt>& externalOffset,
317  const char* local, const Block<uInt>& localOffset,
318  uInt nrpixels);
319 
320  // Get the TSMFile object with the given sequence number.
321  TSMFile* getFile (uInt sequenceNumber);
322 
323  // Open the storage manager for an existing table.
324  virtual void open (uInt nrrow, AipsIO&);
325 
326  // Resync the storage manager with the new file contents.
327  virtual void resync (uInt nrrow);
328 
329  // Reopen all files used in this storage manager for read/write access.
330  virtual void reopenRW();
331 
332  // The data manager will be deleted (because all its columns are
333  // requested to be deleted).
334  // So clean up the things needed (e.g. delete files).
335  virtual void deleteManager();
336 
337  // Create a column in the storage manager on behalf of a table column.
338  // <group>
339  // Create a scalar column.
340  DataManagerColumn* makeScalarColumn (const String& name, int dataType,
341  const String& dataTypeID);
342  // Create a direct array column.
343  DataManagerColumn* makeDirArrColumn (const String& name, int dataType,
344  const String& dataTypeID);
345  // Create an indirect array column.
346  DataManagerColumn* makeIndArrColumn (const String& name, int dataType,
347  const String& dataTypeID);
348  // </group>
349 
350  // The TiledStMan wants to do reallocateColumn.
352 
353  // Reallocate the column object if it is part of this data manager.
354  // It returns a pointer to the new column object.
355  // It is used to remove the indirection of the TSMColumn objects
356  // resulting in only one iso. two virtual column calls to get the data.
358 
359  // Set the shape and tile shape of a hypercube.
360  // By default it throws an "impossible" exception.
361  virtual void setShape (uInt rownr, TSMCube* hypercube,
362  const IPosition& shape,
363  const IPosition& tileShape);
364 
365  // Check the shape to be set for a hypercube.
366  // It checks if it matches predefined (fixed shape) columns
367  // and the shape of already defined coordinate columns.
368  void checkCubeShape (const TSMCube* hypercube,
369  const IPosition& cubeShape) const;
370 
371  // Get the data type of the coordinate column with the given name.
372  // An exception is thrown when the column is unknown.
373  int coordinateDataType (const String& columnName) const;
374 
375  // Initialize the new coordinates for the given cube.
376  void initCoordinates (TSMCube* hypercube);
377 
378  // Get pointer to data column object.
379  const TSMDataColumn* getDataColumn (uInt colnr) const
380  { return dataCols_p[colnr]; }
381 
382 protected:
383  // Set the persistent maximum cache size (in MiB).
384  void setPersMaxCacheSize (uInt nMiB);
385 
386  // Get the bindings of the columns with the given names.
387  // If bound, the pointer to the TSMColumn object is stored in the block.
388  // If mustExist is True, an exception is thrown if the column
389  // is not bound.
390  // It returns the number of bound columns.
391  uInt getBindings (const Vector<String>& columnNames,
392  PtrBlock<TSMColumn*>& colSet,
393  Bool mustExist) const;
394 
395  // Function setup calls this function to allow the derived class
396  // to check specific information. In case of errors, an exception
397  // should be thrown.
398  // By default it does nothing.
399  virtual void setupCheck (const TableDesc& tableDesc,
400  const Vector<String>& dataNames) const;
401 
402  // Get the table description needed for the hypercolumn description.
403  virtual const TableDesc& getDesc() const;
404 
405  // Check if values are given in the record for all columns in
406  // the block. Also check if the data types are correct.
407  // An exception is thrown if something is incorrect.
408  void checkValues (const PtrBlock<TSMColumn*>& colSet,
409  const Record& values) const;
410 
411  // Check if the coordinate values are correct.
412  // This calls checkValues and checks if their shapes match the
413  // hypercube shape.
414  // An exception is thrown if invalid.
415  void checkCoordinates (const PtrBlock<TSMColumn*>& coordColSet,
416  const IPosition& cubeShape,
417  const Record& values) const;
418 
419  // Check if the shapes of FixedShape data and coordinate columns match.
420  // An exception is thrown if not.
421  void checkShapeColumn (const IPosition& shape) const;
422 
423  // Check if the cube shape matches that of defined coordinates.
424  void checkCoordinatesShapes (const TSMCube* hypercube,
425  const IPosition& cubeShape) const;
426 
427  // Check if the hypercube to be added is correctly defined.
428  void checkAddHypercube (const IPosition& cubeShape,
429  const Record& values) const;
430 
431  // Make a new TSMCube object.
432  TSMCube* makeHypercube (const IPosition& cubeShape,
433  const IPosition& tileShape,
434  const Record& values);
435 
436  // Get the index of the hypercube with the given id-values.
437  // If not found, -1 is returned.
438  Int getCubeIndex (const Record& idValues) const;
439 
440  // Determine how many rows need to be added for an extension
441  // (in the last dimension) of a hypercube with the given shape.
442  uInt addedNrrow (const IPosition& shape, uInt incrInLastDim) const;
443 
444  // Flush the caches of all hypercubes.
445  // If data have put and fsync is set, fsync all files.
447 
448  // Let a derived class read the header info.
449  // This is used by the open and resync function.
450  virtual void readHeader (uInt nrrow, Bool firstTime) = 0;
451 
452  // Create the TSM header file.
453  // It creates an AipsIO object for it.
455 
456  // Open the TSM header file.
457  // It creates an AipsIO object for it.
459 
460  // Write the data into the header file.
461  // The given number of TSMCube objects have to be written.
462  void headerFilePut (AipsIO& headerFile, uInt nrCube);
463 
464  // Read the data from the header file.
465  // When done for the first time, setup() is called to initialize
466  // the various variables (using the extraNdim variable).
467  void headerFileGet (AipsIO& headerFile, uInt tabNrrow, Bool firstTime,
468  Int extraNdim);
469 
470  // Close the header file.
471  // It deletes the AipsIO object.
472  void headerFileClose (AipsIO* headerFile);
473 
474  // Set up the TiledStMan variables from the table description.
475  // The argument specifies the number of extra dimensions for the
476  // hypercube compared to the data array (usually 0 or 1).
477  // It is only used if no hypercolumn definition exists.
478  // -1 means that the hypercolumn definition has to be present.
479  void setup (Int extraNdim=-1);
480 
481  // Create a TSMFile object and store its pointer at the given index
482  // in the block.
483  void createFile (uInt index);
484 
485  // Convert the scalar data type to an array data type.
486  // This function is temporary and can disappear when the ColumnDesc
487  // classes use type TpArray*.
488  int arrayDataType (int dataType) const;
489 
490 
491  //# Declare all data members.
492  // The name of the hypercolumn.
494  // The number of rows in the columns.
496  // The assembly of all columns.
498  // The assembly of all data columns.
501  // The assembly of all id columns.
503  // The assembly of all coordinate columns.
505  // The assembly of all TSMFile objects.
506  // The first file is for all non-extensible cubes, while the others
507  // are for one file per extensible cube.
509  // The assembly of all TSMCube objects.
511  // The persistent maximum cache size (in MiB) for a hypercube.
513  // The actual maximum cache size for a hypercube (in MiB).
515  // The dimensionality of the hypercolumn.
517  // The number of vector coordinates.
519  // The fixed cell shape.
521  // Has any data changed since the last flush?
523 
524 private:
525  // Forbid copy constructor.
527 
528  // Forbid assignment.
530 };
531 
532 
534  { return maxCacheSize_p; }
535 
537  { return nrCoordVector_p; }
538 
539 inline uInt TiledStMan::nrow() const
540  { return nrrow_p; }
541 
543  { return cubeSet_p.nelements(); }
544 
546  { dataChanged_p = True; }
547 
548 inline const TSMCube* TiledStMan::getTSMCube (uInt hypercube) const
549  { return const_cast<TiledStMan*>(this)->getTSMCube (hypercube); }
550 
551 inline const TSMCube* TiledStMan::getHypercube (uInt rownr) const
552  { return const_cast<TiledStMan*>(this)->getHypercube (rownr); }
553 
555 {
556  persMaxCacheSize_p = nMiB;
557  maxCacheSize_p = nMiB;
558 }
559 
560 
561 
562 
563 } //# NAMESPACE CASACORE - END
564 
565 #endif
casacore::TableDesc
Define the structure of a Casacore table.
Definition: TableDesc.h:188
casacore::TiledStMan::checkAddHypercube
void checkAddHypercube(const IPosition &cubeShape, const Record &values) const
Check if the hypercube to be added is correctly defined.
casacore::TiledStMan::idColSet_p
PtrBlock< TSMColumn * > idColSet_p
The assembly of all id columns.
Definition: TiledStMan.h:502
casacore::TiledStMan::canReallocateColumns
Bool canReallocateColumns() const
The TiledStMan wants to do reallocateColumn.
casacore::TiledStMan::headerFileClose
void headerFileClose(AipsIO *headerFile)
Close the header file.
casacore::IPosition
A Vector of integers, for indexing into Array<T> objects.
Definition: IPosition.h:120
casacore::TiledStMan::nrCoordVector
uInt nrCoordVector() const
Get the number of coordinate vectors.
Definition: TiledStMan.h:536
casacore::TiledStMan::nhypercubes
uInt nhypercubes() const
Return the number of hypercubes.
Definition: TiledStMan.h:542
casacore::AipsIO
AipsIO is the object persistency mechanism of Casacore
Definition: AipsIO.h:169
casacore::DataManager
Abstract base class for a data manager.
Definition: DataManager.h:225
casacore::TiledStMan::cubeSet_p
PtrBlock< TSMCube * > cubeSet_p
The assembly of all TSMCube objects.
Definition: TiledStMan.h:510
casacore::PtrBlock
A drop-in replacement for Block<T*>.
Definition: Block.h:814
casacore::TiledStMan::nrdim_p
uInt nrdim_p
The dimensionality of the hypercolumn.
Definition: TiledStMan.h:516
casacore::TiledStMan::singleHypercube
virtual TSMCube * singleHypercube()
Test if only one hypercube is used by this storage manager.
casacore::TiledStMan::initCoordinates
void initCoordinates(TSMCube *hypercube)
Initialize the new coordinates for the given cube.
casacore::TiledStMan::persMaxCacheSize_p
uInt persMaxCacheSize_p
The persistent maximum cache size (in MiB) for a hypercube.
Definition: TiledStMan.h:512
casacore::TSMCube
Tiled hypercube in a table.
Definition: TSMCube.h:106
casacore::TiledStMan::dataColSet_p
PtrBlock< TSMColumn * > dataColSet_p
Definition: TiledStMan.h:500
casacore::TiledStMan::hypercubeShape
const IPosition & hypercubeShape(uInt rownr) const
Get the hypercube shape of the data in the given row.
casacore::TiledStMan::getBindings
uInt getBindings(const Vector< String > &columnNames, PtrBlock< TSMColumn * > &colSet, Bool mustExist) const
Get the bindings of the columns with the given names.
casacore::TiledStMan::maximumCacheSize
uInt maximumCacheSize() const
Get the current maximum cache size (in MiB (MibiByte)).
Definition: TiledStMan.h:533
casacore::TiledStMan::checkCoordinates
void checkCoordinates(const PtrBlock< TSMColumn * > &coordColSet, const IPosition &cubeShape, const Record &values) const
Check if the coordinate values are correct.
casacore::TiledStMan::TiledStMan
TiledStMan()
Create a TiledStMan.
casacore::DataManagerColumn
Abstract base class for a column in a data manager.
Definition: DataManager.h:612
casacore::TiledStMan::coordColSet_p
PtrBlock< TSMColumn * > coordColSet_p
The assembly of all coordinate columns.
Definition: TiledStMan.h:504
casacore::TiledStMan::nrCoordVector_p
uInt nrCoordVector_p
The number of vector coordinates.
Definition: TiledStMan.h:518
casacore::TiledStMan::reallocateColumn
DataManagerColumn * reallocateColumn(DataManagerColumn *column)
Reallocate the column object if it is part of this data manager.
casacore::TiledStMan::getTSMCube
TSMCube * getTSMCube(uInt hypercube)
casacore::TiledStMan::open
virtual void open(uInt nrrow, AipsIO &)
Open the storage manager for an existing table.
casacore::TiledStMan::arrayDataType
int arrayDataType(int dataType) const
Convert the scalar data type to an array data type.
casacore::TiledStMan::getHypercube
virtual TSMCube * getHypercube(uInt rownr, IPosition &position)=0
Get the hypercube in which the given row is stored.
casacore::TiledStMan::dataChanged_p
Bool dataChanged_p
Has any data changed since the last flush?
Definition: TiledStMan.h:522
casacore::TiledStMan::setProperties
virtual void setProperties(const Record &spec)
Modify data manager properties.
casacore::TiledStMan::cacheSize
uInt cacheSize(uInt rownr) const
Get the current cache size (in buckets) for the hypercube in the given row.
casacore::TiledStMan::writeTile
void writeTile(char *external, const Block< uInt > &externalOffset, const char *local, const Block< uInt > &localOffset, uInt nrpixels)
Write a tile after converting the data to external format.
casacore::TiledStMan::makeDirArrColumn
DataManagerColumn * makeDirArrColumn(const String &name, int dataType, const String &dataTypeID)
Create a direct array column.
casacore::TiledStMan::colSet_p
PtrBlock< TSMColumn * > colSet_p
The assembly of all columns.
Definition: TiledStMan.h:497
casacore::TiledStMan::setCacheSize
void setCacheSize(uInt rownr, uInt nbuckets, Bool forceSmaller)
Set the cache size for accessing the hypercube containing the given row.
casacore::TiledStMan::canChangeShape
virtual Bool canChangeShape() const
Can the tiled storage manager handle changing array shapes? The default is no (but TiledCellStMan can...
casacore::TiledStMan::makeTileShape
static IPosition makeTileShape(const IPosition &hypercubeShape, const Vector< double > &weight, const Vector< double > &tolerance, uInt maxNrPixelsPerTile=32768)
casacore::TiledStMan::checkValues
void checkValues(const PtrBlock< TSMColumn * > &colSet, const Record &values) const
Check if values are given in the record for all columns in the block.
casacore::TiledStMan::headerFileGet
void headerFileGet(AipsIO &headerFile, uInt tabNrrow, Bool firstTime, Int extraNdim)
Read the data from the header file.
casacore::TiledStMan::canAccessColumn
virtual Bool canAccessColumn(Bool &reask) const
Can the tiled storage manager access an entire column.
casacore::TiledStMan::fixedCellShape_p
IPosition fixedCellShape_p
The fixed cell shape.
Definition: TiledStMan.h:520
casacore::TiledStMan::makeTSMCube
TSMCube * makeTSMCube(TSMFile *file, const IPosition &cubeShape, const IPosition &tileShape, const Record &values, Int64 fileOffset=-1)
Make the correct TSMCube type (depending on tsmOption()).
casacore::TiledStMan::getTSMCube
const TSMCube * getTSMCube(uInt hypercube) const
Get the given hypercube.
Definition: TiledStMan.h:548
casacore::TiledStMan::calcCacheSize
uInt calcCacheSize(uInt rownr, const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath) const
Calculate the cache size (in buckets) for accessing the hypercube containing the given row.
casacore::Double
double Double
Definition: aipstype.h:55
casacore::TSMDataColumn
A data column in Tiled Storage Manager.
Definition: TSMDataColumn.h:97
casacore::TiledStMan::makeScalarColumn
DataManagerColumn * makeScalarColumn(const String &name, int dataType, const String &dataTypeID)
Create a column in the storage manager on behalf of a table column.
casacore::TiledStMan::headerFileOpen
AipsIO * headerFileOpen()
Open the TSM header file.
casacore::TiledStMan
Base class for Tiled Storage Manager classes.
Definition: TiledStMan.h:108
casacore::TiledStMan::defaultTileShape
virtual IPosition defaultTileShape() const
Get the default tile shape.
casacore::uInt
unsigned int uInt
Definition: aipstype.h:51
casacore::TiledStMan::getHypercube
virtual TSMCube * getHypercube(uInt rownr)=0
casacore::shape
TableExprNode shape(const TableExprNode &array)
Function operating on any scalar or array resulting in a Double array containing the shape.
Definition: ExprNode.h:1944
casacore::TiledStMan::setShape
virtual void setShape(uInt rownr, TSMCube *hypercube, const IPosition &shape, const IPosition &tileShape)
Set the shape and tile shape of a hypercube.
casacore::TSMFile
File object for Tiled Storage Manager.
Definition: TSMFile.h:82
casacore::TiledStMan::nrow
uInt nrow() const
Get the nr of rows in this storage manager.
Definition: TiledStMan.h:539
casacore::TiledStMan::dataManagerName
virtual String dataManagerName() const
Get the name given to the storage manager.
casacore::TiledStMan::dataManagerSpec
virtual Record dataManagerSpec() const
Return a record containing data manager specifications.
casacore::TiledStMan::makeHypercube
TSMCube * makeHypercube(const IPosition &cubeShape, const IPosition &tileShape, const Record &values)
Make a new TSMCube object.
casacore::TiledStMan::readTile
void readTile(char *local, const Block< uInt > &localOffset, const char *external, const Block< uInt > &externalOffset, uInt nrpixels)
Read a tile and convert the data to local format.
casacore::TiledStMan::addedNrrow
uInt addedNrrow(const IPosition &shape, uInt incrInLastDim) const
Determine how many rows need to be added for an extension (in the last dimension) of a hypercube with...
casacore::TiledStMan::dataCols_p
PtrBlock< TSMDataColumn * > dataCols_p
The assembly of all data columns.
Definition: TiledStMan.h:499
casacore::TiledStMan::TiledStMan
TiledStMan(const TiledStMan &)
Forbid copy constructor.
casacore::Int
int Int
Definition: aipstype.h:50
casacore
this file contains all the compiler specific defines
Definition: mainpage.dox:28
casacore::TiledStMan::flushCaches
Bool flushCaches(Bool fsync)
Flush the caches of all hypercubes.
casacore::TiledStMan::setup
void setup(Int extraNdim=-1)
Set up the TiledStMan variables from the table description.
casacore::TiledStMan::nrrow_p
uInt nrrow_p
The number of rows in the columns.
Definition: TiledStMan.h:495
casacore::TiledStMan::setCacheSize
void setCacheSize(uInt rownr, const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath, Bool forceSmaller)
Set the cache size using the calcCacheSize function mentioned above.
casacore::TiledStMan::showCacheStatistics
void showCacheStatistics(ostream &os) const
Show the statistics of all caches used.
casacore::True
const Bool True
Definition: aipstype.h:43
casacore::TiledStMan::createFile
void createFile(uInt index)
Create a TSMFile object and store its pointer at the given index in the block.
casacore::TiledStMan::getDesc
virtual const TableDesc & getDesc() const
Get the table description needed for the hypercolumn description.
casacore::TiledStMan::readHeader
virtual void readHeader(uInt nrrow, Bool firstTime)=0
Let a derived class read the header info.
casacore::TiledStMan::reopenRW
virtual void reopenRW()
Reopen all files used in this storage manager for read/write access.
casacore::TiledStMan::setDataChanged
void setDataChanged()
Set the flag to "data has changed since last flush".
Definition: TiledStMan.h:545
casacore::TiledStMan::~TiledStMan
virtual ~TiledStMan()
casacore::TiledStMan::setDataManagerName
void setDataManagerName(const String &newHypercolumnName)
casacore::TiledStMan::setMaximumCacheSize
virtual void setMaximumCacheSize(uInt nMiB)
Set the maximum cache size (in MiB) in a non-persistent way.
casacore::TiledStMan::userSetCache
Bool userSetCache(uInt rownr) const
Determine if the user set the cache size (using setCacheSize).
casacore::Int64
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition: aipsxtype.h:38
casacore::TiledStMan::getFile
TSMFile * getFile(uInt sequenceNumber)
Get the TSMFile object with the given sequence number.
casacore::TiledStMan::canAddRow
Bool canAddRow() const
Does the storage manager allow to add rows? (yes)
casacore::TiledStMan::bucketSize
uInt bucketSize(uInt rownr) const
Get the bucket size (in bytes) of the hypercube in the given row.
casacore::TiledStMan::deleteManager
virtual void deleteManager()
The data manager will be deleted (because all its columns are requested to be deleted).
casacore::TiledStMan::setHypercubeCacheSize
void setHypercubeCacheSize(uInt hypercube, uInt nbuckets, Bool forceSmaller)
Sets the cache size using the hypercube instead of the row number.
casacore::TiledStMan::makeTileShape
static IPosition makeTileShape(const IPosition &hypercubeShape, Double tolerance=0.5, uInt maxNrPixelsPerTile=32768)
Derive the tile shape from the hypercube shape for the given number of pixels per tile.
casacore::TiledStMan::emptyCaches
void emptyCaches()
Empty the caches used by the hypercubes in this storage manager.
casacore::TiledStMan::tileShape
const IPosition & tileShape(uInt rownr) const
Get the tile shape of the data in the given row.
casacore::String
String: the storage and methods of handling collections of characters.
Definition: String.h:223
casacore::TiledStMan::fileSet_p
PtrBlock< TSMFile * > fileSet_p
The assembly of all TSMFile objects.
Definition: TiledStMan.h:508
casacore::TiledStMan::headerFileCreate
AipsIO * headerFileCreate()
Create the TSM header file.
casacore::Bool
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
casacore::Block< uInt >
casacore::TiledStMan::getProperties
virtual Record getProperties() const
Get data manager properties that can be modified.
casacore::TiledStMan::operator=
TiledStMan & operator=(const TiledStMan &)
Forbid assignment.
casacore::TiledStMan::setupCheck
virtual void setupCheck(const TableDesc &tableDesc, const Vector< String > &dataNames) const
Function setup calls this function to allow the derived class to check specific information.
casacore::TiledStMan::getHypercube
const TSMCube * getHypercube(uInt rownr) const
Get the hypercube in which the given row is stored.
Definition: TiledStMan.h:551
casacore::Record
A hierarchical collection of named fields of various types.
Definition: Record.h:181
casacore::TiledStMan::makeIndArrColumn
DataManagerColumn * makeIndArrColumn(const String &name, int dataType, const String &dataTypeID)
Create an indirect array column.
casacore::TiledStMan::checkCoordinatesShapes
void checkCoordinatesShapes(const TSMCube *hypercube, const IPosition &cubeShape) const
Check if the cube shape matches that of defined coordinates.
casacore::Vector< double >
casacore::TiledStMan::getCubeIndex
Int getCubeIndex(const Record &idValues) const
Get the index of the hypercube with the given id-values.
casacore::TiledStMan::hypercolumnName_p
String hypercolumnName_p
The name of the hypercolumn.
Definition: TiledStMan.h:493
casacore::TiledStMan::headerFilePut
void headerFilePut(AipsIO &headerFile, uInt nrCube)
Write the data into the header file.
casacore::TiledStMan::checkShapeColumn
void checkShapeColumn(const IPosition &shape) const
Check if the shapes of FixedShape data and coordinate columns match.
casacore::TiledStMan::resync
virtual void resync(uInt nrrow)
Resync the storage manager with the new file contents.
casacore::TiledStMan::getDataColumn
const TSMDataColumn * getDataColumn(uInt colnr) const
Get pointer to data column object.
Definition: TiledStMan.h:379
casacore::TiledStMan::TiledStMan
TiledStMan(const String &hypercolumnName, uInt maximumCacheSizeMiB)
Create a TiledStMan storage manager.
casacore::TiledStMan::hasMultiFileSupport
virtual Bool hasMultiFileSupport() const
The data manager supports use of MultiFile.
casacore::TiledStMan::coordinateDataType
int coordinateDataType(const String &columnName) const
Get the data type of the coordinate column with the given name.
casacore::TiledStMan::checkCubeShape
void checkCubeShape(const TSMCube *hypercube, const IPosition &cubeShape) const
Check the shape to be set for a hypercube.
casacore::TiledStMan::maxCacheSize_p
uInt maxCacheSize_p
The actual maximum cache size for a hypercube (in MiB).
Definition: TiledStMan.h:514
casacore::TiledStMan::setPersMaxCacheSize
void setPersMaxCacheSize(uInt nMiB)
Set the persistent maximum cache size (in MiB).
Definition: TiledStMan.h:554
casacore::TiledStMan::getLengthOffset
uInt getLengthOffset(uInt nrPixels, Block< uInt > &dataOffset, Block< uInt > &localOffset, uInt &localTileLength) const
Get the length of the data for the given number of pixels.