escript  Revision_
blocktools.h
Go to the documentation of this file.
1 
2 /*****************************************************************************
3 *
4 * Copyright (c) 2014-2016 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Apache License, version 2.0
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
14 *
15 *****************************************************************************/
16 
17 #ifndef __RIPLEY_BLOCKTOOLS_H__
18 #define __RIPLEY_BLOCKTOOLS_H__
19 
20 
21 #include <vector>
22 
23 
24 /* This file contains two main classes for dealing with a large 3D region which
25  * has been divided into a 3D Grid of Blocks (usually to be distributed).
26  * Each block is divided into 27 subblocks. The first and last subblocks in
27  * each dimension are cubes.
28  *
29  * class Block manages a single block. It has methods for copying between a
30  * flat array (storing all the values in the 3D block) and buffers storing
31  * individual subblocks. These buffers can be used to transfer individual
32  * subblocks to other blocks (using some external means).
33  *
34  * class BlockGrid deals with the position of a given block in relation to the
35  * rest of the blocks in the grid. (In an MPI setting, there would be one block
36  * per rank.)
37  * It also requires transfers of subblocks into and out of the block in order
38  * to make the whole (global) array consistent.
39  *
40  * Each block has a region "inset" wide in from each edge which is shared with
41  * neighbouring blocks. Where these regions overlap with another block, the
42  * block closest to block 0,0,0 is used.
43  * Or more precisely, values move left->right then lowy->highy and finally
44  * lowz -> highz.
45  *
46  * Please don't mix external calls into this file, it may be useful to separate
47  * it for debugging purposes.
48  *
49  * Types required:
50  * neighbourID_t - Stores the label of a neighbouring block.
51  * In an MPI setting, this will be the type used refer to
52  * ranks
53  * coord_t - Stores a position of a block in the grid of blocks
54  * (it could be within one dimension or overall in a flat
55  * structure. It is not (necessarily) the same as
56  * neighbourID_t because coord_t should be _unsigned_ and
57  * there is no guarantee that neighbourID_t will even be
58  * an integral type.
59  */
60 
61 #include <esysUtils/Esys_MPI.h>
62 typedef Esys_MPI_rank neighbourID_t; // This should be the MPI_rank type
63 typedef unsigned coord_t; // if we ever get more than 2^32 ranks, we have other problems
64 
65 typedef std::pair<neighbourID_t, int> neighpair;
66 typedef std::vector<neighpair> neighbourvector;
67 
68 
69 typedef struct
70 {
71 public:
72  neighbourID_t sourceID; // ranks involved in communication
74  int tag;
75  unsigned char srcbuffid; // number of buffer to use for coms
76  unsigned char destbuffid;
77 } message;
78 
79 typedef std::vector<message> messvec;
80 
81 
82 class BlockGrid
83 {
84 public:
85  BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz);
86 
88 
89 
90  // generate all incoming com messages for this block.
91  // for each subblock (27 of them), there may be an x, y, z direction to search in
92  void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
93 
94 
95  // generate all outgoing com messages for this block
96  void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
97 
98 private:
102 };
103 
104 
105 
106 /* Do not ask about buffers for sub-block 1,1,1 (also known as #13)
107  They do not exist, such buffers would be:
108  1) big
109  2) unnecessary since the centre sub-block is not sent anywhere
110 
111 Note that this class does not deal with data transfer between blocks
112 Sub-blocks are copied to and from buffers. Other code is required to
113 actually move the data.
114 
115 "dpp" == doubles per point and gives the number of doubles that make up each "point"
116 This is required to calculate offsets and buffer sizes.
117 */
118 
119 class Block
120 {
121 public:
122  // s? specifiy the [local] size (in points) of each dimension
123  Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen,
124  size_t ymidlen, size_t zmidlen, unsigned int dpp=1);
125 
126  ~Block();
127 
128  // Out buffers are loaded with the contents of the flat array and are
129  // to be sent to other blocks
130  double* getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
131  double* getOutBuffer(unsigned char bid);
132 
133  // In buffers are populated from external communications
134  // and copied back to the flat array
135  double* getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
136  double* getInBuffer(unsigned char bid);
137 
138  // return number of doubles in the given block
139  size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz);
140  size_t getBuffSize(unsigned char bid);
141 
142  // where does the subblock specified start in a source array
143  size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz);
144 
145  // debug only
146  void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out);
147 
148  // Copy a 3d region from a flat array into a buffer
149  void copyToBuffer(unsigned char buffid, double* src);
150 
151  // Copy a 3d region from a buffer into a flat array
152  void copyFromBuffer(unsigned char buffid, double* dest);
153 
154 
155  void copyAllToBuffer(double* src);
156 
157  void copyUsedFromBuffer(double* dest);
158 
159  void setUsed(unsigned char buffid);
160 
161 private:
162 
163  // determines the dimensions of each subblock
164  void populateDimsTable();
165  void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
166  void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
167 
168 
169  double* inbuff;
170  double* outbuff;
171  size_t buffoffsets[27]; // offsets of the various blocks within the buffer arrays
172  size_t flatoffsets[27]; // starting point of each block within a flat array
173  bool used[27];
174  size_t dims[27][3]; // dimension of each subblock
175  size_t sx;
176  size_t sy;
177  size_t sz;
178  size_t inset;
179  size_t xmidlen;
180  size_t ymidlen;
181  size_t zmidlen;
182  double* inbuffptr[27];
183  double* outbuffptr[27];
184  const unsigned int dpsize; // number of doubles which make up a point
185 };
186 
187 // Returns the MPI message tag to use for a transfer between the two subblocks
188 int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez,
189  unsigned char targetx, unsigned char targety, unsigned char targetz);
190 
191 // computes the tag based on the destination and the direction it comes from
192 // the booleans indicate whether a negative shift in that direction is required
193 int getTag(unsigned char destx, unsigned char desty, unsigned char destz,
194  bool deltax, bool deltay, bool deltaz);
195 
196 
197 // the booleans indicate whether a negative shift in that direction is required
198 unsigned char getSrcBuffID(unsigned char destx, unsigned char desty,
199  unsigned char destz, bool deltax, bool deltay,
200  bool deltaz);
201 
202 
203 /* Now the 2D versions */
204 
205 // 2D version
207 {
208 public:
209  BlockGrid2(coord_t maxx, coord_t maxy);
210 
211  neighbourID_t getNID(coord_t x, coord_t y) const;
212 
213  // generate all incoming com messages for this block.
214  // for each subblock (9 of them), there may be an x, y direction to search in
215  void generateInNeighbours(coord_t blockx, coord_t blocky, messvec& v);
216 
217  // generate all outgoing com messages for this block
218  void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec& v);
219 
220 private:
223 };
224 
225 // The 2D version - there is no block 4
226 class Block2
227 {
228 public:
229  // s? specifiy the [local] size (in points) of each dimension
230  Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen,
231  size_t ymidlen, unsigned int dpp=1);
232 
233  ~Block2();
234 
235  // Out buffers are loaded with the contents of the flat array and are
236  // to be sent to other blocks
237  double* getOutBuffer(unsigned char subx, unsigned char suby);
238  double* getOutBuffer(unsigned char bid);
239 
240  // In buffers are populated from external communications
241  // and copied back to the flat array
242  double* getInBuffer(unsigned char subx, unsigned char suby);
243  double* getInBuffer(unsigned char bid);
244 
245  // return number of doubles in the given block
246  size_t getBuffSize(unsigned char subx, unsigned char suby);
247  size_t getBuffSize(unsigned char bid);
248 
249  // where does the subblock specified start in a source array
250  size_t startOffset(unsigned char subx, unsigned char suby);
251 
252  // debug only
253  void displayBlock(unsigned char subx, unsigned char suby, bool out);
254 
255  // Copy a 3d region from a flat array into a buffer
256  void copyToBuffer(unsigned char buffid, double* src);
257 
258  // Copy a 3d region from a buffer into a flat array
259  void copyFromBuffer(unsigned char buffid, double* dest);
260 
261 
262  void copyAllToBuffer(double* src);
263 
264  void copyUsedFromBuffer(double* dest);
265 
266  void setUsed(unsigned char buffid);
267 
268 private:
269 
270  // determines the dimensions of each subblock
271  void populateDimsTable();
272  void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen);
273  void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen);
274 
275 
276  double* inbuff;
277  double* outbuff;
278  size_t buffoffsets[9]; // offsets of the various blocks within the buffer arrays
279  size_t flatoffsets[9]; // starting point of each block within a flat array
280  bool used[9];
281  size_t dims[9][2]; // dimension of each subblock
282  size_t sx;
283  size_t sy;
284  size_t inset;
285  size_t xmidlen;
286  size_t ymidlen;
287  double* inbuffptr[9];
288  double* outbuffptr[9];
289  const unsigned int dpsize; // number of doubles which make up a point
290 };
291 
292 
293 // Returns the MPI message tag to use for a transfer between the two subblocks
294 int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety);
295 
296 // computes the tag based on the destination and the direction it comes from
297 // the booleans indicate whether a negative shift in that direction is required
298 int getTag2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
299 
300 
301 // the booleans indicate whether a negative shift in that direction is required
302 unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
303 
304 #endif // __RIPLEY_BLOCKTOOLS_H__
305 
void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition: blocktools.cpp:43
Definition: blocktools.h:69
int tag
Definition: blocktools.h:74
unsigned coord_t
Definition: blocktools.h:63
size_t sx
Definition: blocktools.h:175
double * inbuff
Definition: blocktools.h:169
unsigned char destbuffid
Definition: blocktools.h:76
std::pair< neighbourID_t, int > neighpair
Definition: blocktools.h:65
size_t sx
Definition: blocktools.h:282
const unsigned int dpsize
Definition: blocktools.h:184
size_t inset
Definition: blocktools.h:178
Definition: blocktools.h:226
double * outbuff
Definition: blocktools.h:277
size_t ymidlen
Definition: blocktools.h:286
Definition: blocktools.h:206
Esys_MPI_rank neighbourID_t
Definition: blocktools.h:62
std::vector< neighpair > neighbourvector
Definition: blocktools.h:66
unsigned char getSrcBuffID(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz)
Definition: blocktools.cpp:430
int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez, unsigned char targetx, unsigned char targety, unsigned char targetz)
Definition: blocktools.cpp:412
void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition: blocktools.cpp:82
Definition: blocktools.h:82
double * outbuff
Definition: blocktools.h:170
int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety)
Definition: blocktools2.cpp:364
coord_t zmax
Definition: blocktools.h:101
neighbourID_t sourceID
Definition: blocktools.h:72
const unsigned int dpsize
Definition: blocktools.h:289
double * inbuff
Definition: blocktools.h:276
size_t ymidlen
Definition: blocktools.h:180
neighbourID_t destID
Definition: blocktools.h:73
BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz)
Definition: blocktools.cpp:24
coord_t xmax
Definition: blocktools.h:221
unsigned char srcbuffid
Definition: blocktools.h:75
std::vector< message > messvec
Definition: blocktools.h:79
neighbourID_t getNID(coord_t x, coord_t y, coord_t z) const
Definition: blocktools.cpp:28
size_t xmidlen
Definition: blocktools.h:179
size_t sy
Definition: blocktools.h:283
size_t xmidlen
Definition: blocktools.h:285
int Esys_MPI_rank
Definition: Esys_MPI.h:59
unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay)
Definition: blocktools2.cpp:380
size_t sy
Definition: blocktools.h:176
size_t inset
Definition: blocktools.h:284
coord_t ymax
Definition: blocktools.h:100
coord_t xmax
Definition: blocktools.h:99
size_t zmidlen
Definition: blocktools.h:181
Definition: blocktools.h:119
coord_t ymax
Definition: blocktools.h:222
size_t sz
Definition: blocktools.h:177