3 #ifndef DUNE_GRID_YASPGRID_TORUS_HH 4 #define DUNE_GRID_YASPGRID_TORUS_HH 17 #include <dune/common/binaryfunctions.hh> 18 #include <dune/common/streamoperators.hh> 43 template<
class CollectiveCommunication,
int d>
76 : _comm(comm), _tag(tag)
83 for (
int i=0; i<d; i++)
90 if (inc != _comm.size())
91 DUNE_THROW(Dune::Exception,
"Communicator size and result of the given load balancer do not match!");
128 CollectiveCommunication
comm ()
const 142 for (
int i=d-1; i>=0; i--)
143 if (c[i]<0 || c[i]>=_dims[i])
return false;
151 rank = rank%_comm.size();
152 for (
int i=d-1; i>=0; i--)
154 coord[i] = rank/_increment[i];
155 rank = rank%_increment[i];
163 for (
int i=0; i<d; i++) coord[i] = coord[i]%_dims[i];
165 for (
int i=0; i<d; i++) rank += coord[i]*_increment[i];
173 coord[dir] = (coord[dir]+_dims[dir]+cnt)%_dims[dir];
184 for (
int i=0; i<d; i++)
186 if (coord[i]%2==1) c += power;
191 for (
int i=0; i<d; i++)
193 if (_dims[i]>1 && coord[i]==_dims[i]-1) c += power;
210 for (
int i=0; i<d; ++i)
220 for (
int i=0; i<d; i++)
225 if (coord[i]==0 && periodic[i]==
false)
return false;
230 if (coord[i]==_dims[i]-1 && periodic[i]==
false)
return false;
243 double partition (
int rank, iTupel origin_in, iTupel size_in, iTupel& origin_out, iTupel& size_out)
const 250 for (
int i=0; i<d; i++)
253 int m = size_in[i]/_dims[i];
254 int r = size_in[i]%_dims[i];
258 if (coord[i]<_dims[i]-r)
260 origin_out[i] = origin_in[i] + coord[i]*m;
266 origin_out[i] = origin_in[i] + (_dims[i]-r)*m + (coord[i]-(_dims[i]-r))*(m+1);
271 return maxsize/(sz/_comm.size());
310 iTupel delta=i->delta;
311 for (
int j=0; j<d; ++j)
337 typename std::deque<CommPartner>::const_iterator i;
343 return ProcListIterator(_sendlist.begin());
349 return ProcListIterator(_sendlist.end());
355 return ProcListIterator(_recvlist.begin());
361 return ProcListIterator(_recvlist.end());
369 task.buffer = buffer;
371 if (rank!=_comm.rank())
372 _sendrequests.push_back(task);
374 _localsendrequests.push_back(task);
382 task.buffer = buffer;
384 if (rank!=_comm.rank())
385 _recvrequests.push_back(task);
387 _localrecvrequests.push_back(task);
394 if (_localsendrequests.size()!=_localrecvrequests.size())
396 std::cout <<
"[" <<
rank() <<
"]: ERROR: local sends/receives do not match in exchange!" << std::endl;
399 for (
unsigned int i=0; i<_localsendrequests.size(); i++)
401 if (_localsendrequests[i].size!=_localrecvrequests[i].size)
403 std::cout <<
"[" <<
rank() <<
"]: ERROR: size in local sends/receive does not match in exchange!" << std::endl;
406 memcpy(_localrecvrequests[i].buffer,_localsendrequests[i].buffer,_localsendrequests[i].size);
408 _localsendrequests.clear();
409 _localrecvrequests.clear();
417 for (
unsigned int i=0; i<_sendrequests.size(); i++)
422 MPI_Isend(_sendrequests[i].buffer, _sendrequests[i].size, MPI_BYTE,
423 _sendrequests[i].
rank, _tag, _comm, &(_sendrequests[i].request));
424 _sendrequests[i].flag =
false;
429 for (
unsigned int i=0; i<_recvrequests.size(); i++)
434 MPI_Irecv(_recvrequests[i].buffer, _recvrequests[i].size, MPI_BYTE,
435 _recvrequests[i].
rank, _tag, _comm, &(_recvrequests[i].request));
436 _recvrequests[i].flag =
false;
443 for (
unsigned int i=0; i<_sendrequests.size(); i++)
444 if (!_sendrequests[i].flag)
447 MPI_Test( &(_sendrequests[i].request), &(_sendrequests[i].flag), &status);
448 if (_sendrequests[i].flag)
459 for (
unsigned int i=0; i<_recvrequests.size(); i++)
460 if (!_recvrequests[i].flag)
463 MPI_Test( &(_recvrequests[i].request), &(_recvrequests[i].flag), &status);
464 if (_recvrequests[i].flag)
474 _sendrequests.clear();
475 _recvrequests.clear();
483 _comm.template allreduce<Dune::Max<double>,
double>(&x, &res, 1);
490 s <<
"[" <<
rank() <<
"]: Torus " <<
procs() <<
" processor(s) arranged as " <<
dims() << std::endl;
493 s <<
"[" <<
rank() <<
"]: send to " 494 <<
"rank=" << i.rank()
495 <<
" index=" << i.index()
496 <<
" delta=" << i.delta() <<
" dist=" << i.distance() << std::endl;
500 s <<
"[" <<
rank() <<
"]: recv from " 501 <<
"rank=" << i.rank()
502 <<
" index=" << i.index()
503 <<
" delta=" << i.delta() <<
" dist=" << i.distance() << std::endl;
515 std::fill(delta.begin(), delta.end(), -1);
524 for (
int i=0; i<d; i++)
525 nb[i] = ( me[i]+_dims[i]+delta[i] ) % _dims[i];
531 for (
int i=0; i<d; i++)
537 _recvlist.push_back(cp);
538 cp.index = last-index;
539 _sendlist.push_front(cp);
546 for (
int i=0; i<d; i++)
561 CollectiveCommunication _comm;
566 std::deque<CommPartner> _sendlist;
567 std::deque<CommPartner> _recvlist;
569 mutable std::vector<CommTask> _sendrequests;
570 mutable std::vector<CommTask> _recvrequests;
571 mutable std::vector<CommTask> _localsendrequests;
572 mutable std::vector<CommTask> _localrecvrequests;
577 template <
class CollectiveCommunication,
int d>
578 inline std::ostream& operator<< (std::ostream& s, const Torus<CollectiveCommunication, d> & t)
bool is_neighbor(iTupel delta, std::bitset< d > periodic) const
return true if neighbor with given delta is a neighbor under the given periodicity ...
Definition: torus.hh:216
Include standard header files.
Definition: agrid.hh:58
ProcListIterator sendend() const
end of send list
Definition: torus.hh:347
a base class for the yaspgrid partitioning strategy The name might be irritating. It will probably ch...
Definition: partitioning.hh:23
Torus()
constructor making uninitialized object
Definition: torus.hh:71
int dims(int i) const
return dimensions of torus in direction i
Definition: torus.hh:122
iTupel coord() const
return own coordinates
Definition: torus.hh:104
const iTupel & dims() const
return dimensions of torus
Definition: torus.hh:116
ProcListIterator sendbegin() const
first process in send list
Definition: torus.hh:341
void send(int rank, void *buffer, int size) const
store a send request; buffers are sent in order; handles also local requests with memcpy ...
Definition: torus.hh:365
ProcListIterator(typename std::deque< CommPartner >::const_iterator iter)
make an iterator
Definition: torus.hh:283
Torus(CollectiveCommunication comm, int tag, iTupel size, const YLoadBalance< d > *lb)
make partitioner from communicator and coarse mesh size
Definition: torus.hh:75
int tag() const
return tag used by torus
Definition: torus.hh:134
CollectiveCommunication comm() const
return communicator
Definition: torus.hh:128
void exchange() const
exchange messages stored in request buffers; clear request buffers afterwards
Definition: torus.hh:391
iTupel rank_to_coord(int rank) const
map rank to coordinate in torus using lexicographic ordering
Definition: torus.hh:148
int color(const iTupel &coord) const
assign color to given coordinate
Definition: torus.hh:178
std::array< int, d > iTupel
type used to pass tupels in and out
Definition: torus.hh:47
iTupel delta() const
return distance vector
Definition: torus.hh:295
int neighbors() const
return the number of neighbors, which is
Definition: torus.hh:207
int distance() const
return 1-norm of distance vector
Definition: torus.hh:307
void abs(const DofVectorPointer< int > &dofVector)
Definition: dofvector.hh:326
virtual void loadbalance(const iTupel &, int, iTupel &) const =0
void recv(int rank, void *buffer, int size) const
store a receive request; buffers are received in order; handles also local requests with memcpy ...
Definition: torus.hh:378
This file provides tools to partition YaspGrids. If you want to write your own partitioner, inherit from YLoadBalance and implement the loadbalance() method. You can also browse this file for already available useful partitioners, like YaspFixedSizePartitioner.
double partition(int rank, iTupel origin_in, iTupel size_in, iTupel &origin_out, iTupel &size_out) const
partition the given grid onto the torus and return the piece of the process with given rank; returns ...
Definition: torus.hh:243
int index() const
return index in proclist
Definition: torus.hh:301
int coord_to_rank(iTupel coord) const
map coordinate in torus to rank using lexicographic ordering
Definition: torus.hh:161
ProcListIterator recvend() const
last process in receive list
Definition: torus.hh:359
bool inside(iTupel c) const
return true if coordinate is inside torus
Definition: torus.hh:140
ProcListIterator recvbegin() const
first process in receive list
Definition: torus.hh:353
int color(int rank) const
assign color to given rank
Definition: torus.hh:201
int rank_relative(int rank, int dir, int cnt) const
return rank of process where its coordinate in direction dir has offset cnt (handles periodic case) ...
Definition: torus.hh:170
int rank() const
return own rank
Definition: torus.hh:98
double global_max(double x) const
global max
Definition: torus.hh:480
int procs() const
return number of processes
Definition: torus.hh:110
int rank() const
return rank of neighboring process
Definition: torus.hh:289
void print(std::ostream &s) const
print contents of torus object
Definition: torus.hh:488