escript  Revision_
EsysMPI.h
Go to the documentation of this file.
1 
2 /*****************************************************************************
3 *
4 * Copyright (c) 2003-2020 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Apache License, version 2.0
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014-2017 by Centre for Geoscience Computing (GeoComp)
14 * Development from 2019 by School of Earth and Environmental Sciences
15 **
16 *****************************************************************************/
17 
18 #ifndef __ESCRIPT_ESYSMPI_H__
19 #define __ESCRIPT_ESYSMPI_H__
20 
21 #include <escript/DataTypes.h>
22 
23 #include <ctime>
24 #include <sstream>
25 
26 #include <boost/shared_ptr.hpp>
27 
28 #ifdef _OPENMP
29 #include <omp.h>
30 #endif
31 
32 #ifdef ESYS_MPI
33 #include <mpi.h>
34 
35 #ifdef ESYS_INDEXTYPE_LONG
36 #define MPI_DIM_T MPI_LONG
37 #else
38 #define MPI_DIM_T MPI_INT
39 #endif
40 
41 #else
42  typedef int MPI_Comm;
43  typedef int MPI_Request;
44  typedef int MPI_Op;
45  typedef int MPI_Status;
46  #define MPI_INT 6
47  #define MPI_DOUBLE 11
48  #define MPI_COMM_WORLD 91
49  #define MPI_COMM_NULL 0
50 
51  // MPI_Op replacements for non-MPI - these values are arbitrary
52  #define MPI_SUM 100
53  #define MPI_MIN 101
54  #define MPI_MAX 102
55 
56  #define MPI_OP_NULL 17
57  // end MPI_op
58 
59 #endif // ESYS_MPI
60 
61 namespace escript {
62 
67 inline int getSubWorldTag()
68 {
69  return (('S'<< 24) + ('u' << 16) + ('b' << 8) + 'W')%1010201;
70 }
71 
72 class JMPI_;
73 
74 typedef boost::shared_ptr<JMPI_> JMPI;
75 
78 JMPI makeInfo(MPI_Comm comm, bool owncom=false);
79 
80 class JMPI_
81 {
82 public:
83  ~JMPI_();
84 
87  DataTypes::index_t max_id,
88  DataTypes::index_t* distribution);
89 
92  DataTypes::index_t* offset);
93 
96  inline int mod_rank(int k) const
97  {
98  int out=0;
99 #ifdef ESYS_MPI
100  if (size > 1) {
101  const int q = k/size;
102  if (k > 0) {
103  out=k-size*q;
104  } else if (k < 0) {
105  out=k-size*(q-1);
106  }
107  }
108 #endif
109  return out;
110  }
111 
113  inline std::string appendRankToFileName(const std::string& fileName) const
114  {
115 #ifdef ESYS_MPI
116  if (size > 1) {
117  std::stringstream ss;
118  ss << fileName << '.';
119  ss.fill('0');
120  ss.width(4);
121  ss << rank;
122  return ss.str();
123  }
124 #endif
125  return fileName;
126  }
127 
129  inline int counter() const
130  {
131  return msg_tag_counter;
132  }
133 
135  inline void incCounter(int i=1)
136  {
137  msg_tag_counter+=i;
138  // there is no particular significance here other than being 7 digits
139  // and prime (because why not). It just needs to be big.
140  msg_tag_counter %= 1010201;
141  }
142 
144  inline void setCounter(int value)
145  {
146  msg_tag_counter = value%1010201;
147  }
148 
150  inline bool isValid() const
151  {
152  return comm!=MPI_COMM_NULL;
153  }
154 
155  int size;
156  int rank;
157  MPI_Comm comm;
158 
159 private:
160  JMPI_(MPI_Comm comm, bool owncomm);
161  friend JMPI makeInfo(MPI_Comm comm, bool owncom);
162 #pragma clang diagnostic push
163 #pragma clang diagnostic ignored "-Wunused-private-field"
164  bool ownscomm;
165 #pragma clang diagnostic pop
166  int msg_tag_counter;
167 };
168 
169 // Does not cope with nested calls
170 class NoCOMM_WORLD
171 {
172 public:
173  NoCOMM_WORLD();
174  ~NoCOMM_WORLD();
175  static bool active();
176 };
177 
179 bool checkResult(int input, int& output, const JMPI& comm);
180 
184 bool shipString(const char* src, char** dest, MPI_Comm& comm);
185 
187 inline double gettime()
188 {
189  double out;
190 #ifdef ESYS_MPI
191  out = MPI_Wtime();
192 #else
193 #ifdef _OPENMP
194  out=omp_get_wtime();
195 #else
196  out=((double) clock())/CLOCKS_PER_SEC;
197 #endif
198 #endif
199  return out;
200 }
201 
202 } // namespace escript
203 
204 #endif // __ESCRIPT_ESYSMPI_H__
205 
MPI_Status
int MPI_Status
Definition: EsysMPI.h:44
paso::N
static dim_t N
Definition: SparseMatrix_saveHB.cpp:50
escript::JMPI_
Definition: EsysMPI.h:79
MPI_Op
int MPI_Op
Definition: EsysMPI.h:43
escript::JMPI_::JMPI_
JMPI_(MPI_Comm comm, bool owncomm)
Definition: EsysMPI.cpp:50
escript::getSubWorldTag
int getSubWorldTag()
tag reserved for use by SubWorld code This value should be higher than the modulus used in JMPI_::set...
Definition: EsysMPI.h:66
escript::NoCOMM_WORLD
Definition: EsysMPI.h:169
escript::JMPI_::rank
int rank
Definition: EsysMPI.h:155
escript::checkResult
bool checkResult(int res, int &mres, const JMPI &info)
Everyone puts in their error code and everyone gets the largest one.
Definition: EsysMPI.cpp:122
escript::JMPI_::setDistribution
DataTypes::dim_t setDistribution(DataTypes::index_t min_id, DataTypes::index_t max_id, DataTypes::index_t *distribution)
Definition: EsysMPI.cpp:77
escript::makeInfo
JMPI makeInfo(MPI_Comm comm, bool owncom)
Definition: EsysMPI.cpp:41
MPI_INT
#define MPI_INT
Definition: EsysMPI.h:45
MPI_MAX
#define MPI_MAX
Definition: EsysMPI.h:53
escript::NoCOMM_WORLD::~NoCOMM_WORLD
~NoCOMM_WORLD()
Definition: EsysMPI.cpp:244
MPI_COMM_WORLD
#define MPI_COMM_WORLD
Definition: EsysMPI.h:47
escript::JMPI_::appendRankToFileName
std::string appendRankToFileName(const std::string &fileName) const
appends MPI rank to a file name if MPI size > 1
Definition: EsysMPI.h:112
escript::JMPI_::counter
int counter() const
returns the current value of the message tag counter
Definition: EsysMPI.h:128
escript::DataTypes::dim_t
index_t dim_t
Definition: DataTypes.h:90
escript::JMPI_::makeInfo
friend JMPI makeInfo(MPI_Comm comm, bool owncom)
Definition: EsysMPI.cpp:41
escript::JMPI
boost::shared_ptr< JMPI_ > JMPI
Definition: EsysMPI.h:71
escript::JMPI_::~JMPI_
~JMPI_()
Definition: EsysMPI.cpp:69
escript::JMPI_::mod_rank
int mod_rank(int k) const
Definition: EsysMPI.h:95
escript::JMPI_::msg_tag_counter
int msg_tag_counter
Definition: EsysMPI.h:165
escript::JMPI_::incCounter
void incCounter(int i=1)
increments the message tag counter by i
Definition: EsysMPI.h:134
escript::JMPI_::comm
MPI_Comm comm
Definition: EsysMPI.h:156
escript::EsysException::EsysException
EsysException(const std::string &message)
Constructor which creates an Exception with the given message.
Definition: EsysException.h:63
escript::JMPI_::setCounter
void setCounter(int value)
sets the message tag counter to value
Definition: EsysMPI.h:143
escript::JMPI_::size
int size
Definition: EsysMPI.h:154
escript::gettime
double gettime()
returns the current ticks for timing
Definition: EsysMPI.h:186
escript::DataTypes::index_t
int index_t
type for array/matrix indices used both globally and on each rank
Definition: DataTypes.h:85
EsysException.h
escript
Definition: AbstractContinuousDomain.cpp:23
MPI_COMM_NULL
#define MPI_COMM_NULL
Definition: EsysMPI.h:48
escript::NoCOMM_WORLD::NoCOMM_WORLD
NoCOMM_WORLD()
Definition: EsysMPI.cpp:236
MPI_Request
int MPI_Request
Definition: EsysMPI.h:42
MPI_Comm
int MPI_Comm
Definition: EsysMPI.h:41
escript::shipString
bool shipString(const char *src, char **dest, MPI_Comm &comm)
Definition: EsysMPI.cpp:176
EsysMPI.h
escript::JMPI_::split
void split(DataTypes::dim_t N, DataTypes::dim_t *local_N, DataTypes::index_t *offset)
Definition: EsysMPI.cpp:104
escript::JMPI_::isValid
bool isValid() const
returns true if this has a valid MPI communicator
Definition: EsysMPI.h:149
escript::JMPI_::ownscomm
bool ownscomm
Definition: EsysMPI.h:163
escript::NoCOMM_WORLD::active
static bool active()
Definition: EsysMPI.cpp:249