Teuchos - Trilinos Tools Package  Version of the Day
Teuchos_GlobalMPISession.cpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Teuchos: Common Tools Package
5 // Copyright (2004) Sandia Corporation
6 //
7 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8 // license for use of this work by or on behalf of the U.S. Government.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ***********************************************************************
40 // @HEADER
41 
43 #include "Teuchos_Assert.hpp"
44 
45 // The header file does not at all depend on MPI routines or types,
46 // so we can defer inclusion of mpi.h to here. This also fixes Bug
47 // 5631: https://software.sandia.gov/bugzilla/show_bug.cgi?id=5631
48 #ifdef HAVE_MPI
49 # include "mpi.h"
50 #endif
51 
52 #ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
53 # include "Kokkos_Core.hpp"
54 #endif
55 
56 
57 namespace Teuchos {
58 
59 
60 bool GlobalMPISession::haveMPIState_ = false;
61 bool GlobalMPISession::mpiIsFinalized_ = false;
62 int GlobalMPISession::rank_ = 0 ;
63 int GlobalMPISession::nProc_ = 1 ;
64 
65 
66 GlobalMPISession::GlobalMPISession( int* argc, char*** argv, std::ostream *out )
67 {
68  std::ostringstream oss;
69 
70  // Above is used to create all output before sending to *out to avoid
71  // jumbled parallel output between processors
72 
73 #ifdef HAVE_MPI
74 
75  int mpierr = 0;
76 
77  // Assert that MPI is not already initialized
78  int mpiHasBeenStarted = 0;
79  MPI_Initialized(&mpiHasBeenStarted);
80  if (mpiHasBeenStarted) {
81  if (out) {
82  *out << "GlobalMPISession(): Error, MPI_Intialized() return true,"
83  << " calling std::terminate()!\n"
84  << std::flush;
85  }
86  std::terminate();
87  }
88 
89  // Initialize MPI
90  mpierr = ::MPI_Init(argc, (char ***) argv);
91  if (mpierr != 0) {
92  if (out) {
93  *out << "GlobalMPISession(): Error, MPI_Init() returned error code="
94  << mpierr << "!=0, calling std::terminate()!\n"
95  << std::flush;
96  }
97  std::terminate();
98  }
99 
100  initialize(out); // Get NProc_ and rank_
101 
102  int nameLen;
103  char procName[MPI_MAX_PROCESSOR_NAME];
104  mpierr = ::MPI_Get_processor_name(procName, &nameLen);
105  if (mpierr != 0) {
106  if (out) {
107  *out << "GlobalMPISession(): Error, MPI_Get_processor_name() error code="
108  << mpierr << "!=0, calling std::terminate()!\n"
109  << std::flush;
110  }
111  std::terminate();
112  }
113 
114  oss << "Teuchos::GlobalMPISession::GlobalMPISession(): started processor with name "
115  << procName << " and rank " << rank_ << "!" << std::endl;
116 
117 #else
118 
119  oss << "Teuchos::GlobalMPISession::GlobalMPISession(): started serial run"
120  << std::endl;
121 
122 #endif
123 
124 #ifndef TEUCHOS_SUPPRESS_PROC_STARTUP_BANNER
125 
126  // See if we should suppress the startup banner
127  bool printStartupBanner = true;
128  const std::string suppress_option("--teuchos-suppress-startup-banner");
129  for ( int opt_i = 0; opt_i < *argc; ++opt_i ) {
130  if ( suppress_option == (*argv)[opt_i] ) {
131  // We are suppressing the output!
132  printStartupBanner = false;
133  // Remove this option!
134  // Note that (*argv)[*argc]==0 but convention so we copy it too!
135  for( int i = opt_i; i < *argc; ++i )
136  (*argv)[i] = (*argv)[i+1];
137  --*argc;
138  }
139  }
140  if (out && printStartupBanner) {
141  *out << oss.str() << std::flush;
142  }
143 
144 #endif
145 
146 }
147 
148 
150 {
151 
152 #ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
153  try {
154  Kokkos::finalize_all();
155  }
156  catch (const std::runtime_error& e) {
157  std::cerr << "Kokkos::finalize_all failed:\n"
158  << e.what() << "\n";
159  }
160 #endif
161 
162  haveMPIState_ = false;
163 #ifdef HAVE_MPI
164  const int mpierr = ::MPI_Finalize();
165  mpiIsFinalized_ = (mpierr == 0);
166  if (mpierr != 0)
167  std::cerr << "Error code " << mpierr << " returned from MPI_Finalize()\n";
168 #else
169  mpiIsFinalized_ = true;
170 #endif
171 }
172 
173 
175  justInTimeInitialize();
176  return haveMPIState_;
177 }
178 
179 
181 {
182  return mpiIsFinalized_;
183 }
184 
185 
187 {
188  justInTimeInitialize();
189  return rank_;
190 }
191 
192 
194  justInTimeInitialize();
195  return nProc_;
196 }
197 
198 
200 {
201  justInTimeInitialize();
202 #ifdef HAVE_MPI
203  MPI_Barrier(MPI_COMM_WORLD);
204 #endif
205 }
206 
207 
208 int GlobalMPISession::sum(int localVal)
209 {
210  justInTimeInitialize();
211 #ifdef HAVE_MPI
212  int globalSum = -1;
213  MPI_Allreduce(&localVal, &globalSum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
214  return globalSum;
215 #else
216  return localVal;
217 #endif
218 }
219 
220 
221 void GlobalMPISession::allGather(int localVal, const ArrayView<int> &allVals)
222 {
223  justInTimeInitialize();
224  TEUCHOS_ASSERT_EQUALITY(allVals.size(), getNProc());
225 #ifdef HAVE_MPI
226  MPI_Allgather( &localVal, 1, MPI_INT, allVals.getRawPtr(), 1, MPI_INT,
227  MPI_COMM_WORLD);
228 #else
229  allVals[0] = localVal;
230 #endif
231 }
232 
233 
234 // private
235 
236 
237 void GlobalMPISession::initialize( std::ostream *out )
238 {
239 #ifdef HAVE_MPI
240 
241  if(mpiIsFinalized_) {
242  // MPI has aleady been finalized so we have a serial machine again!
243  rank_ = 0;
244  nProc_ = 1;
245  return;
246  }
247 
248  if(haveMPIState_) {
249  return; // We already have what we need!
250  }
251 
252  // We don't have the state of MPI so the constructor for this class must not
253  // have been called. However, if MPI has been called in another way we
254  // can still get the state of MPI_COMM_WORLD here.
255 
256  int mpiHasBeenStarted = 0;
257  MPI_Initialized(&mpiHasBeenStarted);
258 
259  if(!mpiHasBeenStarted)
260  return; // We have to give up and just leave NProc_ and rank_ at the default values.
261 
262  // Get the state of MPI
263  // Don't throw exceptions here since this part of the code
264  // is used by TEUCHOS_STANDARD_CATCH_STATEMENTS().
265  // See bug #6192 <https://software.sandia.gov/bugzilla/show_bug.cgi?id=6192>.
266  int mpierr = 0;
267  mpierr = ::MPI_Comm_rank( MPI_COMM_WORLD, &rank_ );
268  if (mpierr != 0) {
269  *out << "Error code=" << mpierr << " detected in MPI_Comm_rank()"
270  << std::endl;
271  }
272 
273  mpierr = ::MPI_Comm_size( MPI_COMM_WORLD, &nProc_ );
274  if (mpierr != 0) {
275  *out << "Error code=" << mpierr << " detected in MPI_Comm_size()"
276  << std::endl;
277  }
278 
279  haveMPIState_ = true;
280  mpiIsFinalized_ = false;
281 
282 #endif // HAVE_MPI
283 
284 }
285 
286 
287 void GlobalMPISession::justInTimeInitialize()
288 {
289  if(!haveMPIState_)
290  initialize(&std::cerr);
291 }
292 
293 
294 } // namespace Teuchos
static int getRank()
The rank of the calling process in MPI_COMM_WORLD.
static int getNProc()
The number of processes in MPI_COMM_WORLD.
size_type size() const
The total number of items in the managed array.
static bool mpiIsFinalized()
Return whether MPI was already finalized.
GlobalMPISession(int *argc, char ***argv, std::ostream *out=&std::cout)
Calls MPI_Init() if MPI is enabled.
T * getRawPtr() const
Return a raw pointer to beginning of array or NULL if unsized.
static int sum(int localVal)
Sum a set of integers across processes.
The Teuchos namespace contains all of the classes, structs and enums used by Teuchos, as well as a number of utility routines.
A MPI utilities class, providing methods for initializing, finalizing, and querying the global MPI se...
static void barrier()
Call MPI_Barrier() on MPI_COMM_WORLD.
static bool mpiIsInitialized()
Return whether MPI was initialized.
static void allGather(int localVal, const ArrayView< int > &allVals)
Global all-to-all of a set of integers across processes.
#define TEUCHOS_ASSERT_EQUALITY(val1, val2)
This macro is checks that to numbers are equal and if not then throws an exception with a good error ...
~GlobalMPISession()
Call MPI_Finalize() if MPI is enabled.