Teuchos - Trilinos Tools Package  Version of the Day
Teuchos_MpiReductionOpSetter.cpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Teuchos: Common Tools Package
5 // Copyright (2004) Sandia Corporation
6 //
7 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8 // license for use of this work by or on behalf of the U.S. Government.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ***********************************************************************
40 // @HEADER
41 
43 
44 #ifdef HAVE_MPI
45 # ifdef MPIAPI
46 # define CALL_API MPIAPI
47 # else
48 # define CALL_API
49 # endif
50 
51 //
52 // mfh 23 Nov 2014: My commits over the past day or two attempt to
53 // address Bug 6263. In particular, the code as I found it had the
54 // following issues:
55 //
56 // 1. Static RCP instances (that persist past return of main())
57 // 2. Static MPI_Op datum (that persists past MPI_Finalize())
58 // 3. Code won't work with MPI_THREAD_{SERIALIZED,MULTIPLE},
59 // because it assumes that only one MPI_Op for reductions
60 // is needed at any one time
61 //
62 // I'm neglecting Issue #3 for now and focusing on the first two
63 // issues. #1 goes away if one doesn't use RCPs and handles
64 // deallocation manually (we could also use std::shared_ptr, but that
65 // would require C++11). #2 goes away with the standard idiom of an
66 // MPI_Finalize() hook (attach a (key,value) pair to MPI_COMM_SELF).
67 //
68 
69 extern "C" {
70 
71 // The MPI_Op that implements the reduction or scan operation will
72 // call this function. We only need to create the MPI_Op once
73 // (lazily, on demand). This function in turn will invoke
74 // theReductOp_ (see below), which gets set to the current reduction
75 // operation. Thus, we only never need to create one MPI_Op, but we
76 // swap out the function. This is meant to save overhead in creating
77 // and freeing MPI_Op for each reduction or scan.
78 void CALL_API
79 Teuchos_MPI_reduction_op (void* invec, void* inoutvec,
80  int* len, MPI_Datatype* datatype);
81 } // extern "C"
82 
83 namespace { // anonymous
84 
85 //
86 // theMpiOp_: The MPI_Op singleton that implements the Teuchos
87 // reduction or scan operation. We only need to create the MPI_Op
88 // once (lazily, on demand). When we create the MPI_Op, we stash its
89 // "destructor" in MPI_COMM_SELF so that it gets freed at
90 // MPI_Finalize. (This is a standard MPI idiom.)
91 //
92 // This variable is global, persistent (until MPI_Finalize is called),
93 // and initialized lazily.
94 MPI_Op theMpiOp_ = MPI_OP_NULL;
95 
96 // The current reduction or scan "function." (It's actually a class
97 // instance.)
98 //
99 // This static variable is _NOT_ persistent. It does not need
100 // deallocation.
101 const Teuchos::Details::MpiReductionOpBase* theReductOp_ = NULL;
102 
103 // Free the given MPI_Op, and return the error code returned by MPI_Op_free.
104 int
105 freeMpiOp (MPI_Op* op)
106 {
107  // If this function is called as an MPI_Finalize hook, MPI should
108  // still be initialized at this point, and it should be OK to call
109  // MPI functions. Thus, we don't need to check if MPI is
110  // initialized.
111  int err = MPI_SUCCESS;
112  if (op != NULL) {
113  err = MPI_Op_free (op);
114  if (err == MPI_SUCCESS) {
115  // No externally visible side effects unless the above function succeeded.
116  *op = MPI_OP_NULL;
117  }
118  }
119  return err;
120 }
121 
122 // Free the MPI_Op singleton (theMpiOp_), and return the error code
123 // returned by freeMpiOp(). As a side effect, if freeing succeeds,
124 // set theMpiOp_ to MPI_OP_NULL.
125 //
126 // This is the singleton's "destructor" that we attach to
127 // MPI_COMM_SELF as an MPI_Finalize hook.
128 int
129 freeMpiOpCallback (MPI_Comm, int, void*, void*)
130 {
131  // We don't need any of the arguments to this function, since we're
132  // just freeing the singleton.
133  if (theMpiOp_ == MPI_OP_NULL) {
134  return MPI_SUCCESS;
135  } else {
136  return freeMpiOp (&theMpiOp_);
137  }
138 }
139 
140 // Create the MPI_Op singleton that invokes the
141 // Teuchos_MPI_reduction_op callback. Assign the MPI_Op to theMpiOp_,
142 // and set it up with an MPI_Finalize hook so it gets freed
143 // automatically.
144 void createReductOp ()
145 {
146  // This function has side effects on the global singleton theMpiOp_.
147  // This check ensures that the function is idempotent. We only need
148  // to create the MPI_Op singleton once.
149  if (theMpiOp_ != MPI_OP_NULL) {
150  return; // We've already called this function; we don't have to again.
151  }
152 
153  MPI_Op mpi_op = MPI_OP_NULL;
154 
155  // FIXME (mfh 23 Nov 2014) I found the following comment here:
156  // "Assume op is commutative". That's what it means to pass 1 as
157  // the second argument. I don't know whether it's a good idea to
158  // keep that assumption.
159  int err = MPI_Op_create (&Teuchos_MPI_reduction_op, 1, &mpi_op);
161  err != MPI_SUCCESS, std::runtime_error, "Teuchos::createReductOp: "
162  "MPI_Op_create (for custom reduction operator) failed!");
163 
164  // Use the standard MPI idiom (attach a (key,value) pair to
165  // MPI_COMM_SELF with a "destructor" function) in order that
166  // theMpiOp_ gets freed at MPI_Finalize, if necessary.
167 
168  // 'key' is an output argument of MPI_Comm_create_keyval.
169  int key = MPI_KEYVAL_INVALID;
170  err = MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN, freeMpiOpCallback,
171  &key, NULL);
172  if (err != MPI_SUCCESS) {
173  // Attempt to clean up by freeing the newly created MPI_Op. If
174  // cleaning up fails, just let it slide, since we're already in
175  // trouble if MPI can't create a (key,value) pair.
176  (void) MPI_Op_free (&mpi_op);
178  true, std::runtime_error, "Teuchos::createReductOp: "
179  "MPI_Comm_create_keyval (for custom reduction operator) failed!");
180  }
181  int val = key; // doesn't matter
182 
183  // Attach the attribute to MPI_COMM_SELF.
184  err = MPI_Comm_set_attr (MPI_COMM_SELF, key, &val);
185  if (err != MPI_SUCCESS) {
186  // MPI (versions up to and including 3.0) doesn't promise correct
187  // behavior after any function returns something other than
188  // MPI_SUCCESS. Thus, it's not required to try to free the new
189  // key via MPI_Comm_free_keyval. Furthermore, if something went
190  // wrong with MPI_Comm_set_attr, it's likely that the attribute
191  // mechanism is broken. Thus, it would be unwise to call
192  // MPI_Comm_free_keyval.
193  //
194  // I optimistically assume that the "rest" of MPI is still
195  // working, and attempt to clean up by freeing the newly created
196  // MPI_Op. If cleaning up fails, just let it slide, since we're
197  // already in trouble if MPI can't create a (key,value) pair.
198  (void) MPI_Op_free (&mpi_op);
200  true, std::runtime_error, "Teuchos::createReductOp: "
201  "MPI_Comm_set_attr (for custom reduction operator) failed!");
202  }
203 
204  // It looks weird to "free" the key right away. However, this does
205  // not actually cause the "destructor" to be called. It only gets
206  // called at MPI_FINALIZE. See MPI 3.0 standard, Section 6.7.2,
207  // MPI_COMM_FREE_KEYVAL:
208  //
209  // "Note that it is not erroneous to free an attribute key that is
210  // in use, because the actual free does not transpire until after
211  // all references (in other communicators on the process) to the key
212  // have been freed. These references need to be explicitly freed by
213  // the program, either via calls to MPI_COMM_DELETE_ATTR that free
214  // one attribute instance, or by calls to MPI_COMM_FREE that free
215  // all attribute instances associated with the freed communicator."
216  //
217  // We rely here on the latter mechanism. MPI_FINALIZE calls
218  // MPI_COMM_FREE on MPI_COMM_SELF, so we do not need to call it
219  // explicitly.
220  //
221  // It's not clear what to do if the MPI_* calls above succeeded, but
222  // this call fails (i.e., returns != MPI_SUCCESS). We could throw;
223  // this would make sense to do, because MPI (versions up to and
224  // including 3.0) doesn't promise correct behavior after any MPI
225  // function returns something other than MPI_SUCCESS. We could also
226  // be optimistic and just ignore the return value, hoping that if
227  // the above calls succeeded, then the communicator will get freed
228  // at MPI_FINALIZE, even though the unfreed key may leak memory (see
229  // Bug 6338). I've chosen the latter.
230  (void) MPI_Comm_free_keyval (&key);
231 
232  // The "transaction" succeeded; save the result.
233  theMpiOp_ = mpi_op;
234 }
235 
236 void
237 setReductOp (const Teuchos::Details::MpiReductionOpBase* reductOp)
238 {
239  if (theMpiOp_ == MPI_OP_NULL) {
240  createReductOp ();
241  }
242  theReductOp_ = reductOp;
243 }
244 
245 } // namespace (anonymous)
246 
247 extern "C" {
248 
249 void CALL_API
250 Teuchos_MPI_reduction_op (void* invec,
251  void* inoutvec,
252  int* len,
253  MPI_Datatype* datatype)
254 {
255  if (theReductOp_ != NULL) {
256  theReductOp_->reduce (invec, inoutvec, len, datatype);
257  }
258 }
259 
260 } // extern "C"
261 
262 namespace Teuchos {
263 namespace Details {
264 
265 MPI_Op setMpiReductionOp (const MpiReductionOpBase& reductOp)
266 {
267  setReductOp (&reductOp);
269  (theMpiOp_ == MPI_OP_NULL, std::logic_error, "Teuchos::Details::"
270  "setMpiReductionOp: Failed to create reduction MPI_Op theMpiOp_. "
271  "This should never happen. "
272  "Please report this bug to the Teuchos developers.");
273  return theMpiOp_;
274 }
275 
276 } // namespace Details
277 } // namespace Teuchos
278 
279 #endif // HAVE_MPI
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Macro for throwing an exception with breakpointing to ease debugging.
Namespace of implementation details.
The Teuchos namespace contains all of the classes, structs and enums used by Teuchos, as well as a number of utility routines.
Implementation detail of Teuchos' MPI wrapper.