Tpetra parallel linear algebra  Version of the Day
Tpetra_MpiPlatform.hpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ************************************************************************
40 // @HEADER
41 
42 #ifndef TPETRA_MPIPLATFORM_HPP
43 #define TPETRA_MPIPLATFORM_HPP
44 
45 #include <Tpetra_ConfigDefs.hpp>
46 #include <Tpetra_Core.hpp>
47 #include <Kokkos_DefaultNode.hpp>
48 #include <Teuchos_DefaultMpiComm.hpp>
49 #include <Teuchos_Describable.hpp>
50 
51 namespace Tpetra {
52 
83  template <class Node>
84  class MpiPlatform : public Teuchos::Describable {
85  public:
87 
88 
90  typedef Node NodeType;
91 
93 
95 
106  explicit MpiPlatform (const Teuchos::RCP<NodeType>& node) :
107  comm_ (Teuchos::createMpiComm<int> (Teuchos::opaqueWrapper<MPI_Comm> (MPI_COMM_WORLD))),
108  node_ (node)
109  {
110  // mfh 29 Jun 2014: Don't initialize the Node yet. This ensures
111  // that (new) Kokkos won't get initialized with the wrong
112  // command-line arguments, at least not until getNode() is
113  // called. Initializing Kokkos with the wrong command-line
114  // arguments may result in poor performance due to the wrong
115  // assignment of software threads to hardware execution units.
116  //
117  // if (node_.is_null ()) {
118  // node_ = KokkosClassic::Details::getNode<NodeType> ();
119  // }
120  }
121 
133  MpiPlatform (int* argc,
134  char*** argv,
135  const Teuchos::RCP<NodeType>& node) :
136  comm_ (Teuchos::null),
137  node_ (node)
138  {
139  initialize (argc, argv);
140  comm_ = getDefaultComm ();
141 
142  // mfh 29 Jun 2014: Don't initialize the Node yet. See above note.
143  //
144  // if (node_.is_null ()) {
145  // node_ = KokkosClassic::Details::getNode<Node> ();
146  // }
147  }
148 
169  MpiPlatform (const Teuchos::RCP<NodeType>& node,
170  const Teuchos::RCP<const Teuchos::OpaqueWrapper<MPI_Comm> >& rawMpiComm)
171  : comm_ (Teuchos::null),
172  node_ (node)
173  {
174  TEUCHOS_TEST_FOR_EXCEPTION(
175  rawMpiComm.is_null (), std::invalid_argument, "Tpetra::MpiPlatform "
176  "constructor: The input RCP<OpaqueWrapper<MPI_Comm> > is null. That "
177  "means something different than MPI_COMM_NULL. If you want to give "
178  "MPI_COMM_NULL to this constructor, please wrap MPI_COMM_NULL in a "
179  "nonnull Teuchos::OpaqueWrapper by using the "
180  "Teuchos::opaqueWrapper<MPI_Comm>() nonmember constructor.");
181  comm_ = Teuchos::createMpiComm<int> (rawMpiComm);
182 
183  // mfh 29 Jun 2014: Don't initialize the Node yet. See above note.
184  //
185  // if (node_.is_null ()) {
186  // node_ = KokkosClassic::Details::getNode<NodeType> ();
187  // }
188  }
189 
213  MpiPlatform (int* argc,
214  char*** argv,
215  const Teuchos::RCP<NodeType>& node,
216  const Teuchos::RCP<const Teuchos::OpaqueWrapper<MPI_Comm> >& rawMpiComm)
217  : comm_ (Teuchos::null),
218  node_ (node)
219  {
220  TEUCHOS_TEST_FOR_EXCEPTION(
221  rawMpiComm.is_null (), std::invalid_argument, "Tpetra::MpiPlatform "
222  "constructor: The input RCP<OpaqueWrapper<MPI_Comm> > is null. That "
223  "means something different than MPI_COMM_NULL. If you want to give "
224  "MPI_COMM_NULL to this constructor, please wrap MPI_COMM_NULL in a "
225  "nonnull Teuchos::OpaqueWrapper by using the "
226  "Teuchos::opaqueWrapper<MPI_Comm>() nonmember constructor.");
227  comm_ = Teuchos::createMpiComm<int> (rawMpiComm);
228 
229  // NOTE (mfh 29 Jun 2014): The OpaqueWrapper might wrap the
230  // MPI_Comm in something that calls MPI_Comm_free. Thus, we
231  // can't just ignore it; we have to give it to the comm_ so that
232  // its destructor (which might call MPI_Comm_free) will be
233  // called at the right time. This is why we don't set comm
234  // using getDefaultComm(). This is also why we pass comm_
235  // directly to initialize(): that way there aren't two
236  // references to the raw MPI_Comm floating around, and comm_'s
237  // destructor will get to do the right thing.
238  initialize (argc, argv, comm_);
239 
240  // mfh 29 Jun 2014: Don't initialize the Node yet. See above note.
241  //
242  // if (node_.is_null ()) {
243  // node_ = KokkosClassic::Details::getNode<NodeType> ();
244  // }
245  }
246 
259  MpiPlatform (const Teuchos::RCP<NodeType>& node, MPI_Comm rawMpiComm)
260  : comm_ (Teuchos::createMpiComm<int> (Teuchos::opaqueWrapper<MPI_Comm> (rawMpiComm))),
261  node_ (node)
262  {
263  // mfh 29 Jun 2014: Don't initialize the Node yet. See above note.
264  //
265  // if (node_.is_null ()) {
266  // node_ = KokkosClassic::Details::getNode<NodeType> ();
267  // }
268  }
269 
284  MpiPlatform (int* argc,
285  char*** argv,
286  const Teuchos::RCP<NodeType>& node,
287  MPI_Comm rawMpiComm)
288  : comm_ (Teuchos::null),
289  node_ (node)
290  {
291  initialize (argc, argv, rawMpiComm);
292  comm_ = getDefaultComm ();
293 
294  // mfh 29 Jun 2014: Don't initialize the Node yet. See above note.
295  //
296  // if (node_.is_null ()) {
297  // node_ = KokkosClassic::Details::getNode<NodeType> ();
298  // }
299  }
300 
302  virtual ~MpiPlatform () {}
303 
305 
307 
309  Teuchos::RCP<const Teuchos::Comm<int> > getComm () const {
310  TEUCHOS_TEST_FOR_EXCEPTION(
311  comm_.is_null (), std::logic_error, "Tpetra::MpiPlatform::getComm: "
312  "The default communicator is null. This should never happen. "
313  "Please report this bug to the Tpetra developers.");
314  return comm_;
315  }
316 
324  Teuchos::RCP<NodeType> getNode () const {
325  typedef MpiPlatform<NodeType> this_type;
326  if (node_.is_null ()) {
327  // NOTE (mfh 29 Jun 2014): Creating an instance of one of the
328  // new Kokkos wrapper Nodes _must_ call Kokkos::initialize.
329  // If Kokkos has not been initialized yet, this may result in
330  // Kokkos being initialized correctly, since we have no way to
331  // pass it the command-line arguments at this point. This is
332  // why we should prefer the *Platform constructors that take
333  // argc and argv, since they can (and do) call
334  // Kokkos::initialize (by calling Tpetra::initialize).
335  //
336  // mfh 29 Jun 2014: We're only keeping the *Platform classes
337  // for backwards compatibility anyway, so I don't feel bad
338  // about the const_cast here.
339  const_cast<this_type*> (this)->node_ =
340  KokkosClassic::Details::getNode<NodeType> ();
341  TEUCHOS_TEST_FOR_EXCEPTION(
342  node_.is_null (), std::logic_error, "Tpetra::MpiPlatform::getNode: "
343  "KokkosClassic::Details::getNode<NodeType>() returned null. "
344  "This should never happen. "
345  "Please report this bug to the Tpetra developers.");
346  }
347  return node_;
348  }
349 
351  protected:
353  Teuchos::RCP<const Teuchos::Comm<int> > comm_;
355  Teuchos::RCP<NodeType> node_;
356 
357  private:
359  MpiPlatform (const MpiPlatform<NodeType>& platform);
361  MpiPlatform& operator= (const MpiPlatform<NodeType>& platform);
362  };
363 
370  template <>
372  public Teuchos::Describable {
373  public:
375 
376 
379 
381 
383 
385  MpiPlatform ();
386 
392  MpiPlatform (int* argc, char*** argv);
393 
410  explicit MpiPlatform (const Teuchos::RCP<NodeType>& node);
411 
423  MpiPlatform (int* argc, char*** argv, const Teuchos::RCP<NodeType>& node);
424 
445  MpiPlatform (const Teuchos::RCP<NodeType>& node,
446  const Teuchos::RCP<const Teuchos::OpaqueWrapper<MPI_Comm> >& rawMpiComm);
447 
471  MpiPlatform (int* argc,
472  char*** argv,
473  const Teuchos::RCP<NodeType>& node,
474  const Teuchos::RCP<const Teuchos::OpaqueWrapper<MPI_Comm> >& rawMpiComm);
475 
488  MpiPlatform (const Teuchos::RCP<NodeType>& node, MPI_Comm rawMpiComm);
489 
504  MpiPlatform (int* argc,
505  char*** argv,
506  const Teuchos::RCP<NodeType>& node,
507  MPI_Comm rawMpiComm);
508 
510  virtual ~MpiPlatform ();
511 
513 
515 
517  Teuchos::RCP<const Teuchos::Comm<int> > getComm () const;
518 
526  Teuchos::RCP<NodeType> getNode () const;
527 
529  private:
531  MpiPlatform (const MpiPlatform<NodeType>& platform);
532 
534  MpiPlatform& operator= (const MpiPlatform<NodeType>& platform);
535 
536  protected:
538  RCP<const Teuchos::Comm<int> > comm_;
539 
541  RCP<NodeType> node_;
542  };
543 
544 } // namespace Tpetra
545 
546 #endif // TPETRA_MPIPLATFORM_HPP
void initialize(int *argc, char ***argv)
Initialize Tpetra.
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Implementation of the Platform concept for MPI-based platforms.
MpiPlatform(int *argc, char ***argv, const Teuchos::RCP< NodeType > &node)
Constructor that accepts the same arguments as MPI_Init(), plus a Kokkos Node.
KokkosClassic::DefaultNode::DefaultNodeType node_type
Default value of Node template parameter.
Teuchos::RCP< NodeType > getNode() const
The default Kokkos Node instance.
Teuchos::RCP< NodeType > node_
Kokkos Node object instantiated for the platform.
MpiPlatform(const Teuchos::RCP< NodeType > &node, const Teuchos::RCP< const Teuchos::OpaqueWrapper< MPI_Comm > > &rawMpiComm)
Constructor that accepts a Kokkos Node and a wrapped MPI communicator.
Node NodeType
Kokkos Node type; the template parameter of this class.
virtual ~MpiPlatform()
Destructor (virtual for memory safety of derived classes).
MpiPlatform(int *argc, char ***argv, const Teuchos::RCP< NodeType > &node, const Teuchos::RCP< const Teuchos::OpaqueWrapper< MPI_Comm > > &rawMpiComm)
Constructor that accepts the same arguments as MPI_Init(), plus a Kokkos Node and a wrapped MPI commu...
Teuchos::RCP< const Teuchos::Comm< int > > comm_
Teuchos::Comm object instantiated for the platform.
RCP< NodeType > node_
Kokkos Node object instantiated for the platform.
RCP< const Teuchos::Comm< int > > comm_
Teuchos::Comm object instantiated for the platform.
Functions for initializing and finalizing Tpetra.
MpiPlatform(int *argc, char ***argv, const Teuchos::RCP< NodeType > &node, MPI_Comm rawMpiComm)
Constructor that accepts the same arguments as MPI_Init(), plus a Kokkos Node and a raw MPI communica...
MpiPlatform(const Teuchos::RCP< NodeType > &node)
Constructor that accepts a Kokkos Node.
Teuchos::RCP< const Teuchos::Comm< int > > getDefaultComm()
Get Tpetra&#39;s default communicator.
Definition: Tpetra_Core.cpp:97
MpiPlatform(const Teuchos::RCP< NodeType > &node, MPI_Comm rawMpiComm)
Constructor that accepts a Kokkos Node and a raw MPI communicator.
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const
The Teuchos::Comm instance with which this object was created.
Tpetra::Details::DefaultTypes::node_type NodeType
Kokkos Node type; the template parameter of this class.