Tpetra parallel linear algebra  Version of the Day
Tpetra_MpiPlatform.cpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ************************************************************************
40 // @HEADER
41 
42 #include <Tpetra_ConfigDefs.hpp>
43 
44 // mfh 29 Jun 2014: It makes life easier for Sierra developers if
45 // Trilinos just builds all .cpp files unconditionally, rather than
46 // making the decision whether to build them dependent on CMake
47 // options. Thus, we just exclude all the content of this file if MPI
48 // is not enabled.
49 #ifdef HAVE_TPETRA_MPI
50 # include <Tpetra_MpiPlatform.hpp>
51 
52 namespace Tpetra {
53 
55  MpiPlatform () :
56  comm_ (Teuchos::createMpiComm<int> (Teuchos::opaqueWrapper<MPI_Comm> (MPI_COMM_WORLD))),
57  node_ (Teuchos::null)
58  {
59  // mfh 29 Jun 2014: Don't initialize the Node yet. This ensures
60  // that (new) Kokkos won't get initialized with the wrong
61  // command-line arguments, at least not until getNode() is called.
62  // Initializing Kokkos with the wrong command-line arguments may
63  // result in poor performance due to the wrong assignment of
64  // software threads to hardware execution units.
65  }
66 
67  MpiPlatform<Tpetra::Details::DefaultTypes::node_type>::
68  MpiPlatform (int* argc, char*** argv) :
69  comm_ (Teuchos::null),
70  node_ (Teuchos::null)
71  {
72  initialize (argc, argv);
73  comm_ = getDefaultComm ();
74 
75  // mfh 29 Jun 2014: Don't initialize the Node yet. See above note.
76  //
77  // if (node_.is_null ()) {
78  // node_ = KokkosClassic::DefaultNode::getDefaultNode ();
79  // }
80  }
81 
82  MpiPlatform<Tpetra::Details::DefaultTypes::node_type>::
83  MpiPlatform (const Teuchos::RCP<NodeType>& node) :
84  comm_ (Teuchos::createMpiComm<int> (Teuchos::opaqueWrapper<MPI_Comm> (MPI_COMM_WORLD))),
85  node_ (node)
86  {
87  // mfh 29 Jun 2014: Don't initialize the Node yet. This ensures
88  // that (new) Kokkos won't get initialized with the wrong
89  // command-line arguments, at least not until getNode() is
90  // called. Initializing Kokkos with the wrong command-line
91  // arguments may result in poor performance due to the wrong
92  // assignment of software threads to hardware execution units.
93  //
94  // if (node_.is_null ()) {
95  // node_ = KokkosClassic::DefaultNode::getDefaultNode ();
96  // }
97  }
98 
99  MpiPlatform<Tpetra::Details::DefaultTypes::node_type>::
100  MpiPlatform (int* argc, char*** argv, const Teuchos::RCP<NodeType>& node) :
101  comm_ (Teuchos::null),
102  node_ (node)
103  {
104  initialize (argc, argv);
105  comm_ = getDefaultComm ();
106 
107  // mfh 29 Jun 2014: Don't initialize the Node yet. See above note.
108  //
109  // if (node_.is_null ()) {
110  // node_ = KokkosClassic::DefaultNode::getDefaultNode ();
111  // }
112  }
113 
114  MpiPlatform<Tpetra::Details::DefaultTypes::node_type>::
115  MpiPlatform (const Teuchos::RCP<NodeType>& node,
116  const Teuchos::RCP<const Teuchos::OpaqueWrapper<MPI_Comm> >& rawMpiComm)
117  : comm_ (Teuchos::null),
118  node_ (node)
119  {
120  TEUCHOS_TEST_FOR_EXCEPTION(
121  rawMpiComm.is_null (), std::invalid_argument, "Tpetra::MpiPlatform "
122  "constructor: The input RCP<OpaqueWrapper<MPI_Comm> > is null. That "
123  "means something different than MPI_COMM_NULL. If you want to give "
124  "MPI_COMM_NULL to this constructor, please wrap MPI_COMM_NULL in a "
125  "nonnull Teuchos::OpaqueWrapper by using the "
126  "Teuchos::opaqueWrapper<MPI_Comm>() nonmember constructor.");
127  comm_ = Teuchos::createMpiComm<int> (rawMpiComm);
128 
129  // mfh 29 Jun 2014: Don't initialize the Node yet. See above note.
130  //
131  // if (node_.is_null ()) {
132  // node_ = KokkosClassic::DefaultNode::getDefaultNode ();
133  // }
134  }
135 
136  MpiPlatform<Tpetra::Details::DefaultTypes::node_type>::
137  MpiPlatform (int* argc,
138  char*** argv,
139  const Teuchos::RCP<NodeType>& node,
140  const Teuchos::RCP<const Teuchos::OpaqueWrapper<MPI_Comm> >& rawMpiComm)
141  : comm_ (Teuchos::null),
142  node_ (node)
143  {
144  TEUCHOS_TEST_FOR_EXCEPTION(
145  rawMpiComm.is_null (), std::invalid_argument, "Tpetra::MpiPlatform "
146  "constructor: The input RCP<OpaqueWrapper<MPI_Comm> > is null. That "
147  "means something different than MPI_COMM_NULL. If you want to give "
148  "MPI_COMM_NULL to this constructor, please wrap MPI_COMM_NULL in a "
149  "nonnull Teuchos::OpaqueWrapper by using the "
150  "Teuchos::opaqueWrapper<MPI_Comm>() nonmember constructor.");
151  comm_ = Teuchos::createMpiComm<int> (rawMpiComm);
152 
153  // NOTE (mfh 29 Jun 2014): The OpaqueWrapper might wrap the
154  // MPI_Comm in something that calls MPI_Comm_free. Thus, we
155  // can't just ignore it; we have to give it to the comm_ so that
156  // its destructor (which might call MPI_Comm_free) will be
157  // called at the right time. This is why we don't set comm
158  // using getDefaultComm(). This is also why we pass comm_
159  // directly to initialize(): that way there aren't two
160  // references to the raw MPI_Comm floating around, and comm_'s
161  // destructor will get to do the right thing.
162  initialize (argc, argv, comm_);
163 
164  // mfh 29 Jun 2014: Don't initialize the Node yet. See above note.
165  //
166  // if (node_.is_null ()) {
167  // node_ = KokkosClassic::DefaultNode::getDefaultNode ();
168  // }
169  }
170 
171  MpiPlatform<Tpetra::Details::DefaultTypes::node_type>::
172  MpiPlatform (const Teuchos::RCP<NodeType>& node, MPI_Comm rawMpiComm)
173  : comm_ (Teuchos::createMpiComm<int> (Teuchos::opaqueWrapper<MPI_Comm> (rawMpiComm))),
174  node_ (node)
175  {
176  // mfh 29 Jun 2014: Don't initialize the Node yet. See above note.
177  //
178  // if (node_.is_null ()) {
179  // node_ = KokkosClassic::DefaultNode::getDefaultNode ();
180  // }
181  }
182 
183  MpiPlatform<Tpetra::Details::DefaultTypes::node_type>::
184  MpiPlatform (int* argc,
185  char*** argv,
186  const Teuchos::RCP<NodeType>& node,
187  MPI_Comm rawMpiComm)
188  : comm_ (Teuchos::null),
189  node_ (node)
190  {
191  initialize (argc, argv, rawMpiComm);
192  comm_ = getDefaultComm ();
193 
194  // mfh 29 Jun 2014: Don't initialize the Node yet. See above note.
195  //
196  // if (node_.is_null ()) {
197  // node_ = KokkosClassic::Details::getNode<Node> ();
198  // }
199  }
200 
201  MpiPlatform<Tpetra::Details::DefaultTypes::node_type>::
202  ~MpiPlatform () {}
203 
204  Teuchos::RCP<const Teuchos::Comm<int> >
205  MpiPlatform<Tpetra::Details::DefaultTypes::node_type>::
206  getComm () const {
207  TEUCHOS_TEST_FOR_EXCEPTION(
208  comm_.is_null (), std::logic_error, "Tpetra::MpiPlatform::getComm: "
209  "The default communicator is null. This should never happen. "
210  "Please report this bug to the Tpetra developers.");
211  return comm_;
212  }
213 
214  Teuchos::RCP<MpiPlatform<Tpetra::Details::DefaultTypes::node_type>::NodeType>
215  MpiPlatform<Tpetra::Details::DefaultTypes::node_type>::
216  getNode () const
217  {
218  typedef MpiPlatform<NodeType> this_type;
219  if (node_.is_null ()) {
220  // NOTE (mfh 29 Jun 2014): Creating an instance of one of the
221  // new Kokkos wrapper Nodes _must_ call Kokkos::initialize.
222  // If Kokkos has not been initialized yet, this may result in
223  // Kokkos being initialized correctly, since we have no way to
224  // pass it the command-line arguments at this point. This is
225  // why we should prefer the *Platform constructors that take
226  // argc and argv, since they can (and do) call
227  // Kokkos::initialize (by calling Tpetra::initialize).
228  //
229  // mfh 29 Jun 2014: We're only keeping the *Platform classes
230  // for backwards compatibility anyway, so I don't feel bad
231  // about the const_cast here.
232  const_cast<this_type*> (this)->node_ =
233  KokkosClassic::DefaultNode::getDefaultNode ();
234  TEUCHOS_TEST_FOR_EXCEPTION(
235  node_.is_null (), std::logic_error, "Tpetra::MpiPlatform::getNode: "
236  "KokkosClassic::DefaultNode::getDefaultNode() returned null. "
237  "This should never happen. "
238  "Please report this bug to the Tpetra developers.");
239  }
240  return node_;
241  }
242 
243 } // namespace Tpetra
244 
245 #endif // HAVE_TPETRA_MPI
void initialize(int *argc, char ***argv)
Initialize Tpetra.
Namespace Tpetra contains the class and methods constituting the Tpetra library.
MpiPlatform(const Teuchos::RCP< NodeType > &node)
Constructor that accepts a Kokkos Node.
Teuchos::RCP< const Teuchos::Comm< int > > getDefaultComm()
Get Tpetra&#39;s default communicator.
Definition: Tpetra_Core.cpp:97