Tpetra parallel linear algebra  Version of the Day
Tpetra_Directory_def.hpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ************************************************************************
40 // @HEADER
41 
42 #ifndef TPETRA_DIRECTORY_HPP
43 #define TPETRA_DIRECTORY_HPP
44 
45 #include <Teuchos_as.hpp>
46 #include <Tpetra_ConfigDefs.hpp>
47 #include <Tpetra_Distributor.hpp>
48 #include <Tpetra_Map.hpp>
49 #include <Tpetra_DirectoryImpl.hpp>
51 #include "Tpetra_Directory_decl.hpp"
52 
53 namespace Tpetra {
54 
55  template<class LO, class GO, class NT>
57  impl_ (NULL)
58  {}
59 
60  template<class LO, class GO, class NT>
62  if (impl_ != NULL) {
63  delete impl_;
64  impl_ = NULL;
65  }
66  }
67 
68  template<class LO, class GO, class NT>
69  bool
71  return impl_ != NULL;
72  }
73 
74 
75  template<class LO, class GO, class NT>
76  void
78  initialize (const Map<LO, GO, NT>& map,
79  const Tpetra::Details::TieBreak<LO,GO>& tieBreak)
80  {
81  if (initialized ()) {
82  TEUCHOS_TEST_FOR_EXCEPTION(
83  impl_ == NULL, std::logic_error, "Tpetra::Directory::initialize: "
84  "The Directory claims that it has been initialized, "
85  "but its implementation object has not yet been created. "
86  "Please report this bug to the Tpetra developers.");
87  }
88  else {
89  TEUCHOS_TEST_FOR_EXCEPTION(
90  impl_ != NULL, std::logic_error, "Tpetra::Directory::initialize: "
91  "Directory implementation has already been initialized, "
92  "but initialized() returns false. "
93  "Please report this bug to the Tpetra developers.");
94 
95  // Create an implementation object of the appropriate type,
96  // depending on whether the Map is distributed or replicated,
97  // and contiguous or noncontiguous.
98  //
99  // mfh 06 Apr 2014: When a distributed noncontiguous Directory
100  // takes a TieBreak, all the entries (local indices and process
101  // ranks) owned by the Directory on the calling process pass
102  // through the TieBreak object. This may have side effects,
103  // such as the TieBreak object remembering whether there were
104  // any duplicates on the calling process. We want to extend use
105  // of a TieBreak object to other kinds of Directories. For a
106  // distributed contiguous Directory, the calling process owns
107  // all of the (PID,LID) pairs in the input Map. For a locally
108  // replicated contiguous Directory, Process 0 owns all of the
109  // (PID,LID) pairs in the input Map.
110  //
111  // It may seem silly to pass in a TieBreak when there are no
112  // ties to break. However, the TieBreak object gets to see all
113  // (PID,LID) pairs that the Directory owns on the calling
114  // process, and interface of TieBreak allows side effects.
115  // Users may wish to exploit them regardless of the kind of Map
116  // they pass in.
117  const Details::Directory<LO, GO, NT>* dir = NULL;
118  bool usedTieBreak = false;
119  if (map.isDistributed ()) {
120  if (map.isUniform ()) {
122  }
123  else if (map.isContiguous ()) {
125  }
126  else {
128  usedTieBreak = true;
129  }
130  }
131  else {
133 
134  if (tieBreak.mayHaveSideEffects () && map.getNodeNumElements () != 0) {
135  // We need the second clause in the above test because Map's
136  // interface provides an inclusive range of local indices.
137  const int myRank = map.getComm ()->getRank ();
138  // In a replicated Directory, Process 0 owns all the
139  // Directory's entries. This is an arbitrary assignment; any
140  // one process would do.
141  if (myRank == 0) {
142  std::vector<std::pair<int, LO> > pidLidList (1);
143  const LO minLocInd = map.getMinLocalIndex ();
144  const LO maxLocInd = map.getMaxLocalIndex ();
145  for (LO locInd = minLocInd; locInd <= maxLocInd; ++locInd) {
146  pidLidList[0] = std::make_pair (myRank, locInd);
147  const GO globInd = map.getGlobalElement (locInd);
148  // We don't care about the return value; we just want to
149  // invoke the side effects.
150  (void) tieBreak.selectedIndex (globInd, pidLidList);
151  }
152  }
153  }
154  usedTieBreak = true;
155  } // done with all different Map cases
156 
157  // If we haven't already used the TieBreak object, use it now.
158  // This code appears twice because ReplicatedDirectory is a
159  // special case: we already know what gets replicated.
160  if (! usedTieBreak && tieBreak.mayHaveSideEffects () &&
161  map.getNodeNumElements () != 0) {
162  // We need the third clause in the above test because Map's
163  // interface provides an inclusive range of local indices.
164  std::vector<std::pair<int, LO> > pidLidList (1);
165  const LO minLocInd = map.getMinLocalIndex ();
166  const LO maxLocInd = map.getMaxLocalIndex ();
167  const int myRank = map.getComm ()->getRank ();
168  for (LO locInd = minLocInd; locInd <= maxLocInd; ++locInd) {
169  pidLidList[0] = std::make_pair (myRank, locInd);
170  const GO globInd = map.getGlobalElement (locInd);
171  // We don't care about the return value; we just want to
172  // invoke the side effects.
173  (void) tieBreak.selectedIndex (globInd, pidLidList);
174  }
175  }
176 
177  impl_ = dir;
178  }
179  }
180 
181  template<class LO, class GO, class NT>
182  void
184  {
185  if (initialized ()) {
186  TEUCHOS_TEST_FOR_EXCEPTION(
187  impl_ == NULL, std::logic_error, "Tpetra::Directory::initialize: "
188  "The Directory claims that it has been initialized, "
189  "but its implementation object has not yet been created. "
190  "Please report this bug to the Tpetra developers.");
191  }
192  else {
193  TEUCHOS_TEST_FOR_EXCEPTION(
194  impl_ != NULL, std::logic_error, "Tpetra::Directory::initialize: "
195  "Directory implementation has already been initialized, "
196  "but initialized() returns false. "
197  "Please report this bug to the Tpetra developers.");
198 
199  // Create an implementation object of the appropriate type,
200  // depending on whether the Map is distributed or replicated,
201  // and contiguous or noncontiguous.
202  const Details::Directory<LO, GO, NT>* dir = NULL;
203  if (map.isDistributed ()) {
204  if (map.isUniform ()) {
206  }
207  else if (map.isContiguous ()) {
209  }
210  else {
212  }
213  }
214  else {
216  }
217  TEUCHOS_TEST_FOR_EXCEPTION(
218  dir == NULL, std::logic_error, "Tpetra::Directory::initialize: "
219  "Failed to create Directory implementation. "
220  "Please report this bug to the Tpetra developers.");
221  impl_ = dir;
222  }
223  }
224 
225  template<class LO, class GO, class NT>
229  const Teuchos::ArrayView<const GO>& globalIDs,
230  const Teuchos::ArrayView<int>& nodeIDs) const
231  {
232  if (! initialized ()) {
233  // This const_cast is super wrong, but "mutable" is also a lie,
234  // and Map's interface needs this method to be marked const for
235  // some reason.
236  const_cast<Directory<LO, GO, NT>* > (this)->initialize (map);
237  }
238  const bool computeLIDs = false;
239  return impl_->getEntries (map, globalIDs, nodeIDs, Teuchos::null, computeLIDs);
240  }
241 
242  template<class LO, class GO, class NT>
246  const Teuchos::ArrayView<const GO>& globalIDs,
247  const Teuchos::ArrayView<int>& nodeIDs,
248  const Teuchos::ArrayView<LO>& localIDs) const
249  {
250  if (! initialized ()) {
251  // This const_cast is super wrong, but "mutable" is also a lie,
252  // and Map's interface needs this method to be marked const for
253  // some reason.
254  const_cast<Directory<LO, GO, NT>* > (this)->initialize (map);
255  }
256  const bool computeLIDs = true;
257  return impl_->getEntries (map, globalIDs, nodeIDs, localIDs, computeLIDs);
258  }
259 
260  template<class LO, class GO, class NT>
261  bool Directory<LO, GO, NT>::isOneToOne (const Map<LO, GO, NT>& map) const {
262  if (! initialized ()) {
263  // This const_cast is super wrong, but "mutable" is also a lie,
264  // and Map's interface needs this method to be marked const for
265  // some reason.
266  const_cast<Directory<LO, GO, NT>* > (this)->initialize (map);
267  }
268  return impl_->isOneToOne (* (map.getComm ()));
269  }
270 
271  template<class LO, class GO, class NT>
272  std::string
274  {
275  using Teuchos::TypeNameTraits;
276 
277  std::ostringstream os;
278  os << "Directory"
279  << "<" << TypeNameTraits<LO>::name ()
280  << ", " << TypeNameTraits<GO>::name ()
281  << ", " << TypeNameTraits<NT>::name () << ">";
282  return os.str ();
283  }
284 
285 } // namespace Tpetra
286 
287 //
288 // Explicit instantiation macro
289 //
290 // Must be expanded from within the Tpetra namespace!
291 //
292 
293 #define TPETRA_DIRECTORY_INSTANT(LO,GO,NODE) \
294  \
295  template class Directory< LO , GO , NODE >; \
296 
297 #endif // TPETRA_DIRECTORY_HPP
Interface for breaking ties in ownership.
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Implementation of Directory for a locally replicated Map.
bool isContiguous() const
True if this Map is distributed contiguously, else false.
LookupStatus
Return status of Map remote index lookup (getRemoteIndexList()).
Definition of implementation details of Tpetra::Directory.
bool isUniform() const
Whether the range of global indices is uniform.
Implementation of Directory for a distributed noncontiguous Map.
bool isDistributed() const
Whether this Map is globally distributed or locally replicated.
LocalOrdinal getMinLocalIndex() const
The minimum local index.
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const
Accessors for the Teuchos::Comm and Kokkos Node objects.
Implementation of Directory for a distributed contiguous Map.
bool isOneToOne(const map_type &map) const
Whether the Directory&#39;s input Map is (globally) one to one.
LookupStatus getEntries(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const
GlobalOrdinal getGlobalElement(LocalOrdinal localIndex) const
The global index corresponding to the given local index.
LookupStatus getDirectoryEntries(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs) const
Given a global ID list, return the list of their owning process IDs.
virtual std::size_t selectedIndex(GlobalOrdinal GID, const std::vector< std::pair< int, LocalOrdinal > > &pid_and_lid) const =0
Break any ties in ownership of the given global index GID.
size_t getNodeNumElements() const
The number of elements belonging to the calling process.
Describes a parallel distribution of objects over processes.
Computes the local ID and process ID corresponding to given global IDs.
virtual bool isOneToOne(const Teuchos::Comm< int > &comm) const =0
Whether the Directory&#39;s input Map is (globally) one to one.
Implement mapping from global ID to process ID and local ID.
bool initialized() const
Whether the Directory is initialized.
void initialize(const map_type &map)
Initialize the Directory with its Map.
Directory()
Default constructor: the only one you should use.
Implementation of Directory for a contiguous, uniformly distributed Map.
LocalOrdinal getMaxLocalIndex() const
The maximum local index on the calling process.
std::string description() const
A one-line human-readable description of this object.
virtual bool mayHaveSideEffects() const
Whether selectedIndex() may have side effects.