Tpetra parallel linear algebra  Version of the Day
Tpetra_DistObject_def.hpp
Go to the documentation of this file.
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ************************************************************************
40 // @HEADER
41 
42 #ifndef TPETRA_DISTOBJECT_DEF_HPP
43 #define TPETRA_DISTOBJECT_DEF_HPP
44 
52 
53 #include "Tpetra_Distributor.hpp"
54 
55 namespace Tpetra {
56 
57  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
59  DistObject (const Teuchos::RCP<const map_type>& map) :
60  map_ (map)
61  {
62 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
63  using Teuchos::RCP;
64  using Teuchos::Time;
65  using Teuchos::TimeMonitor;
66 
67  RCP<Time> doXferTimer =
68  TimeMonitor::lookupCounter ("Tpetra::DistObject::doTransfer");
69  if (doXferTimer.is_null ()) {
70  doXferTimer =
71  TimeMonitor::getNewCounter ("Tpetra::DistObject::doTransfer");
72  }
73  doXferTimer_ = doXferTimer;
74 
75  RCP<Time> copyAndPermuteTimer =
76  TimeMonitor::lookupCounter ("Tpetra::DistObject::copyAndPermute");
77  if (copyAndPermuteTimer.is_null ()) {
78  copyAndPermuteTimer =
79  TimeMonitor::getNewCounter ("Tpetra::DistObject::copyAndPermute");
80  }
81  copyAndPermuteTimer_ = copyAndPermuteTimer;
82 
83  RCP<Time> packAndPrepareTimer =
84  TimeMonitor::lookupCounter ("Tpetra::DistObject::packAndPrepare");
85  if (packAndPrepareTimer.is_null ()) {
86  packAndPrepareTimer =
87  TimeMonitor::getNewCounter ("Tpetra::DistObject::packAndPrepare");
88  }
89  packAndPrepareTimer_ = packAndPrepareTimer;
90 
91  RCP<Time> doPostsAndWaitsTimer =
92  TimeMonitor::lookupCounter ("Tpetra::DistObject::doPostsAndWaits");
93  if (doPostsAndWaitsTimer.is_null ()) {
94  doPostsAndWaitsTimer =
95  TimeMonitor::getNewCounter ("Tpetra::DistObject::doPostsAndWaits");
96  }
97  doPostsAndWaitsTimer_ = doPostsAndWaitsTimer;
98 
99  RCP<Time> unpackAndCombineTimer =
100  TimeMonitor::lookupCounter ("Tpetra::DistObject::unpackAndCombine");
101  if (unpackAndCombineTimer.is_null ()) {
102  unpackAndCombineTimer =
103  TimeMonitor::getNewCounter ("Tpetra::DistObject::unpackAndCombine");
104  }
105  unpackAndCombineTimer_ = unpackAndCombineTimer;
106 #endif // HAVE_TPETRA_TRANSFER_TIMERS
107  }
108 
109  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
112  : map_ (rhs.map_)
113  {}
114 
115  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
118  {}
119 
120  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
121  std::string
123  description () const
124  {
125  using Teuchos::TypeNameTraits;
126 
127  std::ostringstream os;
128  os << "\"Tpetra::DistObject\": {"
129  << "Packet: " << TypeNameTraits<packet_type>::name ()
130  << ", LocalOrdinal: " << TypeNameTraits<local_ordinal_type>::name ()
131  << ", GlobalOrdinal: " << TypeNameTraits<global_ordinal_type>::name ()
132  << ", Node: " << TypeNameTraits<Node>::name ();
133  if (this->getObjectLabel () != "") {
134  os << "Label: \"" << this->getObjectLabel () << "\"";
135  }
136  os << "}";
137  return os.str ();
138  }
139 
140  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
141  void
143  describe (Teuchos::FancyOStream &out,
144  const Teuchos::EVerbosityLevel verbLevel) const
145  {
146  using Teuchos::rcpFromRef;
147  using Teuchos::TypeNameTraits;
148  using std::endl;
149  const Teuchos::EVerbosityLevel vl = (verbLevel == Teuchos::VERB_DEFAULT) ?
150  Teuchos::VERB_LOW : verbLevel;
151  Teuchos::RCP<const Teuchos::Comm<int> > comm = this->getMap ()->getComm ();
152  const int myRank = comm.is_null () ? 0 : comm->getRank ();
153  const int numProcs = comm.is_null () ? 1 : comm->getSize ();
154 
155  if (vl != Teuchos::VERB_NONE) {
156  Teuchos::OSTab tab0 (out);
157  if (myRank == 0) {
158  out << "\"Tpetra::DistObject\":" << endl;
159  }
160  Teuchos::OSTab tab1 (out);
161  if (myRank == 0) {
162  out << "Template parameters:" << endl;
163  {
164  Teuchos::OSTab tab2 (out);
165  out << "Packet: " << TypeNameTraits<packet_type>::name () << endl
166  << "LocalOrdinal: " << TypeNameTraits<local_ordinal_type>::name () << endl
167  << "GlobalOrdinal: " << TypeNameTraits<global_ordinal_type>::name () << endl
168  << "Node: " << TypeNameTraits<node_type>::name () << endl;
169  }
170  if (this->getObjectLabel () != "") {
171  out << "Label: \"" << this->getObjectLabel () << "\"" << endl;
172  }
173  } // if myRank == 0
174 
175  // Describe the Map.
176  {
177  if (myRank == 0) {
178  out << "Map:" << endl;
179  }
180  Teuchos::OSTab tab2 (out);
181  map_->describe (out, vl);
182  }
183 
184  // At verbosity > VERB_LOW, each process prints something.
185  if (vl > Teuchos::VERB_LOW) {
186  for (int p = 0; p < numProcs; ++p) {
187  if (myRank == p) {
188  out << "Process " << myRank << ":" << endl;
189  Teuchos::OSTab tab2 (out);
190  out << "Export buffer size (in packets): " << exports_.size ()
191  << endl
192  << "Import buffer size (in packets): " << imports_.size ()
193  << endl;
194  }
195  if (! comm.is_null ()) {
196  comm->barrier (); // give output time to finish
197  comm->barrier ();
198  comm->barrier ();
199  }
200  } // for each process rank p
201  } // if vl > VERB_LOW
202  } // if vl != VERB_NONE
203  }
204 
205  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
206  void
208  removeEmptyProcessesInPlace (const Teuchos::RCP<const map_type>& newMap)
209  {
210  TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,
211  "Tpetra::DistObject::removeEmptyProcessesInPlace: Not implemented");
212  }
213 
214  /* These are provided in base DistObject template
215  template<class DistObjectType>
216  void
217  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
218  const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
219  typename DistObjectType::global_ordinal_type,
220  typename DistObjectType::node_type> >& newMap)
221  {
222  input->removeEmptyProcessesInPlace (newMap);
223  if (newMap.is_null ()) { // my process is excluded
224  input = Teuchos::null;
225  }
226  }
227 
228  template<class DistObjectType>
229  void
230  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input)
231  {
232  using Teuchos::RCP;
233  typedef typename DistObjectType::local_ordinal_type LO;
234  typedef typename DistObjectType::global_ordinal_type GO;
235  typedef typename DistObjectType::node_type NT;
236  typedef Map<LO, GO, NT> map_type;
237 
238  RCP<const map_type> newMap = input->getMap ()->removeEmptyProcesses ();
239  removeEmptyProcessesInPlace<DistObjectType> (input, newMap);
240  }
241  */
242 
243  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
244  void
246  doImport (const SrcDistObject& source,
248  CombineMode CM)
249  {
250  TEUCHOS_TEST_FOR_EXCEPTION(*getMap() != *importer.getTargetMap(),
251  std::invalid_argument, "doImport: The target DistObject's Map is not "
252  "identical to the Import's target Map.");
253 #ifdef HAVE_TPETRA_DEBUG
254  {
255  const this_type* srcDistObj = dynamic_cast<const this_type*> (&source);
256  TEUCHOS_TEST_FOR_EXCEPTION(
257  srcDistObj != NULL && * (srcDistObj->getMap ()) != *importer.getSourceMap(),
258  std::invalid_argument, "doImport: The source is a DistObject, yet its "
259  "Map is not identical to the Import's source Map.");
260  }
261 #endif // HAVE_TPETRA_DEBUG
262  size_t numSameIDs = importer.getNumSameIDs ();
263 
264  typedef Teuchos::ArrayView<const LocalOrdinal> view_type;
265  const view_type exportLIDs = importer.getExportLIDs();
266  const view_type remoteLIDs = importer.getRemoteLIDs();
267  const view_type permuteToLIDs = importer.getPermuteToLIDs();
268  const view_type permuteFromLIDs = importer.getPermuteFromLIDs();
269  this->doTransfer (source, CM, numSameIDs, permuteToLIDs, permuteFromLIDs,
270  remoteLIDs, exportLIDs, importer.getDistributor (),
271  DoForward);
272  }
273 
274  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
275  void
277  doExport (const SrcDistObject& source,
279  CombineMode CM)
280  {
281  TEUCHOS_TEST_FOR_EXCEPTION(
282  *getMap() != *exporter.getTargetMap(), std::invalid_argument,
283  "doExport: The target DistObject's Map is not identical to the Export's "
284  "target Map.");
285 #ifdef HAVE_TPETRA_DEBUG
286  {
287  const this_type* srcDistObj = dynamic_cast<const this_type*> (&source);
288  TEUCHOS_TEST_FOR_EXCEPTION(
289  srcDistObj != NULL && * (srcDistObj->getMap ()) != *exporter.getSourceMap(),
290  std::invalid_argument, "doExport: The source is a DistObject, yet its "
291  "Map is not identical to the Export's source Map.");
292  }
293 #endif // HAVE_TPETRA_DEBUG
294  size_t numSameIDs = exporter.getNumSameIDs();
295 
296  typedef ArrayView<const LocalOrdinal> view_type;
297  view_type exportLIDs = exporter.getExportLIDs();
298  view_type remoteLIDs = exporter.getRemoteLIDs();
299  view_type permuteToLIDs = exporter.getPermuteToLIDs();
300  view_type permuteFromLIDs = exporter.getPermuteFromLIDs();
301  doTransfer (source, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs,
302  exportLIDs, exporter.getDistributor (), DoForward);
303  }
304 
305  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
306  void
308  doImport (const SrcDistObject& source,
310  CombineMode CM)
311  {
312  TEUCHOS_TEST_FOR_EXCEPTION(
313  *getMap() != *exporter.getSourceMap(), std::invalid_argument,
314  "doImport (reverse mode): The target DistObject's Map is not identical "
315  "to the Export's source Map.");
316 #ifdef HAVE_TPETRA_DEBUG
317  {
318  const this_type* srcDistObj = dynamic_cast<const this_type*> (&source);
319  TEUCHOS_TEST_FOR_EXCEPTION(
320  srcDistObj != NULL && * (srcDistObj->getMap ()) != *exporter.getTargetMap(),
321  std::invalid_argument,
322  "doImport (reverse mode): The source is a DistObject, yet its "
323  "Map is not identical to the Export's target Map.");
324  }
325 #endif // HAVE_TPETRA_DEBUG
326  size_t numSameIDs = exporter.getNumSameIDs();
327 
328  typedef ArrayView<const LocalOrdinal> view_type;
329  view_type exportLIDs = exporter.getRemoteLIDs();
330  view_type remoteLIDs = exporter.getExportLIDs();
331  view_type permuteToLIDs = exporter.getPermuteFromLIDs();
332  view_type permuteFromLIDs = exporter.getPermuteToLIDs();
333  doTransfer (source, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs,
334  exportLIDs, exporter.getDistributor (), DoReverse);
335  }
336 
337  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
338  void
340  doExport (const SrcDistObject& source,
342  CombineMode CM)
343  {
344  TEUCHOS_TEST_FOR_EXCEPTION(
345  *getMap() != *importer.getSourceMap(), std::invalid_argument,
346  "doExport (reverse mode): The target object's Map "
347  "is not identical to the Import's source Map.");
348 #ifdef HAVE_TPETRA_DEBUG
349  {
350  const this_type* srcDistObj = dynamic_cast<const this_type*> (&source);
351  TEUCHOS_TEST_FOR_EXCEPTION(
352  srcDistObj != NULL && * (srcDistObj->getMap ()) != *importer.getTargetMap(),
353  std::invalid_argument,
354  "doExport (reverse mode): The source is a DistObject, yet its "
355  "Map is not identical to the Import's target Map.");
356  }
357 #endif // HAVE_TPETRA_DEBUG
358  size_t numSameIDs = importer.getNumSameIDs();
359 
360  typedef ArrayView<const LocalOrdinal> view_type;
361  view_type exportLIDs = importer.getRemoteLIDs();
362  view_type remoteLIDs = importer.getExportLIDs();
363  view_type permuteToLIDs = importer.getPermuteFromLIDs();
364  view_type permuteFromLIDs = importer.getPermuteToLIDs();
365  doTransfer (source, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs,
366  exportLIDs, importer.getDistributor (), DoReverse);
367  }
368 
369  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
370  bool
372  isDistributed () const {
373  return map_->isDistributed ();
374  }
375 
376  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
377  size_t
380  return 0; // default implementation; subclasses may override
381  }
382 
383  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
384  void
387  CombineMode CM,
388  size_t numSameIDs,
389  const Teuchos::ArrayView<const LocalOrdinal>& permuteToLIDs_,
390  const Teuchos::ArrayView<const LocalOrdinal>& permuteFromLIDs_,
391  const Teuchos::ArrayView<const LocalOrdinal>& remoteLIDs_,
392  const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs_,
393  Distributor &distor,
394  ReverseOption revOp)
395  {
396  if (this->useNewInterface()) {
397  doTransferNew(src, CM, numSameIDs, permuteToLIDs_, permuteFromLIDs_, remoteLIDs_, exportLIDs_, distor, revOp);
398  }
399  else {
400  doTransferOld(src, CM, numSameIDs, permuteToLIDs_, permuteFromLIDs_, remoteLIDs_, exportLIDs_, distor, revOp);
401  }
402  }
403 
404  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
405  void
407  doTransferOld (const SrcDistObject& src,
408  CombineMode CM,
409  size_t numSameIDs,
410  const Teuchos::ArrayView<const LocalOrdinal>& permuteToLIDs,
411  const Teuchos::ArrayView<const LocalOrdinal>& permuteFromLIDs,
412  const Teuchos::ArrayView<const LocalOrdinal>& remoteLIDs,
413  const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
414  Distributor &distor,
415  ReverseOption revOp)
416  {
417 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
418  Teuchos::TimeMonitor doXferMon (*doXferTimer_);
419 #endif // HAVE_TPETRA_TRANSFER_TIMERS
420 
421  TEUCHOS_TEST_FOR_EXCEPTION(
422  ! checkSizes (src), std::invalid_argument,
423  "Tpetra::DistObject::doTransfer(): checkSizes() indicates that the "
424  "destination object is not a legal target for redistribution from the "
425  "source object. This probably means that they do not have the same "
426  "dimensions. For example, MultiVectors must have the same number of "
427  "rows and columns.");
428  KokkosClassic::ReadWriteOption rwo = KokkosClassic::ReadWrite;
429  if (CM == INSERT || CM == REPLACE) {
430  const size_t numIDsToWrite = numSameIDs +
431  static_cast<size_t> (permuteToLIDs.size ()) +
432  static_cast<size_t> (remoteLIDs.size ());
433  if (numIDsToWrite == this->getMap ()->getNodeNumElements ()) {
434  // We're overwriting all of our local data in the destination
435  // object, so a write-only view suffices.
436  //
437  // FIXME (mfh 10 Apr 2012) This doesn't make sense for a
438  // CrsMatrix with a dynamic graph. INSERT mode could mean
439  // that we're adding new entries to the object, but we don't
440  // want to get rid of the old ones.
441  rwo = KokkosClassic::WriteOnly;
442  }
443  }
444  // Tell the source to create a read-only view of its data. On a
445  // discrete accelerator such as a GPU, this brings EVERYTHING from
446  // device memory to host memory.
447  //
448  // FIXME (mfh 23 Mar 2012) By passing in the list of GIDs (or
449  // rather, local LIDs to send) and packet counts, createViews()
450  // could create a "sparse view" that only brings in the necessary
451  // data from device to host memory.
452  const this_type* srcDistObj = dynamic_cast<const this_type*> (&src);
453  if (srcDistObj != NULL) {
454  srcDistObj->createViews ();
455  }
456 
457  // Tell the target to create a view of its data. Depending on
458  // rwo, this could be a write-only view or a read-and-write view.
459  // On a discrete accelerator such as a GPU, a write-only view only
460  // requires a transfer from host to device memory. A
461  // read-and-write view requires a two-way transfer. This has the
462  // same problem as createViews(): it transfers EVERYTHING, not
463  // just the necessary data.
464  //
465  // FIXME (mfh 23 Mar 2012) By passing in the list of GIDs (or
466  // rather, local LIDs into which to receive) and packet counts,
467  // createViewsNonConst() could create a "sparse view" that only
468  // transfers the necessary data.
469  this->createViewsNonConst (rwo);
470 
471  if (numSameIDs + permuteToLIDs.size()) {
472 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
473  Teuchos::TimeMonitor copyAndPermuteMon (*copyAndPermuteTimer_);
474 #endif // HAVE_TPETRA_TRANSFER_TIMERS
475  // There is at least one GID to copy or permute.
476  copyAndPermute (src, numSameIDs, permuteToLIDs, permuteFromLIDs);
477  }
478 
479  // The method may return zero even if the implementation actually
480  // does have a constant number of packets per LID. However, if it
481  // returns nonzero, we may use this information to avoid
482  // (re)allocating num{Ex,Im}portPacketsPerLID_. packAndPrepare()
483  // will set this to its final value.
484  //
485  // We only need this if CM != ZERO, but it has to be lifted out of
486  // that scope because there are multiple tests for CM != ZERO.
487  size_t constantNumPackets = this->constantNumberOfPackets ();
488 
489  // We only need to pack communication buffers if the combine mode
490  // is not ZERO. A "ZERO combine mode" means that the results are
491  // the same as if we had received all zeros, and added them to the
492  // existing values. That means we don't need to communicate.
493  if (CM != ZERO) {
494  if (constantNumPackets == 0) {
495  numExportPacketsPerLID_old_.resize (exportLIDs.size ());
496  numImportPacketsPerLID_old_.resize (remoteLIDs.size ());
497  }
498 
499  {
500 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
501  Teuchos::TimeMonitor packAndPrepareMon (*packAndPrepareTimer_);
502 #endif // HAVE_TPETRA_TRANSFER_TIMERS
503  // Ask the source to pack data. Also ask it whether there are a
504  // constant number of packets per element (constantNumPackets is
505  // an output argument). If there are, constantNumPackets will
506  // come back nonzero. Otherwise, the source will fill the
507  // numExportPacketsPerLID_ array.
508  packAndPrepare (src, exportLIDs, exports_old_, numExportPacketsPerLID_old_ (),
509  constantNumPackets, distor);
510  }
511  }
512 
513  // We don't need the source's data anymore, so it can let go of
514  // its views. On an accelerator device with a separate memory
515  // space (like a GPU), this frees host memory, since device memory
516  // has the "master" version of the data.
517  if (srcDistObj != NULL) {
518  srcDistObj->releaseViews ();
519  }
520 
521  // We only need to send data if the combine mode is not ZERO.
522  if (CM != ZERO) {
523  if (constantNumPackets != 0) {
524  // There are a constant number of packets per element. We
525  // already know (from the number of "remote" (incoming)
526  // elements) how many incoming elements we expect, so we can
527  // resize the buffer accordingly.
528  const size_t rbufLen = remoteLIDs.size() * constantNumPackets;
529  if (static_cast<size_t> (imports_old_.size()) != rbufLen) {
530  imports_old_.resize (rbufLen);
531  }
532  }
533 
534  // Do we need to do communication (via doPostsAndWaits)?
535  bool needCommunication = true;
536  if (revOp == DoReverse && ! isDistributed ()) {
537  needCommunication = false;
538  }
539  // FIXME (mfh 30 Jun 2013): Checking whether the source object
540  // is distributed requires a cast to DistObject. If it's not a
541  // DistObject, then I'm not quite sure what to do. Perhaps it
542  // would be more appropriate for SrcDistObject to have an
543  // isDistributed() method. For now, I'll just assume that we
544  // need to do communication unless the cast succeeds and the
545  // source is not distributed.
546  else if (revOp == DoForward && srcDistObj != NULL &&
547  ! srcDistObj->isDistributed ()) {
548  needCommunication = false;
549  }
550 
551  if (needCommunication) {
552  if (revOp == DoReverse) {
553 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
554  Teuchos::TimeMonitor doPostsAndWaitsMon (*doPostsAndWaitsTimer_);
555 #endif // HAVE_TPETRA_TRANSFER_TIMERS
556  if (constantNumPackets == 0) { //variable num-packets-per-LID:
557  distor.doReversePostsAndWaits (numExportPacketsPerLID_old_().getConst(),
558  1,
559  numImportPacketsPerLID_old_());
560  size_t totalImportPackets = 0;
561  for (Array_size_type i = 0; i < numImportPacketsPerLID_old_.size(); ++i) {
562  totalImportPackets += numImportPacketsPerLID_old_[i];
563  }
564  imports_old_.resize(totalImportPackets);
565  distor.doReversePostsAndWaits (exports_old_().getConst(),
566  numExportPacketsPerLID_old_(),
567  imports_old_(),
568  numImportPacketsPerLID_old_());
569  }
570  else {
571  distor.doReversePostsAndWaits (exports_old_ ().getConst (),
572  constantNumPackets,
573  imports_old_ ());
574  }
575  }
576  else { // revOp == DoForward
577 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
578  Teuchos::TimeMonitor doPostsAndWaitsMon (*doPostsAndWaitsTimer_);
579 #endif // HAVE_TPETRA_TRANSFER_TIMERS
580  if (constantNumPackets == 0) { //variable num-packets-per-LID:
581  distor.doPostsAndWaits (numExportPacketsPerLID_old_().getConst(), 1,
582  numImportPacketsPerLID_old_());
583  size_t totalImportPackets = 0;
584  for (Array_size_type i = 0; i < numImportPacketsPerLID_old_.size(); ++i) {
585  totalImportPackets += numImportPacketsPerLID_old_[i];
586  }
587  imports_old_.resize(totalImportPackets);
588  distor.doPostsAndWaits (exports_old_().getConst(),
589  numExportPacketsPerLID_old_(),
590  imports_old_(),
591  numImportPacketsPerLID_old_());
592  }
593  else {
594  distor.doPostsAndWaits (exports_old_ ().getConst (),
595  constantNumPackets,
596  imports_old_ ());
597  }
598  }
599  {
600 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
601  Teuchos::TimeMonitor unpackAndCombineMon (*unpackAndCombineTimer_);
602 #endif // HAVE_TPETRA_TRANSFER_TIMERS
603  unpackAndCombine (remoteLIDs, imports_old_(), numImportPacketsPerLID_old_(),
604  constantNumPackets, distor, CM);
605  }
606  }
607  } // if (CM != ZERO)
608 
609  this->releaseViews ();
610  }
611 
612  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
613  void
615  doTransferNew (const SrcDistObject& src,
616  CombineMode CM,
617  size_t numSameIDs,
618  const Teuchos::ArrayView<const LocalOrdinal>& permuteToLIDs_,
619  const Teuchos::ArrayView<const LocalOrdinal>& permuteFromLIDs_,
620  const Teuchos::ArrayView<const LocalOrdinal>& remoteLIDs_,
621  const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs_,
622  Distributor &distor,
623  ReverseOption revOp)
624  {
625  using Kokkos::Compat::getArrayView;
626  using Kokkos::Compat::getConstArrayView;
627  using Kokkos::Compat::getKokkosViewDeepCopy;
628  using Kokkos::Compat::create_const_view;
629 
630 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
631  Teuchos::TimeMonitor doXferMon (*doXferTimer_);
632 #endif // HAVE_TPETRA_TRANSFER_TIMERS
633 
634  // Convert arguments to Kokkos::View's (involves deep copy to device)
635  //
636  // FIXME (mfh 17 Feb 2014) getKokkosViewDeepCopy _always_ does a
637  // deep copy. It has to, since it returns a managed Kokkos::View,
638  // but the input Teuchos::ArrayView is unmanaged. One way to fix
639  // this would be to change the interface by replacing the
640  // Teuchos::ArrayView inputs with Kokkos::View inputs. This is
641  // the approach taken by Kokkos::DistObjectKA. Some points:
642  //
643  // 1. It's perfectly OK to change the interface of doTransfer.
644  // It should take Kokkos::View by default, and convert to
645  // Teuchos::Array{RCP,View} if needed internally.
646  // 2. Recall that Teuchos::ArrayView is an unmanaged view.
647  // 3. If DistObject ever gets a nonblocking interface, that
648  // interface should take managed views.
649  typedef Kokkos::View<const LocalOrdinal*, execution_space> lo_const_view_type;
650  lo_const_view_type permuteToLIDs =
651  getKokkosViewDeepCopy<execution_space> (permuteToLIDs_);
652  lo_const_view_type permuteFromLIDs =
653  getKokkosViewDeepCopy<execution_space> (permuteFromLIDs_);
654  lo_const_view_type remoteLIDs =
655  getKokkosViewDeepCopy<execution_space> (remoteLIDs_);
656  lo_const_view_type exportLIDs =
657  getKokkosViewDeepCopy<execution_space> (exportLIDs_);
658 
659  TEUCHOS_TEST_FOR_EXCEPTION(
660  ! checkSizes (src), std::invalid_argument,
661  "Tpetra::DistObject::doTransfer(): checkSizes() indicates that the "
662  "destination object is not a legal target for redistribution from the "
663  "source object. This probably means that they do not have the same "
664  "dimensions. For example, MultiVectors must have the same number of "
665  "rows and columns.");
666  KokkosClassic::ReadWriteOption rwo = KokkosClassic::ReadWrite;
667  if (CM == INSERT || CM == REPLACE) {
668  const size_t numIDsToWrite = numSameIDs +
669  static_cast<size_t> (permuteToLIDs.size ()) +
670  static_cast<size_t> (remoteLIDs.size ());
671  if (numIDsToWrite == this->getMap ()->getNodeNumElements ()) {
672  // We're overwriting all of our local data in the destination
673  // object, so a write-only view suffices.
674  //
675  // FIXME (mfh 10 Apr 2012) This doesn't make sense for a
676  // CrsMatrix with a dynamic graph. INSERT mode could mean
677  // that we're adding new entries to the object, but we don't
678  // want to get rid of the old ones.
679  rwo = KokkosClassic::WriteOnly;
680  }
681  }
682 
683  // FIXME (mfh 17 Feb 2014) We're assuming that MPI can read device
684  // memory (that's even pre-UVM), so there is no need to create
685  // host views of the source object's data.
686 
687  // Tell the source to create a read-only view of its data. On a
688  // discrete accelerator such as a GPU, this brings EVERYTHING from
689  // device memory to host memory.
690  //
691  // FIXME (mfh 23 Mar 2012) By passing in the list of GIDs (or
692  // rather, local LIDs to send) and packet counts, createViews()
693  // could create a "sparse view" that only brings in the necessary
694  // data from device to host memory.
695  const this_type* srcDistObj = dynamic_cast<const this_type*> (&src);
696  if (srcDistObj != NULL) {
697  srcDistObj->createViews ();
698  }
699 
700  // FIXME (mfh 17 Feb 2014) We're assuming that MPI can read device
701  // memory (that's even pre-UVM), so there is no need to create
702  // host views of the target object's data.
703 
704  // Tell the target to create a view of its data. Depending on
705  // rwo, this could be a write-only view or a read-and-write view.
706  // On a discrete accelerator such as a GPU, a write-only view only
707  // requires a transfer from host to device memory. A
708  // read-and-write view requires a two-way transfer. This has the
709  // same problem as createViews(): it transfers EVERYTHING, not
710  // just the necessary data.
711  //
712  // FIXME (mfh 23 Mar 2012) By passing in the list of GIDs (or
713  // rather, local LIDs into which to receive) and packet counts,
714  // createViewsNonConst() could create a "sparse view" that only
715  // transfers the necessary data.
716  this->createViewsNonConst (rwo);
717 
718  if (numSameIDs + permuteToLIDs.size()) {
719 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
720  Teuchos::TimeMonitor copyAndPermuteMon (*copyAndPermuteTimer_);
721 #endif // HAVE_TPETRA_TRANSFER_TIMERS
722 
723  // FIXME (mfh 17 Feb 2014) Nobody implements DistObject
724  // subclasses but Tpetra developers anyway, so don't bother with
725  // this "new" business. Just write the interface you want.
726 
727  // There is at least one GID to copy or permute.
728  copyAndPermuteNew (src, numSameIDs, permuteToLIDs, permuteFromLIDs);
729  }
730 
731  // The method may return zero even if the implementation actually
732  // does have a constant number of packets per LID. However, if it
733  // returns nonzero, we may use this information to avoid
734  // (re)allocating num{Ex,Im}portPacketsPerLID_. packAndPrepare()
735  // will set this to its final value.
736  //
737  // We only need this if CM != ZERO, but it has to be lifted out of
738  // that scope because there are multiple tests for CM != ZERO.
739  size_t constantNumPackets = this->constantNumberOfPackets ();
740 
741  // We only need to pack communication buffers if the combine mode
742  // is not ZERO. A "ZERO combine mode" means that the results are
743  // the same as if we had received all zeros, and added them to the
744  // existing values. That means we don't need to communicate.
745  if (CM != ZERO) {
746  if (constantNumPackets == 0) {
747  // FIXME (mfh 17 Feb 2014) Don't "realloc" unless you really
748  // need to. That will avoid a bit of time for reinitializing
749  // the Views' data.
750  Kokkos::Compat::realloc (numExportPacketsPerLID_, exportLIDs.size ());
751  Kokkos::Compat::realloc (numImportPacketsPerLID_, remoteLIDs.size ());
752  }
753 
754  {
755 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
756  Teuchos::TimeMonitor packAndPrepareMon (*packAndPrepareTimer_);
757 #endif // HAVE_TPETRA_TRANSFER_TIMERS
758  // Ask the source to pack data. Also ask it whether there are a
759  // constant number of packets per element (constantNumPackets is
760  // an output argument). If there are, constantNumPackets will
761  // come back nonzero. Otherwise, the source will fill the
762  // numExportPacketsPerLID_ array.
763  packAndPrepareNew (src, exportLIDs, exports_, numExportPacketsPerLID_,
764  constantNumPackets, distor);
765  }
766  }
767 
768  // FIXME (mfh 17 Feb 2014) See comments above; there is no need to
769  // create host views of the source object's data.
770 
771  // We don't need the source's data anymore, so it can let go of
772  // its views. On an accelerator device with a separate memory
773  // space (like a GPU), this frees host memory, since device memory
774  // has the "master" version of the data.
775  if (srcDistObj != NULL) {
776  srcDistObj->releaseViews ();
777  }
778 
779  // We only need to send data if the combine mode is not ZERO.
780  if (CM != ZERO) {
781  if (constantNumPackets != 0) {
782  // There are a constant number of packets per element. We
783  // already know (from the number of "remote" (incoming)
784  // elements) how many incoming elements we expect, so we can
785  // resize the buffer accordingly.
786  const size_t rbufLen = remoteLIDs.size() * constantNumPackets;
787  if (static_cast<size_t> (imports_.size()) != rbufLen) {
788  Kokkos::Compat::realloc (imports_, rbufLen);
789  }
790  }
791 
792  // FIXME (mfh 17 Feb 2014) Why do we need to create mirror
793  // views? Furthermore, if we do need to do this, we should only
794  // do it _once_, since the arrays are constant (they come from
795  // the Import / Export object, which is constant).
796 
797  // Create mirror views of [import|export]PacketsPerLID
798  typename Kokkos::View<size_t*,execution_space>::HostMirror host_numExportPacketsPerLID =
799  Kokkos::create_mirror_view (numExportPacketsPerLID_);
800  typename Kokkos::View<size_t*,execution_space>::HostMirror host_numImportPacketsPerLID =
801  Kokkos::create_mirror_view (numImportPacketsPerLID_);
802 
803  // Copy numExportPacketsPerLID to host
804  Kokkos::deep_copy (host_numExportPacketsPerLID, numExportPacketsPerLID_);
805 
806  // Do we need to do communication (via doPostsAndWaits)?
807  bool needCommunication = true;
808  if (revOp == DoReverse && ! isDistributed ()) {
809  needCommunication = false;
810  }
811  // FIXME (mfh 30 Jun 2013): Checking whether the source object
812  // is distributed requires a cast to DistObject. If it's not a
813  // DistObject, then I'm not quite sure what to do. Perhaps it
814  // would be more appropriate for SrcDistObject to have an
815  // isDistributed() method. For now, I'll just assume that we
816  // need to do communication unless the cast succeeds and the
817  // source is not distributed.
818  else if (revOp == DoForward && srcDistObj != NULL &&
819  ! srcDistObj->isDistributed ()) {
820  needCommunication = false;
821  }
822 
823  // FIXME (mfh 17 Feb 2014) Distributor doesn't actually inspect
824  // the contents of the "exports" or "imports" arrays, other than
825  // to do a deep copy in the (should be technically unnecessary,
826  // but isn't for some odd reason) "self-message" case.
827  // Distributor thus doesn't need host views; it could do just
828  // fine with device views, assuming that MPI knows how to read
829  // device memory (which doesn't even require UVM).
830 
831  if (needCommunication) {
832  if (revOp == DoReverse) {
833 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
834  Teuchos::TimeMonitor doPostsAndWaitsMon (*doPostsAndWaitsTimer_);
835 #endif // HAVE_TPETRA_TRANSFER_TIMERS
836  if (constantNumPackets == 0) { //variable num-packets-per-LID:
837  distor.doReversePostsAndWaits (create_const_view (host_numExportPacketsPerLID),
838  1,
839  host_numImportPacketsPerLID);
840  size_t totalImportPackets = 0;
841  // FIXME (mfh 17 Feb 2014) This would be a good place for
842  // a Kokkos reduction. numImportPacketsPerLID_ has as
843  // many entries as the number of LIDs on the calling
844  // process.
845  for (view_size_type i = 0; i < numImportPacketsPerLID_.size(); ++i) {
846  totalImportPackets += host_numImportPacketsPerLID[i];
847  }
848  // FIXME (mfh 17 Feb 2014) Only realloc if necessary.
849  Kokkos::Compat::realloc (imports_, totalImportPackets);
850  distor.doReversePostsAndWaits (create_const_view (exports_),
851  getArrayView (host_numExportPacketsPerLID),
852  imports_,
853  getArrayView (host_numImportPacketsPerLID));
854  }
855  else {
856  distor.doReversePostsAndWaits (create_const_view (exports_),
857  constantNumPackets,
858  imports_);
859  }
860  }
861  else { // revOp == DoForward
862 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
863  Teuchos::TimeMonitor doPostsAndWaitsMon (*doPostsAndWaitsTimer_);
864 #endif // HAVE_TPETRA_TRANSFER_TIMERS
865  if (constantNumPackets == 0) { //variable num-packets-per-LID:
866  distor.doPostsAndWaits (create_const_view (host_numExportPacketsPerLID), 1,
867  host_numImportPacketsPerLID);
868  size_t totalImportPackets = 0;
869  // FIXME (mfh 17 Feb 2014) This would be a good place for
870  // a Kokkos reduction. numImportPacketsPerLID_ has as
871  // many entries as the number of LIDs on the calling
872  // process.
873  for (view_size_type i = 0; i < numImportPacketsPerLID_.size(); ++i) {
874  totalImportPackets += host_numImportPacketsPerLID[i];
875  }
876  // FIXME (mfh 17 Feb 2014) Only realloc if necessary.
877  Kokkos::Compat::realloc (imports_, totalImportPackets);
878  distor.doPostsAndWaits (create_const_view (exports_),
879  getArrayView (host_numExportPacketsPerLID),
880  imports_,
881  getArrayView (host_numImportPacketsPerLID));
882  }
883  else {
884  distor.doPostsAndWaits (create_const_view (exports_),
885  constantNumPackets,
886  imports_);
887  }
888  }
889 
890  // FIXME (mfh 17 Feb 2014) This array should just live on the
891  // device and stay there. There is no need for a host view,
892  // as long as unpackAndCombine(new) knows what to do with a
893  // device view.
894  //
895  // Copy numImportPacketsPerLID to device
896  Kokkos::deep_copy (numImportPacketsPerLID_, host_numImportPacketsPerLID);
897 
898  {
899 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
900  Teuchos::TimeMonitor unpackAndCombineMon (*unpackAndCombineTimer_);
901 #endif // HAVE_TPETRA_TRANSFER_TIMERS
902  unpackAndCombineNew (remoteLIDs, imports_, numImportPacketsPerLID_,
903  constantNumPackets, distor, CM);
904  }
905  }
906  } // if (CM != ZERO)
907 
908  // FIXME (mfh 17 Dec(??? probably Feb) 2014) We don't have to call
909  // releaseViews() on the destination object any more, since MPI
910  // knows how to read device memory.
911 
912  this->releaseViews ();
913  }
914 
915  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
916  void
918  print (std::ostream &os) const
919  {
920  using Teuchos::FancyOStream;
921  using Teuchos::getFancyOStream;
922  using Teuchos::RCP;
923  using Teuchos::rcpFromRef;
924  using std::endl;
925 
926  RCP<FancyOStream> out = getFancyOStream (rcpFromRef (os));
927  this->describe (*out, Teuchos::VERB_DEFAULT);
928  }
929 
930  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
931  void
933  createViews () const
934  {}
935 
936  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
937  void
939  createViewsNonConst (KokkosClassic::ReadWriteOption /*rwo*/)
940  {}
941 
942  template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node, const bool classic>
943  void
946  {}
947 
948  template<class DistObjectType>
949  void
950  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
951  const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
952  typename DistObjectType::global_ordinal_type,
953  typename DistObjectType::node_type> >& newMap)
954  {
955  input->removeEmptyProcessesInPlace (newMap);
956  if (newMap.is_null ()) { // my process is excluded
957  input = Teuchos::null;
958  }
959  }
960 
961  template<class DistObjectType>
962  void
963  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input)
964  {
965  using Teuchos::RCP;
966  typedef typename DistObjectType::local_ordinal_type LO;
967  typedef typename DistObjectType::global_ordinal_type GO;
968  typedef typename DistObjectType::node_type NT;
969  typedef Map<LO, GO, NT> map_type;
970 
971  RCP<const map_type> newMap = input->getMap ()->removeEmptyProcesses ();
972  removeEmptyProcessesInPlace<DistObjectType> (input, newMap);
973  }
974 
975 // Explicit instantiation macro for general DistObject.
976 #define TPETRA_DISTOBJECT_INSTANT(SCALAR, LO, GO, NODE) \
977  template class DistObject< SCALAR , LO , GO , NODE >;
978 
979 // Explicit instantiation macro for DistObject<char, ...>.
980 // The "SLGN" stuff above doesn't work for Packet=char.
981 #define TPETRA_DISTOBJECT_INSTANT_CHAR(LO, GO, NODE) \
982  template class DistObject< char , LO , GO , NODE >;
983 
984 } // namespace Tpetra
985 
986 #endif // TPETRA_DISTOBJECT_DEF_HPP
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distr...
Teuchos::RCP< const map_type > getSourceMap() const
The source Map used to construct this Export.
ArrayView< const LocalOrdinal > getExportLIDs() const
List of entries in the source Map that will be sent to other processes.
Namespace Tpetra contains the class and methods constituting the Tpetra library.
virtual std::string description() const
One-line descriptiion of this object.
ArrayView< const LocalOrdinal > getRemoteLIDs() const
List of entries in the target Map to receive from other processes.
size_t getNumSameIDs() const
Number of initial identical IDs.
virtual size_t constantNumberOfPackets() const
Whether the implementation&#39;s instance promises always to have a constant number of packets per LID...
virtual void doTransfer(const SrcDistObject &src, CombineMode CM, size_t numSameIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteToLIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteFromLIDs, const Teuchos::ArrayView< const local_ordinal_type > &remoteLIDs, const Teuchos::ArrayView< const local_ordinal_type > &exportLIDs, Distributor &distor, ReverseOption revOp)
Redistribute data across memory images.
virtual void removeEmptyProcessesInPlace(const Teuchos::RCP< const map_type > &newMap)
Remove processes which contain no elements in this object&#39;s Map.
ArrayView< const LocalOrdinal > getPermuteFromLIDs() const
List of local IDs in the source Map that are permuted.
void deep_copy(const LittleBlock< ST2, LO > &dst, const LittleBlock< ST1, LO > &src, typename std::enable_if< std::is_convertible< ST1, ST2 >::value &&!std::is_const< ST2 >::value, int >::type *=NULL)
Copy the LittleBlock src into the LittleBlock dst.
Teuchos_Ordinal Array_size_type
Size type for Teuchos Array objects.
virtual void releaseViews() const
Hook for releasing views.
ArrayView< const LocalOrdinal > getPermuteToLIDs() const
List of local IDs in the target Map that are permuted.
virtual void packAndPrepare(const SrcDistObject &source, const Teuchos::ArrayView< const local_ordinal_type > &exportLIDs, Teuchos::Array< packet_type > &exports, const Teuchos::ArrayView< size_t > &numPacketsPerLID, size_t &constantNumPackets, Distributor &distor)
Perform any packing or preparation required for communication.
virtual bool checkSizes(const SrcDistObject &source)=0
Compare the source and target (this) objects for compatibility.
Insert new values that don&#39;t currently exist.
void doPostsAndWaits(const ArrayView< const Packet > &exports, size_t numPackets, const ArrayView< Packet > &imports)
Execute the (forward) communication plan.
virtual void unpackAndCombine(const Teuchos::ArrayView< const local_ordinal_type > &importLIDs, const Teuchos::ArrayView< const packet_type > &imports, const Teuchos::ArrayView< size_t > &numPacketsPerLID, size_t constantNumPackets, Distributor &distor, CombineMode CM)
Perform any unpacking and combining after communication.
ArrayView< const LocalOrdinal > getPermuteFromLIDs() const
List of local IDs in the source Map that are permuted.
Communication plan for data redistribution from a (possibly) multiply-owned to a uniquely-owned distr...
Map< local_ordinal_type, global_ordinal_type, node_type > map_type
The type of the Map specialization to use with this class.
virtual void createViews() const
Hook for creating a const view.
Sets up and executes a communication plan for a Tpetra DistObject.
CombineMode
Rule for combining data in an Import or Export.
virtual ~DistObject()
Destructor (virtual for memory safety of derived classes).
Teuchos::RCP< const map_type > map_
The Map over which this object is distributed.
virtual void copyAndPermute(const SrcDistObject &source, size_t numSameIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteToLIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteFromLIDs)
Perform copies and permutations that are local to this process.
Teuchos::RCP< const map_type > getTargetMap() const
The target Map used to construct this Export.
bool isDistributed() const
Whether this is a globally distributed object.
ArrayView< const LocalOrdinal > getPermuteToLIDs() const
List of local IDs in the target Map that are permuted.
Abstract base class for objects that can be the source of an Import or Export operation.
Replace existing values with new values.
Replace old values with zero.
ReverseOption
Whether the data transfer should be performed in forward or reverse mode.
void print(std::ostream &os) const
Print this object to the given output stream.
ArrayView< const LocalOrdinal > getExportLIDs() const
List of entries in the source Map that will be sent to other processes.
DistObject(const Teuchos::RCP< const map_type > &map)
Constructor.
void doReversePostsAndWaits(const ArrayView< const Packet > &exports, size_t numPackets, const ArrayView< Packet > &imports)
Execute the reverse communication plan.
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Print a descriptiion of this object to the given output stream.
virtual void createViewsNonConst(KokkosClassic::ReadWriteOption rwo)
Hook for creating a nonconst view.
Describes a parallel distribution of objects over processes.
virtual bool useNewInterface()
Whether lass (???) implements old or new interface.
ArrayView< const LocalOrdinal > getRemoteLIDs() const
List of entries in the target Map to receive from other processes.
Distributor & getDistributor() const
The Distributor that this Export object uses to move data.
size_t getNumSameIDs() const
Number of initial identical IDs.
void doExport(const SrcDistObject &source, const Export< LocalOrdinal, GlobalOrdinal, Node > &exporter, CombineMode CM)
Export data into this object using an Export object ("forward mode").
virtual Teuchos::RCP< const map_type > getMap() const
The Map describing the parallel distribution of this object.
Teuchos::RCP< const map_type > getSourceMap() const
The Source Map used to construct this Import object.
Distributor & getDistributor() const
The Distributor that this Import object uses to move data.
Base class for distributed Tpetra objects that support data redistribution.
Teuchos::RCP< const map_type > getTargetMap() const
The Target Map used to construct this Import object.
void doImport(const SrcDistObject &source, const Import< LocalOrdinal, GlobalOrdinal, Node > &importer, CombineMode CM)
Import data into this object using an Import object ("forward mode").