Tpetra parallel linear algebra  Version of the Day
Tpetra_DistObject_decl.hpp
Go to the documentation of this file.
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ************************************************************************
40 // @HEADER
41 
42 #ifndef TPETRA_DISTOBJECT_DECL_HPP
43 #define TPETRA_DISTOBJECT_DECL_HPP
44 
52 
53 #include "Tpetra_ConfigDefs.hpp"
54 #include "Tpetra_Map.hpp"
55 #include "Tpetra_Import.hpp"
56 #include "Tpetra_Export.hpp"
57 #include "Tpetra_SrcDistObject.hpp"
58 #include "Kokkos_NodeAPIConfigDefs.hpp" // enum KokkosClassic::ReadWriteOption
59 #include "KokkosCompat_ClassicNodeAPI_Wrapper.hpp"
60 #include "Kokkos_ArithTraits.hpp"
61 
62 // #ifndef HAVE_TPETRA_TRANSFER_TIMERS
63 // # define HAVE_TPETRA_TRANSFER_TIMERS 1
64 // #endif // HAVE_TPETRA_TRANSFER_TIMERS
65 
66 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
67 # undef HAVE_TPETRA_TRANSFER_TIMERS
68 #endif // HAVE_TPETRA_TRANSFER_TIMERS
69 
70 
71 namespace Tpetra {
72 
185  template <class Packet,
186  class LocalOrdinal = Details::DefaultTypes::local_ordinal_type,
187  class GlobalOrdinal = Details::DefaultTypes::global_ordinal_type,
189  const bool classic = Node::classic>
190  class DistObject :
191  virtual public SrcDistObject,
192  virtual public Teuchos::Describable
193  {
194  public:
196 
197 
202  typedef typename Kokkos::Details::ArithTraits<Packet>::val_type packet_type;
204  typedef LocalOrdinal local_ordinal_type;
206  typedef GlobalOrdinal global_ordinal_type;
208  typedef Node node_type;
209 
211  typedef typename Node::device_type device_type;
213  typedef typename device_type::execution_space execution_space;
214 
215  private:
216  typedef typename Kokkos::View<packet_type*, device_type>::size_type view_size_type;
218 
219  public:
222 
224 
226 
228  explicit DistObject (const Teuchos::RCP<const map_type>& map);
229 
232 
234  virtual ~DistObject ();
235 
237 
239 
257  void
258  doImport (const SrcDistObject& source,
260  CombineMode CM);
261 
279  void
280  doExport (const SrcDistObject& source,
282  CombineMode CM);
283 
302  void
303  doImport (const SrcDistObject& source,
305  CombineMode CM);
306 
325  void
326  doExport (const SrcDistObject& source,
328  CombineMode CM);
329 
331 
333 
339  bool isDistributed () const;
340 
347  virtual Teuchos::RCP<const map_type> getMap () const { return map_; }
348 
350 
352 
357  void print (std::ostream &os) const;
358 
360 
362 
367  virtual std::string description () const;
368 
373  virtual void
374  describe (Teuchos::FancyOStream &out,
375  const Teuchos::EVerbosityLevel verbLevel =
376  Teuchos::Describable::verbLevel_default) const;
377 
379 
381 
427  virtual void
428  removeEmptyProcessesInPlace (const Teuchos::RCP<const map_type>& newMap);
429 
430  // Forward declaration of nonmember function.
431  template<class PT, class LO, class GO, class NT>
432  friend void
434  const Teuchos::RCP<const Map<LO, GO, NT> >& newMap);
435  // Forward declaration of nonmember function.
436  template<class PT, class LO, class GO, class NT>
437  friend void
440 
441  protected:
450  DoForward, //*!< Perform the transfer in forward mode.
451  DoReverse //*!< Perform the transfer in reverse mode.
452  };
453 
469  virtual size_t constantNumberOfPackets () const;
470 
493  virtual void
494  doTransfer (const SrcDistObject& src,
495  CombineMode CM,
496  size_t numSameIDs,
497  const Teuchos::ArrayView<const local_ordinal_type> &permuteToLIDs,
498  const Teuchos::ArrayView<const local_ordinal_type> &permuteFromLIDs,
499  const Teuchos::ArrayView<const local_ordinal_type> &remoteLIDs,
500  const Teuchos::ArrayView<const local_ordinal_type> &exportLIDs,
501  Distributor &distor,
502  ReverseOption revOp);
503 
504  virtual void
505  doTransferOld (const SrcDistObject& src,
506  CombineMode CM,
507  size_t numSameIDs,
508  const Teuchos::ArrayView<const local_ordinal_type> &permuteToLIDs,
509  const Teuchos::ArrayView<const local_ordinal_type> &permuteFromLIDs,
510  const Teuchos::ArrayView<const local_ordinal_type> &remoteLIDs,
511  const Teuchos::ArrayView<const local_ordinal_type> &exportLIDs,
512  Distributor &distor,
513  ReverseOption revOp);
514 
515  virtual void
516  doTransferNew (const SrcDistObject& src,
517  CombineMode CM,
518  size_t numSameIDs,
519  const Teuchos::ArrayView<const local_ordinal_type> &permuteToLIDs,
520  const Teuchos::ArrayView<const local_ordinal_type> &permuteFromLIDs,
521  const Teuchos::ArrayView<const local_ordinal_type> &remoteLIDs,
522  const Teuchos::ArrayView<const local_ordinal_type> &exportLIDs,
523  Distributor &distor,
524  ReverseOption revOp);
525 
536 
537 
541  virtual bool
542  checkSizes (const SrcDistObject& source) = 0;
543 
545  virtual bool useNewInterface () { return false; }
546 
564  virtual void
566  size_t numSameIDs,
567  const Teuchos::ArrayView<const local_ordinal_type>& permuteToLIDs,
568  const Teuchos::ArrayView<const local_ordinal_type>& permuteFromLIDs)
569  {}
570  virtual void
571  copyAndPermuteNew (const SrcDistObject& source,
572  size_t numSameIDs,
573  const Kokkos::View<const local_ordinal_type*, execution_space> &permuteToLIDs,
574  const Kokkos::View<const local_ordinal_type*, execution_space> &permuteFromLIDs)
575  {}
576 
599  virtual void
601  const Teuchos::ArrayView<const local_ordinal_type>& exportLIDs,
602  Teuchos::Array<packet_type>& exports,
603  const Teuchos::ArrayView<size_t>& numPacketsPerLID,
604  size_t& constantNumPackets,
605  Distributor &distor)
606  {}
607  virtual void
608  packAndPrepareNew (const SrcDistObject& source,
609  const Kokkos::View<const local_ordinal_type*, execution_space> &exportLIDs,
610  Kokkos::View<packet_type*, execution_space> &exports,
611  const Kokkos::View<size_t*, execution_space> &numPacketsPerLID,
612  size_t& constantNumPackets,
613  Distributor &distor)
614  {}
615 
637  virtual void
638  unpackAndCombine (const Teuchos::ArrayView<const local_ordinal_type>& importLIDs,
639  const Teuchos::ArrayView<const packet_type>& imports,
640  const Teuchos::ArrayView<size_t>& numPacketsPerLID,
641  size_t constantNumPackets,
642  Distributor &distor,
643  CombineMode CM)
644  {}
645 
646  virtual void
647  unpackAndCombineNew (
648  const Kokkos::View<const local_ordinal_type*, execution_space> &importLIDs,
649  const Kokkos::View<const packet_type*, execution_space> &imports,
650  const Kokkos::View<size_t*, execution_space> &numPacketsPerLID,
651  size_t constantNumPackets,
652  Distributor &distor,
653  CombineMode CM)
654  {}
656 
663  virtual void createViews () const;
664 
680  virtual void createViewsNonConst (KokkosClassic::ReadWriteOption rwo);
681 
692  virtual void releaseViews () const;
693 
695  Teuchos::RCP<const map_type> map_;
696 
697  private:
699  Kokkos::View<packet_type*, execution_space> imports_;
700 
701  protected:
702  Teuchos::Array<packet_type> imports_old_;
703 
704  private:
715  Kokkos::View<size_t*, execution_space> numImportPacketsPerLID_;
716  typename Kokkos::View<size_t*, execution_space>::HostMirror host_numImportPacketsPerLID_;
717 
718  protected:
719  Teuchos::Array<size_t> numImportPacketsPerLID_old_;
720 
721  private:
723  Kokkos::View<packet_type*, execution_space> exports_;
724 
725  protected:
726  Teuchos::Array<packet_type> exports_old_;
727 
728  private:
739  Kokkos::View<size_t*,execution_space> numExportPacketsPerLID_;
740  protected:
741  Teuchos::Array<size_t> numExportPacketsPerLID_old_;
742 
743 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
744  private:
745  Teuchos::RCP<Teuchos::Time> doXferTimer_;
746  Teuchos::RCP<Teuchos::Time> copyAndPermuteTimer_;
747  Teuchos::RCP<Teuchos::Time> packAndPrepareTimer_;
748  Teuchos::RCP<Teuchos::Time> doPostsAndWaitsTimer_;
749  Teuchos::RCP<Teuchos::Time> unpackAndCombineTimer_;
750 #endif // HAVE_TPETRA_TRANSFER_TIMERS
751 
752  }; // class DistObject
753 
836  template<class DistObjectType>
837  void
838  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
839  const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
840  typename DistObjectType::global_ordinal_type,
841  typename DistObjectType::node_type> >& newMap);
842 
879  template<class DistObjectType>
880  void
881  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input);
882 
883 } // namespace Tpetra
884 
885 #endif // TPETRA_DISTOBJECT_DECL_HPP
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distr...
Namespace Tpetra contains the class and methods constituting the Tpetra library.
virtual std::string description() const
One-line descriptiion of this object.
device_type::execution_space execution_space
The Kokkos execution space.
Node node_type
The Kokkos Node type.
KokkosClassic::DefaultNode::DefaultNodeType node_type
Default value of Node template parameter.
Kokkos::Details::ArithTraits< Packet >::val_type packet_type
The type of each datum being sent or received in an Import or Export.
virtual size_t constantNumberOfPackets() const
Whether the implementation&#39;s instance promises always to have a constant number of packets per LID...
Abstract base class for sources of an Import or Export.
virtual void doTransfer(const SrcDistObject &src, CombineMode CM, size_t numSameIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteToLIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteFromLIDs, const Teuchos::ArrayView< const local_ordinal_type > &remoteLIDs, const Teuchos::ArrayView< const local_ordinal_type > &exportLIDs, Distributor &distor, ReverseOption revOp)
Redistribute data across memory images.
virtual void removeEmptyProcessesInPlace(const Teuchos::RCP< const map_type > &newMap)
Remove processes which contain no elements in this object&#39;s Map.
int local_ordinal_type
Default value of LocalOrdinal template parameter.
Node::device_type device_type
The Kokkos Device type.
virtual void releaseViews() const
Hook for releasing views.
virtual void packAndPrepare(const SrcDistObject &source, const Teuchos::ArrayView< const local_ordinal_type > &exportLIDs, Teuchos::Array< packet_type > &exports, const Teuchos::ArrayView< size_t > &numPacketsPerLID, size_t &constantNumPackets, Distributor &distor)
Perform any packing or preparation required for communication.
GlobalOrdinal global_ordinal_type
The type of global indices.
LocalOrdinal local_ordinal_type
The type of local indices.
virtual bool checkSizes(const SrcDistObject &source)=0
Compare the source and target (this) objects for compatibility.
virtual void unpackAndCombine(const Teuchos::ArrayView< const local_ordinal_type > &importLIDs, const Teuchos::ArrayView< const packet_type > &imports, const Teuchos::ArrayView< size_t > &numPacketsPerLID, size_t constantNumPackets, Distributor &distor, CombineMode CM)
Perform any unpacking and combining after communication.
Communication plan for data redistribution from a (possibly) multiply-owned to a uniquely-owned distr...
Map< local_ordinal_type, global_ordinal_type, node_type > map_type
The type of the Map specialization to use with this class.
virtual void createViews() const
Hook for creating a const view.
Sets up and executes a communication plan for a Tpetra DistObject.
CombineMode
Rule for combining data in an Import or Export.
virtual ~DistObject()
Destructor (virtual for memory safety of derived classes).
Teuchos::RCP< const map_type > map_
The Map over which this object is distributed.
virtual void copyAndPermute(const SrcDistObject &source, size_t numSameIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteToLIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteFromLIDs)
Perform copies and permutations that are local to this process.
bool isDistributed() const
Whether this is a globally distributed object.
Abstract base class for objects that can be the source of an Import or Export operation.
void print(std::ostream &os) const
Print this object to the given output stream.
DistObject(const Teuchos::RCP< const map_type > &map)
Constructor.
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Print a descriptiion of this object to the given output stream.
virtual void createViewsNonConst(KokkosClassic::ReadWriteOption rwo)
Hook for creating a nonconst view.
Describes a parallel distribution of objects over processes.
virtual bool useNewInterface()
Whether lass (???) implements old or new interface.
void doExport(const SrcDistObject &source, const Export< LocalOrdinal, GlobalOrdinal, Node > &exporter, CombineMode CM)
Export data into this object using an Export object ("forward mode").
virtual Teuchos::RCP< const map_type > getMap() const
The Map describing the parallel distribution of this object.
Base class for distributed Tpetra objects that support data redistribution.
void doImport(const SrcDistObject &source, const Import< LocalOrdinal, GlobalOrdinal, Node > &importer, CombineMode CM)
Import data into this object using an Import object ("forward mode").