MueLu  Version of the Day
MueLu_Zoltan2Interface_def.hpp
Go to the documentation of this file.
1 // @HEADER
2 //
3 // ***********************************************************************
4 //
5 // MueLu: A package for multigrid based preconditioning
6 // Copyright 2012 Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact
39 // Jonathan Hu (jhu@sandia.gov)
40 // Andrey Prokopenko (aprokop@sandia.gov)
41 // Ray Tuminaro (rstumin@sandia.gov)
42 //
43 // ***********************************************************************
44 //
45 // @HEADER
46 #ifndef MUELU_ZOLTAN2INTERFACE_DEF_HPP
47 #define MUELU_ZOLTAN2INTERFACE_DEF_HPP
48 
49 #include <sstream>
50 #include <set>
51 
53 #if defined(HAVE_MUELU_ZOLTAN2) && defined(HAVE_MPI)
54 
55 #include <Zoltan2_XpetraMultiVectorAdapter.hpp>
56 #include <Zoltan2_PartitioningProblem.hpp>
57 
58 #include <Teuchos_Utils.hpp>
59 #include <Teuchos_DefaultMpiComm.hpp>
60 #include <Teuchos_OpaqueWrapper.hpp>
61 
62 #include "MueLu_Level.hpp"
63 #include "MueLu_Exceptions.hpp"
64 #include "MueLu_Monitor.hpp"
65 #include "MueLu_Utilities.hpp"
66 
67 namespace MueLu {
68 
69  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
71  defaultZoltan2Params = rcp(new ParameterList());
72  defaultZoltan2Params->set("algorithm", "multijagged");
73  defaultZoltan2Params->set("partitioning_approach", "partition");
74  }
75 
76  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
78  RCP<ParameterList> validParamList = rcp(new ParameterList());
79 
80  validParamList->set< int > ("rowWeight", 0, "Default weight to rows (total weight = nnz + rowWeight");
81 
82  validParamList->set< RCP<const FactoryBase> > ("A", Teuchos::null, "Factory of the matrix A");
83  validParamList->set< RCP<const FactoryBase> > ("Coordinates", Teuchos::null, "Factory of the coordinates");
84  validParamList->set< RCP<const ParameterList> > ("ParameterList", Teuchos::null, "Zoltan2 parameters");
85 
86  return validParamList;
87  }
88 
89 
90  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
92  Input(currentLevel, "A");
93  Input(currentLevel, "Coordinates");
94  }
95 
96  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
98  FactoryMonitor m(*this, "Build", level);
99 
100  RCP<Matrix> A = Get<RCP<Matrix> >(level, "A");
101  RCP<const Map> rowMap = A->getRowMap();
102 
103  typedef Xpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node> dMultiVector;
104  RCP<dMultiVector> coords = Get<RCP<dMultiVector> >(level, "Coordinates");
105  RCP<const Map> map = coords->getMap();
106  GO numElements = map->getNodeNumElements();
107 
108  LO blkSize = A->GetFixedBlockSize();
109 
110  // Check that the number of local coordinates is consistent with the #rows in A
111  TEUCHOS_TEST_FOR_EXCEPTION(rowMap->getNodeNumElements()/blkSize != coords->getLocalLength(), Exceptions::Incompatible,
112  "Coordinate vector length (" + toString(coords->getLocalLength()) << " is incompatible with number of block rows in A ("
113  + toString(rowMap->getNodeNumElements()/blkSize) + "The vector length should be the same as the number of mesh points.");
114 #ifdef HAVE_MUELU_DEBUG
115  GO indexBase = rowMap->getIndexBase();
116  GetOStream(Runtime0) << "Checking consistence of row and coordinates maps" << std::endl;
117  // Make sure that logical blocks in row map coincide with logical nodes in coordinates map
118  ArrayView<const GO> rowElements = rowMap->getNodeElementList();
119  ArrayView<const GO> coordsElements = map ->getNodeElementList();
120  for (LO i = 0; i < Teuchos::as<LO>(numElements); i++)
121  TEUCHOS_TEST_FOR_EXCEPTION((coordsElements[i]-indexBase)*blkSize + indexBase != rowElements[i*blkSize],
122  Exceptions::RuntimeError, "i = " << i << ", coords GID = " << coordsElements[i]
123  << ", row GID = " << rowElements[i*blkSize] << ", blkSize = " << blkSize << std::endl);
124 #endif
125 
126  GO numParts = level.Get<GO>("number of partitions");
127  if (numParts == 1) {
128  // Single processor, decomposition is trivial: all zeros
129  RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap, true);
130  Set(level, "Partition", decomposition);
131  return;
132  }
133 
134  const ParameterList& pL = GetParameterList();
135 
136  RCP<const ParameterList> providedList = pL.get<RCP<const ParameterList> >("ParameterList");
137  ParameterList Zoltan2Params;
138  if (providedList != Teuchos::null)
139  Zoltan2Params = *providedList;
140 
141  // Merge defalt Zoltan2 parameters with user provided
142  // If default and user parameters contain the same parameter name, user one is always preferred
143  for (ParameterList::ConstIterator param = defaultZoltan2Params->begin(); param != defaultZoltan2Params->end(); param++) {
144  const std::string& pName = defaultZoltan2Params->name(param);
145  if (!Zoltan2Params.isParameter(pName))
146  Zoltan2Params.set(pName, defaultZoltan2Params->get<std::string>(pName));
147  }
148  Zoltan2Params.set("num_global_parts", Teuchos::as<int>(numParts));
149 
150  GetOStream(Runtime0) << "Zoltan2 parameters:\n----------\n" << Zoltan2Params << "----------" << std::endl;
151 
152  const std::string& algo = Zoltan2Params.get<std::string>("algorithm");
153  TEUCHOS_TEST_FOR_EXCEPTION(algo != "multijagged" && algo != "rcb", Exceptions::RuntimeError,
154  "Unknown partitioning algorithm: \"" << algo << "\"");
155 
156  typedef Zoltan2::XpetraMultiVectorAdapter<dMultiVector> InputAdapterType;
157  typedef Zoltan2::PartitioningProblem<InputAdapterType> ProblemType;
158 
159  int rowWeight = pL.get<int>("rowWeight");
160  GetOStream(Runtime0) << "Using weights formula: nnz + " << rowWeight << std::endl;
161 
162  Array<double> weightsPerRow(numElements);
163  for (LO i = 0; i < numElements; i++) {
164  weightsPerRow[i] = 0.0;
165 
166  for (LO j = 0; j < blkSize; j++) {
167  weightsPerRow[i] += A->getNumEntriesInLocalRow(i*blkSize+j);
168  // Zoltan2 pqJagged gets as good partitioning as Zoltan RCB in terms of nnz
169  // but Zoltan also gets a good partioning in rows, which sometimes does not
170  // happen for Zoltan2. So here is an attempt to get a better row partitioning
171  // without significantly screwing up nnz partitioning
172  // NOTE: no good heuristic here, the value was chosen almost randomly
173  weightsPerRow[i] += rowWeight;
174  }
175  }
176 
177  std::vector<int> strides;
178  std::vector<const double*> weights(1, weightsPerRow.getRawPtr());
179 
180  RCP<const Teuchos::MpiComm<int> > dupMpiComm = rcp_dynamic_cast<const Teuchos::MpiComm<int> >(rowMap->getComm()->duplicate());
181  RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > zoltanComm = dupMpiComm->getRawMpiComm();
182 
183  InputAdapterType adapter(coords, weights, strides);
184  RCP<ProblemType> problem(new ProblemType(&adapter, &Zoltan2Params, (*zoltanComm)()));
185 
186  {
187  SubFactoryMonitor m1(*this, "Zoltan2 " + toString(algo), level);
188  problem->solve();
189  }
190 
191  RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = Xpetra::VectorFactory<GO,LO,GO,NO>::Build(rowMap, false);
192  ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
193 
194  const typename InputAdapterType::part_t * parts = problem->getSolution().getPartListView();
195 
196  for (GO i = 0; i < numElements; i++) {
197  int partNum = parts[i];
198 
199  for (LO j = 0; j < blkSize; j++)
200  decompEntries[i*blkSize + j] = partNum;
201  }
202 
203  Set(level, "Partition", decomposition);
204  }
205 
206 } //namespace MueLu
207 
208 #endif //if defined(HAVE_MUELU_ZOLTAN2) && defined(HAVE_MPI)
209 
210 #endif // MUELU_ZOLTAN2INTERFACE_DEF_HPP
std::string toString(const T &what)
Little helper function to convert non-string types to strings.
Timer to be used in factories. Similar to Monitor but with additional timers.
One-liner description of what is happening.
Namespace for MueLu classes and methods.
Exception throws to report incompatible objects (like maps).
Class that holds all level-specific information.
Definition: MueLu_Level.hpp:99
Timer to be used in factories. Similar to SubMonitor but adds a timer level by level.
void DeclareInput(Level &level) const
Specifies the data that this class needs, and the factories that generate that data.
void Build(Level &level) const
Build an object with this factory.
Exception throws to report errors in the internal logical of the program.
RCP< const ParameterList > GetValidParameterList() const
Return a const parameter list of valid parameters that setParameterList() will accept.