46 #ifndef MUELU_ZOLTANINTERFACE_DEF_HPP 47 #define MUELU_ZOLTANINTERFACE_DEF_HPP 50 #if defined(HAVE_MUELU_ZOLTAN) && defined(HAVE_MPI) 52 #include <Teuchos_Utils.hpp> 53 #include <Teuchos_DefaultMpiComm.hpp> 54 #include <Teuchos_OpaqueWrapper.hpp> 59 #include "MueLu_Utilities.hpp" 63 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
65 RCP<ParameterList> validParamList = rcp(
new ParameterList());
67 validParamList->set< RCP<const FactoryBase> >(
"A", Teuchos::null,
"Factory of the matrix A");
68 validParamList->set< RCP<const FactoryBase> >(
"Coordinates", Teuchos::null,
"Factory of the coordinates");
70 return validParamList;
74 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
76 Input(currentLevel,
"A");
77 Input(currentLevel,
"Coordinates");
80 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
84 RCP<Matrix> A = Get< RCP<Matrix> > (level,
"A");
85 RCP<const Map> rowMap = A->getRowMap();
87 typedef Xpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node> double_multivector_type;
88 RCP<double_multivector_type> Coords = Get< RCP<double_multivector_type> >(level,
"Coordinates");
89 size_t dim = Coords->getNumVectors();
91 GO numParts = level.Get<GO>(
"number of partitions");
95 RCP<Xpetra::Vector<GO, LO, GO, NO> > decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap,
true);
96 Set(level,
"Partition", decomposition);
100 float zoltanVersion_;
101 Zoltan_Initialize(0, NULL, &zoltanVersion_);
103 RCP<const Teuchos::MpiComm<int> > dupMpiComm = rcp_dynamic_cast<
const Teuchos::MpiComm<int> >(rowMap->getComm()->duplicate());
104 RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > zoltanComm = dupMpiComm->getRawMpiComm();
106 RCP<Zoltan> zoltanObj_ = rcp(
new Zoltan((*zoltanComm)()));
107 if (zoltanObj_ == Teuchos::null)
114 if ((rv = zoltanObj_->Set_Param(
"num_gid_entries",
"1")) != ZOLTAN_OK)
116 if ((rv = zoltanObj_->Set_Param(
"num_lid_entries",
"0") ) != ZOLTAN_OK)
118 if ((rv = zoltanObj_->Set_Param(
"obj_weight_dim",
"1") ) != ZOLTAN_OK)
121 if (GetVerbLevel() &
Statistics1) zoltanObj_->Set_Param(
"debug_level",
"1");
122 else zoltanObj_->Set_Param(
"debug_level",
"0");
124 zoltanObj_->Set_Param(
"num_global_partitions",
toString(numParts));
126 zoltanObj_->Set_Num_Obj_Fn(GetLocalNumberOfRows, (
void *) &*A);
127 zoltanObj_->Set_Obj_List_Fn(GetLocalNumberOfNonzeros, (
void *) &*A);
128 zoltanObj_->Set_Num_Geom_Fn(GetProblemDimension, (
void *) &dim);
129 zoltanObj_->Set_Geom_Multi_Fn(GetProblemGeometry, (
void *) Coords.get());
132 ZOLTAN_ID_PTR import_gids = NULL;
133 ZOLTAN_ID_PTR import_lids = NULL;
134 int *import_procs = NULL;
135 int *import_to_part = NULL;
136 ZOLTAN_ID_PTR export_gids = NULL;
137 ZOLTAN_ID_PTR export_lids = NULL;
138 int *export_procs = NULL;
139 int *export_to_part = NULL;
148 rv = zoltanObj_->LB_Partition(newDecomp, num_gid_entries, num_lid_entries,
149 num_imported, import_gids, import_lids, import_procs, import_to_part,
150 num_exported, export_gids, export_lids, export_procs, export_to_part);
151 if (rv == ZOLTAN_FATAL)
157 RCP<Xpetra::Vector<GO, LO, GO, NO> > decomposition;
159 decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap,
false);
160 ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
162 int mypid = rowMap->getComm()->getRank();
163 for (
typename ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i)
166 LO blockSize = A->GetFixedBlockSize();
167 for (
int i = 0; i < num_exported; ++i) {
170 LO localEl = rowMap->getLocalElement(export_gids[i]);
171 int partNum = export_to_part[i];
172 for (LO j = 0; j < blockSize; ++j)
173 decompEntries[localEl + j] = partNum;
177 Set(level,
"Partition", decomposition);
179 zoltanObj_->LB_Free_Part(&import_gids, &import_lids, &import_procs, &import_to_part);
180 zoltanObj_->LB_Free_Part(&export_gids, &export_lids, &export_procs, &export_to_part);
188 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
191 *ierr = ZOLTAN_FATAL;
194 Matrix *A = (Matrix*) data;
197 LO blockSize = A->GetFixedBlockSize();
200 return A->getRowMap()->getNodeNumElements() / blockSize;
207 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
210 ZOLTAN_ID_PTR lids,
int wgtDim,
float *weights,
int *ierr) {
211 if (data == NULL || NumGidEntries < 1) {
212 *ierr = ZOLTAN_FATAL;
218 Matrix *A = (Matrix*) data;
219 RCP<const Map> map = A->getRowMap();
221 LO blockSize = A->GetFixedBlockSize();
224 size_t numElements = map->getNodeNumElements();
225 ArrayView<const GO> mapGIDs = map->getNodeElementList();
227 if (blockSize == 1) {
228 for (
size_t i = 0; i < numElements; i++) {
229 gids[i] = as<ZOLTAN_ID_TYPE>(mapGIDs[i]);
230 weights[i] = A->getNumEntriesInLocalRow(i);
234 LO numBlockElements = numElements / blockSize;
236 for (LO i = 0; i < numBlockElements; i++) {
239 gids[i] = as<ZOLTAN_ID_TYPE>(mapGIDs[i*blockSize]);
241 for (LO j = 0; j < blockSize; j++)
242 weights[i] += A->getNumEntriesInLocalRow(i*blockSize+j);
252 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
256 int dim = *((
int*)data);
266 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
269 ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
int dim,
double *coordinates,
int *ierr)
272 *ierr = ZOLTAN_FATAL;
276 typedef Xpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node> double_multivector_type;
277 double_multivector_type *Coords = (double_multivector_type*) data;
279 if (dim != Teuchos::as<int>(Coords->getNumVectors())) {
281 *ierr = ZOLTAN_FATAL;
285 TEUCHOS_TEST_FOR_EXCEPTION(numObjectIDs != Teuchos::as<int>(Coords->getLocalLength()),
Exceptions::Incompatible,
"Length of coordinates must be the same as the number of objects");
287 ArrayRCP<ArrayRCP<const double> > CoordsData(dim);
288 for (
int j = 0; j < dim; ++j)
289 CoordsData[j] = Coords->getData(j);
291 size_t numElements = Coords->getLocalLength();
292 for (
size_t i = 0; i < numElements; ++i)
293 for (
int j = 0; j < dim; ++j)
294 coordinates[i*dim+j] = (
double) CoordsData[j][i];
302 #endif //if defined(HAVE_MUELU_ZOLTAN) && defined(HAVE_MPI) 304 #endif // MUELU_ZOLTANINTERFACE_DEF_HPP
std::string toString(const T &what)
Little helper function to convert non-string types to strings.
Timer to be used in factories. Similar to Monitor but with additional timers.
static int GetProblemDimension(void *data, int *ierr)
Namespace for MueLu classes and methods.
static void GetProblemGeometry(void *data, int numGIDEntries, int numLIDEntries, int numObjectIDs, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int dim, double *coordinates, int *ierr)
Exception throws to report incompatible objects (like maps).
void Build(Level &level) const
Build an object with this factory.
Class that holds all level-specific information.
Timer to be used in factories. Similar to SubMonitor but adds a timer level by level.
static void GetLocalNumberOfNonzeros(void *data, int NumGidEntries, int NumLidEntries, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int wgtDim, float *weights, int *ierr)
void DeclareInput(Level &level) const
Specifies the data that this class needs, and the factories that generate that data.
Exception throws to report errors in the internal logical of the program.
RCP< const ParameterList > GetValidParameterList() const
Return a const parameter list of valid parameters that setParameterList() will accept.
static int GetLocalNumberOfRows(void *data, int *ierr)