Zoltan2
APFMeshAdapterTest.cpp
Go to the documentation of this file.
1 // @HEADER
2 //
3 // ***********************************************************************
4 //
5 // Zoltan2: A package of combinatorial algorithms for scientific computing
6 // Copyright 2012 Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact Karen Devine (kddevin@sandia.gov)
39 // Erik Boman (egboman@sandia.gov)
40 // Siva Rajamanickam (srajama@sandia.gov)
41 //
42 // ***********************************************************************
43 //
44 // @HEADER
45 
53 /**************************************************************/
54 /* Includes */
55 /**************************************************************/
56 
58 #include <Zoltan2_Environment.hpp>
62 //Tpetra includes
63 #include "Tpetra_DefaultPlatform.hpp"
64 
65 // Teuchos includes
66 #include "Teuchos_RCP.hpp"
67 #include "Teuchos_GlobalMPISession.hpp"
68 #include "Teuchos_XMLParameterListHelpers.hpp"
69 #include "Teuchos_Hashtable.hpp"
70 
71 // SCOREC includes
72 #ifdef HAVE_ZOLTAN2_PARMA
73 #include <parma.h>
74 #include <apf.h>
75 #include <apfMesh.h>
76 #include <apfMDS.h>
77 #include <apfMesh2.h>
78 #include <PCU.h>
79 #include <gmi_mesh.h>
80 #endif
81 
82 #include <set>
83 
84 using namespace std;
85 using Teuchos::ParameterList;
86 using Teuchos::RCP;
87 using Teuchos::ArrayView;
88 
89 /*********************************************************/
90 /* Typedefs */
91 /*********************************************************/
92 //Tpetra typedefs
93 typedef Tpetra::DefaultPlatform::DefaultPlatformType Platform;
94 
95 
96 // Computes and prints ghost metrics (takes in a Hyper graph model)
97 template <typename Adapter>
99  typedef typename Adapter::gno_t gno_t;
100  typedef typename Adapter::lno_t lno_t;
101  typedef typename Adapter::scalar_t scalar_t;
103 
104  ArrayView<const gno_t> Ids;
105  ArrayView<input_t> wgts;
106  mdl.getEdgeList(Ids,wgts);
107  ArrayView<bool> isOwner;
108  mdl.getOwnedList(isOwner);
109  size_t numOwned = mdl.getLocalNumOwnedVertices();
110  ArrayView<const gno_t> pins;
111  ArrayView<const lno_t> offsets;
112  mdl.getPinList(pins,offsets,wgts);
113 
114  std::set<gno_t> gids;
115  for (size_t i=0;i<mdl.getLocalNumVertices();i++) {
116  if (isOwner[i])
117  gids.insert(Ids[i]);
118  }
119  std::set<gno_t> ghosts;
120  gno_t num_ghosts=0;
121  for (size_t i=0;i<mdl.getLocalNumPins();i++) {
122  gno_t pin = pins[i];
123  if (gids.find(pin)==gids.end()) {
124  num_ghosts++;
125  if (ghosts.find(pin)==ghosts.end())
126  ghosts.insert(pin);
127  }
128  }
129  std::cout<< "[METRIC] " << PCU_Comm_Self() << " Total number of ghosts in the hypergraph: " << num_ghosts << "\n"
130  << "[METRIC] " << PCU_Comm_Self() << " Number of unique ghosts: " << ghosts.size() << "\n";
131  gno_t unique_ghosts =ghosts.size();
132  gno_t owned_and_ghosts =unique_ghosts+numOwned;
133  gno_t max_o_and_g,min_o_and_g;
134  gno_t max_ghosts,max_u_ghosts;
135  gno_t min_ghosts,min_u_ghosts;
136  max_ghosts = min_ghosts = num_ghosts;
137  max_u_ghosts = min_u_ghosts = unique_ghosts;
138  max_o_and_g = min_o_and_g = owned_and_ghosts;
139  double avg_ghosts,avg_u_ghosts,avg_o_and_g;
140  PCU_Add_Ints(&num_ghosts,1);
141  PCU_Add_Ints(&unique_ghosts,1);
142  PCU_Add_Ints(&owned_and_ghosts,1);
143  PCU_Max_Ints(&max_ghosts,1);
144  PCU_Max_Ints(&max_u_ghosts,1);
145  PCU_Max_Ints(&max_o_and_g,1);
146  PCU_Min_Ints(&min_ghosts,1);
147  PCU_Min_Ints(&min_u_ghosts,1);
148  PCU_Min_Ints(&min_o_and_g,1);
149  avg_ghosts = num_ghosts*1.0/PCU_Comm_Peers();
150  avg_u_ghosts = unique_ghosts*1.0/PCU_Comm_Peers();
151  avg_o_and_g = owned_and_ghosts*1.0/PCU_Comm_Peers();
152  if (!PCU_Comm_Self())
153  std::cout<< "[METRIC] Global ghosts in the hypergraph (tot max min avg imb): "
154  << num_ghosts<<" "<<max_ghosts<<" "<<min_ghosts<<" "<<avg_ghosts<<" "
155  <<max_ghosts/avg_ghosts << "\n"
156  << "[METRIC] Global unique ghosts (tot max min avg imb): "
157  << unique_ghosts<<" "<<max_u_ghosts<<" "<<min_u_ghosts<<" "<<avg_u_ghosts<<" "
158  <<max_u_ghosts/avg_u_ghosts << "\n"
159  << "[METRIC] Global owned and ghosts (tot max min avg imb): "
160  << owned_and_ghosts<<" "<<max_o_and_g<<" "<<min_o_and_g<<" "<<avg_o_and_g<<" "
161  <<max_o_and_g/avg_o_and_g << "\n";
162 
163 }
164 
165 /*****************************************************************************/
166 /******************************** MAIN ***************************************/
167 /*****************************************************************************/
168 
169 int main(int narg, char *arg[]) {
170 
171  Teuchos::GlobalMPISession mpiSession(&narg, &arg,0);
172  Platform &platform = Tpetra::DefaultPlatform::getDefaultPlatform();
173  RCP<const Teuchos::Comm<int> > CommT = platform.getComm();
174 
175  int me = CommT->getRank();
176  //int numProcs = CommT->getSize();
177 
178  if (me == 0){
179  cout
180  << "====================================================================\n"
181  << "| |\n"
182  << "| Example: Partition APF Mesh |\n"
183  << "| |\n"
184  << "| Questions? Contact Karen Devine (kddevin@sandia.gov), |\n"
185  << "| Erik Boman (egboman@sandia.gov), |\n"
186  << "| Siva Rajamanickam (srajama@sandia.gov). |\n"
187  << "| |\n"
188  << "| Zoltan2's website: http://trilinos.sandia.gov/packages/zoltan2 |\n"
189  << "| Trilinos website: http://trilinos.sandia.gov |\n"
190  << "| |\n"
191  << "====================================================================\n";
192  }
193 
194 
195 #ifdef HAVE_MPI
196  if (me == 0) {
197  cout << "PARALLEL executable \n";
198  }
199 #else
200  if (me == 0) {
201  cout << "SERIAL executable \n";
202  }
203 #endif
204 
205  /***************************************************************************/
206  /******************************* GET INPUTS ********************************/
207  /***************************************************************************/
208 
209  // default values for command-line arguments
210  std::string meshFileName("4/");
211  std::string modelFileName("torus.dmg");
212  std::string action("parma");
213  std::string parma_method("VtxElm");
214  std::string output_loc("");
215  int nParts = CommT->getSize();
216  double imbalance = 1.1;
217  int layers=2;
218  int ghost_metric=false;
219  // Read run-time options.
220  Teuchos::CommandLineProcessor cmdp (false, false);
221  cmdp.setOption("meshfile", &meshFileName,
222  "Mesh file with APF specifications (.smb file(s))");
223  cmdp.setOption("modelfile", &modelFileName,
224  "Model file with APF specifications (.dmg file)");
225  cmdp.setOption("action", &action,
226  "Method to use: mj, scotch, zoltan_rcb, parma or color");
227  cmdp.setOption("parma_method", &parma_method,
228  "Method to use: Vertex, Element, VtxElm, VtxEdgeElm, Ghost, Shape, or Centroid ");
229  cmdp.setOption("nparts", &nParts,
230  "Number of parts to create");
231  cmdp.setOption("imbalance", &imbalance,
232  "Target imbalance for the partitioning method");
233  cmdp.setOption("output", &output_loc,
234  "Location of new partitioned apf mesh. Ex: 4/torus.smb");
235  cmdp.setOption("layers", &layers,
236  "Number of layers for ghosting");
237  cmdp.setOption("ghost_metric", &ghost_metric,
238  "0 does not compute ghost metric otherwise compute both before and after");
239  cmdp.parse(narg, arg);
240 
241  /***************************************************************************/
242  /********************** GET CELL TOPOLOGY **********************************/
243  /***************************************************************************/
244 
245  // Get dimensions
246  //int dim = 3;
247 
248  /***************************************************************************/
249  /***************************** GENERATE MESH *******************************/
250  /***************************************************************************/
251 
252 #ifdef HAVE_ZOLTAN2_PARMA
253 
254  if (me == 0) cout << "Generating mesh ... \n\n";
255 
256  //Setup for SCOREC
257  PCU_Comm_Init();
258 
259  // Generate mesh with MDS
260  gmi_register_mesh();
261  apf::Mesh2* m = apf::loadMdsMesh(modelFileName.c_str(),meshFileName.c_str());
262  apf::verify(m);
263 
264  //Data for APF MeshAdapter
265  std::string primary="region";
266  std::string adjacency="face";
267  if (m->getDimension()==2) {
268  primary="face";
269  adjacency="edge";
270  }
271  bool needSecondAdj=false;
272 
273  // Set parameters for partitioning
274  if (me == 0) cout << "Creating parameter list ... \n\n";
275 
276  Teuchos::ParameterList params("test params");
277  params.set("timer_output_stream" , "std::cout");
278 
279  bool do_partitioning = false;
280  if (action == "mj") {
281  do_partitioning = true;
282  params.set("debug_level", "basic_status");
283  params.set("imbalance_tolerance", imbalance);
284  params.set("num_global_parts", nParts);
285  params.set("algorithm", "multijagged");
286  params.set("rectilinear", "yes");
287  }
288  else if (action == "scotch") {
289  do_partitioning = true;
290  params.set("debug_level", "no_status");
291  params.set("imbalance_tolerance", imbalance);
292  params.set("num_global_parts", nParts);
293  params.set("partitioning_approach", "partition");
294  params.set("objects_to_partition","mesh_elements");
295  params.set("algorithm", "scotch");
296  needSecondAdj=true;
297  }
298  else if (action == "zoltan_rcb") {
299  do_partitioning = true;
300  params.set("debug_level", "verbose_detailed_status");
301  params.set("imbalance_tolerance", imbalance);
302  params.set("num_global_parts", nParts);
303  params.set("partitioning_approach", "partition");
304  params.set("algorithm", "zoltan");
305  }
306  else if (action == "parma") {
307  do_partitioning = true;
308  params.set("debug_level", "no_status");
309  params.set("imbalance_tolerance", imbalance);
310  params.set("algorithm", "parma");
311  Teuchos::ParameterList &pparams = params.sublist("parma_parameters",false);
312  pparams.set("parma_method",parma_method);
313  pparams.set("step_size",1.1);
314  if (parma_method=="Ghost") {
315  pparams.set("ghost_layers",layers);
316  pparams.set("ghost_bridge",m->getDimension()-1);
317  }
318  params.set("compute_metrics","yes");
319  adjacency="vertex";
320  }
321  else if (action=="zoltan_hg") {
322  do_partitioning = true;
323  params.set("debug_level", "no_status");
324  params.set("imbalance_tolerance", imbalance);
325  params.set("algorithm", "zoltan");
326  params.set("num_global_parts", nParts);
327  Teuchos::ParameterList &zparams = params.sublist("zoltan_parameters",false);
328  zparams.set("LB_METHOD","HYPERGRAPH");
329  zparams.set("LB_APPROACH","PARTITION");
330  //params.set("compute_metrics","yes");
331  adjacency="vertex";
332  }
333  else if (action=="hg_ghost") {
334  do_partitioning = true;
335  params.set("debug_level", "no_status");
336  params.set("imbalance_tolerance", imbalance);
337  params.set("algorithm", "zoltan");
338  params.set("num_global_parts", nParts);
339  params.set("hypergraph_model_type","ghosting");
340  params.set("ghost_layers",layers);
341  Teuchos::ParameterList &zparams = params.sublist("zoltan_parameters",false);
342  zparams.set("LB_METHOD","HYPERGRAPH");
343  zparams.set("LB_APPROACH","PARTITION");
344  zparams.set("PHG_EDGE_SIZE_THRESHOLD", "1.0");
345  primary="vertex";
346  adjacency="edge";
347  needSecondAdj=true;
348  }
349  else if (action == "color") {
350  params.set("debug_level", "verbose_detailed_status");
351  params.set("debug_output_file", "kdd");
352  params.set("debug_procs", "all");
353  }
354  Parma_PrintPtnStats(m,"before");
355 
356  // Creating mesh adapter
357  if (me == 0) cout << "Creating mesh adapter ... \n\n";
358  typedef Zoltan2::APFMeshAdapter<apf::Mesh2*> inputAdapter_t;
359  typedef Zoltan2::MeshAdapter<apf::Mesh2*> baseMeshAdapter_t;
360 
361  double time_1=PCU_Time();
362  inputAdapter_t ia(*CommT, m,primary,adjacency,needSecondAdj);
363  double time_2=PCU_Time();
364 
365 
366  inputAdapter_t::scalar_t* arr = new inputAdapter_t::scalar_t[ia.getLocalNumOf(ia.getPrimaryEntityType())];
367  for (size_t i=0;i<ia.getLocalNumOf(ia.getPrimaryEntityType());i++) {
368  arr[i]=PCU_Comm_Self()+1;
369  }
370 
371  const inputAdapter_t::scalar_t* weights=arr;
372  ia.setWeights(ia.getPrimaryEntityType(),weights,1);
373 
374 
375  if (ghost_metric) {
376  const baseMeshAdapter_t *base_ia = dynamic_cast<const baseMeshAdapter_t*>(&ia);
377  Zoltan2::modelFlag_t graphFlags_;
378  RCP<Zoltan2::Environment> env;
379  try{
380  env = rcp(new Zoltan2::Environment(params, Teuchos::DefaultComm<int>::getComm()));
381  }
383 
384  RCP<const Zoltan2::Environment> envConst = Teuchos::rcp_const_cast<const Zoltan2::Environment>(env);
385 
386  RCP<const baseMeshAdapter_t> baseInputAdapter_(base_ia,false);
387  Zoltan2::HyperGraphModel<inputAdapter_t> model(baseInputAdapter_,envConst,CommT,
388  graphFlags_,Zoltan2::HYPEREDGE_CENTRIC);
389  PrintGhostMetrics(model);
390  }
391 
392  // create Partitioning problem
393  double time_3 = PCU_Time();
394  if (do_partitioning) {
395  if (me == 0) cout << "Creating partitioning problem ... \n\n";
396 
397  Zoltan2::PartitioningProblem<inputAdapter_t> problem(&ia, &params, CommT);
398 
399  // call the partitioner
400  if (me == 0) cout << "Calling the partitioner ... \n\n";
401 
402  problem.solve();
403 
404 
405 
406  if (me==0) cout << "Applying Solution to Mesh\n\n";
407  apf::Mesh2** new_mesh = &m;
408  ia.applyPartitioningSolution(m,new_mesh,problem.getSolution());
409 
410 
411 
412  if (!me) problem.printMetrics(cout);
413  }
414  else {
415  if (me == 0) cout << "Creating coloring problem ... \n\n";
416 
417  Zoltan2::ColoringProblem<inputAdapter_t> problem(&ia, &params);
418 
419  // call the partitioner
420  if (me == 0) cout << "Calling the coloring algorithm ... \n\n";
421 
422  problem.solve();
423 
424  problem.printTimers();
425 
426 
427  }
428 
429  double time_4=PCU_Time();
430 
431  //Destroy the adapter
432  ia.destroy();
433  delete [] arr;
434  //Parma_PrintPtnStats(m,"after");
435 
436  if (ghost_metric) {
437  inputAdapter_t ia2(*CommT, m,primary,adjacency,true);
438  const baseMeshAdapter_t *base_ia = dynamic_cast<const baseMeshAdapter_t*>(&ia2);
439 
440  Zoltan2::modelFlag_t graphFlags_;
441  RCP<Zoltan2::Environment> env;
442  try{
443  env = rcp(new Zoltan2::Environment(params, Teuchos::DefaultComm<int>::getComm()));
444  }
446  RCP<const Zoltan2::Environment> envConst = Teuchos::rcp_const_cast<const Zoltan2::Environment>(env);
447  RCP<const baseMeshAdapter_t> baseInputAdapter_(base_ia,false);
448  Zoltan2::HyperGraphModel<inputAdapter_t> model(baseInputAdapter_, envConst, CommT,
449  graphFlags_,Zoltan2::HYPEREDGE_CENTRIC);
450 
451  PrintGhostMetrics(model);
452  ia2.destroy();
453  }
454 
455  if (output_loc!="") {
456  m->writeNative(output_loc.c_str());
457  }
458 
459  // delete mesh
460  if (me == 0) cout << "Deleting the mesh ... \n\n";
461  time_4-=time_3;
462  time_2-=time_1;
463  PCU_Max_Doubles(&time_2,1);
464  PCU_Max_Doubles(&time_4,1);
465  if (!me) {
466  std::cout<<"\nConstruction time: "<<time_2<<"\n"
467  <<"Problem time: " << time_4<<"\n\n";
468  }
469  //Delete the APF Mesh
470  m->destroyNative();
471  apf::destroyMesh(m);
472  //End communications
473  PCU_Comm_Free();
474 #endif
475 
476  if (me == 0)
477  std::cout << "PASS" << std::endl;
478 
479  return 0;
480 
481 }
482 /*****************************************************************************/
483 /********************************* END MAIN **********************************/
484 /*****************************************************************************/
ColoringProblem sets up coloring problems for the user.
Defines the ColoringProblem class.
#define Z2_FORWARD_EXCEPTIONS
Forward an exception back through call stack.
void solve(bool updateInputData=true)
Direct the problem to create a solution.
MeshAdapter defines the interface for mesh input.
std::bitset< NUM_MODEL_FLAGS > modelFlag_t
size_t getPinList(ArrayView< const gno_t > &pinIds, ArrayView< const lno_t > &offsets, ArrayView< input_t > &wgts) const
Sets pointers to this process&#39; pins global Ids based on the centric view given by getCentricView() ...
static ArrayRCP< ArrayRCP< zscalar_t > > weights
Defines the APFMeshAdapter class.
size_t getOwnedList(ArrayView< bool > &isOwner) const
Sets pointer to the ownership of this processes vertices.
Tpetra::DefaultPlatform::DefaultPlatformType Platform
size_t getEdgeList(ArrayView< const gno_t > &Ids, ArrayView< input_t > &wgts) const
Sets pointers to this process&#39; hyperedge Ids and their weights.
void PrintGhostMetrics(Zoltan2::HyperGraphModel< Adapter > &mdl)
void printMetrics(std::ostream &os) const
Print the array of metrics.
The StridedData class manages lists of weights or coordinates.
Tpetra::DefaultPlatform::DefaultPlatformType Platform
int main(int narg, char *arg[])
void printTimers() const
Return the communicator passed to the problem.
The user parameters, debug, timing and memory profiling output objects, and error checking methods...
size_t getLocalNumPins() const
Returns the local number of pins.
const PartitioningSolution< Adapter > & getSolution()
Get the solution to the problem.
PartitioningProblem sets up partitioning problems for the user.
Defines the HyperGraphModel interface.
Defines the Environment class.
#define nParts
size_t getLocalNumVertices() const
Returns the number vertices on this process.
Defines the PartitioningProblem class.
HyperGraphModel defines the interface required for hyper graph models.
void solve(bool updateInputData=true)
Direct the problem to create a solution.
size_t getLocalNumOwnedVertices() const
Returns the number vertices on this process that are owned.