Zoltan2
Zoltan2_ComparisonHelper.hpp
Go to the documentation of this file.
1 // @HEADER
2 //
3 // ***********************************************************************
4 //
5 // Zoltan2: A package of combinatorial algorithms for scientific computing
6 // Copyright 2012 Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact Karen Devine (kddevin@sandia.gov)
39 // Erik Boman (egboman@sandia.gov)
40 // Siva Rajamanickam (srajama@sandia.gov)
41 //
42 // ***********************************************************************
43 //
44 // @HEADER
45 
50 #pragma once
51 
52 #include "Zoltan2_TestHelpers.hpp"
53 #include <AdapterForTests.hpp>
57 
58 #include <Teuchos_DefaultComm.hpp>
59 #include <Teuchos_Time.hpp>
60 
61 #include <sstream>
62 #include <string>
63 #include <map>
64 #include <iostream>
65 
66 using Teuchos::Comm;
67 using Teuchos::RCP;
68 using Teuchos::ParameterList;
69 using Teuchos::Time;
70 using std::cout;
71 using std::endl;
72 using std::string;
73 using std::map;
74 using std::pair;
75 using std::ostringstream;
76 using Teuchos::reduceAll;
77 
81 {
82 
83 public:
84 
86  typedef AdapterForTests::basic_id_t basic_id_t; // basic_identifier_type
91 
99 
100 
104  {
105  if(adapter_kind == "XpetraCrsGraph")
106  delete reinterpret_cast<xcrsGraph_t *>(adapter.getRawPtr())->getCoordinateInput();
107  if(adapter_kind == "XpetraCrsMatrix")
108  delete reinterpret_cast<xcrsMatrix_t *>(adapter.getRawPtr())->getCoordinateInput();
109  }
110  /* \brief Add a timer by name to the comparison sources timers map.
111  * \param name is the name of the timer to be defined
112  */
113  void addTimer(const std::string &name)
114  {
115  timers.insert(std::pair<const std::string &, RCP<Time> >(name,rcp(new Time(name))));
116  timers[name]->enable();
117  }
118 
119  RCP<basic_problem_t> problem;
120  RCP<basic_id_t> adapter;
121  string problem_kind;
122  string adapter_kind;
123  std::map<const std::string, RCP<Time> > timers;
124 };
125 
129 {
130 
131 public:
132 
134  typedef AdapterForTests::basic_id_t basic_id_t; // basic_identifier_type
139 
147 
149 
150  /* \brief Compare the solutions, metrics or timers of two Zoltan2 solutions.
151  * \param pList is a parameter list defining the comparison
152  * \param comm is the process communicator
153  */
154  void Compare(const ParameterList &pList, const RCP<const Comm<int> > &comm);
155 
156  /* \brief Add a new source by name to the comparison source map.
157  * \param name is the name of the new source
158  * \param source a problem source that to be used for comparison to another source
159  */
160  void AddSource(const string &name, ComparisonSource * source);
161 
162  /* \brief Return the total number of saved sources.
163  */
164  size_t getNumberOfSources() const
165  {
166  return this->sources.size();
167  }
168 
169 private:
170  map<const string,RCP<const ComparisonSource> > sources;
171 
172 
173  /* \brief Method called to compare two solutions
174  * \param p1 is the name of problem 1
175  * \param p2 is the name of problem 2
176  * \param comm is the process communicator
177  */
178  void CompareSolutions(const string &p1,
179  const string &p2,
180  const RCP<const Comm<int> > &comm);
181 
182  /* \brief Method called to compare two paritioning solutions
183  * \param sourceA is a ptr to problem A's comparison source
184  * \param sourceB is a ptr to problem B's comparison source
185  * \param comm is the process communicator
186  */
187  void ComparePartitionSolutions(const ComparisonSource * sourceA,
188  const ComparisonSource * sourceB,
189  const RCP<const Comm<int> > &comm);
190 
191  /* \brief Method called to compare two coloring solutions
192  * \param sourceA is a ptr to problem A's comparison source
193  * \param sourceB is a ptr to problem B's comparison source
194  * \param comm is the process communicator
195  */
196  void CompareColoringSolutions(const ComparisonSource * sourceA,
197  const ComparisonSource * sourceB,
198  const RCP<const Comm<int> > &comm);
199 
200  /* \brief Method called to compare two ordering solutions
201  * \param sourceA is a ptr to problem A's comparison source
202  * \param sourceB is a ptr to problem B's comparison source
203  * \param comm is the process communicator
204  */
205  void CompareOrderingSolutions(const ComparisonSource * sourceA,
206  const ComparisonSource * sourceB,
207  const RCP<const Comm<int> > &comm);
208 
209  /* \brief Method called to compare the metrics/timers of two problems.
210  * \param metricsPlist is a parameter list defining the comparison
211  * \param comm is the process communicator
212  */
213  void CompareMetrics(const ParameterList &metricsPlist,
214  const RCP<const Comm<int> > &comm);
215 
216  /* \brief Method that compares two metrics and returns a pass/fail message.
217  * \param[in] comm is the process communicator
218  * \param[in] metric is the metric to be compared to a reference metric
219  * \param[in] ref_metric is the reference metric for comparison
220  * \param[in] metricPlist is the parameter list defining the metric tolerances
221  * \param[out] msg is a returned pass/fail message
222  *
223  * \return boolean value indicated pass/fail status
224  */
225  static bool
226  metricComparisonTest(const RCP<const Comm<int> > &comm,
227  const metric_t & metric,
228  const metric_t &ref_metric,
229  const Teuchos::ParameterList & metricPlist,
230  ostringstream &msg);
231 
232  /* \brief Method that compares two timers and returns a pass/fail message.
233  * \param[in] comm is the process communicator
234  * \param[in] time is the timer data to be compared to a reference metric
235  * \param[in] ref_time is the reference timer for comparison
236  * \param[in] metricPlist is the parameter list defining the timer tolerances
237  * \param[out] msg is a returned pass/fail message
238  *
239  * \return boolean value indicated pass/fail status
240  */
241  static bool
242  timerComparisonTest(const RCP<const Comm<int> > &comm,
243  const double time,
244  const double ref_time,
245  const Teuchos::ParameterList & metricPlist,
246  ostringstream &msg);
247 
248  /* \brief Method for inserting an array of metrics into a map
249  * param[in] metrics an array of metric objects
250  *
251  * \return a map with metrics assigned to keys assigned by name
252  */
253  static std::map<const string, const metric_t>
254  metricArrayToMap(const ArrayRCP<const metric_t> &metrics);
255 
256  /* \brief Method for inserting data from all timers to a map of clocked times
257  * param[in] timers a map of timers
258  *
259  * \return a map with clocked times from timers
260  */
261  static std::map<const string, const double>
262  timerDataToMap(const map<const std::string, RCP<Time> > &timers);
263 
264 
265  /* \brief Method for extracting all methods to compare from a parameter list
266  * param[in] plist a parameter list defining 1 or more metric/timer comparisons
267  *
268  * \return a queue of metric comparison definitions
269  */
270  static std::queue<ParameterList>
271  getMetricsToCompare(const ParameterList & pList);
272 
273  static void
274  reduceWithMessage(const RCP<const Comm<int> > &comm,const std::string &msg_in,
275  int &local_status, std::ostringstream &msg);
276 
277 };
278 
279 
280 void ComparisonHelper::AddSource(const string &name, ComparisonSource * source)
281 {
282  typedef std::pair<const string &, RCP<const ComparisonSource> > pair_t;
283  this->sources.insert(pair_t(name, RCP<ComparisonSource>(source)));
284 }
285 
286 void ComparisonHelper::Compare(const ParameterList &pList, const RCP<const Comm<int> > &comm)
287 {
288  if(pList.isParameter("A") && pList.isParameter("B"))
289  {
290  // comparing solutions
291 
292  string pA = pList.get<string>("A");
293  if(this->sources.find(pA) == this->sources.end())
294  {
295  cout << "\nProblem: " + pA + ", was not saved for comparison.";
296  cout << "\nThis typically indicates that an error occured while running the problem.";
297  cout << "\nSolution comparison FAILED." << endl;
298  return;
299  }
300 
301  string pB = pList.get<string>("B");
302  if(this->sources.find(pB) == this->sources.end())
303  {
304  cout << "\nProblem: " + pB + ", was not saved for comparison.";
305  cout << "\nThis typically indicates that an error occured while running the problem.";
306  cout << "\nSolution comparison FAILED." << endl;
307 
308  return;
309  }
310 
311  this->CompareSolutions(pA, pB, comm);
312  }else if (pList.isParameter("Problem") && pList.isParameter("Reference"))
313  {
314  // comparing metrics/timers
315  string prb = pList.get<string>("Problem");
316  if(this->sources.find(prb) == this->sources.end())
317  {
318  cout << "\nProblem: " + prb + ", was not saved for comparison.";
319  cout << "\nThis typically indicates that an error occured while running the problem.";
320  cout << "\nMetric comparison FAILED." << endl;
321  return;
322  }
323 
324  string ref = pList.get<string>("Reference");
325  if(this->sources.find(ref) == this->sources.end())
326  {
327  cout << "\nReference: " + ref + ", was not saved for comparison.";
328  cout << "\nThis typically indicates that an error occured while running the problem.";
329  cout << "\nMetric comparison FAILED." << endl;
330  return;
331  }
332 
333  this->CompareMetrics(pList,
334  comm);
335 
336  }else if (pList.isParameter("A") || pList.isParameter("B"))
337  {
338  if(comm->getRank() == 0)
339  {
340  cout << "Problem A or Problem B is not specified -- check input.";
341  cout <<"\nSolution comparison FAILED." << endl;
342  }
343 
344  }else if (pList.isParameter("Problem") || pList.isParameter("Reference"))
345  {
346  if(comm->getRank() == 0)
347  {
348  cout << "Problem or reference is not specified -- check input.";
349  cout <<"\nMetric comparison FAILED." << endl;
350  }
351 
352  }
353 
354 }
355 
356 void ComparisonHelper::CompareSolutions(const string &p1,
357  const string &p2,
358  const RCP<const Comm<int> > &comm)
359 {
360 
361  if(comm->getRank() == 0) printf("\nComparing: %s and %s\n",p1.c_str(),p2.c_str());
362  auto A = this->sources[p1];
363  auto B = this->sources[p2];
364  if(A->problem_kind != B->problem_kind)
365  {
366  cout << "Problem A and B are of a different kind and cannot be compared.";
367  cout <<"\nSolution comparison FAILED." << endl;
368  }else{
369 
370  if(A->problem_kind == "partitioning")
371  {
372  this->ComparePartitionSolutions(A.getRawPtr(), B.getRawPtr(), comm);
373 
374  }else if(A->problem_kind == "coloring")
375  {
376  this->CompareColoringSolutions(A.getRawPtr(), B.getRawPtr(), comm);
377 
378  }else if(A->problem_kind == "ordering"){
379 
380  this->CompareOrderingSolutions(A.getRawPtr(), B.getRawPtr(), comm);
381 
382  }else{
383  cout << "Problem kind not recognized. Check spelling.";
384  cout <<"\nSolution comparison FAILED." << endl;
385  }
386  }
387 
388 }
389 
390 void
391 ComparisonHelper::reduceWithMessage(const RCP<const Comm<int> > &comm, const std::string &msg_in,
392  int &local_status, std::ostringstream &msg)
393 {
394  comm->barrier();
395  int global_buff;
396  Teuchos::Ptr<int> global(&global_buff);
397  reduceAll<int,int>(*comm.get(), Teuchos::EReductionType::REDUCE_MAX, local_status , global);
398 
399  local_status = *global;
400  if(local_status == 1)
401  {
402  msg << msg_in;
403  }
404 
405 }
406 
407 void ComparisonHelper::ComparePartitionSolutions(const ComparisonSource * sourceA,
408  const ComparisonSource * sourceB,
409  const RCP<const Comm<int> > &comm)
410 {
411  int rank = comm->getRank();
412  ostringstream status;
413  int failed = 0;
414 
415  if(!sourceA->problem.getRawPtr()){ failed = 1;}
416  ComparisonHelper::reduceWithMessage(comm,
417  "Solution A is NULL. Solution comparison FAILED.",
418  failed,
419  status);
420 
421  if(!failed && !sourceB->problem.getRawPtr()){ failed = 1;}
422  ComparisonHelper::reduceWithMessage(comm,
423  "Solution B is NULL. Solution comparison FAILED.",
424  failed,
425  status);
426 
427  if(!failed)
428  {
429  // typedef Zoltan2::PartitioningSolution<basic_id_t> partitioning_solution_t; // BDD unused
430  // have some solutions lets compare them
431  if(basic_problem_t * problem_a = reinterpret_cast<basic_problem_t *>(sourceA->problem.getRawPtr()))
432  {
433  if(basic_problem_t * problem_b = reinterpret_cast<basic_problem_t *>(sourceB->problem.getRawPtr()))
434  {
435  auto solution_a = problem_a->getSolution();
436  auto solution_b = problem_b->getSolution();
437 
438  if(sourceA->adapter->getLocalNumIDs() != sourceB->adapter->getLocalNumIDs()){failed = 1;}
439  ComparisonHelper::reduceWithMessage(comm,
440  "Number of parts in Solution A != Solution B. \
441  Partitioning solution comparison FAILED.",
442  failed,
443  status);
444 
445  if(!failed)
446  {
447  for(size_t i = 0; i < sourceA->adapter->getLocalNumIDs(); i++)
448  {
449  if(solution_a.getPartListView()[i] != solution_b.getPartListView()[i])
450  {
451  if(!failed){ failed = 1; }
452  }
453  }
454 
455  ComparisonHelper::reduceWithMessage(comm,
456  "Partitioning solution comparison FAILED.",
457  failed,
458  status);
459  }
460  }else{
461  failed = 1;
462  ComparisonHelper::reduceWithMessage(comm,
463  "Solution sets A and B are from different problem types. \
464  Solution comparison FAILED.",
465  failed,
466  status);
467  }
468 
469  }else{
470  failed = 1;
471  ComparisonHelper::reduceWithMessage(comm,
472  "Could not cast solution A to valid problem type. \
473  Solution comparison FAILED.",
474  failed,
475  status);
476  }
477  }
478 
479  if(!failed)
480  {
481  status << "Solution sets A and B are the same. ";
482  status << "Solution set comparison PASSED.";
483  }
484 
485 
486  if(rank == 0)
487  {
488  cout << status.str() << endl;
489  }
490 
491 }
492 
493 
494 void ComparisonHelper::CompareColoringSolutions(const ComparisonSource * sourceA,
495  const ComparisonSource * sourceB,
496  const RCP<const Comm<int> > &comm)
497 {
498  int rank = comm->getRank();
499  ostringstream status;
500  int failed = 0;
501 
502  if(!sourceA->problem.getRawPtr())
503  {
504  failed = 1;
505  }
506  ComparisonHelper::reduceWithMessage(comm,
507  "Solution A is NULL. Solution comparison FAILED.",
508  failed,
509  status);
510 
511  if(!failed && !sourceB->problem.getRawPtr())
512  {
513  failed = 1;
514  }
515  ComparisonHelper::reduceWithMessage(comm,
516  "Solution B is NULL. Solution comparison FAILED.",
517  failed,
518  status);
519 
520  if(!failed)
521  {
522  // have some solutions lets compare them
523  typedef Zoltan2::ColoringProblem<basic_id_t> coloring_problem_t; //BDD unused
524  // have some solutions lets compare them
525  if(coloring_problem_t * problem_a = reinterpret_cast<coloring_problem_t *>(sourceA->problem.getRawPtr()))
526  {
527  if(coloring_problem_t * problem_b = reinterpret_cast<coloring_problem_t *>(sourceB->problem.getRawPtr()))
528  {
529  auto solution_a = problem_a->getSolution();
530  auto solution_b = problem_b->getSolution();
531 
532  if(solution_a->getNumColors() != solution_b->getNumColors())
533  {
534  failed = 1;
535  }
536  ComparisonHelper::reduceWithMessage(comm,
537  "Number of colors for Solution A != Solution B. \
538  Coloring solution comparison FAILED.",
539  failed,
540  status);
541 
542  if(!failed)
543  {
544  if(solution_a->getColorsSize() != solution_b->getColorsSize())
545  {
546  failed = 1;
547  }
548  ComparisonHelper::reduceWithMessage(comm,
549  "Size of colors array for Solution A != Solution B. \
550  Coloring solution comparison FAILED.",
551  failed,
552  status);
553 
554  }
555 
556  if(!failed)
557  {
558  for(size_t i = 0; i < solution_a->getColorsSize(); i++)
559  {
560  if(solution_a->getColors()[i] != solution_b->getColors()[i])
561  {
562  // fail
563  if(!failed) failed = 1;
564  }
565  }
566  ComparisonHelper::reduceWithMessage(comm,
567  "Coloring solution comparison FAILED.",
568  failed,
569  status);
570  }
571  }else{
572  failed = 1;
573  ComparisonHelper::reduceWithMessage(comm,
574  "Solution sets A and B are from different problem types. \
575  Solution comparison FAILED.",
576  failed,
577  status);
578  }
579 
580  }else{
581  failed = 1;
582  ComparisonHelper::reduceWithMessage(comm,
583  "Could not cast solution A to valid problem type. \
584  Solution comparison FAILED.",
585  failed,
586  status);
587  }
588  }
589 
590  if(!failed)
591  {
592  status << "Solution sets A and B are the same. ";
593  status << "Solution set comparison PASSED.";
594  }
595 
596  if(rank == 0)
597  {
598  cout << status.str() << endl;
599  }
600 
601 }
602 
603 void ComparisonHelper::CompareOrderingSolutions(const ComparisonSource * sourceA,
604  const ComparisonSource * sourceB,
605  const RCP<const Comm<int> > &comm)
606 {
607  int rank = comm->getRank();
608  ostringstream status;
609  int failed = 0;
610 
611  if(!sourceA->problem.getRawPtr()){ failed = 1;}
612  ComparisonHelper::reduceWithMessage(comm,
613  "Solution A is NULL. Solution comparison FAILED.",
614  failed,
615  status);
616 
617  if(!failed && !sourceB->problem.getRawPtr()){ failed = 1;}
618  ComparisonHelper::reduceWithMessage(comm,
619  "Solution B is NULL. Solution comparison FAILED.",
620  failed,
621  status);
622 
623  // if(!failed) //BDD, finish implementation when ordering problem metrics defined
624  // {
625  // // have some solutions lets compare them
626  // typedef Zoltan2::OrderingProblem<basic_id_t> ordering_problem_t;
627  // // have some solutions lets compare them
628  // if(ordering_problem_t * problem_a = reinterpret_cast<ordering_problem_t *>(sourceA->problem.getRawPtr()))
629  // {
630  // if(ordering_problem_t * problem_b = reinterpret_cast<ordering_problem_t *>(sourceB->problem.getRawPtr()))
631  // {
632  //
633  // }else{
634  // status << "Solution sets A and B are from different problem types. ";
635  // status << "Solution comparison FAILED.";
636  // failed = true;
637  // }
638  //
639  //
640  // }else{
641  // if(rank == 0)
642  // {
643  // status << "Could not cast solution A to valid problem type. ";
644  // status << "Solution comparison FAILED.";
645  // }
646  // }
647  // }
648 
649 
650  if(!failed)
651  {
652  status << "Solution sets A and B are the same. ";
653  status << "Solution set comparison PASSED.";
654  }
655 
656  if(rank == 0)
657  {
658  cout << status.str() << endl;
659  }
660 
661 }
662 
663 // compare metrics
664 void ComparisonHelper::CompareMetrics(const ParameterList &metricsPlist,
665  const RCP<const Comm<int> > &comm)
666 {
667 
668  int rank = comm->getRank();
669 
670  //get sources for problema nd reference
671  const string prb_name = metricsPlist.get<string>("Problem");
672  const string ref_name = metricsPlist.get<string>("Reference");
673  if(rank == 0)
674  {
675  cout << "\nMetric/Timer comparison of: " << prb_name << " and ";
676  cout << ref_name <<" (reference source)\n";
677  }
678 
679  // get sources
680  RCP<const ComparisonSource> sourcePrb = this->sources[prb_name];
681  RCP<const ComparisonSource> sourceRef = this->sources[ref_name];
682 
683  // get problems
684  auto problem = sourcePrb.get()->problem.get();
685  auto reference = sourceRef.get()->problem.get();
686 
687  // get metrics
688  std::map<const string, const metric_t> prb_metrics = this->metricArrayToMap(problem->getMetrics());
689  std::map<const string, const metric_t> ref_metrics = this->metricArrayToMap(reference->getMetrics());
690 
691  // get timing data
692  std::map< const string, const double> prb_timers = this->timerDataToMap(sourcePrb->timers);
693  std::map< const string, const double> ref_timers = this->timerDataToMap(sourceRef->timers);
694 
695  // get all of the metrics to be tested
696  std::queue<ParameterList> metrics = ComparisonHelper::getMetricsToCompare(metricsPlist);
697 
698  // run comparison
699  int all_tests_pass = 1;
700  string metric_name;
701  while(!metrics.empty())
702  {
703  // print their names...
704  ostringstream msg;
705  metric_name = metrics.front().name();
706 
707  if(prb_metrics.find(metric_name) != prb_metrics.end() &&
708  ref_metrics.find(metric_name) != ref_metrics.end())
709  {
710  if(rank == 0) cout << "\ncomparing metric: " << metric_name << endl;
711  if(!ComparisonHelper::metricComparisonTest(comm,
712  prb_metrics[metric_name],
713  ref_metrics[metric_name],
714  metrics.front(), msg))
715  {
716  all_tests_pass = 0;
717  }
718  if(rank == 0) cout << msg.str() << endl;
719 
720  }
721  else if(prb_timers.find(metric_name) != prb_timers.end() &&
722  ref_timers.find(metric_name) != ref_timers.end())
723  {
724  if(rank == 0) cout << "\ncomparing timer: " << metric_name << endl;
725  if(!ComparisonHelper::timerComparisonTest(comm,
726  prb_timers.at(metric_name),
727  ref_timers.at(metric_name),
728  metrics.front(), msg))
729  {
730  all_tests_pass = 0;
731  }
732 
733  if(rank == 0) cout << msg.str() << endl;
734  }
735 
736  metrics.pop();
737  }
738 
739 
740  if(rank == 0)
741  {
742  if(all_tests_pass == 1) cout << "\nAll metric/timer comparisons PASSED." << endl;
743  else cout << "\nMetric/timer metric comparisons FAILED." << endl;
744  }
745 }
746 
747 std::map<const string, const ComparisonHelper::metric_t>
748 ComparisonHelper::metricArrayToMap(const ArrayRCP<const ComparisonHelper::metric_t> &metrics)
749 {
750  typedef std::pair<const string,const metric_t> pair_t;
751  std::map<const string, const metric_t> metric_map;
752  ArrayRCP<const ComparisonHelper::metric_t>::size_type idx;
753  for(idx = 0; idx < metrics.size(); idx++)
754  {
755  metric_map.insert(pair_t(metrics[idx].getName(),metrics[idx]));
756  }
757 
758  return metric_map;
759 
760 }
761 
762 std::map<const string, const double>
763 ComparisonHelper::timerDataToMap(const map<const std::string, RCP<Time> > &timers)
764 {
765  typedef std::pair<const string,const double> pair_t;
766  std::map<const string, const double> time_data;
767  for(auto &i : timers)
768  {
769  time_data.insert(pair_t(i.first, i.second->totalElapsedTime()));
770  }
771 
772  return time_data;
773 }
774 
775 bool
776 ComparisonHelper::metricComparisonTest(const RCP<const Comm<int> > &comm,
777  const Zoltan2::MetricValues<zscalar_t> & metric,
778  const Zoltan2::MetricValues<zscalar_t> & ref_metric,
779  const Teuchos::ParameterList & metricPlist,
780  ostringstream &msg)
781 {
782  // run a comparison of min and max agains a given metric
783  // return an error message on failure
784  bool pass = true;
785  string test_name = metricPlist.name() + " test";
786  double local_ref_value = ref_metric.getMaxImbalance()/ref_metric.getAvgImbalance();
787  double local_value = metric.getMaxImbalance()/metric.getAvgImbalance();
788 
789  // reduce problem metric
790  double value;
791  Teuchos::Ptr<double> global(&value);
792  comm->barrier();
793  reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,local_value,global);
794 
795  // reduce reference metric
796  double ref_value;
797  Teuchos::Ptr<double> globalRef(&ref_value);
798  comm->barrier();
799  reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,local_ref_value,globalRef);
800 
801  // want to reduce value to max value for all procs
802 
803  if (metricPlist.isParameter("lower"))
804  {
805  double min = metricPlist.get<double>("lower")*ref_value;
806 
807  if(value < min)
808  {
809  msg << test_name << " FAILED: Minimum imbalance per part, "
810  << value << ", less than specified allowable minimum, " << min << ".\n";
811  pass = false;
812  }else{
813  msg << test_name << " PASSED: Minimum imbalance per part, "
814  << value << ", greater than specified allowable minimum, " << min << ".\n";
815  }
816  }
817 
818  if(metricPlist.isParameter("upper" ) && pass != false) {
819 
820  double max = metricPlist.get<double>("upper") * ref_value;
821  if (value > max)
822  {
823  msg << test_name << " FAILED: Maximum imbalance per part, "
824  << value << ", greater than specified allowable maximum, " << max << ".\n";
825  pass = false;
826  }else{
827  msg << test_name << " PASSED: Maximum imbalance per part, "
828  << value << ", less than specified allowable maximum, " << max << ".\n";
829  }
830 
831  }
832 
833  return pass;
834 }
835 // BDD, to do: print metrics even for pass
836 // reduce max metric to process 0
837 // print only on process 0 --- duh.
838 bool ComparisonHelper::timerComparisonTest(const RCP<const Comm<int> > &comm,
839  const double time,
840  const double ref_time,
841  const Teuchos::ParameterList & metricPlist,
842  ostringstream &msg)
843 {
844  // Reduce time from test
845  double global_time;
846  Teuchos::Ptr<double> global(&global_time);
847  comm->barrier();
848  reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,time,global);
849 
850  // Reduce time from reference
851  double global_ref_time;
852  Teuchos::Ptr<double> globalRef(&global_ref_time);
853  comm->barrier();
854  reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,ref_time,globalRef);
855 
856 
857  // run a comparison of min and max agains a given metric
858  // return an error message on failure
859  bool pass = true;
860  string test_name = metricPlist.name() + " test";
861  if (metricPlist.isParameter("lower"))
862  {
863  double min = metricPlist.get<double>("lower")*global_ref_time;
864 
865  if(global_time < min)
866  {
867  msg << test_name << " FAILED: Minimum time, "
868  << time <<
869  "[s], less than specified allowable minimum time, " << min <<"[s]"<< ".\n";
870  pass = false;
871  }else{
872  msg << test_name << " PASSED: Minimum time, "
873  << time <<
874  "[s], greater than specified allowable minimum time, " << min <<"[s]"<< ".\n";
875  }
876  }
877 
878  if(metricPlist.isParameter("upper" ) && pass != false) {
879 
880  double max = metricPlist.get<double>("upper") * global_ref_time;
881  if (global_time > max)
882  {
883  msg << test_name << " FAILED: Maximum time, "
884  << global_time <<
885  "[s], greater than specified allowable maximum time, " << max <<"[s]"<< ".\n";
886  pass = false;
887  }else{
888  msg << test_name << " PASSED: Maximum time, "
889  << global_time <<
890  "[s], less than specified allowable maximum time, " << max <<"[s]"<< ".\n";
891  }
892 
893  }
894 
895  return pass;
896 }
897 
898 std::queue<ParameterList>
899 ComparisonHelper::getMetricsToCompare(const ParameterList &pList)
900 {
901  // extract all of the metrics to be testd
902  std::queue<ParameterList> metrics;
903  for(auto it = pList.begin(); it != pList.end(); ++it)
904  {
905  if(pList.isSublist(it->first))
906  {
907  metrics.push(pList.sublist(it->first));
908  }
909  }
910 
911  return metrics;
912 }
913 
RCP< basic_problem_t > problem
Zoltan2::Problem< base_t > problem_t
scalar_t getMaxImbalance() const
Get the imbalance of the most imbalanced part. This is what we normally call the imbalance of a parti...
ColoringProblem sets up coloring problems for the user.
Defines the ColoringProblem class.
AdapterForTests::basic_id_t basic_id_t
AdapterForTests::xcrsGraph_adapter xcrsGraph_t
Provides access for Zoltan2 to Xpetra::CrsMatrix data.
AdapterForTests::base_adapter_t base_t
Zoltan2::Problem< base_t > problem_t
ColoringSolution< Adapter > * getSolution()
Get the solution to the problem.
AdapterForTests::xcrsGraph_adapter xcrsGraph_t
AdapterForTests::xcrsMatrix_adapter xcrsMatrix_t
Zoltan2::PartitioningProblem< xcrsMatrix_t > xcrsMatrix_problem_t
Zoltan2::PartitioningProblem< base_t > partioning_problem_t
void Compare(const ParameterList &pList, const RCP< const Comm< int > > &comm)
Provides access for Zoltan2 to Xpetra::CrsGraph data.
common code used by tests
AdapterForTests::xpetra_mv_adapter xpetra_mv_t
void AddSource(const string &name, ComparisonSource *source)
Zoltan2::PartitioningProblem< xcrsGraph_t > xcrsGraph_problem_t
Zoltan2::PartitioningProblem< base_t > partioning_problem_t
scalar_t getAvgImbalance() const
Get the average of the part imbalances.
AdapterForTests::basic_vector_adapter basic_vector_t
This class represents a collection of global Identifiers and their associated weights, if any.
Zoltan2::PartitioningProblem< basic_vector_t > basicVector_problem_t
Zoltan2::PartitioningProblem< xcrsGraph_t > xcrsGraph_problem_t
Zoltan2::PartitioningProblem< basic_id_t > basic_problem_t
size_t getNumberOfSources() const
Problem base class from which other classes (PartitioningProblem, ColoringProblem, OrderingProblem, MatchingProblem, etc.) derive.
BasicVectorAdapter represents a vector (plus optional weights) supplied by the user as pointers to st...
Zoltan2::PartitioningProblem< basic_vector_t > basicVector_problem_t
const Zoltan2::MetricValues< zscalar_t > metric_t
AdapterForTests::basic_vector_adapter basic_vector_t
An adapter for Xpetra::MultiVector.
A class for comparing solutions, metrics, and timing data of Zoltan2 problems.
std::map< const std::string, RCP< Time > > timers
Defines the OrderingProblem class.
AdapterForTests::basic_id_t basic_id_t
PartitioningProblem sets up partitioning problems for the user.
static const std::string pass
BaseAdapter defines methods required by all Adapters.
AdapterForTests::xcrsMatrix_adapter xcrsMatrix_t
A class containing the metrics for one measurable item.
Zoltan2::PartitioningProblem< xpetra_mv_t > xpetra_mv_problem_t
Zoltan2::PartitioningProblem< basic_id_t > basic_problem_t
A class used to save problem solutions and timers.
AdapterForTests::xpetra_mv_adapter xpetra_mv_t
Defines the PartitioningProblem class.
AdapterForTests::base_adapter_t base_t
void addTimer(const std::string &name)
Zoltan2::PartitioningProblem< xpetra_mv_t > xpetra_mv_problem_t
Zoltan2::PartitioningProblem< xcrsMatrix_t > xcrsMatrix_problem_t
Generate Adapter for testing purposes.