42 #ifndef TEUCHOS_MPI_COMM_HPP 43 #define TEUCHOS_MPI_COMM_HPP 53 #ifdef HAVE_TEUCHOS_MPI 55 #include "Teuchos_Comm.hpp" 56 #include "Teuchos_CommUtilities.hpp" 58 #include "Teuchos_OpaqueWrapper.hpp" 60 #include "Teuchos_SerializationTraitsHelpers.hpp" 61 #include "Teuchos_Workspace.hpp" 64 #include "Teuchos_Assert.hpp" 71 #ifdef TEUCHOS_MPI_COMM_DUMP 72 # include "Teuchos_VerboseObject.hpp" 79 mpiErrorCodeToString (
const int err);
95 void safeCommFree (MPI_Comm* comm);
101 int setCommErrhandler (MPI_Comm comm, MPI_Errhandler handler);
105 #ifdef TEUCHOS_MPI_COMM_DUMP 106 template<
typename Ordinal,
typename T>
108 const std::string &funcName,
const std::string &buffName
109 ,
const Ordinal bytes,
const T buff[]
116 <<
"\n" << funcName <<
"::" << buffName <<
":\n";
118 for( Ordinal i = 0; i < bytes; ++i ) {
119 *out << buffName <<
"[" << i <<
"] = '" << buff[i] <<
"'\n";
123 #endif // TEUCHOS_MPI_COMM_DUMP 136 template<
class OrdinalType>
137 class MpiCommStatus :
public CommStatus<OrdinalType> {
139 MpiCommStatus (MPI_Status status) : status_ (status) {}
142 virtual ~MpiCommStatus() {}
145 OrdinalType getSourceRank () {
return status_.MPI_SOURCE; }
148 OrdinalType getTag () {
return status_.MPI_TAG; }
151 OrdinalType getError () {
return status_.MPI_ERROR; }
164 template<
class OrdinalType>
165 inline RCP<MpiCommStatus<OrdinalType> >
166 mpiCommStatus (MPI_Status rawMpiStatus)
168 return rcp (
new MpiCommStatus<OrdinalType> (rawMpiStatus));
186 template<
class OrdinalType>
187 class MpiCommRequestBase :
public CommRequest<OrdinalType> {
190 MpiCommRequestBase () :
191 rawMpiRequest_ (MPI_REQUEST_NULL)
195 MpiCommRequestBase (MPI_Request rawMpiRequest) :
196 rawMpiRequest_ (rawMpiRequest)
206 MPI_Request releaseRawMpiRequest()
208 MPI_Request tmp_rawMpiRequest = rawMpiRequest_;
209 rawMpiRequest_ = MPI_REQUEST_NULL;
210 return tmp_rawMpiRequest;
214 bool isNull()
const {
215 return rawMpiRequest_ == MPI_REQUEST_NULL;
223 RCP<CommStatus<OrdinalType> > wait () {
224 MPI_Status rawMpiStatus;
227 const int err = MPI_Wait (&rawMpiRequest_, &rawMpiStatus);
229 err != MPI_SUCCESS, std::runtime_error,
230 "Teuchos: MPI_Wait() failed with error \"" 231 << mpiErrorCodeToString (err));
233 return mpiCommStatus<OrdinalType> (rawMpiStatus);
240 RCP<CommStatus<OrdinalType> > cancel () {
241 if (rawMpiRequest_ == MPI_REQUEST_NULL) {
245 int err = MPI_Cancel (&rawMpiRequest_);
247 err != MPI_SUCCESS, std::runtime_error,
248 "Teuchos: MPI_Cancel failed with the following error: " 249 << mpiErrorCodeToString (err));
256 err = MPI_Wait (&rawMpiRequest_, &status);
258 "Teuchos::MpiCommStatus::cancel: MPI_Wait failed with the following " 259 "error: " << mpiErrorCodeToString (err));
260 return mpiCommStatus<OrdinalType> (status);
265 virtual ~MpiCommRequestBase () {
266 if (rawMpiRequest_ != MPI_REQUEST_NULL) {
269 const int err = MPI_Cancel (&rawMpiRequest_);
270 if (err == MPI_SUCCESS) {
287 (void) MPI_Wait (&rawMpiRequest_, MPI_STATUS_IGNORE);
294 MPI_Request rawMpiRequest_;
312 template<
class OrdinalType>
313 class MpiCommRequest :
public MpiCommRequestBase<OrdinalType> {
317 MpiCommRequestBase<OrdinalType> (MPI_REQUEST_NULL),
322 MpiCommRequest (MPI_Request rawMpiRequest,
324 MpiCommRequestBase<OrdinalType> (rawMpiRequest),
325 numBytes_ (numBytesInMessage)
338 virtual ~MpiCommRequest () {}
353 template<
class OrdinalType>
354 inline RCP<MpiCommRequest<OrdinalType> >
355 mpiCommRequest (MPI_Request rawMpiRequest,
358 return rcp (
new MpiCommRequest<OrdinalType> (rawMpiRequest, numBytes));
376 template<
typename Ordinal>
377 class MpiComm :
public Comm<Ordinal> {
402 explicit MpiComm (MPI_Comm rawMpiComm);
418 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm);
437 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm,
438 const int defaultTag);
456 MpiComm (
const MpiComm<Ordinal>& other);
459 RCP<const OpaqueWrapper<MPI_Comm> > getRawMpiComm ()
const {
527 void setErrorHandler (
const RCP<
const OpaqueWrapper<MPI_Errhandler> >& errHandler);
534 virtual int getRank()
const;
537 virtual int getSize()
const;
540 virtual void barrier()
const;
543 virtual void broadcast(
544 const int rootRank,
const Ordinal bytes,
char buffer[]
549 gather (
const Ordinal sendBytes,
const char sendBuffer[],
550 const Ordinal recvBytes,
char recvBuffer[],
551 const int root)
const;
553 virtual void gatherAll(
554 const Ordinal sendBytes,
const char sendBuffer[]
555 ,
const Ordinal recvBytes,
char recvBuffer[]
559 const ValueTypeReductionOp<Ordinal,char> &reductOp
560 ,
const Ordinal bytes,
const char sendBuffer[],
char globalReducts[]
563 TEUCHOS_DEPRECATED
virtual void reduceAllAndScatter(
564 const ValueTypeReductionOp<Ordinal,char> &reductOp
565 ,
const Ordinal sendBytes,
const char sendBuffer[]
566 ,
const Ordinal recvCounts[],
char myGlobalReducts[]
570 const ValueTypeReductionOp<Ordinal,char> &reductOp
571 ,
const Ordinal bytes,
const char sendBuffer[],
char scanReducts[]
575 const Ordinal bytes,
const char sendBuffer[],
const int destRank
579 send (
const Ordinal bytes,
580 const char sendBuffer[],
582 const int tag)
const;
585 const Ordinal bytes,
const char sendBuffer[],
const int destRank
589 ssend (
const Ordinal bytes,
590 const char sendBuffer[],
592 const int tag)
const;
595 const int sourceRank,
const Ordinal bytes,
char recvBuffer[]
599 const ArrayView<const char> &sendBuffer,
605 const char sendBuffer[],
607 const int tag)
const;
609 virtual RCP<CommRequest<Ordinal> > isend(
610 const ArrayView<const char> &sendBuffer,
614 virtual RCP<CommRequest<Ordinal> >
615 isend (
const ArrayView<const char> &sendBuffer,
617 const int tag)
const;
619 virtual RCP<CommRequest<Ordinal> >
ireceive(
620 const ArrayView<char> &Buffer,
624 virtual RCP<CommRequest<Ordinal> >
625 ireceive (
const ArrayView<char> &Buffer,
626 const int sourceRank,
627 const int tag)
const;
629 virtual void waitAll(
630 const ArrayView<RCP<CommRequest<Ordinal> > > &requests
634 waitAll (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
635 const ArrayView<RCP<CommStatus<Ordinal> > >& statuses)
const;
637 virtual RCP<CommStatus<Ordinal> >
638 wait (
const Ptr<RCP<CommRequest<Ordinal> > >& request)
const;
640 virtual RCP< Comm<Ordinal> > duplicate()
const;
642 virtual RCP< Comm<Ordinal> > split(
const int color,
const int key)
const;
644 virtual RCP< Comm<Ordinal> > createSubcommunicator(
645 const ArrayView<const int>& ranks)
const;
652 std::string description()
const;
658 static int const minTag_ = 26000;
659 static int const maxTag_ = 26099;
666 int getTag ()
const {
return tag_; }
673 void setupMembersFromComm();
674 static int tagCounter_;
683 RCP<const OpaqueWrapper<MPI_Comm> > rawMpiComm_;
701 RCP<const OpaqueWrapper<MPI_Errhandler> > customErrorHandler_;
703 void assertRank(
const int rank,
const std::string &rankName)
const;
708 #ifdef TEUCHOS_MPI_COMM_DUMP 710 static bool show_dump;
711 #endif // TEUCHOS_MPI_COMM_DUMP 729 template<
typename Ordinal>
730 RCP<MpiComm<Ordinal> >
732 const RCP<
const OpaqueWrapper<MPI_Comm> > &rawMpiComm
763 template<
typename Ordinal>
765 getRawMpiComm(
const Comm<Ordinal> &comm);
775 template<
typename Ordinal>
776 int MpiComm<Ordinal>::tagCounter_ = MpiComm<Ordinal>::minTag_;
782 template<
typename Ordinal>
784 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm)
787 rawMpiComm.get () == NULL, std::invalid_argument,
788 "Teuchos::MpiComm constructor: The input RCP is null.");
790 *rawMpiComm == MPI_COMM_NULL, std::invalid_argument,
791 "Teuchos::MpiComm constructor: The given MPI_Comm is MPI_COMM_NULL.");
793 rawMpiComm_ = rawMpiComm;
807 setupMembersFromComm ();
811 template<
typename Ordinal>
813 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm,
814 const int defaultTag)
817 rawMpiComm.get () == NULL, std::invalid_argument,
818 "Teuchos::MpiComm constructor: The input RCP is null.");
820 *rawMpiComm == MPI_COMM_NULL, std::invalid_argument,
821 "Teuchos::MpiComm constructor: The given MPI_Comm is MPI_COMM_NULL.");
823 rawMpiComm_ = rawMpiComm;
825 int err = MPI_Comm_size (*rawMpiComm_, &size_);
827 "Teuchos::MpiComm constructor: MPI_Comm_size failed with " 828 "error \"" << mpiErrorCodeToString (err) <<
"\".");
830 err = MPI_Comm_rank (*rawMpiComm_, &rank_);
832 "Teuchos::MpiComm constructor: MPI_Comm_rank failed with " 833 "error \"" << mpiErrorCodeToString (err) <<
"\".");
838 template<
typename Ordinal>
839 MpiComm<Ordinal>::MpiComm (MPI_Comm rawMpiComm)
842 std::invalid_argument,
"Teuchos::MpiComm constructor: The given MPI_Comm " 843 "is MPI_COMM_NULL.");
847 rawMpiComm_ = opaqueWrapper<MPI_Comm> (rawMpiComm);
861 setupMembersFromComm ();
865 template<
typename Ordinal>
866 MpiComm<Ordinal>::MpiComm (
const MpiComm<Ordinal>& other) :
867 rawMpiComm_ (opaqueWrapper<MPI_Comm> (MPI_COMM_NULL))
870 RCP<const OpaqueWrapper<MPI_Comm> > origCommPtr = other.getRawMpiComm ();
872 "Teuchos::MpiComm copy constructor: " 873 "The input's getRawMpiComm() method returns null.");
874 MPI_Comm origComm = *origCommPtr;
876 "Teuchos::MpiComm copy constructor: " 877 "The input's raw MPI_Comm is MPI_COMM_NULL.");
885 rawMpiComm_ = origCommPtr;
889 const int err = MPI_Comm_dup (origComm, &newComm);
891 "Teuchos::MpiComm copy constructor: MPI_Comm_dup failed with " 892 "the following error: " << mpiErrorCodeToString (err));
894 rawMpiComm_ = opaqueWrapper (newComm, details::safeCommFree);
897 setupMembersFromComm ();
901 template<
typename Ordinal>
902 void MpiComm<Ordinal>::setupMembersFromComm ()
904 int err = MPI_Comm_size (*rawMpiComm_, &size_);
906 "Teuchos::MpiComm constructor: MPI_Comm_size failed with " 907 "error \"" << mpiErrorCodeToString (err) <<
"\".");
908 err = MPI_Comm_rank (*rawMpiComm_, &rank_);
910 "Teuchos::MpiComm constructor: MPI_Comm_rank failed with " 911 "error \"" << mpiErrorCodeToString (err) <<
"\".");
914 if (tagCounter_ > maxTag_) {
915 tagCounter_ = minTag_;
917 tag_ = tagCounter_++;
927 MPI_Bcast (&tag_, 1, MPI_INT, 0, *rawMpiComm_);
931 template<
typename Ordinal>
934 setErrorHandler (
const RCP<
const OpaqueWrapper<MPI_Errhandler> >& errHandler)
937 const int err = details::setCommErrhandler (*getRawMpiComm (), *errHandler);
939 "Teuchos::MpiComm: Setting the MPI_Comm's error handler failed with " 940 "error \"" << mpiErrorCodeToString (err) <<
"\".");
944 customErrorHandler_ = errHandler;
951 template<
typename Ordinal>
952 int MpiComm<Ordinal>::getRank()
const 958 template<
typename Ordinal>
959 int MpiComm<Ordinal>::getSize()
const 965 template<
typename Ordinal>
966 void MpiComm<Ordinal>::barrier()
const 968 TEUCHOS_COMM_TIME_MONITOR(
971 const int err = MPI_Barrier (*rawMpiComm_);
973 "Teuchos::MpiComm::barrier: MPI_Barrier failed with error \"" 974 << mpiErrorCodeToString (err) <<
"\".");
978 template<
typename Ordinal>
979 void MpiComm<Ordinal>::broadcast(
980 const int rootRank,
const Ordinal bytes,
char buffer[]
983 TEUCHOS_COMM_TIME_MONITOR(
986 const int err = MPI_Bcast (buffer, bytes, MPI_CHAR, rootRank, *rawMpiComm_);
988 "Teuchos::MpiComm::broadcast: MPI_Bcast failed with error \"" 989 << mpiErrorCodeToString (err) <<
"\".");
993 template<
typename Ordinal>
994 void MpiComm<Ordinal>::gatherAll(
995 const Ordinal sendBytes,
const char sendBuffer[],
996 const Ordinal recvBytes,
char recvBuffer[]
999 TEUCHOS_COMM_TIME_MONITOR(
1004 MPI_Allgather (const_cast<char *>(sendBuffer), sendBytes, MPI_CHAR,
1005 recvBuffer, sendBytes, MPI_CHAR, *rawMpiComm_);
1010 "Teuchos::MpiComm::gatherAll: MPI_Allgather failed with error \"" 1011 << mpiErrorCodeToString (err) <<
"\".");
1015 template<
typename Ordinal>
1017 MpiComm<Ordinal>::gather (
const Ordinal sendBytes,
1018 const char sendBuffer[],
1019 const Ordinal recvBytes,
1021 const int root)
const 1025 TEUCHOS_COMM_TIME_MONITOR(
1029 MPI_Gather (const_cast<char *> (sendBuffer), sendBytes, MPI_CHAR,
1030 recvBuffer, sendBytes, MPI_CHAR, root, *rawMpiComm_);
1032 "Teuchos::MpiComm::gather: MPI_Gather failed with error \"" 1033 << mpiErrorCodeToString (err) <<
"\".");
1037 template<
typename Ordinal>
1040 reduceAll (
const ValueTypeReductionOp<Ordinal,char> &reductOp,
1041 const Ordinal bytes,
1042 const char sendBuffer[],
1043 char globalReducts[])
const 1045 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::reduceAll(...)" );
1046 int err = MPI_SUCCESS;
1048 Details::MpiReductionOp<Ordinal> opWrap (reductOp);
1049 MPI_Op op = Details::setMpiReductionOp (opWrap);
1058 MPI_Datatype char_block;
1059 err = MPI_Type_contiguous (bytes, MPI_CHAR, &char_block);
1061 err != MPI_SUCCESS, std::runtime_error,
"Teuchos::reduceAll: " 1062 "MPI_Type_contiguous failed with error \"" << mpiErrorCodeToString (err)
1064 err = MPI_Type_commit (&char_block);
1066 err != MPI_SUCCESS, std::runtime_error,
"Teuchos::reduceAll: " 1067 "MPI_Type_commit failed with error \"" << mpiErrorCodeToString (err)
1070 err = MPI_Allreduce (const_cast<char*> (sendBuffer), globalReducts, 1,
1071 char_block, op, *rawMpiComm_);
1072 if (err != MPI_SUCCESS) {
1077 (void) MPI_Type_free (&char_block);
1079 true, std::runtime_error,
"Teuchos::reduceAll (MPI, custom op): " 1080 "MPI_Allreduce failed with error \"" << mpiErrorCodeToString (err)
1083 err = MPI_Type_free (&char_block);
1085 err != MPI_SUCCESS, std::runtime_error,
"Teuchos::reduceAll: " 1086 "MPI_Type_free failed with error \"" << mpiErrorCodeToString (err)
1091 template<
typename Ordinal>
1092 TEUCHOS_DEPRECATED
void MpiComm<Ordinal>::reduceAllAndScatter(
1093 const ValueTypeReductionOp<Ordinal,char> &reductOp
1094 ,
const Ordinal sendBytes,
const char sendBuffer[]
1095 ,
const Ordinal recvCounts[],
char myGlobalReducts[]
1101 TEUCHOS_COMM_TIME_MONITOR(
1105 #ifdef TEUCHOS_DEBUG 1106 Ordinal sumRecvBytes = 0;
1107 for( Ordinal i = 0; i < size_; ++i ) {
1108 sumRecvBytes += recvCounts[i];
1111 #endif // TEUCHOS_DEBUG 1113 #ifdef TEUCHOS_MPI_COMM_DUMP 1115 dumpBuffer<Ordinal,char>(
1116 "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
1117 "sendBuffer", sendBytes, sendBuffer );
1118 dumpBuffer<Ordinal,Ordinal>(
1119 "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
1120 "recvCounts", as<Ordinal>(size_), recvCounts );
1121 dumpBuffer<Ordinal,char>(
1122 "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
1123 "myGlobalReducts", as<char>(recvCounts[rank_]), myGlobalReducts );
1125 #endif // TEUCHOS_MPI_COMM_DUMP 1129 const bool Ordinal_is_int =
typeid(int)==
typeid(Ordinal);
1130 Workspace<int> ws_int_recvCounts(wss,Ordinal_is_int?0:size_);
1131 const int *int_recvCounts = 0;
1132 if(Ordinal_is_int) {
1133 int_recvCounts =
reinterpret_cast<const int*
>(recvCounts);
1140 std::copy(recvCounts, recvCounts+size_, &ws_int_recvCounts[0]);
1141 int_recvCounts = &ws_int_recvCounts[0];
1144 Details::MpiReductionOp<Ordinal> opWrap (reductOp);
1145 MPI_Op op = Details::setMpiReductionOp (opWrap);
1147 MPI_Reduce_scatter (const_cast<char*> (sendBuffer), myGlobalReducts,
1148 const_cast<int*> (int_recvCounts), MPI_CHAR,
1151 err != MPI_SUCCESS, std::runtime_error,
"Teuchos::MpiComm::" 1152 "reduceAllAndScatter: MPI_Reduce_scatter failed with error \"" 1153 << mpiErrorCodeToString (err) <<
"\".");
1157 template<
typename Ordinal>
1158 void MpiComm<Ordinal>::scan(
1159 const ValueTypeReductionOp<Ordinal,char> &reductOp
1160 ,
const Ordinal bytes,
const char sendBuffer[],
char scanReducts[]
1163 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::scan(...)" );
1165 Details::MpiReductionOp<Ordinal> opWrap (reductOp);
1166 MPI_Op op = Details::setMpiReductionOp (opWrap);
1168 MPI_Scan (const_cast<char*> (sendBuffer), scanReducts, bytes, MPI_CHAR,
1171 "Teuchos::MpiComm::scan: MPI_Scan() failed with error \"" 1172 << mpiErrorCodeToString (err) <<
"\".");
1176 template<
typename Ordinal>
1178 MpiComm<Ordinal>::send (
const Ordinal bytes,
1179 const char sendBuffer[],
1180 const int destRank)
const 1182 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::send(...)" );
1184 #ifdef TEUCHOS_MPI_COMM_DUMP 1186 dumpBuffer<Ordinal,char>(
1187 "Teuchos::MpiComm<Ordinal>::send(...)" 1188 ,
"sendBuffer", bytes, sendBuffer
1191 #endif // TEUCHOS_MPI_COMM_DUMP 1193 const int err = MPI_Send (const_cast<char*>(sendBuffer), bytes, MPI_CHAR,
1194 destRank, tag_, *rawMpiComm_);
1196 "Teuchos::MpiComm::send: MPI_Send() failed with error \"" 1197 << mpiErrorCodeToString (err) <<
"\".");
1201 template<
typename Ordinal>
1203 MpiComm<Ordinal>::send (
const Ordinal bytes,
1204 const char sendBuffer[],
1206 const int tag)
const 1208 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::send(...)" );
1209 const int err = MPI_Send (const_cast<char*> (sendBuffer), bytes, MPI_CHAR,
1210 destRank, tag, *rawMpiComm_);
1212 "Teuchos::MpiComm::send: MPI_Send() failed with error \"" 1213 << mpiErrorCodeToString (err) <<
"\".");
1217 template<
typename Ordinal>
1219 MpiComm<Ordinal>::ssend (
const Ordinal bytes,
1220 const char sendBuffer[],
1221 const int destRank)
const 1223 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::ssend(...)" );
1225 #ifdef TEUCHOS_MPI_COMM_DUMP 1227 dumpBuffer<Ordinal,char>(
1228 "Teuchos::MpiComm<Ordinal>::send(...)" 1229 ,
"sendBuffer", bytes, sendBuffer
1232 #endif // TEUCHOS_MPI_COMM_DUMP 1234 const int err = MPI_Ssend (const_cast<char*>(sendBuffer), bytes, MPI_CHAR,
1235 destRank, tag_, *rawMpiComm_);
1237 "Teuchos::MpiComm::send: MPI_Ssend() failed with error \"" 1238 << mpiErrorCodeToString (err) <<
"\".");
1241 template<
typename Ordinal>
1243 MpiComm<Ordinal>::ssend (
const Ordinal bytes,
1244 const char sendBuffer[],
1246 const int tag)
const 1248 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::ssend(...)" );
1250 MPI_Ssend (const_cast<char*>(sendBuffer), bytes, MPI_CHAR,
1251 destRank, tag, *rawMpiComm_);
1253 "Teuchos::MpiComm::send: MPI_Ssend() failed with error \"" 1254 << mpiErrorCodeToString (err) <<
"\".");
1257 template<
typename Ordinal>
1258 void MpiComm<Ordinal>::readySend(
1259 const ArrayView<const char> &sendBuffer,
1263 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::readySend" );
1265 #ifdef TEUCHOS_MPI_COMM_DUMP 1267 dumpBuffer<Ordinal,char>(
1268 "Teuchos::MpiComm<Ordinal>::readySend(...)" 1269 ,
"sendBuffer", bytes, sendBuffer
1272 #endif // TEUCHOS_MPI_COMM_DUMP 1275 MPI_Rsend (const_cast<char*>(sendBuffer.getRawPtr()), sendBuffer.size(),
1276 MPI_CHAR, destRank, tag_, *rawMpiComm_);
1278 "Teuchos::MpiComm::readySend: MPI_Rsend() failed with error \"" 1279 << mpiErrorCodeToString (err) <<
"\".");
1283 template<
typename Ordinal>
1284 void MpiComm<Ordinal>::
1285 readySend (
const Ordinal bytes,
1286 const char sendBuffer[],
1288 const int tag)
const 1290 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::readySend" );
1292 MPI_Rsend (const_cast<char*> (sendBuffer), bytes,
1293 MPI_CHAR, destRank, tag, *rawMpiComm_);
1295 "Teuchos::MpiComm::readySend: MPI_Rsend() failed with error \"" 1296 << mpiErrorCodeToString (err) <<
"\".");
1300 template<
typename Ordinal>
1302 MpiComm<Ordinal>::receive (
const int sourceRank,
1303 const Ordinal bytes,
1304 char recvBuffer[])
const 1306 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::receive(...)" );
1311 const int theSrcRank = (sourceRank < 0) ? MPI_ANY_SOURCE : sourceRank;
1314 const int err = MPI_Recv (recvBuffer, bytes, MPI_CHAR, theSrcRank, tag_,
1315 *rawMpiComm_, &status);
1317 "Teuchos::MpiComm::receive: MPI_Recv() failed with error \"" 1318 << mpiErrorCodeToString (err) <<
"\".");
1320 #ifdef TEUCHOS_MPI_COMM_DUMP 1322 dumpBuffer<Ordinal,char> (
"Teuchos::MpiComm<Ordinal>::receive(...)",
1323 "recvBuffer", bytes, recvBuffer);
1325 #endif // TEUCHOS_MPI_COMM_DUMP 1328 return status.MPI_SOURCE;
1332 template<
typename Ordinal>
1333 RCP<CommRequest<Ordinal> >
1334 MpiComm<Ordinal>::isend (
const ArrayView<const char> &sendBuffer,
1335 const int destRank)
const 1338 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::isend(...)" );
1340 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1342 MPI_Isend (const_cast<char*> (sendBuffer.getRawPtr ()),
1343 as<Ordinal> (sendBuffer.size ()), MPI_CHAR,
1344 destRank, tag_, *rawMpiComm_, &rawMpiRequest);
1346 "Teuchos::MpiComm::isend: MPI_Isend() failed with error \"" 1347 << mpiErrorCodeToString (err) <<
"\".");
1349 return mpiCommRequest<Ordinal> (rawMpiRequest, sendBuffer.size ());
1353 template<
typename Ordinal>
1354 RCP<CommRequest<Ordinal> >
1356 isend (
const ArrayView<const char> &sendBuffer,
1358 const int tag)
const 1361 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::isend(...)" );
1363 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1365 MPI_Isend (const_cast<char*> (sendBuffer.getRawPtr ()),
1366 as<Ordinal> (sendBuffer.size ()), MPI_CHAR,
1367 destRank, tag, *rawMpiComm_, &rawMpiRequest);
1369 "Teuchos::MpiComm::isend: MPI_Isend() failed with error \"" 1370 << mpiErrorCodeToString (err) <<
"\".");
1372 return mpiCommRequest<Ordinal> (rawMpiRequest, sendBuffer.size ());
1376 template<
typename Ordinal>
1377 RCP<CommRequest<Ordinal> >
1378 MpiComm<Ordinal>::ireceive (
const ArrayView<char> &recvBuffer,
1379 const int sourceRank)
const 1381 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::ireceive(...)" );
1386 const int theSrcRank = (sourceRank < 0) ? MPI_ANY_SOURCE : sourceRank;
1388 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1390 MPI_Irecv (const_cast<char*>(recvBuffer.getRawPtr()), recvBuffer.size(),
1391 MPI_CHAR, theSrcRank, tag_, *rawMpiComm_, &rawMpiRequest);
1393 "Teuchos::MpiComm::ireceive: MPI_Irecv() failed with error \"" 1394 << mpiErrorCodeToString (err) <<
"\".");
1396 return mpiCommRequest<Ordinal> (rawMpiRequest, recvBuffer.size());
1399 template<
typename Ordinal>
1400 RCP<CommRequest<Ordinal> >
1401 MpiComm<Ordinal>::ireceive (
const ArrayView<char> &recvBuffer,
1402 const int sourceRank,
1403 const int tag)
const 1405 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::ireceive(...)" );
1410 const int theSrcRank = (sourceRank < 0) ? MPI_ANY_SOURCE : sourceRank;
1412 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1414 MPI_Irecv (const_cast<char*> (recvBuffer.getRawPtr ()), recvBuffer.size (),
1415 MPI_CHAR, theSrcRank, tag, *rawMpiComm_, &rawMpiRequest);
1417 "Teuchos::MpiComm::ireceive: MPI_Irecv() failed with error \"" 1418 << mpiErrorCodeToString (err) <<
"\".");
1420 return mpiCommRequest<Ordinal> (rawMpiRequest, recvBuffer.size ());
1425 template<
typename Ordinal>
1427 waitAllImpl (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
1428 const ArrayView<MPI_Status>& rawMpiStatuses)
1430 typedef typename ArrayView<RCP<CommRequest<Ordinal> > >::size_type size_type;
1431 const size_type count = requests.size();
1435 std::logic_error,
"Teuchos::MpiComm's waitAllImpl: rawMpiStatus.size() = " 1436 << rawMpiStatuses.size() <<
" != requests.size() = " << requests.size()
1437 <<
". Please report this bug to the Tpetra developers.");
1450 bool someNullRequests =
false;
1451 Array<MPI_Request> rawMpiRequests (count, MPI_REQUEST_NULL);
1452 for (
int i = 0; i < count; ++i) {
1453 RCP<CommRequest<Ordinal> > request = requests[i];
1455 RCP<MpiCommRequestBase<Ordinal> > mpiRequest =
1456 rcp_dynamic_cast<MpiCommRequestBase<Ordinal> > (request);
1464 rawMpiRequests[i] = mpiRequest->releaseRawMpiRequest();
1467 rawMpiRequests[i] = MPI_REQUEST_NULL;
1468 someNullRequests =
true;
1481 const int err = MPI_Waitall (count, rawMpiRequests.getRawPtr(),
1482 rawMpiStatuses.getRawPtr());
1492 if (err != MPI_SUCCESS) {
1493 if (err == MPI_ERR_IN_STATUS) {
1501 Array<std::pair<size_type, int> > errorLocationsAndCodes;
1502 for (size_type k = 0; k < rawMpiStatuses.size(); ++k) {
1503 const int curErr = rawMpiStatuses[k].MPI_ERROR;
1504 if (curErr != MPI_SUCCESS) {
1505 errorLocationsAndCodes.push_back (std::make_pair (k, curErr));
1508 const size_type numErrs = errorLocationsAndCodes.size();
1513 std::ostringstream os;
1514 os <<
"Teuchos::MpiComm::waitAll: MPI_Waitall() failed with error \"" 1515 << mpiErrorCodeToString (err) <<
"\". Of the " << count
1516 <<
" total request" << (count != 1 ?
"s" :
"") <<
", " << numErrs
1517 <<
" failed. Here are the indices of the failed requests, and the " 1518 "error codes extracted from their returned MPI_Status objects:" 1520 for (size_type k = 0; k < numErrs; ++k) {
1521 const size_type errInd = errorLocationsAndCodes[k].first;
1522 os <<
"Request " << errInd <<
": MPI_ERROR = " 1523 << mpiErrorCodeToString (rawMpiStatuses[errInd].MPI_ERROR)
1526 if (someNullRequests) {
1527 os <<
" On input to MPI_Waitall, there was at least one MPI_" 1528 "Request that was MPI_REQUEST_NULL. MPI_Waitall should not " 1529 "normally fail in that case, but we thought we should let you know " 1538 std::ostringstream os;
1539 os <<
"Teuchos::MpiComm::waitAll: MPI_Waitall() failed with error \"" 1540 << mpiErrorCodeToString (err) <<
"\".";
1541 if (someNullRequests) {
1542 os <<
" On input to MPI_Waitall, there was at least one MPI_Request " 1543 "that was MPI_REQUEST_NULL. MPI_Waitall should not normally fail in " 1544 "that case, but we thought we should let you know regardless.";
1552 std::fill (requests.begin(), requests.end(), null);
1558 template<
typename Ordinal>
1560 waitAllImpl (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests)
1562 typedef typename ArrayView<RCP<CommRequest<Ordinal> > >::size_type size_type;
1563 const size_type count = requests.size ();
1576 bool someNullRequests =
false;
1577 Array<MPI_Request> rawMpiRequests (count, MPI_REQUEST_NULL);
1578 for (
int i = 0; i < count; ++i) {
1579 RCP<CommRequest<Ordinal> > request = requests[i];
1580 if (! request.is_null ()) {
1581 RCP<MpiCommRequestBase<Ordinal> > mpiRequest =
1582 rcp_dynamic_cast<MpiCommRequestBase<Ordinal> > (request);
1590 rawMpiRequests[i] = mpiRequest->releaseRawMpiRequest ();
1593 rawMpiRequests[i] = MPI_REQUEST_NULL;
1594 someNullRequests =
true;
1604 const int err = MPI_Waitall (count, rawMpiRequests.getRawPtr(),
1605 MPI_STATUSES_IGNORE);
1615 if (err != MPI_SUCCESS) {
1616 std::ostringstream os;
1617 os <<
"Teuchos::MpiComm::waitAll: MPI_Waitall() failed with error \"" 1618 << mpiErrorCodeToString (err) <<
"\".";
1619 if (someNullRequests) {
1620 os << std::endl <<
"On input to MPI_Waitall, there was at least one " 1621 "MPI_Request that was MPI_REQUEST_NULL. MPI_Waitall should not " 1622 "normally fail in that case, but we thought we should let you know " 1633 std::fill (requests.begin(), requests.end(), null);
1640 template<
typename Ordinal>
1643 waitAll (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests)
const 1645 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::waitAll(requests)" );
1648 waitAllImpl<Ordinal> (requests);
1652 template<
typename Ordinal>
1655 waitAll (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
1656 const ArrayView<RCP<CommStatus<Ordinal> > >& statuses)
const 1658 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::waitAll(requests, statuses)" );
1660 typedef typename ArrayView<RCP<CommRequest<Ordinal> > >::size_type size_type;
1661 const size_type count = requests.size();
1664 std::invalid_argument,
"Teuchos::MpiComm::waitAll: requests.size() = " 1665 << count <<
" != statuses.size() = " << statuses.size() <<
".");
1667 Array<MPI_Status> rawMpiStatuses (count);
1668 waitAllImpl<Ordinal> (requests, rawMpiStatuses());
1671 for (size_type i = 0; i < count; ++i) {
1672 statuses[i] = mpiCommStatus<Ordinal> (rawMpiStatuses[i]);
1677 template<
typename Ordinal>
1678 RCP<CommStatus<Ordinal> >
1679 MpiComm<Ordinal>::wait (
const Ptr<RCP<CommRequest<Ordinal> > >& request)
const 1681 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::wait(...)" );
1687 RCP<CommStatus<Ordinal> > status = (*request)->wait ();
1695 template<
typename Ordinal>
1696 RCP< Comm<Ordinal> >
1697 MpiComm<Ordinal>::duplicate()
const 1699 MPI_Comm origRawComm = *rawMpiComm_;
1700 MPI_Comm newRawComm = MPI_COMM_NULL;
1701 const int err = MPI_Comm_dup (origRawComm, &newRawComm);
1703 "::MpiComm::duplicate: MPI_Comm_dup failed with the following error: " 1704 << mpiErrorCodeToString (err));
1709 RCP<OpaqueWrapper<MPI_Comm> > wrapped =
1710 opaqueWrapper<MPI_Comm> (newRawComm, details::safeCommFree);
1714 RCP<MpiComm<Ordinal> > newComm =
1715 rcp (
new MpiComm<Ordinal> (wrapped.getConst (), minTag_));
1716 return rcp_implicit_cast<Comm<Ordinal> > (newComm);
1720 template<
typename Ordinal>
1721 RCP< Comm<Ordinal> >
1722 MpiComm<Ordinal>::split(
const int color,
const int key)
const 1725 const int splitReturn =
1726 MPI_Comm_split (*rawMpiComm_,
1727 color < 0 ? MPI_UNDEFINED : color,
1731 splitReturn != MPI_SUCCESS,
1733 "Teuchos::MpiComm::split: Failed to create communicator with color " 1734 << color <<
"and key " << key <<
". MPI_Comm_split failed with error \"" 1735 << mpiErrorCodeToString (splitReturn) <<
"\".");
1736 if (newComm == MPI_COMM_NULL) {
1737 return RCP< Comm<Ordinal> >();
1739 RCP<const OpaqueWrapper<MPI_Comm> > wrapped =
1740 opaqueWrapper<MPI_Comm> (newComm, details::safeCommFree);
1745 return rcp (
new MpiComm<Ordinal> (wrapped, minTag_));
1750 template<
typename Ordinal>
1751 RCP< Comm<Ordinal> >
1752 MpiComm<Ordinal>::createSubcommunicator(
const ArrayView<const int> &ranks)
const 1754 int err = MPI_SUCCESS;
1757 MPI_Group thisGroup;
1758 err = MPI_Comm_group (*rawMpiComm_, &thisGroup);
1760 "Failed to obtain the current communicator's group. " 1761 "MPI_Comm_group failed with error \"" 1762 << mpiErrorCodeToString (err) <<
"\".");
1770 err = MPI_Group_incl (thisGroup, ranks.size(),
1771 const_cast<int*
> (ranks.getRawPtr ()), &newGroup);
1773 "Failed to create subgroup. MPI_Group_incl failed with error \"" 1774 << mpiErrorCodeToString (err) <<
"\".");
1779 err = MPI_Comm_create (*rawMpiComm_, newGroup, &newComm);
1781 "Failed to create subcommunicator. MPI_Comm_create failed with error \"" 1782 << mpiErrorCodeToString (err) <<
"\".");
1789 (void) MPI_Group_free (&newGroup);
1790 (void) MPI_Group_free (&thisGroup);
1795 err = MPI_Group_free (&newGroup);
1797 "Failed to free subgroup. MPI_Group_free failed with error \"" 1798 << mpiErrorCodeToString (err) <<
"\".");
1799 err = MPI_Group_free (&thisGroup);
1801 "Failed to free subgroup. MPI_Group_free failed with error \"" 1802 << mpiErrorCodeToString (err) <<
"\".");
1804 if (newComm == MPI_COMM_NULL) {
1805 return RCP<Comm<Ordinal> > ();
1807 using Teuchos::details::safeCommFree;
1808 typedef OpaqueWrapper<MPI_Comm> ow_type;
1809 RCP<const ow_type> wrapper =
1810 rcp_implicit_cast<
const ow_type> (opaqueWrapper (newComm, safeCommFree));
1815 return rcp (
new MpiComm<Ordinal> (wrapper, minTag_));
1823 template<
typename Ordinal>
1824 std::string MpiComm<Ordinal>::description()
const 1826 std::ostringstream oss;
1832 <<
",rawMpiComm="<<
static_cast<MPI_Comm
>(*rawMpiComm_)
1838 #ifdef TEUCHOS_MPI_COMM_DUMP 1839 template<
typename Ordinal>
1840 bool MpiComm<Ordinal>::show_dump =
false;
1847 template<
typename Ordinal>
1848 void MpiComm<Ordinal>::assertRank(
const int rank,
const std::string &rankName)
const 1851 ! ( 0 <= rank && rank < size_ ), std::logic_error
1852 ,
"Error, "<<rankName<<
" = " << rank <<
" is not < 0 or is not" 1853 " in the range [0,"<<size_-1<<
"]!" 1861 template<
typename Ordinal>
1863 Teuchos::createMpiComm(
1864 const RCP<
const OpaqueWrapper<MPI_Comm> > &rawMpiComm
1867 if( rawMpiComm.get()!=NULL && *rawMpiComm != MPI_COMM_NULL )
1868 return rcp(
new MpiComm<Ordinal>(rawMpiComm));
1869 return Teuchos::null;
1873 template<
typename Ordinal>
1875 Teuchos::getRawMpiComm(
const Comm<Ordinal> &comm)
1878 dyn_cast<
const MpiComm<Ordinal> >(comm).getRawMpiComm()
1883 #endif // HAVE_TEUCHOS_MPI 1884 #endif // TEUCHOS_MPI_COMM_HPP bool is_null(const std::shared_ptr< T > &p)
Returns true if p.get()==NULL.
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Macro for throwing an exception with breakpointing to ease debugging.
Ordinal size_type
Type representing the number of elements in an ArrayRCP or view thereof.
T_To & dyn_cast(T_From &from)
Dynamic casting utility function meant to replace dynamic_cast<T&> by throwing a better documented er...
Teuchos header file which uses auto-configuration information to include necessary C++ headers...
Tabbing class for helping to create formated, indented output for a basic_FancyOStream object...
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Deprecated.
Teuchos implementation details.
TEUCHOS_DEPRECATED void scan(const Comm< Ordinal > &comm, const EReductionType reductType, const Packet &send, Packet *scanReduct)
Deprecated.
static RCP< FancyOStream > getDefaultOStream()
Get the default output stream object.
TEUCHOS_DEPRECATED void reduceAll(const Comm< Ordinal > &comm, const EReductionType reductType, const Packet &send, Packet *globalReduct)
Deprecated .
void send(const Packet sendBuffer[], const Ordinal count, const int destRank, const int tag, const Comm< Ordinal > &comm)
Variant of send() that takes a tag (and restores the correct order of arguments). ...
RCP< CommRequest< Ordinal > > ireceive(const ArrayRCP< Packet > &recvBuffer, const int sourceRank, const int tag, const Comm< Ordinal > &comm)
Variant of ireceive that takes a tag argument (and restores the correct order of arguments).
Defines basic traits for the ordinal field type.
The Teuchos namespace contains all of the classes, structs and enums used by Teuchos, as well as a number of utility routines.
TypeTo as(const TypeFrom &t)
Convert from one value type to another.
void ssend(const Packet sendBuffer[], const Ordinal count, const int destRank, const int tag, const Comm< Ordinal > &comm)
Variant of ssend() that takes a tag (and restores the correct order of arguments).
static std::string name()
Returns name of this ordinal type.
Smart reference counting pointer class for automatic garbage collection.
Implementation detail of Teuchos' MPI wrapper.
#define TEUCHOS_ASSERT_EQUALITY(val1, val2)
This macro is checks that to numbers are equal and if not then throws an exception with a good error ...
Defines basic traits returning the name of a type in a portable and readable way. ...
Definition of Teuchos::as, for conversions between types.
void readySend(const Packet sendBuffer[], const Ordinal count, const int destRank, const int tag, const Comm< Ordinal > &comm)
Variant of readySend() that accepts a message tag.
#define TEUCHOS_TEST_FOR_EXCEPT(throw_exception_test)
This macro is designed to be a short version of TEUCHOS_TEST_FOR_EXCEPTION() that is easier to call...
TEUCHOSCORE_LIB_DLL_EXPORT Teuchos::RCP< WorkspaceStore > get_default_workspace_store()
Get the global workspace object set by set_default_workspace_store().
std::string typeName(const T &t)
Template function for returning the concrete type name of a passed-in object.