Actual source code: sectioncopy.c

  1: #include <petscsys.h>
  2: #include <private/vecimpl.h>
  3: #include <sieve/BasicCommunication.hh>

  5: typedef struct _n_PetscOverlap *PetscOverlap;
  6: struct _n_PetscOverlap
  7: {
  8:   MPI_Comm  comm;
  9:   PetscInt  numRanks;     // Number of partner processes
 10:   PetscInt *ranks;        // MPI Rank of each partner process
 11:   PetscInt *pointsOffset; // Offset into points array for each partner process
 12:   PetscInt *points;       // Points array for each partner process, in sorted order
 13:   PetscInt *remotePoints; // Remote points array for each partner process
 14: };

 16: PetscErrorCode PetscOverlapCreate(MPI_Comm comm, PetscOverlap *o)
 17: {

 21:   PetscNew(struct _n_PetscOverlap, o);
 22:   (*o)->comm         = comm;
 23:   (*o)->numRanks     = 0;
 24:   (*o)->ranks        = PETSC_NULL;
 25:   (*o)->pointsOffset = PETSC_NULL;
 26:   (*o)->points       = PETSC_NULL;
 27:   (*o)->remotePoints = PETSC_NULL;
 28:   return(0);
 29: }

 31: PetscErrorCode PetscOverlapDestroy(PetscOverlap *o)
 32: {

 36:   if (!*o) return(0);
 37:   PetscFree((*o)->ranks);
 38:   PetscFree((*o)->pointsOffset);
 39:   PetscFree((*o)->points);
 40:   PetscFree((*o)->remotePoints);
 41:   PetscFree((*o));
 42:   *o = PETSC_NULL;
 43:   return(0);
 44: }

 46: PetscErrorCode PetscOverlapGetNumRanks(PetscOverlap overlap, PetscInt *numRanks)
 47: {
 50:   *numRanks = overlap->numRanks;
 51:   return(0);
 52: };

 54: PetscErrorCode PetscOverlapGetRank(PetscOverlap overlap, PetscInt r, PetscInt *rank)
 55: {
 58:   if (r < 0 || r >= overlap->numRanks) {SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid rank index %d should be in [%d, %d)", r, 0, overlap->numRanks);}
 59:   *rank = overlap->ranks[r];
 60:   return(0);
 61: };

 63: // TODO: Replace this with binary search
 64: PetscErrorCode PetscOverlapGetRankIndex(PetscOverlap overlap, PetscInt rank, PetscInt *r)
 65: {
 66:   PetscInt p;

 70:   for(p = 0; p < overlap->numRanks; ++p) {
 71:     if (overlap->ranks[p] == rank) {
 72:       *r = p;
 73:       return(0);
 74:     }
 75:   }
 76:   SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid rank %d was not contained in this overlap", rank);
 77: };

 79: PetscErrorCode PetscOverlapGetNumPoints(PetscOverlap overlap, PetscInt r, PetscInt *numPoints)
 80: {
 83:   if (r < 0 || r >= overlap->numRanks) {SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid rank index %d should be in [%d, %d)", r, 0, overlap->numRanks);}
 84:   *numPoints = overlap->pointsOffset[r+1] - overlap->pointsOffset[r];
 85:   return(0);
 86: };

 88: // These should be sorted
 89: PetscErrorCode PetscOverlapGetPoints(PetscOverlap overlap, PetscInt r, const PetscInt **points)
 90: {
 93:   if (r < 0 || r >= overlap->numRanks) {SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid rank index %d should be in [%d, %d)", r, 0, overlap->numRanks);}
 94:   *points = &overlap->points[overlap->pointsOffset[r]];
 95:   return(0);
 96: };

 98: // These cannot be sorted
 99: PetscErrorCode PetscOverlapGetRemotePoints(PetscOverlap overlap, PetscInt r, const PetscInt **remotePoints)
100: {
103:   if (r < 0 || r >= overlap->numRanks) {SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid rank index %d should be in [%d, %d)", r, 0, overlap->numRanks);}
104:   *remotePoints = &overlap->remotePoints[overlap->pointsOffset[r]];
105:   return(0);
106: };

108: PetscErrorCode PetscOverlapGetNumPointsByRank(PetscOverlap overlap, PetscInt rank, PetscInt *numPoints)
109: {
110:   PetscInt       r;

115:   PetscOverlapGetRankIndex(overlap, rank, &r);
116:   *numPoints = overlap->pointsOffset[r+1] - overlap->pointsOffset[r];
117:   return(0);
118: };

120: // These should be sorted
121: PetscErrorCode PetscOverlapGetPointsByRank(PetscOverlap overlap, PetscInt rank, const PetscInt **points)
122: {
123:   PetscInt       r;

128:   PetscOverlapGetRankIndex(overlap, rank, &r);
129:   *points = &overlap->points[overlap->pointsOffset[r]];
130:   return(0);
131: };

133: typedef PetscInt point_type;

135: // Copies the PetscUniformSection from one process to another, specified by the overlap
136: //   Moves the chart interval [pStart, pEnd) and the numDof
137: PetscErrorCode PetscCopySection(PetscOverlap sendOverlap, PetscOverlap recvOverlap, const PetscUniformSection sendSection, const PetscUniformSection recvSection) {
138:   PetscInt       numSendRanks; // This is sendOverlap->base()
139:   PetscInt       numRecvRanks; // This is recvOverlap->cap()
140:   const int      debug = 0;
141:   point_type   **sendPoints;
142:   point_type   **recvPoints;
144:   ALE::MPIMover<point_type> pMover(sendSection->comm, debug);

147:   PetscOverlapGetNumRanks(sendOverlap, &numSendRanks);
148:   PetscOverlapGetNumRanks(recvOverlap, &numRecvRanks);
149:   PetscMalloc(numSendRanks * sizeof(point_type *), &sendPoints);
150:   for(PetscInt r = 0; r < numSendRanks; ++r) {
151:     PetscInt    rank;
152:     point_type *v;

154:     PetscOverlapGetRank(sendOverlap, r, &rank);
155:     PetscMalloc(3 * sizeof(point_type), &v);
156:     v[0] = sendSection->pStart;
157:     v[1] = sendSection->pEnd;
158:     v[2] = sendSection->numDof;
159:     sendPoints[r] = v;
160:     pMover.send(rank, 3, sendPoints[r]);
161:     if (debug) {PetscPrintf(PETSC_COMM_SELF, "[%d]Sending chart (%d, %d, %d) to process %d\n", pMover.commRank(), v[0], v[1], v[2], rank);}
162:   }

164:   PetscMalloc(numRecvRanks * sizeof(point_type *), &recvPoints);
165:   for(PetscInt r = 0; r < numRecvRanks; ++r) {
166:     PetscInt    rank;
167:     point_type *v;

169:     PetscOverlapGetRank(recvOverlap, r, &rank);
170:     PetscMalloc(3 * sizeof(point_type), &v);
171:     recvPoints[r] = v;
172:     pMover.recv(rank, 3, recvPoints[r]);
173:   }
174:   pMover.start();
175:   pMover.end();
176:   point_type min = -1;
177:   point_type max = -1;
178:   point_type dof = -1;

180:   if (!numRecvRanks) {min = max = 0;}
181:   for(PetscInt r = 0; r < numRecvRanks; ++r) {
182:     const point_type *v      = recvPoints[r];
183:     point_type        newMin = v[0];
184:     point_type        newMax = v[1];
185:     PetscInt          rank;

187:     PetscOverlapGetRank(recvOverlap, r, &rank);
188:     if (debug) {PetscPrintf(PETSC_COMM_SELF, "[%d]Received chart (%d, %d, %d) from process %d\n", pMover.commRank(), v[0],v[1], v[2], rank);}
189:     // Take the union of all charts
190:     if (min < 0) {
191:       min = newMin;
192:       max = newMax;
193:     } else {
194:       min = std::min(min, newMin);
195:       max = std::max(max, newMax);
196:     }
197:     if (dof < 0) {
198:       dof = v[2];
199:     } else {
200:       if (dof != v[2]) {SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of dof %d from rank %d should be %d", v[2], rank, dof);}
201:     }
202:   }
203:   recvSection->pStart = min;
204:   recvSection->pEnd   = max;
205:   recvSection->numDof = dof;
206:   for(PetscInt r = 0; r < numSendRanks; ++r) {
207:     PetscFree(sendPoints[r]);
208:   }
209:   for(PetscInt r = 0; r < numRecvRanks; ++r) {
210:     PetscFree(recvPoints[r]);
211:   }
212:   return(0);
213: };

215: // Full Sequence:
216: //   PetscCopySection(sOv, rOv, sSec, sStor, rSec, rStor)
217: //   --> PetscCopySection(sOv, rOv, sSec->atlasLayout, sSec->atlasDof, rSec->atlasLayout, rSec->atlasDof)
218: //       --> PetscCopyChart(sOv, rOv, sSec->atlasLayout, rSec->atlasLayout)
219: //       --> Copy sSec->atlasDof to rSec->atlasDof
220: //   --> Copy sStor to rStor
221: //
222: // RecvSection
223: //   1) Usually has scattered values, so using an interval chart is wrong
224: //   2) Must differentiate between data from different ranks (wait for fuse() process to merge)
225: //      Should we combine the fuse step here?

227: // Copies the PetscSection from one process to another, specified by the overlap
228: //   Use MPI_DATATYPE_NULL for a default
229: template<typename section_type, typename send_value_type, typename recv_value_type>
230: PetscErrorCode PetscCopySection(PetscOverlap sendOverlap, PetscOverlap recvOverlap, section_type sendSection, section_type recvSection, send_value_type *sendStorage, recv_value_type **recvStorage) {
231:   PetscInt         numSendRanks; // This is sendOverlap->base()
232:   PetscInt         numRecvRanks; // This is recvOverlap->cap()
233:   PetscInt         recvSize;
234:   send_value_type *sendValues;
235:   send_value_type *recvValues;
236:   const int        debug = 1;
237:   PetscErrorCode   ierr;

240:   if (!sendStorage) {
241:     PetscCopySection(sendOverlap, recvOverlap, sendSection, recvSection);
242:     return(0);
243:   } else {
244:     PetscCopySection(sendOverlap, recvOverlap, sendSection->atlasLayout, recvSection->atlasLayout, sendSection->atlasDof, &recvSection->atlasDof);
245:   }
246:   ALE::MPIMover<send_value_type> vMover(sendSection->comm, debug);

248:   PetscOverlapGetNumRanks(sendOverlap, &numSendRanks);
249:   PetscOverlapGetNumRanks(recvOverlap, &numRecvRanks);
250:   PetscMalloc(numSendRanks * sizeof(send_value_type *), &sendValues);
251:   for(PetscInt r = 0; r < numSendRanks; ++r) {
252:     PetscInt        rank;
253:     PetscInt        numPoints;
254:     const PetscInt *points; // This is sendOverlap->cone(rank)
255:     PetscInt        numVals   = 0;

257:     PetscOverlapGetRank(sendOverlap, r, &rank);
258:     PetscOverlapGetNumPoints(sendOverlap, r, &numPoints);
259:     PetscOverlapGetPoints(sendOverlap, r, &points);
260:     for(PetscInt p = 0; p < numPoints; ++p) {
261:       PetscInt fDim;

263:       PetscSectionGetDof(sendSection, points[p], &fDim);
264:       numVals += fDim;
265:     }
266:     send_value_type *v;
267:     PetscInt         k = 0;

269:     PetscMalloc(numVals * sizeof(send_value_type), &v);
270:     for(PetscInt p = 0; p < numPoints; ++p) {
271:       PetscInt fDim, off;

273:       PetscSectionGetDof(sendSection, points[p], &fDim);
274:       PetscSectionGetOffset(sendSection, points[p], &off);
275:       for(PetscInt i = 0; i < fDim; ++i, ++k) {
276:         v[k] = sendStorage[off+i];
277:       }
278:     }
279:     sendValues[r] = v;
280:     vMover.send(r, numVals, v);
281:   }
282:   PetscSectionSetUp(recvSection);
283:   PetscSectionGetStorageSize(recvSection, &recvSize);
284:   PetscMalloc(recvSize * sizeof(recv_value_type), &recvStorage);

286:   PetscMalloc(numRecvRanks * sizeof(send_value_type *), &recvValues);
287:   for(PetscInt r = 0; r < numRecvRanks; ++r) {
288:     PetscInt        rank;
289:     PetscInt        numPoints;
290:     const PetscInt *points;       // This is recvOverlap->support(rank)
291:     const PetscInt *remotePoints; // This is recvOverlap->support(rank).color()
292:     PetscInt        numVals      = 0;

294:     PetscOverlapGetRank(recvOverlap, r, &rank);
295:     PetscOverlapGetNumPoints(recvOverlap, r, &numPoints);
296:     PetscOverlapGetPoints(recvOverlap, r, &points);
297:     PetscOverlapGetRemotePoints(recvOverlap, r, &remotePoints);
298:     for(PetscInt p = 0; p < numPoints; ++p) {
299:       PetscInt fDim;

301:       /// TODO
302:       ///PetscSectionGetDof(recvSection, recv_point_type(rank, remotePoints[p]), &fDim);
303:       numVals += fDim;
304:     }
305:     send_value_type *v;

307:     PetscMalloc(numVals * sizeof(send_value_type), &v);
308:     recvValues[r] = v;
309:     vMover.recv(rank, numVals, v);
310:   }
311:   vMover.start();
312:   vMover.end();

314:   for(PetscInt r = 0; r < numRecvRanks; ++r) {
315:     PetscInt         rank;
316:     PetscInt         numPoints;
317:     const PetscInt  *points;       // This is recvOverlap->support(rank)
318:     const PetscInt  *remotePoints; // This is recvOverlap->support(rank).color()
319:     send_value_type *v = recvValues[r];
320:     PetscInt         p, k = 0;
321:     point_type      *sortedPoints;

323:     PetscMalloc(numPoints * sizeof(point_type), &sortedPoints);
324:     for(p = 0; p < numPoints; ++p) {
325:       sortedPoints[p] = remotePoints[p];
326:     }
327:     PetscSortInt(numPoints, sortedPoints);

329:     PetscOverlapGetRank(recvOverlap, r, &rank);
330:     PetscOverlapGetNumPoints(recvOverlap, r, &numPoints);
331:     PetscOverlapGetPoints(recvOverlap, r, &points);
332:     PetscOverlapGetRemotePoints(recvOverlap, r, &remotePoints);
333:     for(p = 0; p < numPoints; ++p) {
334:       const int fDim(0), off(0);

336:       /// TODO
337:       ///PetscSectionGetDof(recvSection, recv_point_type(rank, sortedPoints[p]), &fDim);
338:       ///PetscSectionGetOffset(recvSection, recv_point_type(rank, sortedPoints[p]), &off);
339:       for(PetscInt i = 0; i < fDim; ++i, ++k) {
340:         recvStorage[off+i] = (recv_value_type) v[k];
341:       }
342:     }
343:     PetscFree(sortedPoints);
344:   }
345:   for(PetscInt r = 0; r < numSendRanks; ++r) {
346:     PetscFree(sendValues[r]);
347:   }
348:   for(PetscInt r = 0; r < numRecvRanks; ++r) {
349:     PetscFree(recvValues[r]);
350:   }
351:   return(0);
352: };

354: #if 0
355: // Copies the PetscSection from one process to another, specified by the overlap
356: //   Use MPI_DATATYPE_NULL for a default
357: template<typename send_value_type, typename recv_value_type>
358: PetscErrorCode PetscCopySection(PetscOverlap sendOverlap, PetscOverlap recvOverlap, PetscSection sendSection, PetscSection recvSection, const MPI_Datatype datatype, send_value_type *sendStorage, recv_value_type **recvStorage) {
359:   PetscInt         numSendRanks; // This is sendOverlap->base()
360:   PetscInt         numRecvRanks; // This is recvOverlap->cap()
361:   PetscInt         recvSize;
362:   send_value_type *sendValues;
363:   send_value_type *recvValues;
364:   const int        debug = 1;
365:   PetscErrorCode   ierr;
366:   ALE::MPIMover<send_value_type> vMover(sendSection->atlasLayout.comm, datatype, MPI_UNDEFINED, debug);

368:   PetscCopyChart(sendOverlap, recvOverlap, &sendSection->atlasLayout, &recvSection->atlasLayout);

370:   PetscOverlapGetNumRanks(sendOverlap, &numSendRanks);
371:   PetscOverlapGetNumRanks(recvOverlap, &numRecvRanks);
372:   PetscMalloc(numSendRanks * sizeof(send_value_type *), &sendValues);
373:   for(PetscInt r = 0; r < numSendRanks; ++r) {
374:     PetscInt        rank;
375:     PetscInt        numPoints;
376:     const PetscInt *points; // This is sendOverlap->cone(rank)
377:     PetscInt        numVals   = 0;

379:     PetscOverlapGetRank(sendOverlap, r, &rank);
380:     PetscOverlapGetNumPoints(sendOverlap, r, &numPoints);
381:     PetscOverlapGetPoints(sendOverlap, r, &points);
382:     for(PetscInt p = 0; p < numPoints; ++p) {
383:       PetscInt fDim;

385:       PetscSectionGetDof(sendSection, points[p], &fDim);
386:       numVals += fDim;
387:     }
388:     send_value_type *v;
389:     PetscInt         k = 0;

391:     PetscMalloc(numVals * sizeof(send_value_type), &v);
392:     for(PetscInt p = 0; p < numPoints; ++p) {
393:       PetscInt fDim, off;

395:       PetscSectionGetDof(sendSection, points[p], &fDim);
396:       PetscSectionGetOffset(sendSection, points[p], &off);
397:       for(PetscInt i = 0; i < fDim; ++i, ++k) {
398:         v[k] = sendStorage[off+i];
399:       }
400:     }
401:     sendValues[r] = v;
402:     vMover.send(r, numVals, v);
403:   }
404:   PetscSectionSetUp(recvSection);
405:   PetscSectionGetStorageSize(recvSection, &recvSize);
406:   PetscMalloc(recvSize * sizeof(recv_value_type), &recvStorage);

408:   PetscMalloc(numRecvRanks * sizeof(send_value_type *), &recvValues);
409:   for(PetscInt r = 0; r < numRecvRanks; ++r) {
410:     PetscInt        rank;
411:     PetscInt        numPoints;
412:     const PetscInt *points;       // This is recvOverlap->support(rank)
413:     const PetscInt *remotePoints; // This is recvOverlap->support(rank).color()
414:     PetscInt        numVals      = 0;

416:     PetscOverlapGetRank(recvOverlap, r, &rank);
417:     PetscOverlapGetNumPoints(recvOverlap, r, &numPoints);
418:     PetscOverlapGetPoints(recvOverlap, r, &points);
419:     PetscOverlapGetRemotePoints(recvOverlap, r, &remotePoints);
420:     for(PetscInt p = 0; p < numPoints; ++p) {
421:       PetscInt fDim;

423:       PetscSectionGetDof(recvSection, recv_point_type(rank, remotePoints[p]), &fDim);
424:       numVals += fDim;
425:     }
426:     send_value_type *v;

428:     PetscMalloc(numVals * sizeof(send_value_type), &v);
429:     recvValues[r] = v;
430:     vMover.recv(rank, numVals, v);
431:   }
432:   vMover.start();
433:   vMover.end();

435:   for(PetscInt r = 0; r < numRecvRanks; ++r) {
436:     PetscInt         rank;
437:     PetscInt         numPoints;
438:     const PetscInt  *points;       // This is recvOverlap->support(rank)
439:     const PetscInt  *remotePoints; // This is recvOverlap->support(rank).color()
440:     send_value_type *v = recvValues[r];
441:     PetscInt         p, k = 0;
442:     point_type      *sortedPoints;

444:     PetscMalloc(numPoints * sizeof(point_type), &sortedPoints);
445:     for(p = 0; p < numPoints; ++p) {
446:       sortedPoints[p] = remotePoints[p];
447:     }
448:     PetscSortInt(numPoints, sortedPoints);

450:     PetscOverlapGetRank(recvOverlap, r, &rank);
451:     PetscOverlapGetNumPoints(recvOverlap, r, &numPoints);
452:     PetscOverlapGetPoints(recvOverlap, r, &points);
453:     PetscOverlapGetRemotePoints(recvOverlap, r, &remotePoints);
454:     for(p = 0; p < numPoints; ++p) {
455:       const int fDim, off;

457:       PetscSectionGetDof(recvSection, recv_point_type(rank, sortedPoints[p]), &fDim);
458:       PetscSectionGetOff(recvSection, recv_point_type(rank, sortedPoints[p]), &off);
459:       for(PetscInt i = 0; i < fDim; ++i, ++k) {
460:         recvStorage[off+i] = (recv_value_type) v[k];
461:       }
462:     }
463:     PetscFree(sortedPoints);
464:   }
465:   for(PetscInt r = 0; r < numSendRanks; ++r) {
466:     PetscFree(sendValues[r]);
467:   }
468:   for(PetscInt r = 0; r < numRecvRanks; ++r) {
469:     PetscFree(recvValues[r]);
470:   }
471: };
472: #endif