Actual source code: sectioncopy.c
1: #include <petscsys.h>
2: #include <private/vecimpl.h>
3: #include <sieve/BasicCommunication.hh>
5: typedef struct _n_PetscOverlap *PetscOverlap;
6: struct _n_PetscOverlap
7: {
8: MPI_Comm comm;
9: PetscInt numRanks; // Number of partner processes
10: PetscInt *ranks; // MPI Rank of each partner process
11: PetscInt *pointsOffset; // Offset into points array for each partner process
12: PetscInt *points; // Points array for each partner process, in sorted order
13: PetscInt *remotePoints; // Remote points array for each partner process
14: };
16: PetscErrorCode PetscOverlapCreate(MPI_Comm comm, PetscOverlap *o)
17: {
21: PetscNew(struct _n_PetscOverlap, o);
22: (*o)->comm = comm;
23: (*o)->numRanks = 0;
24: (*o)->ranks = PETSC_NULL;
25: (*o)->pointsOffset = PETSC_NULL;
26: (*o)->points = PETSC_NULL;
27: (*o)->remotePoints = PETSC_NULL;
28: return(0);
29: }
31: PetscErrorCode PetscOverlapDestroy(PetscOverlap *o)
32: {
36: if (!*o) return(0);
37: PetscFree((*o)->ranks);
38: PetscFree((*o)->pointsOffset);
39: PetscFree((*o)->points);
40: PetscFree((*o)->remotePoints);
41: PetscFree((*o));
42: *o = PETSC_NULL;
43: return(0);
44: }
46: PetscErrorCode PetscOverlapGetNumRanks(PetscOverlap overlap, PetscInt *numRanks)
47: {
50: *numRanks = overlap->numRanks;
51: return(0);
52: };
54: PetscErrorCode PetscOverlapGetRank(PetscOverlap overlap, PetscInt r, PetscInt *rank)
55: {
58: if (r < 0 || r >= overlap->numRanks) {SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid rank index %d should be in [%d, %d)", r, 0, overlap->numRanks);}
59: *rank = overlap->ranks[r];
60: return(0);
61: };
63: // TODO: Replace this with binary search
64: PetscErrorCode PetscOverlapGetRankIndex(PetscOverlap overlap, PetscInt rank, PetscInt *r)
65: {
66: PetscInt p;
70: for(p = 0; p < overlap->numRanks; ++p) {
71: if (overlap->ranks[p] == rank) {
72: *r = p;
73: return(0);
74: }
75: }
76: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid rank %d was not contained in this overlap", rank);
77: };
79: PetscErrorCode PetscOverlapGetNumPoints(PetscOverlap overlap, PetscInt r, PetscInt *numPoints)
80: {
83: if (r < 0 || r >= overlap->numRanks) {SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid rank index %d should be in [%d, %d)", r, 0, overlap->numRanks);}
84: *numPoints = overlap->pointsOffset[r+1] - overlap->pointsOffset[r];
85: return(0);
86: };
88: // These should be sorted
89: PetscErrorCode PetscOverlapGetPoints(PetscOverlap overlap, PetscInt r, const PetscInt **points)
90: {
93: if (r < 0 || r >= overlap->numRanks) {SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid rank index %d should be in [%d, %d)", r, 0, overlap->numRanks);}
94: *points = &overlap->points[overlap->pointsOffset[r]];
95: return(0);
96: };
98: // These cannot be sorted
99: PetscErrorCode PetscOverlapGetRemotePoints(PetscOverlap overlap, PetscInt r, const PetscInt **remotePoints)
100: {
103: if (r < 0 || r >= overlap->numRanks) {SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid rank index %d should be in [%d, %d)", r, 0, overlap->numRanks);}
104: *remotePoints = &overlap->remotePoints[overlap->pointsOffset[r]];
105: return(0);
106: };
108: PetscErrorCode PetscOverlapGetNumPointsByRank(PetscOverlap overlap, PetscInt rank, PetscInt *numPoints)
109: {
110: PetscInt r;
115: PetscOverlapGetRankIndex(overlap, rank, &r);
116: *numPoints = overlap->pointsOffset[r+1] - overlap->pointsOffset[r];
117: return(0);
118: };
120: // These should be sorted
121: PetscErrorCode PetscOverlapGetPointsByRank(PetscOverlap overlap, PetscInt rank, const PetscInt **points)
122: {
123: PetscInt r;
128: PetscOverlapGetRankIndex(overlap, rank, &r);
129: *points = &overlap->points[overlap->pointsOffset[r]];
130: return(0);
131: };
133: typedef PetscInt point_type;
135: // Copies the PetscUniformSection from one process to another, specified by the overlap
136: // Moves the chart interval [pStart, pEnd) and the numDof
137: PetscErrorCode PetscCopySection(PetscOverlap sendOverlap, PetscOverlap recvOverlap, const PetscUniformSection sendSection, const PetscUniformSection recvSection) {
138: PetscInt numSendRanks; // This is sendOverlap->base()
139: PetscInt numRecvRanks; // This is recvOverlap->cap()
140: const int debug = 0;
141: point_type **sendPoints;
142: point_type **recvPoints;
144: ALE::MPIMover<point_type> pMover(sendSection->comm, debug);
147: PetscOverlapGetNumRanks(sendOverlap, &numSendRanks);
148: PetscOverlapGetNumRanks(recvOverlap, &numRecvRanks);
149: PetscMalloc(numSendRanks * sizeof(point_type *), &sendPoints);
150: for(PetscInt r = 0; r < numSendRanks; ++r) {
151: PetscInt rank;
152: point_type *v;
154: PetscOverlapGetRank(sendOverlap, r, &rank);
155: PetscMalloc(3 * sizeof(point_type), &v);
156: v[0] = sendSection->pStart;
157: v[1] = sendSection->pEnd;
158: v[2] = sendSection->numDof;
159: sendPoints[r] = v;
160: pMover.send(rank, 3, sendPoints[r]);
161: if (debug) {PetscPrintf(PETSC_COMM_SELF, "[%d]Sending chart (%d, %d, %d) to process %d\n", pMover.commRank(), v[0], v[1], v[2], rank);}
162: }
164: PetscMalloc(numRecvRanks * sizeof(point_type *), &recvPoints);
165: for(PetscInt r = 0; r < numRecvRanks; ++r) {
166: PetscInt rank;
167: point_type *v;
169: PetscOverlapGetRank(recvOverlap, r, &rank);
170: PetscMalloc(3 * sizeof(point_type), &v);
171: recvPoints[r] = v;
172: pMover.recv(rank, 3, recvPoints[r]);
173: }
174: pMover.start();
175: pMover.end();
176: point_type min = -1;
177: point_type max = -1;
178: point_type dof = -1;
180: if (!numRecvRanks) {min = max = 0;}
181: for(PetscInt r = 0; r < numRecvRanks; ++r) {
182: const point_type *v = recvPoints[r];
183: point_type newMin = v[0];
184: point_type newMax = v[1];
185: PetscInt rank;
187: PetscOverlapGetRank(recvOverlap, r, &rank);
188: if (debug) {PetscPrintf(PETSC_COMM_SELF, "[%d]Received chart (%d, %d, %d) from process %d\n", pMover.commRank(), v[0],v[1], v[2], rank);}
189: // Take the union of all charts
190: if (min < 0) {
191: min = newMin;
192: max = newMax;
193: } else {
194: min = std::min(min, newMin);
195: max = std::max(max, newMax);
196: }
197: if (dof < 0) {
198: dof = v[2];
199: } else {
200: if (dof != v[2]) {SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of dof %d from rank %d should be %d", v[2], rank, dof);}
201: }
202: }
203: recvSection->pStart = min;
204: recvSection->pEnd = max;
205: recvSection->numDof = dof;
206: for(PetscInt r = 0; r < numSendRanks; ++r) {
207: PetscFree(sendPoints[r]);
208: }
209: for(PetscInt r = 0; r < numRecvRanks; ++r) {
210: PetscFree(recvPoints[r]);
211: }
212: return(0);
213: };
215: // Full Sequence:
216: // PetscCopySection(sOv, rOv, sSec, sStor, rSec, rStor)
217: // --> PetscCopySection(sOv, rOv, sSec->atlasLayout, sSec->atlasDof, rSec->atlasLayout, rSec->atlasDof)
218: // --> PetscCopyChart(sOv, rOv, sSec->atlasLayout, rSec->atlasLayout)
219: // --> Copy sSec->atlasDof to rSec->atlasDof
220: // --> Copy sStor to rStor
221: //
222: // RecvSection
223: // 1) Usually has scattered values, so using an interval chart is wrong
224: // 2) Must differentiate between data from different ranks (wait for fuse() process to merge)
225: // Should we combine the fuse step here?
227: // Copies the PetscSection from one process to another, specified by the overlap
228: // Use MPI_DATATYPE_NULL for a default
229: template<typename section_type, typename send_value_type, typename recv_value_type>
230: PetscErrorCode PetscCopySection(PetscOverlap sendOverlap, PetscOverlap recvOverlap, section_type sendSection, section_type recvSection, send_value_type *sendStorage, recv_value_type **recvStorage) {
231: PetscInt numSendRanks; // This is sendOverlap->base()
232: PetscInt numRecvRanks; // This is recvOverlap->cap()
233: PetscInt recvSize;
234: send_value_type *sendValues;
235: send_value_type *recvValues;
236: const int debug = 1;
237: PetscErrorCode ierr;
240: if (!sendStorage) {
241: PetscCopySection(sendOverlap, recvOverlap, sendSection, recvSection);
242: return(0);
243: } else {
244: PetscCopySection(sendOverlap, recvOverlap, sendSection->atlasLayout, recvSection->atlasLayout, sendSection->atlasDof, &recvSection->atlasDof);
245: }
246: ALE::MPIMover<send_value_type> vMover(sendSection->comm, debug);
248: PetscOverlapGetNumRanks(sendOverlap, &numSendRanks);
249: PetscOverlapGetNumRanks(recvOverlap, &numRecvRanks);
250: PetscMalloc(numSendRanks * sizeof(send_value_type *), &sendValues);
251: for(PetscInt r = 0; r < numSendRanks; ++r) {
252: PetscInt rank;
253: PetscInt numPoints;
254: const PetscInt *points; // This is sendOverlap->cone(rank)
255: PetscInt numVals = 0;
257: PetscOverlapGetRank(sendOverlap, r, &rank);
258: PetscOverlapGetNumPoints(sendOverlap, r, &numPoints);
259: PetscOverlapGetPoints(sendOverlap, r, &points);
260: for(PetscInt p = 0; p < numPoints; ++p) {
261: PetscInt fDim;
263: PetscSectionGetDof(sendSection, points[p], &fDim);
264: numVals += fDim;
265: }
266: send_value_type *v;
267: PetscInt k = 0;
269: PetscMalloc(numVals * sizeof(send_value_type), &v);
270: for(PetscInt p = 0; p < numPoints; ++p) {
271: PetscInt fDim, off;
273: PetscSectionGetDof(sendSection, points[p], &fDim);
274: PetscSectionGetOffset(sendSection, points[p], &off);
275: for(PetscInt i = 0; i < fDim; ++i, ++k) {
276: v[k] = sendStorage[off+i];
277: }
278: }
279: sendValues[r] = v;
280: vMover.send(r, numVals, v);
281: }
282: PetscSectionSetUp(recvSection);
283: PetscSectionGetStorageSize(recvSection, &recvSize);
284: PetscMalloc(recvSize * sizeof(recv_value_type), &recvStorage);
286: PetscMalloc(numRecvRanks * sizeof(send_value_type *), &recvValues);
287: for(PetscInt r = 0; r < numRecvRanks; ++r) {
288: PetscInt rank;
289: PetscInt numPoints;
290: const PetscInt *points; // This is recvOverlap->support(rank)
291: const PetscInt *remotePoints; // This is recvOverlap->support(rank).color()
292: PetscInt numVals = 0;
294: PetscOverlapGetRank(recvOverlap, r, &rank);
295: PetscOverlapGetNumPoints(recvOverlap, r, &numPoints);
296: PetscOverlapGetPoints(recvOverlap, r, &points);
297: PetscOverlapGetRemotePoints(recvOverlap, r, &remotePoints);
298: for(PetscInt p = 0; p < numPoints; ++p) {
299: PetscInt fDim;
301: /// TODO
302: ///PetscSectionGetDof(recvSection, recv_point_type(rank, remotePoints[p]), &fDim);
303: numVals += fDim;
304: }
305: send_value_type *v;
307: PetscMalloc(numVals * sizeof(send_value_type), &v);
308: recvValues[r] = v;
309: vMover.recv(rank, numVals, v);
310: }
311: vMover.start();
312: vMover.end();
314: for(PetscInt r = 0; r < numRecvRanks; ++r) {
315: PetscInt rank;
316: PetscInt numPoints;
317: const PetscInt *points; // This is recvOverlap->support(rank)
318: const PetscInt *remotePoints; // This is recvOverlap->support(rank).color()
319: send_value_type *v = recvValues[r];
320: PetscInt p, k = 0;
321: point_type *sortedPoints;
323: PetscMalloc(numPoints * sizeof(point_type), &sortedPoints);
324: for(p = 0; p < numPoints; ++p) {
325: sortedPoints[p] = remotePoints[p];
326: }
327: PetscSortInt(numPoints, sortedPoints);
329: PetscOverlapGetRank(recvOverlap, r, &rank);
330: PetscOverlapGetNumPoints(recvOverlap, r, &numPoints);
331: PetscOverlapGetPoints(recvOverlap, r, &points);
332: PetscOverlapGetRemotePoints(recvOverlap, r, &remotePoints);
333: for(p = 0; p < numPoints; ++p) {
334: const int fDim(0), off(0);
336: /// TODO
337: ///PetscSectionGetDof(recvSection, recv_point_type(rank, sortedPoints[p]), &fDim);
338: ///PetscSectionGetOffset(recvSection, recv_point_type(rank, sortedPoints[p]), &off);
339: for(PetscInt i = 0; i < fDim; ++i, ++k) {
340: recvStorage[off+i] = (recv_value_type) v[k];
341: }
342: }
343: PetscFree(sortedPoints);
344: }
345: for(PetscInt r = 0; r < numSendRanks; ++r) {
346: PetscFree(sendValues[r]);
347: }
348: for(PetscInt r = 0; r < numRecvRanks; ++r) {
349: PetscFree(recvValues[r]);
350: }
351: return(0);
352: };
354: #if 0
355: // Copies the PetscSection from one process to another, specified by the overlap
356: // Use MPI_DATATYPE_NULL for a default
357: template<typename send_value_type, typename recv_value_type>
358: PetscErrorCode PetscCopySection(PetscOverlap sendOverlap, PetscOverlap recvOverlap, PetscSection sendSection, PetscSection recvSection, const MPI_Datatype datatype, send_value_type *sendStorage, recv_value_type **recvStorage) {
359: PetscInt numSendRanks; // This is sendOverlap->base()
360: PetscInt numRecvRanks; // This is recvOverlap->cap()
361: PetscInt recvSize;
362: send_value_type *sendValues;
363: send_value_type *recvValues;
364: const int debug = 1;
365: PetscErrorCode ierr;
366: ALE::MPIMover<send_value_type> vMover(sendSection->atlasLayout.comm, datatype, MPI_UNDEFINED, debug);
368: PetscCopyChart(sendOverlap, recvOverlap, &sendSection->atlasLayout, &recvSection->atlasLayout);
370: PetscOverlapGetNumRanks(sendOverlap, &numSendRanks);
371: PetscOverlapGetNumRanks(recvOverlap, &numRecvRanks);
372: PetscMalloc(numSendRanks * sizeof(send_value_type *), &sendValues);
373: for(PetscInt r = 0; r < numSendRanks; ++r) {
374: PetscInt rank;
375: PetscInt numPoints;
376: const PetscInt *points; // This is sendOverlap->cone(rank)
377: PetscInt numVals = 0;
379: PetscOverlapGetRank(sendOverlap, r, &rank);
380: PetscOverlapGetNumPoints(sendOverlap, r, &numPoints);
381: PetscOverlapGetPoints(sendOverlap, r, &points);
382: for(PetscInt p = 0; p < numPoints; ++p) {
383: PetscInt fDim;
385: PetscSectionGetDof(sendSection, points[p], &fDim);
386: numVals += fDim;
387: }
388: send_value_type *v;
389: PetscInt k = 0;
391: PetscMalloc(numVals * sizeof(send_value_type), &v);
392: for(PetscInt p = 0; p < numPoints; ++p) {
393: PetscInt fDim, off;
395: PetscSectionGetDof(sendSection, points[p], &fDim);
396: PetscSectionGetOffset(sendSection, points[p], &off);
397: for(PetscInt i = 0; i < fDim; ++i, ++k) {
398: v[k] = sendStorage[off+i];
399: }
400: }
401: sendValues[r] = v;
402: vMover.send(r, numVals, v);
403: }
404: PetscSectionSetUp(recvSection);
405: PetscSectionGetStorageSize(recvSection, &recvSize);
406: PetscMalloc(recvSize * sizeof(recv_value_type), &recvStorage);
408: PetscMalloc(numRecvRanks * sizeof(send_value_type *), &recvValues);
409: for(PetscInt r = 0; r < numRecvRanks; ++r) {
410: PetscInt rank;
411: PetscInt numPoints;
412: const PetscInt *points; // This is recvOverlap->support(rank)
413: const PetscInt *remotePoints; // This is recvOverlap->support(rank).color()
414: PetscInt numVals = 0;
416: PetscOverlapGetRank(recvOverlap, r, &rank);
417: PetscOverlapGetNumPoints(recvOverlap, r, &numPoints);
418: PetscOverlapGetPoints(recvOverlap, r, &points);
419: PetscOverlapGetRemotePoints(recvOverlap, r, &remotePoints);
420: for(PetscInt p = 0; p < numPoints; ++p) {
421: PetscInt fDim;
423: PetscSectionGetDof(recvSection, recv_point_type(rank, remotePoints[p]), &fDim);
424: numVals += fDim;
425: }
426: send_value_type *v;
428: PetscMalloc(numVals * sizeof(send_value_type), &v);
429: recvValues[r] = v;
430: vMover.recv(rank, numVals, v);
431: }
432: vMover.start();
433: vMover.end();
435: for(PetscInt r = 0; r < numRecvRanks; ++r) {
436: PetscInt rank;
437: PetscInt numPoints;
438: const PetscInt *points; // This is recvOverlap->support(rank)
439: const PetscInt *remotePoints; // This is recvOverlap->support(rank).color()
440: send_value_type *v = recvValues[r];
441: PetscInt p, k = 0;
442: point_type *sortedPoints;
444: PetscMalloc(numPoints * sizeof(point_type), &sortedPoints);
445: for(p = 0; p < numPoints; ++p) {
446: sortedPoints[p] = remotePoints[p];
447: }
448: PetscSortInt(numPoints, sortedPoints);
450: PetscOverlapGetRank(recvOverlap, r, &rank);
451: PetscOverlapGetNumPoints(recvOverlap, r, &numPoints);
452: PetscOverlapGetPoints(recvOverlap, r, &points);
453: PetscOverlapGetRemotePoints(recvOverlap, r, &remotePoints);
454: for(p = 0; p < numPoints; ++p) {
455: const int fDim, off;
457: PetscSectionGetDof(recvSection, recv_point_type(rank, sortedPoints[p]), &fDim);
458: PetscSectionGetOff(recvSection, recv_point_type(rank, sortedPoints[p]), &off);
459: for(PetscInt i = 0; i < fDim; ++i, ++k) {
460: recvStorage[off+i] = (recv_value_type) v[k];
461: }
462: }
463: PetscFree(sortedPoints);
464: }
465: for(PetscInt r = 0; r < numSendRanks; ++r) {
466: PetscFree(sendValues[r]);
467: }
468: for(PetscInt r = 0; r < numRecvRanks; ++r) {
469: PetscFree(recvValues[r]);
470: }
471: };
472: #endif