Actual source code: fdda.c
1:
2: #include <private/daimpl.h> /*I "petscdmda.h" I*/
3: #include <petscmat.h> /*I "petscmat.h" I*/
4: #include <private/matimpl.h>
11: /*
12: For ghost i that may be negative or greater than the upper bound this
13: maps it into the 0:m-1 range using periodicity
14: */
15: #define SetInRange(i,m) ((i < 0) ? m+i:((i >= m) ? i-m:i))
19: static PetscErrorCode DMDASetBlockFills_Private(PetscInt *dfill,PetscInt w,PetscInt **rfill)
20: {
22: PetscInt i,j,nz,*fill;
25: if (!dfill) return(0);
27: /* count number nonzeros */
28: nz = 0;
29: for (i=0; i<w; i++) {
30: for (j=0; j<w; j++) {
31: if (dfill[w*i+j]) nz++;
32: }
33: }
34: PetscMalloc((nz + w + 1)*sizeof(PetscInt),&fill);
35: /* construct modified CSR storage of nonzero structure */
36: nz = w + 1;
37: for (i=0; i<w; i++) {
38: fill[i] = nz;
39: for (j=0; j<w; j++) {
40: if (dfill[w*i+j]) {
41: fill[nz] = j;
42: nz++;
43: }
44: }
45: }
46: fill[w] = nz;
47:
48: *rfill = fill;
49: return(0);
50: }
54: /*@
55: DMDASetBlockFills - Sets the fill pattern in each block for a multi-component problem
56: of the matrix returned by DMGetMatrix().
58: Logically Collective on DMDA
60: Input Parameter:
61: + da - the distributed array
62: . dfill - the fill pattern in the diagonal block (may be PETSC_NULL, means use dense block)
63: - ofill - the fill pattern in the off-diagonal blocks
66: Level: developer
68: Notes: This only makes sense when you are doing multicomponent problems but using the
69: MPIAIJ matrix format
71: The format for dfill and ofill is a 2 dimensional dof by dof matrix with 1 entries
72: representing coupling and 0 entries for missing coupling. For example
73: $ dfill[9] = {1, 0, 0,
74: $ 1, 1, 0,
75: $ 0, 1, 1}
76: means that row 0 is coupled with only itself in the diagonal block, row 1 is coupled with
77: itself and row 0 (in the diagonal block) and row 2 is coupled with itself and row 1 (in the
78: diagonal block).
80: DMDASetGetMatrix() allows you to provide general code for those more complicated nonzero patterns then
81: can be represented in the dfill, ofill format
83: Contributed by Glenn Hammond
85: .seealso DMGetMatrix(), DMDASetGetMatrix(), DMDASetMatPreallocateOnly()
87: @*/
88: PetscErrorCode DMDASetBlockFills(DM da,PetscInt *dfill,PetscInt *ofill)
89: {
90: DM_DA *dd = (DM_DA*)da->data;
94: DMDASetBlockFills_Private(dfill,dd->w,&dd->dfill);
95: DMDASetBlockFills_Private(ofill,dd->w,&dd->ofill);
96: return(0);
97: }
102: PetscErrorCode DMGetColoring_DA(DM da,ISColoringType ctype,const MatType mtype,ISColoring *coloring)
103: {
104: PetscErrorCode ierr;
105: PetscInt dim,m,n,p,nc;
106: DMDABoundaryType bx,by,bz;
107: MPI_Comm comm;
108: PetscMPIInt size;
109: PetscBool isBAIJ;
110: DM_DA *dd = (DM_DA*)da->data;
113: /*
114: m
115: ------------------------------------------------------
116: | |
117: | |
118: | ---------------------- |
119: | | | |
120: n | yn | | |
121: | | | |
122: | .--------------------- |
123: | (xs,ys) xn |
124: | . |
125: | (gxs,gys) |
126: | |
127: -----------------------------------------------------
128: */
130: /*
131: nc - number of components per grid point
132: col - number of colors needed in one direction for single component problem
133:
134: */
135: DMDAGetInfo(da,&dim,0,0,0,&m,&n,&p,&nc,0,&bx,&by,&bz,0);
137: PetscObjectGetComm((PetscObject)da,&comm);
138: MPI_Comm_size(comm,&size);
139: if (ctype == IS_COLORING_GHOSTED){
140: if (size == 1) {
141: ctype = IS_COLORING_GLOBAL;
142: } else if (dim > 1){
143: if ((m==1 && bx == DMDA_BOUNDARY_PERIODIC) || (n==1 && by == DMDA_BOUNDARY_PERIODIC) || (p==1 && bz == DMDA_BOUNDARY_PERIODIC)){
144: SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"IS_COLORING_GHOSTED cannot be used for periodic boundary condition having both ends of the domain on the same process");
145: }
146: }
147: }
149: /* Tell the DMDA it has 1 degree of freedom per grid point so that the coloring for BAIJ
150: matrices is for the blocks, not the individual matrix elements */
151: PetscStrcmp(mtype,MATBAIJ,&isBAIJ);
152: if (!isBAIJ) {PetscStrcmp(mtype,MATMPIBAIJ,&isBAIJ);}
153: if (!isBAIJ) {PetscStrcmp(mtype,MATSEQBAIJ,&isBAIJ);}
154: if (isBAIJ) {
155: dd->w = 1;
156: dd->xs = dd->xs/nc;
157: dd->xe = dd->xe/nc;
158: dd->Xs = dd->Xs/nc;
159: dd->Xe = dd->Xe/nc;
160: }
162: /*
163: We do not provide a getcoloring function in the DMDA operations because
164: the basic DMDA does not know about matrices. We think of DMDA as being more
165: more low-level then matrices.
166: */
167: if (dim == 1) {
168: DMGetColoring_DA_1d_MPIAIJ(da,ctype,coloring);
169: } else if (dim == 2) {
170: DMGetColoring_DA_2d_MPIAIJ(da,ctype,coloring);
171: } else if (dim == 3) {
172: DMGetColoring_DA_3d_MPIAIJ(da,ctype,coloring);
173: } else SETERRQ1(((PetscObject)da)->comm,PETSC_ERR_SUP,"Not done for %D dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
174: if (isBAIJ) {
175: dd->w = nc;
176: dd->xs = dd->xs*nc;
177: dd->xe = dd->xe*nc;
178: dd->Xs = dd->Xs*nc;
179: dd->Xe = dd->Xe*nc;
180: }
181: return(0);
182: }
184: /* ---------------------------------------------------------------------------------*/
188: PetscErrorCode DMGetColoring_DA_2d_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
189: {
190: PetscErrorCode ierr;
191: PetscInt xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny,m,n,M,N,dim,s,k,nc,col;
192: PetscInt ncolors;
193: MPI_Comm comm;
194: DMDABoundaryType bx,by;
195: DMDAStencilType st;
196: ISColoringValue *colors;
197: DM_DA *dd = (DM_DA*)da->data;
200: /*
201: nc - number of components per grid point
202: col - number of colors needed in one direction for single component problem
203:
204: */
205: DMDAGetInfo(da,&dim,&m,&n,0,&M,&N,0,&nc,&s,&bx,&by,0,&st);
206: col = 2*s + 1;
207: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
208: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
209: PetscObjectGetComm((PetscObject)da,&comm);
211: /* special case as taught to us by Paul Hovland */
212: if (st == DMDA_STENCIL_STAR && s == 1) {
213: DMGetColoring_DA_2d_5pt_MPIAIJ(da,ctype,coloring);
214: } else {
216: if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)){
217: SETERRQ2(((PetscObject)da)->comm,PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X (%d) is divisible\n\
218: by 2*stencil_width + 1 (%d)\n", m, col);
219: }
220: if (by == DMDA_BOUNDARY_PERIODIC && (n % col)){
221: SETERRQ2(((PetscObject)da)->comm,PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y (%d) is divisible\n\
222: by 2*stencil_width + 1 (%d)\n", n, col);
223: }
224: if (ctype == IS_COLORING_GLOBAL) {
225: if (!dd->localcoloring) {
226: PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
227: ii = 0;
228: for (j=ys; j<ys+ny; j++) {
229: for (i=xs; i<xs+nx; i++) {
230: for (k=0; k<nc; k++) {
231: colors[ii++] = k + nc*((i % col) + col*(j % col));
232: }
233: }
234: }
235: ncolors = nc + nc*(col-1 + col*(col-1));
236: ISColoringCreate(comm,ncolors,nc*nx*ny,colors,&dd->localcoloring);
237: }
238: *coloring = dd->localcoloring;
239: } else if (ctype == IS_COLORING_GHOSTED) {
240: if (!dd->ghostedcoloring) {
241: PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
242: ii = 0;
243: for (j=gys; j<gys+gny; j++) {
244: for (i=gxs; i<gxs+gnx; i++) {
245: for (k=0; k<nc; k++) {
246: /* the complicated stuff is to handle periodic boundaries */
247: colors[ii++] = k + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col));
248: }
249: }
250: }
251: ncolors = nc + nc*(col - 1 + col*(col-1));
252: ISColoringCreate(comm,ncolors,nc*gnx*gny,colors,&dd->ghostedcoloring);
253: /* PetscIntView(ncolors,(PetscInt *)colors,0); */
255: ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
256: }
257: *coloring = dd->ghostedcoloring;
258: } else SETERRQ1(((PetscObject)da)->comm,PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
259: }
260: ISColoringReference(*coloring);
261: return(0);
262: }
264: /* ---------------------------------------------------------------------------------*/
268: PetscErrorCode DMGetColoring_DA_3d_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
269: {
270: PetscErrorCode ierr;
271: PetscInt xs,ys,nx,ny,i,j,gxs,gys,gnx,gny,m,n,p,dim,s,k,nc,col,zs,gzs,ii,l,nz,gnz,M,N,P;
272: PetscInt ncolors;
273: MPI_Comm comm;
274: DMDABoundaryType bx,by,bz;
275: DMDAStencilType st;
276: ISColoringValue *colors;
277: DM_DA *dd = (DM_DA*)da->data;
280: /*
281: nc - number of components per grid point
282: col - number of colors needed in one direction for single component problem
283:
284: */
285: DMDAGetInfo(da,&dim,&m,&n,&p,&M,&N,&P,&nc,&s,&bx,&by,&bz,&st);
286: col = 2*s + 1;
287: if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)){
288: SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
289: by 2*stencil_width + 1\n");
290: }
291: if (by == DMDA_BOUNDARY_PERIODIC && (n % col)){
292: SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
293: by 2*stencil_width + 1\n");
294: }
295: if (bz == DMDA_BOUNDARY_PERIODIC && (p % col)){
296: SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
297: by 2*stencil_width + 1\n");
298: }
300: DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
301: DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
302: PetscObjectGetComm((PetscObject)da,&comm);
304: /* create the coloring */
305: if (ctype == IS_COLORING_GLOBAL) {
306: if (!dd->localcoloring) {
307: PetscMalloc(nc*nx*ny*nz*sizeof(ISColoringValue),&colors);
308: ii = 0;
309: for (k=zs; k<zs+nz; k++) {
310: for (j=ys; j<ys+ny; j++) {
311: for (i=xs; i<xs+nx; i++) {
312: for (l=0; l<nc; l++) {
313: colors[ii++] = l + nc*((i % col) + col*(j % col) + col*col*(k % col));
314: }
315: }
316: }
317: }
318: ncolors = nc + nc*(col-1 + col*(col-1)+ col*col*(col-1));
319: ISColoringCreate(comm,ncolors,nc*nx*ny*nz,colors,&dd->localcoloring);
320: }
321: *coloring = dd->localcoloring;
322: } else if (ctype == IS_COLORING_GHOSTED) {
323: if (!dd->ghostedcoloring) {
324: PetscMalloc(nc*gnx*gny*gnz*sizeof(ISColoringValue),&colors);
325: ii = 0;
326: for (k=gzs; k<gzs+gnz; k++) {
327: for (j=gys; j<gys+gny; j++) {
328: for (i=gxs; i<gxs+gnx; i++) {
329: for (l=0; l<nc; l++) {
330: /* the complicated stuff is to handle periodic boundaries */
331: colors[ii++] = l + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col) + col*col*(SetInRange(k,p) % col));
332: }
333: }
334: }
335: }
336: ncolors = nc + nc*(col-1 + col*(col-1)+ col*col*(col-1));
337: ISColoringCreate(comm,ncolors,nc*gnx*gny*gnz,colors,&dd->ghostedcoloring);
338: ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
339: }
340: *coloring = dd->ghostedcoloring;
341: } else SETERRQ1(((PetscObject)da)->comm,PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
342: ISColoringReference(*coloring);
343: return(0);
344: }
346: /* ---------------------------------------------------------------------------------*/
350: PetscErrorCode DMGetColoring_DA_1d_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
351: {
352: PetscErrorCode ierr;
353: PetscInt xs,nx,i,i1,gxs,gnx,l,m,M,dim,s,nc,col;
354: PetscInt ncolors;
355: MPI_Comm comm;
356: DMDABoundaryType bx;
357: ISColoringValue *colors;
358: DM_DA *dd = (DM_DA*)da->data;
361: /*
362: nc - number of components per grid point
363: col - number of colors needed in one direction for single component problem
364:
365: */
366: DMDAGetInfo(da,&dim,&m,0,0,&M,0,0,&nc,&s,&bx,0,0,0);
367: col = 2*s + 1;
369: if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)) {
370: SETERRQ2(((PetscObject)da)->comm,PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points %d is divisible\n\
371: by 2*stencil_width + 1 %d\n",(int)m,(int)col);
372: }
374: DMDAGetCorners(da,&xs,0,0,&nx,0,0);
375: DMDAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
376: PetscObjectGetComm((PetscObject)da,&comm);
378: /* create the coloring */
379: if (ctype == IS_COLORING_GLOBAL) {
380: if (!dd->localcoloring) {
381: PetscMalloc(nc*nx*sizeof(ISColoringValue),&colors);
382: i1 = 0;
383: for (i=xs; i<xs+nx; i++) {
384: for (l=0; l<nc; l++) {
385: colors[i1++] = l + nc*(i % col);
386: }
387: }
388: ncolors = nc + nc*(col-1);
389: ISColoringCreate(comm,ncolors,nc*nx,colors,&dd->localcoloring);
390: }
391: *coloring = dd->localcoloring;
392: } else if (ctype == IS_COLORING_GHOSTED) {
393: if (!dd->ghostedcoloring) {
394: PetscMalloc(nc*gnx*sizeof(ISColoringValue),&colors);
395: i1 = 0;
396: for (i=gxs; i<gxs+gnx; i++) {
397: for (l=0; l<nc; l++) {
398: /* the complicated stuff is to handle periodic boundaries */
399: colors[i1++] = l + nc*(SetInRange(i,m) % col);
400: }
401: }
402: ncolors = nc + nc*(col-1);
403: ISColoringCreate(comm,ncolors,nc*gnx,colors,&dd->ghostedcoloring);
404: ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
405: }
406: *coloring = dd->ghostedcoloring;
407: } else SETERRQ1(((PetscObject)da)->comm,PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
408: ISColoringReference(*coloring);
409: return(0);
410: }
414: PetscErrorCode DMGetColoring_DA_2d_5pt_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
415: {
416: PetscErrorCode ierr;
417: PetscInt xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny,m,n,dim,s,k,nc;
418: PetscInt ncolors;
419: MPI_Comm comm;
420: DMDABoundaryType bx,by;
421: ISColoringValue *colors;
422: DM_DA *dd = (DM_DA*)da->data;
425: /*
426: nc - number of components per grid point
427: col - number of colors needed in one direction for single component problem
428:
429: */
430: DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,0);
431: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
432: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
433: PetscObjectGetComm((PetscObject)da,&comm);
435: if (bx == DMDA_BOUNDARY_PERIODIC && (m % 5)){
436: SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
437: by 5\n");
438: }
439: if (by == DMDA_BOUNDARY_PERIODIC && (n % 5)){
440: SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
441: by 5\n");
442: }
444: /* create the coloring */
445: if (ctype == IS_COLORING_GLOBAL) {
446: if (!dd->localcoloring) {
447: PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
448: ii = 0;
449: for (j=ys; j<ys+ny; j++) {
450: for (i=xs; i<xs+nx; i++) {
451: for (k=0; k<nc; k++) {
452: colors[ii++] = k + nc*((3*j+i) % 5);
453: }
454: }
455: }
456: ncolors = 5*nc;
457: ISColoringCreate(comm,ncolors,nc*nx*ny,colors,&dd->localcoloring);
458: }
459: *coloring = dd->localcoloring;
460: } else if (ctype == IS_COLORING_GHOSTED) {
461: if (!dd->ghostedcoloring) {
462: PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
463: ii = 0;
464: for (j=gys; j<gys+gny; j++) {
465: for (i=gxs; i<gxs+gnx; i++) {
466: for (k=0; k<nc; k++) {
467: colors[ii++] = k + nc*((3*SetInRange(j,n) + SetInRange(i,m)) % 5);
468: }
469: }
470: }
471: ncolors = 5*nc;
472: ISColoringCreate(comm,ncolors,nc*gnx*gny,colors,&dd->ghostedcoloring);
473: ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
474: }
475: *coloring = dd->ghostedcoloring;
476: } else SETERRQ1(((PetscObject)da)->comm,PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
477: return(0);
478: }
480: /* =========================================================================== */
493: /*@
494: MatSetDM - Sets the DMDA that is to be used by the HYPRE_StructMatrix PETSc matrix
496: Logically Collective on Mat
498: Input Parameters:
499: + mat - the matrix
500: - da - the da
502: Level: intermediate
504: @*/
505: PetscErrorCode MatSetDM(Mat mat,DM da)
506: {
512: PetscTryMethod(mat,"MatSetDM_C",(Mat,DM),(mat,da));
513: return(0);
514: }
519: PetscErrorCode MatView_MPI_DA(Mat A,PetscViewer viewer)
520: {
521: DM da;
523: const char *prefix;
524: Mat Anatural;
525: AO ao;
526: PetscInt rstart,rend,*petsc,i;
527: IS is;
528: MPI_Comm comm;
531: PetscObjectGetComm((PetscObject)A,&comm);
532: PetscObjectQuery((PetscObject)A,"DM",(PetscObject*)&da);
533: if (!da) SETERRQ(((PetscObject)A)->comm,PETSC_ERR_ARG_WRONG,"Matrix not generated from a DMDA");
535: DMDAGetAO(da,&ao);
536: MatGetOwnershipRange(A,&rstart,&rend);
537: PetscMalloc((rend-rstart)*sizeof(PetscInt),&petsc);
538: for (i=rstart; i<rend; i++) petsc[i-rstart] = i;
539: AOApplicationToPetsc(ao,rend-rstart,petsc);
540: ISCreateGeneral(comm,rend-rstart,petsc,PETSC_OWN_POINTER,&is);
542: /* call viewer on natural ordering */
543: MatGetSubMatrix(A,is,is,MAT_INITIAL_MATRIX,&Anatural);
544: ISDestroy(&is);
545: PetscObjectGetOptionsPrefix((PetscObject)A,&prefix);
546: PetscObjectSetOptionsPrefix((PetscObject)Anatural,prefix);
547: PetscObjectSetName((PetscObject)Anatural,((PetscObject)A)->name);
548: MatView(Anatural,viewer);
549: MatDestroy(&Anatural);
550: return(0);
551: }
557: PetscErrorCode MatLoad_MPI_DA(Mat A,PetscViewer viewer)
558: {
559: DM da;
561: Mat Anatural,Aapp;
562: AO ao;
563: PetscInt rstart,rend,*app,i;
564: IS is;
565: MPI_Comm comm;
568: PetscObjectGetComm((PetscObject)A,&comm);
569: PetscObjectQuery((PetscObject)A,"DM",(PetscObject*)&da);
570: if (!da) SETERRQ(((PetscObject)A)->comm,PETSC_ERR_ARG_WRONG,"Matrix not generated from a DMDA");
572: /* Load the matrix in natural ordering */
573: MatCreate(((PetscObject)A)->comm,&Anatural);
574: MatSetType(Anatural,((PetscObject)A)->type_name);
575: MatSetSizes(Anatural,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);
576: MatLoad(Anatural,viewer);
578: /* Map natural ordering to application ordering and create IS */
579: DMDAGetAO(da,&ao);
580: MatGetOwnershipRange(Anatural,&rstart,&rend);
581: PetscMalloc((rend-rstart)*sizeof(PetscInt),&app);
582: for (i=rstart; i<rend; i++) app[i-rstart] = i;
583: AOPetscToApplication(ao,rend-rstart,app);
584: ISCreateGeneral(comm,rend-rstart,app,PETSC_OWN_POINTER,&is);
586: /* Do permutation and replace header */
587: MatGetSubMatrix(Anatural,is,is,MAT_INITIAL_MATRIX,&Aapp);
588: MatHeaderReplace(A,Aapp);
589: ISDestroy(&is);
590: MatDestroy(&Anatural);
591: return(0);
592: }
597: PetscErrorCode DMGetMatrix_DA(DM da, const MatType mtype,Mat *J)
598: {
600: PetscInt dim,dof,nx,ny,nz,dims[3],starts[3],M,N,P;
601: Mat A;
602: MPI_Comm comm;
603: const MatType Atype;
604: void (*aij)(void)=PETSC_NULL,(*baij)(void)=PETSC_NULL,(*sbaij)(void)=PETSC_NULL;
605: MatType ttype[256];
606: PetscBool flg;
607: PetscMPIInt size;
608: DM_DA *dd = (DM_DA*)da->data;
611: #ifndef PETSC_USE_DYNAMIC_LIBRARIES
612: MatInitializePackage(PETSC_NULL);
613: #endif
614: if (!mtype) mtype = MATAIJ;
615: PetscStrcpy((char*)ttype,mtype);
616: PetscOptionsBegin(((PetscObject)da)->comm,((PetscObject)da)->prefix,"DMDA options","Mat");
617: PetscOptionsList("-da_mat_type","Matrix type","MatSetType",MatList,mtype,(char*)ttype,256,&flg);
618: PetscOptionsEnd();
620: /*
621: m
622: ------------------------------------------------------
623: | |
624: | |
625: | ---------------------- |
626: | | | |
627: n | ny | | |
628: | | | |
629: | .--------------------- |
630: | (xs,ys) nx |
631: | . |
632: | (gxs,gys) |
633: | |
634: -----------------------------------------------------
635: */
637: /*
638: nc - number of components per grid point
639: col - number of colors needed in one direction for single component problem
640:
641: */
642: DMDAGetInfo(da,&dim,&M,&N,&P,0,0,0,&dof,0,0,0,0,0);
643: DMDAGetCorners(da,0,0,0,&nx,&ny,&nz);
644: PetscObjectGetComm((PetscObject)da,&comm);
645: MatCreate(comm,&A);
646: MatSetSizes(A,dof*nx*ny*nz,dof*nx*ny*nz,dof*M*N*P,dof*M*N*P);
647: MatSetType(A,(const MatType)ttype);
648: MatSetDM(A,da);
649: MatSetFromOptions(A);
650: MatGetType(A,&Atype);
651: /*
652: We do not provide a getmatrix function in the DMDA operations because
653: the basic DMDA does not know about matrices. We think of DMDA as being more
654: more low-level than matrices. This is kind of cheating but, cause sometimes
655: we think of DMDA has higher level than matrices.
657: We could switch based on Atype (or mtype), but we do not since the
658: specialized setting routines depend only the particular preallocation
659: details of the matrix, not the type itself.
660: */
661: PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
662: if (!aij) {
663: PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
664: }
665: if (!aij) {
666: PetscObjectQueryFunction((PetscObject)A,"MatMPIBAIJSetPreallocation_C",&baij);
667: if (!baij) {
668: PetscObjectQueryFunction((PetscObject)A,"MatSeqBAIJSetPreallocation_C",&baij);
669: }
670: if (!baij){
671: PetscObjectQueryFunction((PetscObject)A,"MatMPISBAIJSetPreallocation_C",&sbaij);
672: if (!sbaij) {
673: PetscObjectQueryFunction((PetscObject)A,"MatSeqSBAIJSetPreallocation_C",&sbaij);
674: }
675: }
676: }
677: if (aij) {
678: if (dim == 1) {
679: DMGetMatrix_DA_1d_MPIAIJ(da,A);
680: } else if (dim == 2) {
681: if (dd->ofill) {
682: DMGetMatrix_DA_2d_MPIAIJ_Fill(da,A);
683: } else {
684: DMGetMatrix_DA_2d_MPIAIJ(da,A);
685: }
686: } else if (dim == 3) {
687: if (dd->ofill) {
688: DMGetMatrix_DA_3d_MPIAIJ_Fill(da,A);
689: } else {
690: DMGetMatrix_DA_3d_MPIAIJ(da,A);
691: }
692: }
693: } else if (baij) {
694: if (dim == 2) {
695: DMGetMatrix_DA_2d_MPIBAIJ(da,A);
696: } else if (dim == 3) {
697: DMGetMatrix_DA_3d_MPIBAIJ(da,A);
698: } else {
699: SETERRQ3(((PetscObject)da)->comm,PETSC_ERR_SUP,"Not implemented for %D dimension and Matrix Type: %s in %D dimension!\n" \
700: "Send mail to petsc-maint@mcs.anl.gov for code",dim,Atype,dim);
701: }
702: } else if (sbaij) {
703: if (dim == 2) {
704: DMGetMatrix_DA_2d_MPISBAIJ(da,A);
705: } else if (dim == 3) {
706: DMGetMatrix_DA_3d_MPISBAIJ(da,A);
707: } else {
708: SETERRQ3(((PetscObject)da)->comm,PETSC_ERR_SUP,"Not implemented for %D dimension and Matrix Type: %s in %D dimension!\n" \
709: "Send mail to petsc-maint@mcs.anl.gov for code",dim,Atype,dim);
710: }
711: } else {
712: ISLocalToGlobalMapping ltog,ltogb;
713: DMGetLocalToGlobalMapping(da,<og);
714: DMGetLocalToGlobalMappingBlock(da,<ogb);
715: MatSetLocalToGlobalMapping(A,ltog,ltog);
716: MatSetLocalToGlobalMappingBlock(A,ltogb,ltogb);
717: }
718: DMDAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
719: MatSetStencil(A,dim,dims,starts,dof);
720: PetscObjectCompose((PetscObject)A,"DM",(PetscObject)da);
721: MPI_Comm_size(comm,&size);
722: if (size > 1) {
723: /* change viewer to display matrix in natural ordering */
724: MatShellSetOperation(A, MATOP_VIEW, (void (*)(void)) MatView_MPI_DA);
725: MatShellSetOperation(A, MATOP_LOAD, (void (*)(void)) MatLoad_MPI_DA);
726: }
727: *J = A;
728: return(0);
729: }
731: /* ---------------------------------------------------------------------------------*/
734: PetscErrorCode DMGetMatrix_DA_2d_MPIAIJ(DM da,Mat J)
735: {
736: PetscErrorCode ierr;
737: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny,m,n,dim,s,*cols = PETSC_NULL,k,nc,*rows = PETSC_NULL,col,cnt,l,p;
738: PetscInt lstart,lend,pstart,pend,*dnz,*onz;
739: MPI_Comm comm;
740: PetscScalar *values;
741: DMDABoundaryType bx,by;
742: ISLocalToGlobalMapping ltog,ltogb;
743: DMDAStencilType st;
746: /*
747: nc - number of components per grid point
748: col - number of colors needed in one direction for single component problem
749:
750: */
751: DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
752: col = 2*s + 1;
753: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
754: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
755: PetscObjectGetComm((PetscObject)da,&comm);
757: PetscMalloc2(nc,PetscInt,&rows,col*col*nc*nc,PetscInt,&cols);
758: DMGetLocalToGlobalMapping(da,<og);
759: DMGetLocalToGlobalMappingBlock(da,<ogb);
760:
761: /* determine the matrix preallocation information */
762: MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
763: for (i=xs; i<xs+nx; i++) {
765: pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
766: pend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
768: for (j=ys; j<ys+ny; j++) {
769: slot = i - gxs + gnx*(j - gys);
771: lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
772: lend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
774: cnt = 0;
775: for (k=0; k<nc; k++) {
776: for (l=lstart; l<lend+1; l++) {
777: for (p=pstart; p<pend+1; p++) {
778: if ((st == DMDA_STENCIL_BOX) || (!l || !p)) { /* entries on star have either l = 0 or p = 0 */
779: cols[cnt++] = k + nc*(slot + gnx*l + p);
780: }
781: }
782: }
783: rows[k] = k + nc*(slot);
784: }
785: MatPreallocateSetLocal(ltog,nc,rows,ltog,cnt,cols,dnz,onz);
786: }
787: }
788: MatSeqAIJSetPreallocation(J,0,dnz);
789: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
790: MatSetBlockSize(J,nc);
791: MatPreallocateFinalize(dnz,onz);
793: MatSetLocalToGlobalMapping(J,ltog,ltog);
794: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
796: /*
797: For each node in the grid: we get the neighbors in the local (on processor ordering
798: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
799: PETSc ordering.
800: */
801: if (!da->prealloc_only) {
802: PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
803: PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
804: for (i=xs; i<xs+nx; i++) {
805:
806: pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
807: pend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
808:
809: for (j=ys; j<ys+ny; j++) {
810: slot = i - gxs + gnx*(j - gys);
811:
812: lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
813: lend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
815: cnt = 0;
816: for (k=0; k<nc; k++) {
817: for (l=lstart; l<lend+1; l++) {
818: for (p=pstart; p<pend+1; p++) {
819: if ((st == DMDA_STENCIL_BOX) || (!l || !p)) { /* entries on star have either l = 0 or p = 0 */
820: cols[cnt++] = k + nc*(slot + gnx*l + p);
821: }
822: }
823: }
824: rows[k] = k + nc*(slot);
825: }
826: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
827: }
828: }
829: PetscFree(values);
830: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
831: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
832: }
833: PetscFree2(rows,cols);
834: return(0);
835: }
839: PetscErrorCode DMGetMatrix_DA_2d_MPIAIJ_Fill(DM da,Mat J)
840: {
841: PetscErrorCode ierr;
842: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
843: PetscInt m,n,dim,s,*cols,k,nc,row,col,cnt,l,p;
844: PetscInt lstart,lend,pstart,pend,*dnz,*onz;
845: DM_DA *dd = (DM_DA*)da->data;
846: PetscInt ifill_col,*ofill = dd->ofill, *dfill = dd->dfill;
847: MPI_Comm comm;
848: PetscScalar *values;
849: DMDABoundaryType bx,by;
850: ISLocalToGlobalMapping ltog,ltogb;
851: DMDAStencilType st;
854: /*
855: nc - number of components per grid point
856: col - number of colors needed in one direction for single component problem
857:
858: */
859: DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
860: col = 2*s + 1;
861: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
862: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
863: PetscObjectGetComm((PetscObject)da,&comm);
865: PetscMalloc(col*col*nc*nc*sizeof(PetscInt),&cols);
866: DMGetLocalToGlobalMapping(da,<og);
867: DMGetLocalToGlobalMappingBlock(da,<ogb);
868:
869: /* determine the matrix preallocation information */
870: MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
871: for (i=xs; i<xs+nx; i++) {
873: pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
874: pend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
876: for (j=ys; j<ys+ny; j++) {
877: slot = i - gxs + gnx*(j - gys);
879: lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
880: lend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
882: for (k=0; k<nc; k++) {
883: cnt = 0;
884: for (l=lstart; l<lend+1; l++) {
885: for (p=pstart; p<pend+1; p++) {
886: if (l || p) {
887: if ((st == DMDA_STENCIL_BOX) || (!l || !p)) { /* entries on star */
888: for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
889: cols[cnt++] = ofill[ifill_col] + nc*(slot + gnx*l + p);
890: }
891: } else {
892: if (dfill) {
893: for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
894: cols[cnt++] = dfill[ifill_col] + nc*(slot + gnx*l + p);
895: } else {
896: for (ifill_col=0; ifill_col<nc; ifill_col++)
897: cols[cnt++] = ifill_col + nc*(slot + gnx*l + p);
898: }
899: }
900: }
901: }
902: row = k + nc*(slot);
903: MatPreallocateSetLocal(ltog,1,&row,ltog,cnt,cols,dnz,onz);
904: }
905: }
906: }
907: MatSeqAIJSetPreallocation(J,0,dnz);
908: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
909: MatPreallocateFinalize(dnz,onz);
910: MatSetLocalToGlobalMapping(J,ltog,ltog);
911: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
913: /*
914: For each node in the grid: we get the neighbors in the local (on processor ordering
915: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
916: PETSc ordering.
917: */
918: if (!da->prealloc_only) {
919: PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
920: PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
921: for (i=xs; i<xs+nx; i++) {
922:
923: pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
924: pend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
925:
926: for (j=ys; j<ys+ny; j++) {
927: slot = i - gxs + gnx*(j - gys);
928:
929: lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
930: lend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
932: for (k=0; k<nc; k++) {
933: cnt = 0;
934: for (l=lstart; l<lend+1; l++) {
935: for (p=pstart; p<pend+1; p++) {
936: if (l || p) {
937: if ((st == DMDA_STENCIL_BOX) || (!l || !p)) { /* entries on star */
938: for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
939: cols[cnt++] = ofill[ifill_col] + nc*(slot + gnx*l + p);
940: }
941: } else {
942: if (dfill) {
943: for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
944: cols[cnt++] = dfill[ifill_col] + nc*(slot + gnx*l + p);
945: } else {
946: for (ifill_col=0; ifill_col<nc; ifill_col++)
947: cols[cnt++] = ifill_col + nc*(slot + gnx*l + p);
948: }
949: }
950: }
951: }
952: row = k + nc*(slot);
953: MatSetValuesLocal(J,1,&row,cnt,cols,values,INSERT_VALUES);
954: }
955: }
956: }
957: PetscFree(values);
958: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
959: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
960: }
961: PetscFree(cols);
962: return(0);
963: }
965: /* ---------------------------------------------------------------------------------*/
969: PetscErrorCode DMGetMatrix_DA_3d_MPIAIJ(DM da,Mat J)
970: {
971: PetscErrorCode ierr;
972: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
973: PetscInt m,n,dim,s,*cols = PETSC_NULL,k,nc,*rows = PETSC_NULL,col,cnt,l,p,*dnz = PETSC_NULL,*onz = PETSC_NULL;
974: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
975: MPI_Comm comm;
976: PetscScalar *values;
977: DMDABoundaryType bx,by,bz;
978: ISLocalToGlobalMapping ltog,ltogb;
979: DMDAStencilType st;
982: /*
983: nc - number of components per grid point
984: col - number of colors needed in one direction for single component problem
985:
986: */
987: DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
988: col = 2*s + 1;
990: DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
991: DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
992: PetscObjectGetComm((PetscObject)da,&comm);
994: PetscMalloc2(nc,PetscInt,&rows,col*col*col*nc*nc,PetscInt,&cols);
995: DMGetLocalToGlobalMapping(da,<og);
996: DMGetLocalToGlobalMappingBlock(da,<ogb);
998: /* determine the matrix preallocation information */
999: MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
1000: for (i=xs; i<xs+nx; i++) {
1001: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1002: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1003: for (j=ys; j<ys+ny; j++) {
1004: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1005: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1006: for (k=zs; k<zs+nz; k++) {
1007: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1008: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1009:
1010: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1011:
1012: cnt = 0;
1013: for (l=0; l<nc; l++) {
1014: for (ii=istart; ii<iend+1; ii++) {
1015: for (jj=jstart; jj<jend+1; jj++) {
1016: for (kk=kstart; kk<kend+1; kk++) {
1017: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1018: cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1019: }
1020: }
1021: }
1022: }
1023: rows[l] = l + nc*(slot);
1024: }
1025: MatPreallocateSetLocal(ltog,nc,rows,ltog,cnt,cols,dnz,onz);
1026: }
1027: }
1028: }
1029: MatSeqAIJSetPreallocation(J,0,dnz);
1030: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
1031: MatPreallocateFinalize(dnz,onz);
1032: MatSetBlockSize(J,nc);
1033: MatSetLocalToGlobalMapping(J,ltog,ltog);
1034: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1036: /*
1037: For each node in the grid: we get the neighbors in the local (on processor ordering
1038: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1039: PETSc ordering.
1040: */
1041: if (!da->prealloc_only) {
1042: PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
1043: PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
1044: for (i=xs; i<xs+nx; i++) {
1045: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1046: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1047: for (j=ys; j<ys+ny; j++) {
1048: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1049: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1050: for (k=zs; k<zs+nz; k++) {
1051: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1052: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1053:
1054: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1055:
1056: cnt = 0;
1057: for (l=0; l<nc; l++) {
1058: for (ii=istart; ii<iend+1; ii++) {
1059: for (jj=jstart; jj<jend+1; jj++) {
1060: for (kk=kstart; kk<kend+1; kk++) {
1061: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1062: cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1063: }
1064: }
1065: }
1066: }
1067: rows[l] = l + nc*(slot);
1068: }
1069: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
1070: }
1071: }
1072: }
1073: PetscFree(values);
1074: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1075: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1076: }
1077: PetscFree2(rows,cols);
1078: return(0);
1079: }
1081: /* ---------------------------------------------------------------------------------*/
1085: PetscErrorCode DMGetMatrix_DA_1d_MPIAIJ(DM da,Mat J)
1086: {
1087: PetscErrorCode ierr;
1088: PetscInt xs,nx,i,i1,slot,gxs,gnx;
1089: PetscInt m,dim,s,*cols = PETSC_NULL,nc,*rows = PETSC_NULL,col,cnt,l;
1090: PetscInt istart,iend;
1091: PetscScalar *values;
1092: DMDABoundaryType bx;
1093: ISLocalToGlobalMapping ltog,ltogb;
1096: /*
1097: nc - number of components per grid point
1098: col - number of colors needed in one direction for single component problem
1099:
1100: */
1101: DMDAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&bx,0,0,0);
1102: col = 2*s + 1;
1104: DMDAGetCorners(da,&xs,0,0,&nx,0,0);
1105: DMDAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
1107: MatSeqAIJSetPreallocation(J,col*nc,0);
1108: MatMPIAIJSetPreallocation(J,col*nc,0,col*nc,0);
1109: MatSetBlockSize(J,nc);
1110: PetscMalloc2(nc,PetscInt,&rows,col*nc*nc,PetscInt,&cols);
1111:
1112: DMGetLocalToGlobalMapping(da,<og);
1113: DMGetLocalToGlobalMappingBlock(da,<ogb);
1114: MatSetLocalToGlobalMapping(J,ltog,ltog);
1115: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1116:
1117: /*
1118: For each node in the grid: we get the neighbors in the local (on processor ordering
1119: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1120: PETSc ordering.
1121: */
1122: if (!da->prealloc_only) {
1123: PetscMalloc(col*nc*nc*sizeof(PetscScalar),&values);
1124: PetscMemzero(values,col*nc*nc*sizeof(PetscScalar));
1125: for (i=xs; i<xs+nx; i++) {
1126: istart = PetscMax(-s,gxs - i);
1127: iend = PetscMin(s,gxs + gnx - i - 1);
1128: slot = i - gxs;
1129:
1130: cnt = 0;
1131: for (l=0; l<nc; l++) {
1132: for (i1=istart; i1<iend+1; i1++) {
1133: cols[cnt++] = l + nc*(slot + i1);
1134: }
1135: rows[l] = l + nc*(slot);
1136: }
1137: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
1138: }
1139: PetscFree(values);
1140: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1141: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1142: }
1143: PetscFree2(rows,cols);
1144: return(0);
1145: }
1149: PetscErrorCode DMGetMatrix_DA_2d_MPIBAIJ(DM da,Mat J)
1150: {
1151: PetscErrorCode ierr;
1152: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1153: PetscInt m,n,dim,s,*cols,nc,col,cnt,*dnz,*onz;
1154: PetscInt istart,iend,jstart,jend,ii,jj;
1155: MPI_Comm comm;
1156: PetscScalar *values;
1157: DMDABoundaryType bx,by;
1158: DMDAStencilType st;
1159: ISLocalToGlobalMapping ltog,ltogb;
1162: /*
1163: nc - number of components per grid point
1164: col - number of colors needed in one direction for single component problem
1165: */
1166: DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
1167: col = 2*s + 1;
1169: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
1170: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
1171: PetscObjectGetComm((PetscObject)da,&comm);
1173: PetscMalloc(col*col*nc*nc*sizeof(PetscInt),&cols);
1175: DMGetLocalToGlobalMapping(da,<og);
1176: DMGetLocalToGlobalMappingBlock(da,<ogb);
1178: /* determine the matrix preallocation information */
1179: MatPreallocateInitialize(comm,nx*ny,nx*ny,dnz,onz);
1180: for (i=xs; i<xs+nx; i++) {
1181: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1182: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1183: for (j=ys; j<ys+ny; j++) {
1184: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1185: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1186: slot = i - gxs + gnx*(j - gys);
1188: /* Find block columns in block row */
1189: cnt = 0;
1190: for (ii=istart; ii<iend+1; ii++) {
1191: for (jj=jstart; jj<jend+1; jj++) {
1192: if (st == DMDA_STENCIL_BOX || !ii || !jj) { /* BOX or on the STAR */
1193: cols[cnt++] = slot + ii + gnx*jj;
1194: }
1195: }
1196: }
1197: MatPreallocateSetLocal(ltogb,1,&slot,ltogb,cnt,cols,dnz,onz);
1198: }
1199: }
1200: MatSeqBAIJSetPreallocation(J,nc,0,dnz);
1201: MatMPIBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1202: MatPreallocateFinalize(dnz,onz);
1204: MatSetLocalToGlobalMapping(J,ltog,ltog);
1205: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1207: /*
1208: For each node in the grid: we get the neighbors in the local (on processor ordering
1209: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1210: PETSc ordering.
1211: */
1212: if (!da->prealloc_only) {
1213: PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
1214: PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
1215: for (i=xs; i<xs+nx; i++) {
1216: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1217: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1218: for (j=ys; j<ys+ny; j++) {
1219: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1220: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1221: slot = i - gxs + gnx*(j - gys);
1222: cnt = 0;
1223: for (ii=istart; ii<iend+1; ii++) {
1224: for (jj=jstart; jj<jend+1; jj++) {
1225: if (st == DMDA_STENCIL_BOX || !ii || !jj) { /* BOX or on the STAR */
1226: cols[cnt++] = slot + ii + gnx*jj;
1227: }
1228: }
1229: }
1230: MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1231: }
1232: }
1233: PetscFree(values);
1234: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1235: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1236: }
1237: PetscFree(cols);
1238: return(0);
1239: }
1243: PetscErrorCode DMGetMatrix_DA_3d_MPIBAIJ(DM da,Mat J)
1244: {
1245: PetscErrorCode ierr;
1246: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1247: PetscInt m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1248: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1249: MPI_Comm comm;
1250: PetscScalar *values;
1251: DMDABoundaryType bx,by,bz;
1252: DMDAStencilType st;
1253: ISLocalToGlobalMapping ltog,ltogb;
1256: /*
1257: nc - number of components per grid point
1258: col - number of colors needed in one direction for single component problem
1259:
1260: */
1261: DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
1262: col = 2*s + 1;
1264: DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1265: DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1266: PetscObjectGetComm((PetscObject)da,&comm);
1268: PetscMalloc(col*col*col*sizeof(PetscInt),&cols);
1270: DMGetLocalToGlobalMapping(da,<og);
1271: DMGetLocalToGlobalMappingBlock(da,<ogb);
1273: /* determine the matrix preallocation information */
1274: MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1275: for (i=xs; i<xs+nx; i++) {
1276: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1277: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1278: for (j=ys; j<ys+ny; j++) {
1279: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1280: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1281: for (k=zs; k<zs+nz; k++) {
1282: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1283: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1285: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1287: /* Find block columns in block row */
1288: cnt = 0;
1289: for (ii=istart; ii<iend+1; ii++) {
1290: for (jj=jstart; jj<jend+1; jj++) {
1291: for (kk=kstart; kk<kend+1; kk++) {
1292: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1293: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1294: }
1295: }
1296: }
1297: }
1298: MatPreallocateSetLocal(ltogb,1,&slot,ltogb,cnt,cols,dnz,onz);
1299: }
1300: }
1301: }
1302: MatSeqBAIJSetPreallocation(J,nc,0,dnz);
1303: MatMPIBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1304: MatPreallocateFinalize(dnz,onz);
1306: MatSetLocalToGlobalMapping(J,ltog,ltog);
1307: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1309: /*
1310: For each node in the grid: we get the neighbors in the local (on processor ordering
1311: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1312: PETSc ordering.
1313: */
1314: if (!da->prealloc_only) {
1315: PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1316: PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1317: for (i=xs; i<xs+nx; i++) {
1318: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1319: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1320: for (j=ys; j<ys+ny; j++) {
1321: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1322: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1323: for (k=zs; k<zs+nz; k++) {
1324: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1325: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1326:
1327: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1328:
1329: cnt = 0;
1330: for (ii=istart; ii<iend+1; ii++) {
1331: for (jj=jstart; jj<jend+1; jj++) {
1332: for (kk=kstart; kk<kend+1; kk++) {
1333: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1334: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1335: }
1336: }
1337: }
1338: }
1339: MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1340: }
1341: }
1342: }
1343: PetscFree(values);
1344: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1345: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1346: }
1347: PetscFree(cols);
1348: return(0);
1349: }
1353: /*
1354: This helper is for of SBAIJ preallocation, to discard the lower-triangular values which are difficult to
1355: identify in the local ordering with periodic domain.
1356: */
1357: static PetscErrorCode L2GFilterUpperTriangular(ISLocalToGlobalMapping ltog,PetscInt *row,PetscInt *cnt,PetscInt col[])
1358: {
1360: PetscInt i,n;
1363: ISLocalToGlobalMappingApply(ltog,1,row,row);
1364: ISLocalToGlobalMappingApply(ltog,*cnt,col,col);
1365: for (i=0,n=0; i<*cnt; i++) {
1366: if (col[i] >= *row) col[n++] = col[i];
1367: }
1368: *cnt = n;
1369: return(0);
1370: }
1374: PetscErrorCode DMGetMatrix_DA_2d_MPISBAIJ(DM da,Mat J)
1375: {
1376: PetscErrorCode ierr;
1377: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1378: PetscInt m,n,dim,s,*cols,nc,col,cnt,*dnz,*onz;
1379: PetscInt istart,iend,jstart,jend,ii,jj;
1380: MPI_Comm comm;
1381: PetscScalar *values;
1382: DMDABoundaryType bx,by;
1383: DMDAStencilType st;
1384: ISLocalToGlobalMapping ltog,ltogb;
1387: /*
1388: nc - number of components per grid point
1389: col - number of colors needed in one direction for single component problem
1390: */
1391: DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
1392: col = 2*s + 1;
1394: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
1395: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
1396: PetscObjectGetComm((PetscObject)da,&comm);
1398: PetscMalloc(col*col*nc*nc*sizeof(PetscInt),&cols);
1400: DMGetLocalToGlobalMapping(da,<og);
1401: DMGetLocalToGlobalMappingBlock(da,<ogb);
1403: /* determine the matrix preallocation information */
1404: MatPreallocateSymmetricInitialize(comm,nx*ny,nx*ny,dnz,onz);
1405: for (i=xs; i<xs+nx; i++) {
1406: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1407: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1408: for (j=ys; j<ys+ny; j++) {
1409: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1410: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1411: slot = i - gxs + gnx*(j - gys);
1413: /* Find block columns in block row */
1414: cnt = 0;
1415: for (ii=istart; ii<iend+1; ii++) {
1416: for (jj=jstart; jj<jend+1; jj++) {
1417: if (st == DMDA_STENCIL_BOX || !ii || !jj) {
1418: cols[cnt++] = slot + ii + gnx*jj;
1419: }
1420: }
1421: }
1422: L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1423: MatPreallocateSymmetricSet(slot,cnt,cols,dnz,onz);
1424: }
1425: }
1426: MatSeqSBAIJSetPreallocation(J,nc,0,dnz);
1427: MatMPISBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1428: MatPreallocateFinalize(dnz,onz);
1430: MatSetLocalToGlobalMapping(J,ltog,ltog);
1431: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1433: /*
1434: For each node in the grid: we get the neighbors in the local (on processor ordering
1435: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1436: PETSc ordering.
1437: */
1438: if (!da->prealloc_only) {
1439: PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
1440: PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
1441: for (i=xs; i<xs+nx; i++) {
1442: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1443: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1444: for (j=ys; j<ys+ny; j++) {
1445: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1446: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1447: slot = i - gxs + gnx*(j - gys);
1449: /* Find block columns in block row */
1450: cnt = 0;
1451: for (ii=istart; ii<iend+1; ii++) {
1452: for (jj=jstart; jj<jend+1; jj++) {
1453: if (st == DMDA_STENCIL_BOX || !ii || !jj) {
1454: cols[cnt++] = slot + ii + gnx*jj;
1455: }
1456: }
1457: }
1458: L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1459: MatSetValuesBlocked(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1460: }
1461: }
1462: PetscFree(values);
1463: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1464: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1465: }
1466: PetscFree(cols);
1467: return(0);
1468: }
1472: PetscErrorCode DMGetMatrix_DA_3d_MPISBAIJ(DM da,Mat J)
1473: {
1474: PetscErrorCode ierr;
1475: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1476: PetscInt m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1477: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1478: MPI_Comm comm;
1479: PetscScalar *values;
1480: DMDABoundaryType bx,by,bz;
1481: DMDAStencilType st;
1482: ISLocalToGlobalMapping ltog,ltogb;
1485: /*
1486: nc - number of components per grid point
1487: col - number of colors needed in one direction for single component problem
1488: */
1489: DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
1490: col = 2*s + 1;
1492: DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1493: DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1494: PetscObjectGetComm((PetscObject)da,&comm);
1496: /* create the matrix */
1497: PetscMalloc(col*col*col*sizeof(PetscInt),&cols);
1499: DMGetLocalToGlobalMapping(da,<og);
1500: DMGetLocalToGlobalMappingBlock(da,<ogb);
1502: /* determine the matrix preallocation information */
1503: MatPreallocateSymmetricInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1504: for (i=xs; i<xs+nx; i++) {
1505: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1506: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1507: for (j=ys; j<ys+ny; j++) {
1508: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1509: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1510: for (k=zs; k<zs+nz; k++) {
1511: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1512: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1514: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1516: /* Find block columns in block row */
1517: cnt = 0;
1518: for (ii=istart; ii<iend+1; ii++) {
1519: for (jj=jstart; jj<jend+1; jj++) {
1520: for (kk=kstart; kk<kend+1; kk++) {
1521: if ((st == DMDA_STENCIL_BOX) || (!ii && !jj) || (!jj && !kk) || (!ii && !kk)) {
1522: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1523: }
1524: }
1525: }
1526: }
1527: L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1528: MatPreallocateSymmetricSet(slot,cnt,cols,dnz,onz);
1529: }
1530: }
1531: }
1532: MatSeqSBAIJSetPreallocation(J,nc,0,dnz);
1533: MatMPISBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1534: MatPreallocateFinalize(dnz,onz);
1536: MatSetLocalToGlobalMapping(J,ltog,ltog);
1537: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1539: /*
1540: For each node in the grid: we get the neighbors in the local (on processor ordering
1541: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1542: PETSc ordering.
1543: */
1544: if (!da->prealloc_only) {
1545: PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1546: PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1547: for (i=xs; i<xs+nx; i++) {
1548: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1549: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1550: for (j=ys; j<ys+ny; j++) {
1551: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1552: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1553: for (k=zs; k<zs+nz; k++) {
1554: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1555: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1556:
1557: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1558:
1559: cnt = 0;
1560: for (ii=istart; ii<iend+1; ii++) {
1561: for (jj=jstart; jj<jend+1; jj++) {
1562: for (kk=kstart; kk<kend+1; kk++) {
1563: if ((st == DMDA_STENCIL_BOX) || (!ii && !jj) || (!jj && !kk) || (!ii && !kk)) {
1564: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1565: }
1566: }
1567: }
1568: }
1569: L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1570: MatSetValuesBlocked(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1571: }
1572: }
1573: }
1574: PetscFree(values);
1575: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1576: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1577: }
1578: PetscFree(cols);
1579: return(0);
1580: }
1582: /* ---------------------------------------------------------------------------------*/
1586: PetscErrorCode DMGetMatrix_DA_3d_MPIAIJ_Fill(DM da,Mat J)
1587: {
1588: PetscErrorCode ierr;
1589: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1590: PetscInt m,n,dim,s,*cols,k,nc,row,col,cnt,l,p,*dnz,*onz;
1591: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1592: DM_DA *dd = (DM_DA*)da->data;
1593: PetscInt ifill_col,*dfill = dd->dfill,*ofill = dd->ofill;
1594: MPI_Comm comm;
1595: PetscScalar *values;
1596: DMDABoundaryType bx,by,bz;
1597: ISLocalToGlobalMapping ltog,ltogb;
1598: DMDAStencilType st;
1601: /*
1602: nc - number of components per grid point
1603: col - number of colors needed in one direction for single component problem
1604:
1605: */
1606: DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
1607: col = 2*s + 1;
1608: if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)){
1609: SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
1610: by 2*stencil_width + 1\n");
1611: }
1612: if (by == DMDA_BOUNDARY_PERIODIC && (n % col)){
1613: SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
1614: by 2*stencil_width + 1\n");
1615: }
1616: if (bz == DMDA_BOUNDARY_PERIODIC && (p % col)){
1617: SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
1618: by 2*stencil_width + 1\n");
1619: }
1621: DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1622: DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1623: PetscObjectGetComm((PetscObject)da,&comm);
1625: PetscMalloc(col*col*col*nc*sizeof(PetscInt),&cols);
1626: DMGetLocalToGlobalMapping(da,<og);
1627: DMGetLocalToGlobalMappingBlock(da,<ogb);
1629: /* determine the matrix preallocation information */
1630: MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
1633: for (i=xs; i<xs+nx; i++) {
1634: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1635: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1636: for (j=ys; j<ys+ny; j++) {
1637: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1638: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1639: for (k=zs; k<zs+nz; k++) {
1640: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1641: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1642:
1643: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1644:
1645: for (l=0; l<nc; l++) {
1646: cnt = 0;
1647: for (ii=istart; ii<iend+1; ii++) {
1648: for (jj=jstart; jj<jend+1; jj++) {
1649: for (kk=kstart; kk<kend+1; kk++) {
1650: if (ii || jj || kk) {
1651: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1652: for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1653: cols[cnt++] = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1654: }
1655: } else {
1656: if (dfill) {
1657: for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1658: cols[cnt++] = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1659: } else {
1660: for (ifill_col=0; ifill_col<nc; ifill_col++)
1661: cols[cnt++] = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1662: }
1663: }
1664: }
1665: }
1666: }
1667: row = l + nc*(slot);
1668: MatPreallocateSetLocal(ltog,1,&row,ltog,cnt,cols,dnz,onz);
1669: }
1670: }
1671: }
1672: }
1673: MatSeqAIJSetPreallocation(J,0,dnz);
1674: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
1675: MatPreallocateFinalize(dnz,onz);
1676: MatSetLocalToGlobalMapping(J,ltog,ltog);
1677: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1679: /*
1680: For each node in the grid: we get the neighbors in the local (on processor ordering
1681: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1682: PETSc ordering.
1683: */
1684: if (!da->prealloc_only) {
1685: PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
1686: PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
1687: for (i=xs; i<xs+nx; i++) {
1688: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1689: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1690: for (j=ys; j<ys+ny; j++) {
1691: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1692: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1693: for (k=zs; k<zs+nz; k++) {
1694: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1695: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1696:
1697: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1698:
1699: for (l=0; l<nc; l++) {
1700: cnt = 0;
1701: for (ii=istart; ii<iend+1; ii++) {
1702: for (jj=jstart; jj<jend+1; jj++) {
1703: for (kk=kstart; kk<kend+1; kk++) {
1704: if (ii || jj || kk) {
1705: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1706: for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1707: cols[cnt++] = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1708: }
1709: } else {
1710: if (dfill) {
1711: for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1712: cols[cnt++] = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1713: } else {
1714: for (ifill_col=0; ifill_col<nc; ifill_col++)
1715: cols[cnt++] = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1716: }
1717: }
1718: }
1719: }
1720: }
1721: row = l + nc*(slot);
1722: MatSetValuesLocal(J,1,&row,cnt,cols,values,INSERT_VALUES);
1723: }
1724: }
1725: }
1726: }
1727: PetscFree(values);
1728: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1729: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1730: }
1731: PetscFree(cols);
1732: return(0);
1733: }