Actual source code: asm.c

  2: /*
  3:   This file defines an additive Schwarz preconditioner for any Mat implementation.

  5:   Note that each processor may have any number of subdomains. But in order to 
  6:   deal easily with the VecScatter(), we treat each processor as if it has the
  7:   same number of subdomains.

  9:        n - total number of true subdomains on all processors
 10:        n_local_true - actual number of subdomains on this processor
 11:        n_local = maximum over all processors of n_local_true
 12: */
 13: #include <private/pcimpl.h>     /*I "petscpc.h" I*/

 15: typedef struct {
 16:   PetscInt   n, n_local, n_local_true;
 17:   PetscInt   overlap;             /* overlap requested by user */
 18:   KSP        *ksp;                /* linear solvers for each block */
 19:   VecScatter *restriction;        /* mapping from global to subregion */
 20:   VecScatter *localization;       /* mapping from overlapping to non-overlapping subregion */
 21:   VecScatter *prolongation;       /* mapping from subregion to global */
 22:   Vec        *x,*y,*y_local;      /* work vectors */
 23:   IS         *is;                 /* index set that defines each overlapping subdomain */
 24:   IS         *is_local;           /* index set that defines each non-overlapping subdomain, may be NULL */
 25:   Mat        *mat,*pmat;          /* mat is not currently used */
 26:   PCASMType  type;                /* use reduced interpolation, restriction or both */
 27:   PetscBool  type_set;            /* if user set this value (so won't change it for symmetric problems) */
 28:   PetscBool  same_local_solves;   /* flag indicating whether all local solvers are same */
 29:   PetscBool  sort_indices;        /* flag to sort subdomain indices */
 30: } PC_ASM;

 34: static PetscErrorCode PCView_ASM(PC pc,PetscViewer viewer)
 35: {
 36:   PC_ASM         *osm = (PC_ASM*)pc->data;
 38:   PetscMPIInt    rank;
 39:   PetscInt       i,bsz;
 40:   PetscBool      iascii,isstring;
 41:   PetscViewer    sviewer;


 45:   PetscTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
 46:   PetscTypeCompare((PetscObject)viewer,PETSCVIEWERSTRING,&isstring);
 47:   if (iascii) {
 48:     char overlaps[256] = "user-defined overlap",blocks[256] = "total subdomain blocks not yet set";
 49:     if (osm->overlap >= 0) {PetscSNPrintf(overlaps,sizeof overlaps,"amount of overlap = %D",osm->overlap);}
 50:     if (osm->n > 0) {PetscSNPrintf(blocks,sizeof blocks,"total subdomain blocks = %D",osm->n);}
 51:     PetscViewerASCIIPrintf(viewer,"  Additive Schwarz: %s, %s\n",blocks,overlaps);
 52:     PetscViewerASCIIPrintf(viewer,"  Additive Schwarz: restriction/interpolation type - %s\n",PCASMTypes[osm->type]);
 53:     MPI_Comm_rank(((PetscObject)pc)->comm,&rank);
 54:     if (osm->same_local_solves) {
 55:       if (osm->ksp) {
 56:         PetscViewerASCIIPrintf(viewer,"  Local solve is same for all blocks, in the following KSP and PC objects:\n");
 57:         PetscViewerGetSingleton(viewer,&sviewer);
 58:         if (!rank) {
 59:           PetscViewerASCIIPushTab(viewer);
 60:           KSPView(osm->ksp[0],sviewer);
 61:           PetscViewerASCIIPopTab(viewer);
 62:         }
 63:         PetscViewerRestoreSingleton(viewer,&sviewer);
 64:       }
 65:     } else {
 66:       PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);
 67:       PetscViewerASCIISynchronizedPrintf(viewer,"  [%d] number of local blocks = %D\n",(int)rank,osm->n_local_true);
 68:       PetscViewerFlush(viewer);
 69:       PetscViewerASCIIPrintf(viewer,"  Local solve info for each block is in the following KSP and PC objects:\n");
 70:       PetscViewerASCIIPushTab(viewer);
 71:       PetscViewerASCIIPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
 72:       for (i=0; i<osm->n_local; i++) {
 73:         PetscViewerGetSingleton(viewer,&sviewer);
 74:         if (i < osm->n_local_true) {
 75:           ISGetLocalSize(osm->is[i],&bsz);
 76:           PetscViewerASCIISynchronizedPrintf(sviewer,"[%d] local block number %D, size = %D\n",(int)rank,i,bsz);
 77:           KSPView(osm->ksp[i],sviewer);
 78:           PetscViewerASCIISynchronizedPrintf(sviewer,"- - - - - - - - - - - - - - - - - -\n");
 79:         }
 80:         PetscViewerRestoreSingleton(viewer,&sviewer);
 81:       }
 82:       PetscViewerASCIIPopTab(viewer);
 83:       PetscViewerFlush(viewer);
 84:       PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);
 85:     }
 86:   } else if (isstring) {
 87:     PetscViewerStringSPrintf(viewer," blocks=%D, overlap=%D, type=%s",osm->n,osm->overlap,PCASMTypes[osm->type]);
 88:     PetscViewerGetSingleton(viewer,&sviewer);
 89:     if (osm->ksp) {KSPView(osm->ksp[0],sviewer);}
 90:     PetscViewerRestoreSingleton(viewer,&sviewer);
 91:   } else {
 92:     SETERRQ1(((PetscObject)pc)->comm,PETSC_ERR_SUP,"Viewer type %s not supported for PCASM",((PetscObject)viewer)->type_name);
 93:   }
 94:   return(0);
 95: }

 99: static PetscErrorCode PCASMPrintSubdomains(PC pc)
100: {
101:   PC_ASM         *osm  = (PC_ASM*)pc->data;
102:   const char     *prefix;
103:   char           fname[PETSC_MAX_PATH_LEN+1];
104:   PetscViewer    viewer;
105:   PetscInt       i,j,nidx;
106:   const PetscInt *idx;

110:   PCGetOptionsPrefix(pc,&prefix);
111:   PetscOptionsGetString(prefix,"-pc_asm_print_subdomains",fname,PETSC_MAX_PATH_LEN,PETSC_NULL);
112:   if (fname[0] == 0) { PetscStrcpy(fname,"stdout"); };
113:   PetscViewerASCIIOpen(((PetscObject)pc)->comm,fname,&viewer);
114:   PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);
115:   for (i=0;i<osm->n_local_true;i++) {
116:     ISGetLocalSize(osm->is[i],&nidx);
117:     ISGetIndices(osm->is[i],&idx);
118:     for (j=0; j<nidx; j++) {
119:       PetscViewerASCIISynchronizedPrintf(viewer,"%D ",idx[j]);
120:     }
121:     ISRestoreIndices(osm->is[i],&idx);
122:     PetscViewerASCIISynchronizedPrintf(viewer,"\n");
123:     if (osm->is_local) {
124:       ISGetLocalSize(osm->is_local[i],&nidx);
125:       ISGetIndices(osm->is_local[i],&idx);
126:       for (j=0; j<nidx; j++) {
127:         PetscViewerASCIISynchronizedPrintf(viewer,"%D ",idx[j]);
128:       }
129:       ISRestoreIndices(osm->is_local[i],&idx);
130:       PetscViewerASCIISynchronizedPrintf(viewer,"\n");
131:     }
132:   }
133:   PetscViewerFlush(viewer);
134:   PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);
135:   PetscViewerDestroy(&viewer);
136:   return(0);
137: }

141: static PetscErrorCode PCSetUp_ASM(PC pc)
142: {
143:   PC_ASM         *osm  = (PC_ASM*)pc->data;
145:   PetscBool      symset,flg;
146:   PetscInt       i,m,m_local,firstRow,lastRow;
147:   PetscMPIInt    size;
148:   MatReuse       scall = MAT_REUSE_MATRIX;
149:   IS             isl;
150:   KSP            ksp;
151:   PC             subpc;
152:   const char     *prefix,*pprefix;
153:   Vec            vec;

156:   if (!pc->setupcalled) {

158:     if (!osm->type_set) {
159:       MatIsSymmetricKnown(pc->pmat,&symset,&flg);
160:       if (symset && flg) { osm->type = PC_ASM_BASIC; }
161:     }

163:     if (osm->n == PETSC_DECIDE && osm->n_local_true < 1) {
164:       /* no subdomains given, use one per processor */
165:       osm->n_local = osm->n_local_true = 1;
166:       MPI_Comm_size(((PetscObject)pc)->comm,&size);
167:       osm->n = size;
168:     } else if (osm->n == PETSC_DECIDE) {
169:       /* determine global number of subdomains */
170:       PetscInt inwork[2],outwork[2];
171:       inwork[0] = inwork[1] = osm->n_local_true;
172:       MPI_Allreduce(inwork,outwork,1,MPIU_2INT,PetscMaxSum_Op,((PetscObject)pc)->comm);
173:       osm->n_local = outwork[0];
174:       osm->n       = outwork[1];
175:     }

177:     if (!osm->is){ /* create the index sets */
178:       PCASMCreateSubdomains(pc->pmat,osm->n_local_true,&osm->is);
179:     }
180:     if (osm->n_local_true > 1 && !osm->is_local) {
181:       PetscMalloc(osm->n_local_true*sizeof(IS),&osm->is_local);
182:       for (i=0; i<osm->n_local_true; i++) {
183:         if (osm->overlap > 0) { /* With positive overlap, osm->is[i] will be modified */
184:           ISDuplicate(osm->is[i],&osm->is_local[i]);
185:           ISCopy(osm->is[i],osm->is_local[i]);
186:         } else {
187:           PetscObjectReference((PetscObject)osm->is[i]);
188:           osm->is_local[i] = osm->is[i];
189:         }
190:       }
191:     }
192:     PCGetOptionsPrefix(pc,&prefix);
193:     flg  = PETSC_FALSE;
194:     PetscOptionsGetBool(prefix,"-pc_asm_print_subdomains",&flg,PETSC_NULL);
195:     if (flg) { PCASMPrintSubdomains(pc); }

197:     if (osm->overlap > 0) {
198:       /* Extend the "overlapping" regions by a number of steps */
199:       MatIncreaseOverlap(pc->pmat,osm->n_local_true,osm->is,osm->overlap);
200:     }
201:     if (osm->sort_indices) {
202:       for (i=0; i<osm->n_local_true; i++) {
203:         ISSort(osm->is[i]);
204:         if (osm->is_local) {
205:           ISSort(osm->is_local[i]);
206:         }
207:       }
208:     }

210:     /* Create the local work vectors and scatter contexts */
211:     MatGetVecs(pc->pmat,&vec,0);
212:     PetscMalloc(osm->n_local*sizeof(VecScatter),&osm->restriction);
213:     if (osm->is_local) {PetscMalloc(osm->n_local*sizeof(VecScatter),&osm->localization);}
214:     PetscMalloc(osm->n_local*sizeof(VecScatter),&osm->prolongation);
215:     PetscMalloc(osm->n_local*sizeof(Vec),&osm->x);
216:     PetscMalloc(osm->n_local*sizeof(Vec),&osm->y);
217:     PetscMalloc(osm->n_local*sizeof(Vec),&osm->y_local);
218:     VecGetOwnershipRange(vec, &firstRow, &lastRow);
219:     for (i=0; i<osm->n_local_true; ++i, firstRow += m_local) {
220:       ISGetLocalSize(osm->is[i],&m);
221:       VecCreateSeq(PETSC_COMM_SELF,m,&osm->x[i]);
222:       ISCreateStride(PETSC_COMM_SELF,m,0,1,&isl);
223:       VecScatterCreate(vec,osm->is[i],osm->x[i],isl,&osm->restriction[i]);
224:       ISDestroy(&isl);
225:       VecDuplicate(osm->x[i],&osm->y[i]);
226:       if (osm->is_local) {
227:         ISLocalToGlobalMapping ltog;
228:         IS                     isll;
229:         const PetscInt         *idx_local;
230:         PetscInt               *idx,nout;

232:         ISLocalToGlobalMappingCreateIS(osm->is[i],&ltog);
233:         ISGetLocalSize(osm->is_local[i],&m_local);
234:         ISGetIndices(osm->is_local[i], &idx_local);
235:         PetscMalloc(m_local*sizeof(PetscInt),&idx);
236:         ISGlobalToLocalMappingApply(ltog,IS_GTOLM_DROP,m_local,idx_local,&nout,idx);
237:         ISLocalToGlobalMappingDestroy(&ltog);
238:         if (nout != m_local) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"is_local not a subset of is");
239:         ISRestoreIndices(osm->is_local[i], &idx_local);
240:         ISCreateGeneral(PETSC_COMM_SELF,m_local,idx,PETSC_OWN_POINTER,&isll);
241:         ISCreateStride(PETSC_COMM_SELF,m_local,0,1,&isl);
242:         VecCreateSeq(PETSC_COMM_SELF,m_local,&osm->y_local[i]);
243:         VecScatterCreate(osm->y[i],isll,osm->y_local[i],isl,&osm->localization[i]);
244:         ISDestroy(&isll);

246:         VecScatterCreate(vec,osm->is_local[i],osm->y_local[i],isl,&osm->prolongation[i]);
247:         ISDestroy(&isl);
248:       } else {
249:         VecGetLocalSize(vec,&m_local);
250:         osm->y_local[i] = osm->y[i];
251:         PetscObjectReference((PetscObject) osm->y[i]);
252:         osm->prolongation[i] = osm->restriction[i];
253:         PetscObjectReference((PetscObject) osm->restriction[i]);
254:       }
255:     }
256:     if (firstRow != lastRow) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB, "Specified ASM subdomain sizes were invalid: %d != %d", firstRow, lastRow);
257:     for (i=osm->n_local_true; i<osm->n_local; i++) {
258:       VecCreateSeq(PETSC_COMM_SELF,0,&osm->x[i]);
259:       VecDuplicate(osm->x[i],&osm->y[i]);
260:       VecDuplicate(osm->x[i],&osm->y_local[i]);
261:       ISCreateStride(PETSC_COMM_SELF,0,0,1,&isl);
262:       VecScatterCreate(vec,isl,osm->x[i],isl,&osm->restriction[i]);
263:       if (osm->is_local) {
264:         VecScatterCreate(osm->y[i],isl,osm->y_local[i],isl,&osm->localization[i]);
265:         VecScatterCreate(vec,isl,osm->x[i],isl,&osm->prolongation[i]);
266:       } else {
267:         osm->prolongation[i] = osm->restriction[i];
268:         PetscObjectReference((PetscObject) osm->restriction[i]);
269:       }
270:       ISDestroy(&isl);
271:     }
272:     VecDestroy(&vec);

274:     if (!osm->ksp) {
275:       /* Create the local solvers */
276:       PetscMalloc(osm->n_local_true*sizeof(KSP *),&osm->ksp);
277:       for (i=0; i<osm->n_local_true; i++) {
278:         KSPCreate(PETSC_COMM_SELF,&ksp);
279:         PetscLogObjectParent(pc,ksp);
280:         PetscObjectIncrementTabLevel((PetscObject)ksp,(PetscObject)pc,1);
281:         KSPSetType(ksp,KSPPREONLY);
282:         KSPGetPC(ksp,&subpc);
283:         PCGetOptionsPrefix(pc,&prefix);
284:         KSPSetOptionsPrefix(ksp,prefix);
285:         KSPAppendOptionsPrefix(ksp,"sub_");
286:         osm->ksp[i] = ksp;
287:       }
288:     }
289:     scall = MAT_INITIAL_MATRIX;
290:   } else {
291:     /* 
292:        Destroy the blocks from the previous iteration
293:     */
294:     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
295:       MatDestroyMatrices(osm->n_local_true,&osm->pmat);
296:       scall = MAT_INITIAL_MATRIX;
297:     }
298:   }

300:   /* 
301:      Extract out the submatrices
302:   */
303:   MatGetSubMatrices(pc->pmat,osm->n_local_true,osm->is,osm->is,scall,&osm->pmat);
304:   if (scall == MAT_INITIAL_MATRIX) {
305:     PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
306:     for (i=0; i<osm->n_local_true; i++) {
307:       PetscLogObjectParent(pc,osm->pmat[i]);
308:       PetscObjectSetOptionsPrefix((PetscObject)osm->pmat[i],pprefix);
309:     }
310:   }

312:   /* Return control to the user so that the submatrices can be modified (e.g., to apply
313:      different boundary conditions for the submatrices than for the global problem) */
314:   PCModifySubMatrices(pc,osm->n_local_true,osm->is,osm->is,osm->pmat,pc->modifysubmatricesP);

316:   /* 
317:      Loop over subdomains putting them into local ksp
318:   */
319:   for (i=0; i<osm->n_local_true; i++) {
320:     KSPSetOperators(osm->ksp[i],osm->pmat[i],osm->pmat[i],pc->flag);
321:     if (!pc->setupcalled) {
322:       KSPSetFromOptions(osm->ksp[i]);
323:     }
324:   }

326:   return(0);
327: }

331: static PetscErrorCode PCSetUpOnBlocks_ASM(PC pc)
332: {
333:   PC_ASM         *osm = (PC_ASM*)pc->data;
335:   PetscInt       i;

338:   for (i=0; i<osm->n_local_true; i++) {
339:     KSPSetUp(osm->ksp[i]);
340:   }
341:   return(0);
342: }

346: static PetscErrorCode PCApply_ASM(PC pc,Vec x,Vec y)
347: {
348:   PC_ASM         *osm = (PC_ASM*)pc->data;
350:   PetscInt       i,n_local = osm->n_local,n_local_true = osm->n_local_true;
351:   ScatterMode    forward = SCATTER_FORWARD,reverse = SCATTER_REVERSE;

354:   /*
355:      Support for limiting the restriction or interpolation to only local 
356:      subdomain values (leaving the other values 0). 
357:   */
358:   if (!(osm->type & PC_ASM_RESTRICT)) {
359:     forward = SCATTER_FORWARD_LOCAL;
360:     /* have to zero the work RHS since scatter may leave some slots empty */
361:     for (i=0; i<n_local_true; i++) {
362:       VecZeroEntries(osm->x[i]);
363:     }
364:   }
365:   if (!(osm->type & PC_ASM_INTERPOLATE)) {
366:     reverse = SCATTER_REVERSE_LOCAL;
367:   }

369:   for (i=0; i<n_local; i++) {
370:     VecScatterBegin(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
371:   }
372:   VecZeroEntries(y);
373:   /* do the local solves */
374:   for (i=0; i<n_local_true; i++) {
375:     VecScatterEnd(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
376:     KSPSolve(osm->ksp[i],osm->x[i],osm->y[i]);
377:     if (osm->localization) {
378:       VecScatterBegin(osm->localization[i],osm->y[i],osm->y_local[i],INSERT_VALUES,forward);
379:       VecScatterEnd(osm->localization[i],osm->y[i],osm->y_local[i],INSERT_VALUES,forward);
380:     }
381:     VecScatterBegin(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
382:   }
383:   /* handle the rest of the scatters that do not have local solves */
384:   for (i=n_local_true; i<n_local; i++) {
385:     VecScatterEnd(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
386:     VecScatterBegin(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
387:   }
388:   for (i=0; i<n_local; i++) {
389:     VecScatterEnd(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
390:   }
391:   return(0);
392: }

396: static PetscErrorCode PCApplyTranspose_ASM(PC pc,Vec x,Vec y)
397: {
398:   PC_ASM         *osm = (PC_ASM*)pc->data;
400:   PetscInt       i,n_local = osm->n_local,n_local_true = osm->n_local_true;
401:   ScatterMode    forward = SCATTER_FORWARD,reverse = SCATTER_REVERSE;

404:   /*
405:      Support for limiting the restriction or interpolation to only local 
406:      subdomain values (leaving the other values 0).

408:      Note: these are reversed from the PCApply_ASM() because we are applying the 
409:      transpose of the three terms 
410:   */
411:   if (!(osm->type & PC_ASM_INTERPOLATE)) {
412:     forward = SCATTER_FORWARD_LOCAL;
413:     /* have to zero the work RHS since scatter may leave some slots empty */
414:     for (i=0; i<n_local_true; i++) {
415:       VecZeroEntries(osm->x[i]);
416:     }
417:   }
418:   if (!(osm->type & PC_ASM_RESTRICT)) {
419:     reverse = SCATTER_REVERSE_LOCAL;
420:   }

422:   for (i=0; i<n_local; i++) {
423:     VecScatterBegin(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
424:   }
425:   VecZeroEntries(y);
426:   /* do the local solves */
427:   for (i=0; i<n_local_true; i++) {
428:     VecScatterEnd(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
429:     KSPSolveTranspose(osm->ksp[i],osm->x[i],osm->y[i]);
430:     if (osm->localization) {
431:       VecScatterBegin(osm->localization[i],osm->y[i],osm->y_local[i],INSERT_VALUES,forward);
432:       VecScatterEnd(osm->localization[i],osm->y[i],osm->y_local[i],INSERT_VALUES,forward);
433:     }
434:     VecScatterBegin(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
435:   }
436:   /* handle the rest of the scatters that do not have local solves */
437:   for (i=n_local_true; i<n_local; i++) {
438:     VecScatterEnd(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
439:     VecScatterBegin(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
440:   }
441:   for (i=0; i<n_local; i++) {
442:     VecScatterEnd(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
443:   }
444:   return(0);
445: }

449: static PetscErrorCode PCReset_ASM(PC pc)
450: {
451:   PC_ASM         *osm = (PC_ASM*)pc->data;
453:   PetscInt       i;

456:   if (osm->ksp) {
457:     for (i=0; i<osm->n_local_true; i++) {
458:       KSPReset(osm->ksp[i]);
459:     }
460:   }
461:   if (osm->pmat) {
462:     if (osm->n_local_true > 0) {
463:       MatDestroyMatrices(osm->n_local_true,&osm->pmat);
464:     }
465:   }
466:   if (osm->restriction) {
467:     for (i=0; i<osm->n_local; i++) {
468:       VecScatterDestroy(&osm->restriction[i]);
469:       if (osm->localization) {VecScatterDestroy(&osm->localization[i]);}
470:       VecScatterDestroy(&osm->prolongation[i]);
471:       VecDestroy(&osm->x[i]);
472:       VecDestroy(&osm->y[i]);
473:       VecDestroy(&osm->y_local[i]);
474:     }
475:     PetscFree(osm->restriction);
476:     if (osm->localization) {PetscFree(osm->localization);}
477:     PetscFree(osm->prolongation);
478:     PetscFree(osm->x);
479:     PetscFree(osm->y);
480:     PetscFree(osm->y_local);
481:   }
482:   if (osm->is) {
483:     PCASMDestroySubdomains(osm->n_local_true,osm->is,osm->is_local);
484:     osm->is       = 0;
485:     osm->is_local = 0;
486:   }
487:   return(0);
488: }

492: static PetscErrorCode PCDestroy_ASM(PC pc)
493: {
494:   PC_ASM         *osm = (PC_ASM*)pc->data;
496:   PetscInt       i;

499:   PCReset_ASM(pc);
500:   if (osm->ksp) {
501:     for (i=0; i<osm->n_local_true; i++) {
502:       KSPDestroy(&osm->ksp[i]);
503:     }
504:     PetscFree(osm->ksp);
505:   }
506:   PetscFree(pc->data);
507:   return(0);
508: }

512: static PetscErrorCode PCSetFromOptions_ASM(PC pc)
513: {
514:   PC_ASM         *osm = (PC_ASM*)pc->data;
516:   PetscInt       blocks,ovl;
517:   PetscBool      symset,flg;
518:   PCASMType      asmtype;

521:   /* set the type to symmetric if matrix is symmetric */
522:   if (!osm->type_set && pc->pmat) {
523:     MatIsSymmetricKnown(pc->pmat,&symset,&flg);
524:     if (symset && flg) { osm->type = PC_ASM_BASIC; }
525:   }
526:   PetscOptionsHead("Additive Schwarz options");
527:     PetscOptionsInt("-pc_asm_blocks","Number of subdomains","PCASMSetTotalSubdomains",osm->n,&blocks,&flg);
528:     if (flg) {PCASMSetTotalSubdomains(pc,blocks,PETSC_NULL,PETSC_NULL); }
529:     PetscOptionsInt("-pc_asm_overlap","Number of grid points overlap","PCASMSetOverlap",osm->overlap,&ovl,&flg);
530:     if (flg) {PCASMSetOverlap(pc,ovl); }
531:     flg  = PETSC_FALSE;
532:     PetscOptionsEnum("-pc_asm_type","Type of restriction/extension","PCASMSetType",PCASMTypes,(PetscEnum)osm->type,(PetscEnum*)&asmtype,&flg);
533:     if (flg) {PCASMSetType(pc,asmtype); }
534:   PetscOptionsTail();
535:   return(0);
536: }

538: /*------------------------------------------------------------------------------------*/

543: PetscErrorCode  PCASMSetLocalSubdomains_ASM(PC pc,PetscInt n,IS is[],IS is_local[])
544: {
545:   PC_ASM         *osm = (PC_ASM*)pc->data;
547:   PetscInt       i;

550:   if (n < 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Each process must have 1 or more blocks, n = %D",n);
551:   if (pc->setupcalled && (n != osm->n_local_true || is)) SETERRQ(((PetscObject)pc)->comm,PETSC_ERR_ARG_WRONGSTATE,"PCASMSetLocalSubdomains() should be called before calling PCSetUp().");

553:   if (!pc->setupcalled) {
554:     if (is) {
555:       for (i=0; i<n; i++) {PetscObjectReference((PetscObject)is[i]);}
556:     }
557:     if (is_local) {
558:       for (i=0; i<n; i++) {PetscObjectReference((PetscObject)is_local[i]);}
559:     }
560:     if (osm->is) {
561:       PCASMDestroySubdomains(osm->n_local_true,osm->is,osm->is_local);
562:     }
563:     osm->n_local_true = n;
564:     osm->is           = 0;
565:     osm->is_local     = 0;
566:     if (is) {
567:       PetscMalloc(n*sizeof(IS),&osm->is);
568:       for (i=0; i<n; i++) { osm->is[i] = is[i]; }
569:       /* Flag indicating that the user has set overlapping subdomains so PCASM should not increase their size. */
570:       osm->overlap = -1;
571:     }
572:     if (is_local) {
573:       PetscMalloc(n*sizeof(IS),&osm->is_local);
574:       for (i=0; i<n; i++) { osm->is_local[i] = is_local[i]; }
575:     }
576:   }
577:   return(0);
578: }

584: PetscErrorCode  PCASMSetTotalSubdomains_ASM(PC pc,PetscInt N,IS *is,IS *is_local)
585: {
586:   PC_ASM         *osm = (PC_ASM*)pc->data;
588:   PetscMPIInt    rank,size;
589:   PetscInt       n;

592:   if (N < 1) SETERRQ1(((PetscObject)pc)->comm,PETSC_ERR_ARG_OUTOFRANGE,"Number of total blocks must be > 0, N = %D",N);
593:   if (is || is_local) SETERRQ(((PetscObject)pc)->comm,PETSC_ERR_SUP,"Use PCASMSetLocalSubdomains() to set specific index sets\n\they cannot be set globally yet.");

595:   /*
596:      Split the subdomains equally among all processors
597:   */
598:   MPI_Comm_rank(((PetscObject)pc)->comm,&rank);
599:   MPI_Comm_size(((PetscObject)pc)->comm,&size);
600:   n = N/size + ((N % size) > rank);
601:   if (!n) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Process %d must have at least one block: total processors %d total blocks %D",(int)rank,(int)size,N);
602:   if (pc->setupcalled && n != osm->n_local_true) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"PCASMSetTotalSubdomains() should be called before PCSetUp().");
603:   if (!pc->setupcalled) {
604:     if (osm->is) {
605:       PCASMDestroySubdomains(osm->n_local_true,osm->is,osm->is_local);
606:     }
607:     osm->n_local_true = n;
608:     osm->is           = 0;
609:     osm->is_local     = 0;
610:   }
611:   return(0);
612: }

618: PetscErrorCode  PCASMSetOverlap_ASM(PC pc,PetscInt ovl)
619: {
620:   PC_ASM *osm = (PC_ASM*)pc->data;

623:   if (ovl < 0) SETERRQ(((PetscObject)pc)->comm,PETSC_ERR_ARG_OUTOFRANGE,"Negative overlap value requested");
624:   if (pc->setupcalled && ovl != osm->overlap) SETERRQ(((PetscObject)pc)->comm,PETSC_ERR_ARG_WRONGSTATE,"PCASMSetOverlap() should be called before PCSetUp().");
625:   if (!pc->setupcalled) {
626:     osm->overlap = ovl;
627:   }
628:   return(0);
629: }

635: PetscErrorCode  PCASMSetType_ASM(PC pc,PCASMType type)
636: {
637:   PC_ASM *osm = (PC_ASM*)pc->data;

640:   osm->type     = type;
641:   osm->type_set = PETSC_TRUE;
642:   return(0);
643: }

649: PetscErrorCode  PCASMSetSortIndices_ASM(PC pc,PetscBool  doSort)
650: {
651:   PC_ASM *osm = (PC_ASM*)pc->data;

654:   osm->sort_indices = doSort;
655:   return(0);
656: }

662: PetscErrorCode  PCASMGetSubKSP_ASM(PC pc,PetscInt *n_local,PetscInt *first_local,KSP **ksp)
663: {
664:   PC_ASM         *osm = (PC_ASM*)pc->data;

668:   if (osm->n_local_true < 1) SETERRQ(((PetscObject)pc)->comm,PETSC_ERR_ORDER,"Need to call PCSetUP() on PC (or KSPSetUp() on the outer KSP object) before calling here");

670:   if (n_local) {
671:     *n_local = osm->n_local_true;
672:   }
673:   if (first_local) {
674:     MPI_Scan(&osm->n_local_true,first_local,1,MPIU_INT,MPI_SUM,((PetscObject)pc)->comm);
675:     *first_local -= osm->n_local_true;
676:   }
677:   if (ksp) {
678:     /* Assume that local solves are now different; not necessarily
679:        true though!  This flag is used only for PCView_ASM() */
680:     *ksp                   = osm->ksp;
681:     osm->same_local_solves = PETSC_FALSE;
682:   }
683:   return(0);
684: }


690: /*@C
691:     PCASMSetLocalSubdomains - Sets the local subdomains (for this processor only) for the additive Schwarz preconditioner. 

693:     Collective on PC 

695:     Input Parameters:
696: +   pc - the preconditioner context
697: .   n - the number of subdomains for this processor (default value = 1)
698: .   is - the index set that defines the subdomains for this processor
699:          (or PETSC_NULL for PETSc to determine subdomains)
700: -   is_local - the index sets that define the local part of the subdomains for this processor
701:          (or PETSC_NULL to use the default of 1 subdomain per process)

703:     Notes:
704:     The IS numbering is in the parallel, global numbering of the vector for both is and is_local

706:     By default the ASM preconditioner uses 1 block per processor.  

708:     Use PCASMSetTotalSubdomains() to set the subdomains for all processors.

710:     Level: advanced

712: .keywords: PC, ASM, set, local, subdomains, additive Schwarz

714: .seealso: PCASMSetTotalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
715:           PCASMCreateSubdomains2D(), PCASMGetLocalSubdomains()
716: @*/
717: PetscErrorCode  PCASMSetLocalSubdomains(PC pc,PetscInt n,IS is[],IS is_local[])
718: {

723:   PetscTryMethod(pc,"PCASMSetLocalSubdomains_C",(PC,PetscInt,IS[],IS[]),(pc,n,is,is_local));
724:   return(0);
725: }

729: /*@C
730:     PCASMSetTotalSubdomains - Sets the subdomains for all processor for the 
731:     additive Schwarz preconditioner.  Either all or no processors in the
732:     PC communicator must call this routine, with the same index sets.

734:     Collective on PC

736:     Input Parameters:
737: +   pc - the preconditioner context
738: .   n - the number of subdomains for all processors
739: .   is - the index sets that define the subdomains for all processor
740:          (or PETSC_NULL for PETSc to determine subdomains)
741: -   is_local - the index sets that define the local part of the subdomains for this processor
742:          (or PETSC_NULL to use the default of 1 subdomain per process)

744:     Options Database Key:
745:     To set the total number of subdomain blocks rather than specify the
746:     index sets, use the option
747: .    -pc_asm_blocks <blks> - Sets total blocks

749:     Notes:
750:     Currently you cannot use this to set the actual subdomains with the argument is.

752:     By default the ASM preconditioner uses 1 block per processor.  

754:     These index sets cannot be destroyed until after completion of the
755:     linear solves for which the ASM preconditioner is being used.

757:     Use PCASMSetLocalSubdomains() to set local subdomains.

759:     The IS numbering is in the parallel, global numbering of the vector for both is and is_local

761:     Level: advanced

763: .keywords: PC, ASM, set, total, global, subdomains, additive Schwarz

765: .seealso: PCASMSetLocalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
766:           PCASMCreateSubdomains2D()
767: @*/
768: PetscErrorCode  PCASMSetTotalSubdomains(PC pc,PetscInt N,IS is[],IS is_local[])
769: {

774:   PetscTryMethod(pc,"PCASMSetTotalSubdomains_C",(PC,PetscInt,IS[],IS[]),(pc,N,is,is_local));
775:   return(0);
776: }

780: /*@
781:     PCASMSetOverlap - Sets the overlap between a pair of subdomains for the
782:     additive Schwarz preconditioner.  Either all or no processors in the
783:     PC communicator must call this routine. 

785:     Logically Collective on PC

787:     Input Parameters:
788: +   pc  - the preconditioner context
789: -   ovl - the amount of overlap between subdomains (ovl >= 0, default value = 1)

791:     Options Database Key:
792: .   -pc_asm_overlap <ovl> - Sets overlap

794:     Notes:
795:     By default the ASM preconditioner uses 1 block per processor.  To use
796:     multiple blocks per perocessor, see PCASMSetTotalSubdomains() and
797:     PCASMSetLocalSubdomains() (and the option -pc_asm_blocks <blks>).

799:     The overlap defaults to 1, so if one desires that no additional
800:     overlap be computed beyond what may have been set with a call to
801:     PCASMSetTotalSubdomains() or PCASMSetLocalSubdomains(), then ovl
802:     must be set to be 0.  In particular, if one does not explicitly set
803:     the subdomains an application code, then all overlap would be computed
804:     internally by PETSc, and using an overlap of 0 would result in an ASM 
805:     variant that is equivalent to the block Jacobi preconditioner.  

807:     Note that one can define initial index sets with any overlap via
808:     PCASMSetTotalSubdomains() or PCASMSetLocalSubdomains(); the routine
809:     PCASMSetOverlap() merely allows PETSc to extend that overlap further
810:     if desired.

812:     Level: intermediate

814: .keywords: PC, ASM, set, overlap

816: .seealso: PCASMSetTotalSubdomains(), PCASMSetLocalSubdomains(), PCASMGetSubKSP(),
817:           PCASMCreateSubdomains2D(), PCASMGetLocalSubdomains()
818: @*/
819: PetscErrorCode  PCASMSetOverlap(PC pc,PetscInt ovl)
820: {

826:   PetscTryMethod(pc,"PCASMSetOverlap_C",(PC,PetscInt),(pc,ovl));
827:   return(0);
828: }

832: /*@
833:     PCASMSetType - Sets the type of restriction and interpolation used
834:     for local problems in the additive Schwarz method.

836:     Logically Collective on PC

838:     Input Parameters:
839: +   pc  - the preconditioner context
840: -   type - variant of ASM, one of
841: .vb
842:       PC_ASM_BASIC       - full interpolation and restriction
843:       PC_ASM_RESTRICT    - full restriction, local processor interpolation
844:       PC_ASM_INTERPOLATE - full interpolation, local processor restriction
845:       PC_ASM_NONE        - local processor restriction and interpolation
846: .ve

848:     Options Database Key:
849: .   -pc_asm_type [basic,restrict,interpolate,none] - Sets ASM type

851:     Level: intermediate

853: .keywords: PC, ASM, set, type

855: .seealso: PCASMSetTotalSubdomains(), PCASMSetTotalSubdomains(), PCASMGetSubKSP(),
856:           PCASMCreateSubdomains2D()
857: @*/
858: PetscErrorCode  PCASMSetType(PC pc,PCASMType type)
859: {

865:   PetscTryMethod(pc,"PCASMSetType_C",(PC,PCASMType),(pc,type));
866:   return(0);
867: }

871: /*@
872:     PCASMSetSortIndices - Determines whether subdomain indices are sorted.

874:     Logically Collective on PC

876:     Input Parameters:
877: +   pc  - the preconditioner context
878: -   doSort - sort the subdomain indices

880:     Level: intermediate

882: .keywords: PC, ASM, set, type

884: .seealso: PCASMSetLocalSubdomains(), PCASMSetTotalSubdomains(), PCASMGetSubKSP(),
885:           PCASMCreateSubdomains2D()
886: @*/
887: PetscErrorCode  PCASMSetSortIndices(PC pc,PetscBool  doSort)
888: {

894:   PetscTryMethod(pc,"PCASMSetSortIndices_C",(PC,PetscBool),(pc,doSort));
895:   return(0);
896: }

900: /*@C
901:    PCASMGetSubKSP - Gets the local KSP contexts for all blocks on
902:    this processor.
903:    
904:    Collective on PC iff first_local is requested

906:    Input Parameter:
907: .  pc - the preconditioner context

909:    Output Parameters:
910: +  n_local - the number of blocks on this processor or PETSC_NULL
911: .  first_local - the global number of the first block on this processor or PETSC_NULL,
912:                  all processors must request or all must pass PETSC_NULL
913: -  ksp - the array of KSP contexts

915:    Note:  
916:    After PCASMGetSubKSP() the array of KSPes is not to be freed

918:    Currently for some matrix implementations only 1 block per processor 
919:    is supported.
920:    
921:    You must call KSPSetUp() before calling PCASMGetSubKSP().

923:    Level: advanced

925: .keywords: PC, ASM, additive Schwarz, get, sub, KSP, context

927: .seealso: PCASMSetTotalSubdomains(), PCASMSetTotalSubdomains(), PCASMSetOverlap(),
928:           PCASMCreateSubdomains2D(),
929: @*/
930: PetscErrorCode  PCASMGetSubKSP(PC pc,PetscInt *n_local,PetscInt *first_local,KSP *ksp[])
931: {

936:   PetscUseMethod(pc,"PCASMGetSubKSP_C",(PC,PetscInt*,PetscInt*,KSP **),(pc,n_local,first_local,ksp));
937:   return(0);
938: }

940: /* -------------------------------------------------------------------------------------*/
941: /*MC
942:    PCASM - Use the (restricted) additive Schwarz method, each block is (approximately) solved with 
943:            its own KSP object.

945:    Options Database Keys:
946: +  -pc_asm_truelocal - Activates PCASMSetUseTrueLocal()
947: .  -pc_asm_blocks <blks> - Sets total blocks
948: .  -pc_asm_overlap <ovl> - Sets overlap
949: -  -pc_asm_type [basic,restrict,interpolate,none] - Sets ASM type

951:      IMPORTANT: If you run with, for example, 3 blocks on 1 processor or 3 blocks on 3 processors you 
952:       will get a different convergence rate due to the default option of -pc_asm_type restrict. Use
953:       -pc_asm_type basic to use the standard ASM. 

955:    Notes: Each processor can have one or more blocks, but a block cannot be shared by more
956:      than one processor. Defaults to one block per processor.

958:      To set options on the solvers for each block append -sub_ to all the KSP, and PC
959:         options database keys. For example, -sub_pc_type ilu -sub_pc_factor_levels 1 -sub_ksp_type preonly
960:         
961:      To set the options on the solvers separate for each block call PCASMGetSubKSP()
962:          and set the options directly on the resulting KSP object (you can access its PC
963:          with KSPGetPC())


966:    Level: beginner

968:    Concepts: additive Schwarz method

970:     References:
971:     An additive variant of the Schwarz alternating method for the case of many subregions
972:     M Dryja, OB Widlund - Courant Institute, New York University Technical report

974:     Domain Decompositions: Parallel Multilevel Methods for Elliptic Partial Differential Equations, 
975:     Barry Smith, Petter Bjorstad, and William Gropp, Cambridge University Press, ISBN 0-521-49589-X.

977: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types), PC,
978:            PCBJACOBI, PCASMSetUseTrueLocal(), PCASMGetSubKSP(), PCASMSetLocalSubdomains(),
979:            PCASMSetTotalSubdomains(), PCSetModifySubmatrices(), PCASMSetOverlap(), PCASMSetType()

981: M*/

986: PetscErrorCode  PCCreate_ASM(PC pc)
987: {
989:   PC_ASM         *osm;

992:   PetscNewLog(pc,PC_ASM,&osm);
993:   osm->n                 = PETSC_DECIDE;
994:   osm->n_local           = 0;
995:   osm->n_local_true      = 0;
996:   osm->overlap           = 1;
997:   osm->ksp               = 0;
998:   osm->restriction       = 0;
999:   osm->localization      = 0;
1000:   osm->prolongation      = 0;
1001:   osm->x                 = 0;
1002:   osm->y                 = 0;
1003:   osm->y_local           = 0;
1004:   osm->is                = 0;
1005:   osm->is_local          = 0;
1006:   osm->mat               = 0;
1007:   osm->pmat              = 0;
1008:   osm->type              = PC_ASM_RESTRICT;
1009:   osm->same_local_solves = PETSC_TRUE;
1010:   osm->sort_indices      = PETSC_TRUE;

1012:   pc->data                   = (void*)osm;
1013:   pc->ops->apply             = PCApply_ASM;
1014:   pc->ops->applytranspose    = PCApplyTranspose_ASM;
1015:   pc->ops->setup             = PCSetUp_ASM;
1016:   pc->ops->reset             = PCReset_ASM;
1017:   pc->ops->destroy           = PCDestroy_ASM;
1018:   pc->ops->setfromoptions    = PCSetFromOptions_ASM;
1019:   pc->ops->setuponblocks     = PCSetUpOnBlocks_ASM;
1020:   pc->ops->view              = PCView_ASM;
1021:   pc->ops->applyrichardson   = 0;

1023:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetLocalSubdomains_C","PCASMSetLocalSubdomains_ASM",
1024:                     PCASMSetLocalSubdomains_ASM);
1025:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetTotalSubdomains_C","PCASMSetTotalSubdomains_ASM",
1026:                     PCASMSetTotalSubdomains_ASM);
1027:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetOverlap_C","PCASMSetOverlap_ASM",
1028:                     PCASMSetOverlap_ASM);
1029:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetType_C","PCASMSetType_ASM",
1030:                     PCASMSetType_ASM);
1031:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetSortIndices_C","PCASMSetSortIndices_ASM",
1032:                     PCASMSetSortIndices_ASM);
1033:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMGetSubKSP_C","PCASMGetSubKSP_ASM",
1034:                     PCASMGetSubKSP_ASM);
1035:   return(0);
1036: }


1042: /*@C
1043:    PCASMCreateSubdomains - Creates the index sets for the overlapping Schwarz
1044:    preconditioner for a any problem on a general grid.

1046:    Collective

1048:    Input Parameters:
1049: +  A - The global matrix operator
1050: -  n - the number of local blocks

1052:    Output Parameters:
1053: .  outis - the array of index sets defining the subdomains

1055:    Level: advanced

1057:    Note: this generates nonoverlapping subdomains; the PCASM will generate the overlap
1058:     from these if you use PCASMSetLocalSubdomains()

1060:     In the Fortran version you must provide the array outis[] already allocated of length n.

1062: .keywords: PC, ASM, additive Schwarz, create, subdomains, unstructured grid

1064: .seealso: PCASMSetLocalSubdomains(), PCASMDestroySubdomains()
1065: @*/
1066: PetscErrorCode  PCASMCreateSubdomains(Mat A, PetscInt n, IS* outis[])
1067: {
1068:   MatPartitioning           mpart;
1069:   const char                *prefix;
1070:   PetscErrorCode            (*f)(Mat,Mat*);
1071:   PetscMPIInt               size;
1072:   PetscInt                  i,j,rstart,rend,bs;
1073:   PetscBool                 isbaij = PETSC_FALSE,foundpart = PETSC_FALSE;
1074:   Mat                       Ad = PETSC_NULL, adj;
1075:   IS                        ispart,isnumb,*is;
1076:   PetscErrorCode            ierr;

1081:   if (n < 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"number of local blocks must be > 0, n = %D",n);

1083:   /* Get prefix, row distribution, and block size */
1084:   MatGetOptionsPrefix(A,&prefix);
1085:   MatGetOwnershipRange(A,&rstart,&rend);
1086:   MatGetBlockSize(A,&bs);
1087:   if (rstart/bs*bs != rstart || rend/bs*bs != rend) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"bad row distribution [%D,%D) for matrix block size %D",rstart,rend,bs);

1089:   /* Get diagonal block from matrix if possible */
1090:   MPI_Comm_size(((PetscObject)A)->comm,&size);
1091:   PetscObjectQueryFunction((PetscObject)A,"MatGetDiagonalBlock_C",(void (**)(void))&f);
1092:   if (f) {
1093:     MatGetDiagonalBlock(A,&Ad);
1094:   } else if (size == 1) {
1095:     Ad = A;
1096:   }
1097:   if (Ad) {
1098:     PetscTypeCompare((PetscObject)Ad,MATSEQBAIJ,&isbaij);
1099:     if (!isbaij) {PetscTypeCompare((PetscObject)Ad,MATSEQSBAIJ,&isbaij);}
1100:   }
1101:   if (Ad && n > 1) {
1102:     PetscBool  match,done;
1103:     /* Try to setup a good matrix partitioning if available */
1104:     MatPartitioningCreate(PETSC_COMM_SELF,&mpart);
1105:     PetscObjectSetOptionsPrefix((PetscObject)mpart,prefix);
1106:     MatPartitioningSetFromOptions(mpart);
1107:     PetscTypeCompare((PetscObject)mpart,MATPARTITIONINGCURRENT,&match);
1108:     if (!match) {
1109:       PetscTypeCompare((PetscObject)mpart,MATPARTITIONINGSQUARE,&match);
1110:     }
1111:     if (!match) { /* assume a "good" partitioner is available */
1112:       PetscInt na,*ia,*ja;
1113:       MatGetRowIJ(Ad,0,PETSC_TRUE,isbaij,&na,&ia,&ja,&done);
1114:       if (done) {
1115:         /* Build adjacency matrix by hand. Unfortunately a call to
1116:            MatConvert(Ad,MATMPIADJ,MAT_INITIAL_MATRIX,&adj) will
1117:            remove the block-aij structure and we cannot expect
1118:            MatPartitioning to split vertices as we need */
1119:         PetscInt i,j,*row,len,nnz,cnt,*iia=0,*jja=0;
1120:         nnz = 0;
1121:         for (i=0; i<na; i++) { /* count number of nonzeros */
1122:           len = ia[i+1] - ia[i];
1123:           row = ja + ia[i];
1124:           for (j=0; j<len; j++) {
1125:             if (row[j] == i) { /* don't count diagonal */
1126:               len--; break;
1127:             }
1128:           }
1129:           nnz += len;
1130:         }
1131:         PetscMalloc((na+1)*sizeof(PetscInt),&iia);
1132:         PetscMalloc((nnz)*sizeof(PetscInt),&jja);
1133:         nnz    = 0;
1134:         iia[0] = 0;
1135:         for (i=0; i<na; i++) { /* fill adjacency */
1136:           cnt = 0;
1137:           len = ia[i+1] - ia[i];
1138:           row = ja + ia[i];
1139:           for (j=0; j<len; j++) {
1140:             if (row[j] != i) { /* if not diagonal */
1141:               jja[nnz+cnt++] = row[j];
1142:             }
1143:           }
1144:           nnz += cnt;
1145:           iia[i+1] = nnz;
1146:         }
1147:         /* Partitioning of the adjacency matrix */
1148:         MatCreateMPIAdj(PETSC_COMM_SELF,na,na,iia,jja,PETSC_NULL,&adj);
1149:         MatPartitioningSetAdjacency(mpart,adj);
1150:         MatPartitioningSetNParts(mpart,n);
1151:         MatPartitioningApply(mpart,&ispart);
1152:         ISPartitioningToNumbering(ispart,&isnumb);
1153:         MatDestroy(&adj);
1154:         foundpart = PETSC_TRUE;
1155:       }
1156:       MatRestoreRowIJ(Ad,0,PETSC_TRUE,isbaij,&na,&ia,&ja,&done);
1157:     }
1158:     MatPartitioningDestroy(&mpart);
1159:   }

1161:   PetscMalloc(n*sizeof(IS),&is);
1162:   *outis = is;

1164:   if (!foundpart) {

1166:     /* Partitioning by contiguous chunks of rows */

1168:     PetscInt mbs   = (rend-rstart)/bs;
1169:     PetscInt start = rstart;
1170:     for (i=0; i<n; i++) {
1171:       PetscInt count = (mbs/n + ((mbs % n) > i)) * bs;
1172:       ISCreateStride(PETSC_COMM_SELF,count,start,1,&is[i]);
1173:       start += count;
1174:     }

1176:   } else {

1178:     /* Partitioning by adjacency of diagonal block  */

1180:     const PetscInt *numbering;
1181:     PetscInt       *count,nidx,*indices,*newidx,start=0;
1182:     /* Get node count in each partition */
1183:     PetscMalloc(n*sizeof(PetscInt),&count);
1184:     ISPartitioningCount(ispart,n,count);
1185:     if (isbaij && bs > 1) { /* adjust for the block-aij case */
1186:       for (i=0; i<n; i++) count[i] *= bs;
1187:     }
1188:     /* Build indices from node numbering */
1189:     ISGetLocalSize(isnumb,&nidx);
1190:     PetscMalloc(nidx*sizeof(PetscInt),&indices);
1191:     for (i=0; i<nidx; i++) indices[i] = i; /* needs to be initialized */
1192:     ISGetIndices(isnumb,&numbering);
1193:     PetscSortIntWithPermutation(nidx,numbering,indices);
1194:     ISRestoreIndices(isnumb,&numbering);
1195:     if (isbaij && bs > 1) { /* adjust for the block-aij case */
1196:       PetscMalloc(nidx*bs*sizeof(PetscInt),&newidx);
1197:       for (i=0; i<nidx; i++)
1198:         for (j=0; j<bs; j++)
1199:           newidx[i*bs+j] = indices[i]*bs + j;
1200:       PetscFree(indices);
1201:       nidx   *= bs;
1202:       indices = newidx;
1203:     }
1204:     /* Shift to get global indices */
1205:     for (i=0; i<nidx; i++) indices[i] += rstart;

1207:     /* Build the index sets for each block */
1208:     for (i=0; i<n; i++) {
1209:       ISCreateGeneral(PETSC_COMM_SELF,count[i],&indices[start],PETSC_COPY_VALUES,&is[i]);
1210:       ISSort(is[i]);
1211:       start += count[i];
1212:     }

1214:     PetscFree(count);
1215:     PetscFree(indices);
1216:     ISDestroy(&isnumb);
1217:     ISDestroy(&ispart);

1219:   }

1221:   return(0);
1222: }

1226: /*@C
1227:    PCASMDestroySubdomains - Destroys the index sets created with
1228:    PCASMCreateSubdomains(). Should be called after setting subdomains
1229:    with PCASMSetLocalSubdomains().

1231:    Collective

1233:    Input Parameters:
1234: +  n - the number of index sets
1235: .  is - the array of index sets
1236: -  is_local - the array of local index sets, can be PETSC_NULL

1238:    Level: advanced

1240: .keywords: PC, ASM, additive Schwarz, create, subdomains, unstructured grid

1242: .seealso: PCASMCreateSubdomains(), PCASMSetLocalSubdomains()
1243: @*/
1244: PetscErrorCode  PCASMDestroySubdomains(PetscInt n, IS is[], IS is_local[])
1245: {
1246:   PetscInt       i;
1249:   if (n <= 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"n must be > 0: n = %D",n);
1251:   for (i=0; i<n; i++) { ISDestroy(&is[i]); }
1252:   PetscFree(is);
1253:   if (is_local) {
1255:     for (i=0; i<n; i++) { ISDestroy(&is_local[i]); }
1256:     PetscFree(is_local);
1257:   }
1258:   return(0);
1259: }

1263: /*@
1264:    PCASMCreateSubdomains2D - Creates the index sets for the overlapping Schwarz 
1265:    preconditioner for a two-dimensional problem on a regular grid.

1267:    Not Collective

1269:    Input Parameters:
1270: +  m, n - the number of mesh points in the x and y directions
1271: .  M, N - the number of subdomains in the x and y directions
1272: .  dof - degrees of freedom per node
1273: -  overlap - overlap in mesh lines

1275:    Output Parameters:
1276: +  Nsub - the number of subdomains created
1277: .  is - array of index sets defining overlapping (if overlap > 0) subdomains
1278: -  is_local - array of index sets defining non-overlapping subdomains

1280:    Note:
1281:    Presently PCAMSCreateSubdomains2d() is valid only for sequential
1282:    preconditioners.  More general related routines are
1283:    PCASMSetTotalSubdomains() and PCASMSetLocalSubdomains().

1285:    Level: advanced

1287: .keywords: PC, ASM, additive Schwarz, create, subdomains, 2D, regular grid

1289: .seealso: PCASMSetTotalSubdomains(), PCASMSetLocalSubdomains(), PCASMGetSubKSP(),
1290:           PCASMSetOverlap()
1291: @*/
1292: PetscErrorCode  PCASMCreateSubdomains2D(PetscInt m,PetscInt n,PetscInt M,PetscInt N,PetscInt dof,PetscInt overlap,PetscInt *Nsub,IS **is,IS **is_local)
1293: {
1294:   PetscInt       i,j,height,width,ystart,xstart,yleft,yright,xleft,xright,loc_outer;
1296:   PetscInt       nidx,*idx,loc,ii,jj,count;

1299:   if (dof != 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP," ");

1301:   *Nsub = N*M;
1302:   PetscMalloc((*Nsub)*sizeof(IS*),is);
1303:   PetscMalloc((*Nsub)*sizeof(IS*),is_local);
1304:   ystart = 0;
1305:   loc_outer = 0;
1306:   for (i=0; i<N; i++) {
1307:     height = n/N + ((n % N) > i); /* height of subdomain */
1308:     if (height < 2) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Too many N subdomains for mesh dimension n");
1309:     yleft  = ystart - overlap; if (yleft < 0) yleft = 0;
1310:     yright = ystart + height + overlap; if (yright > n) yright = n;
1311:     xstart = 0;
1312:     for (j=0; j<M; j++) {
1313:       width = m/M + ((m % M) > j); /* width of subdomain */
1314:       if (width < 2) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Too many M subdomains for mesh dimension m");
1315:       xleft  = xstart - overlap; if (xleft < 0) xleft = 0;
1316:       xright = xstart + width + overlap; if (xright > m) xright = m;
1317:       nidx   = (xright - xleft)*(yright - yleft);
1318:       PetscMalloc(nidx*sizeof(PetscInt),&idx);
1319:       loc    = 0;
1320:       for (ii=yleft; ii<yright; ii++) {
1321:         count = m*ii + xleft;
1322:         for (jj=xleft; jj<xright; jj++) {
1323:           idx[loc++] = count++;
1324:         }
1325:       }
1326:       ISCreateGeneral(PETSC_COMM_SELF,nidx,idx,PETSC_COPY_VALUES,(*is)+loc_outer);
1327:       if (overlap == 0) {
1328:         PetscObjectReference((PetscObject)(*is)[loc_outer]);
1329:         (*is_local)[loc_outer] = (*is)[loc_outer];
1330:       } else {
1331:         for (loc=0,ii=ystart; ii<ystart+height; ii++) {
1332:           for (jj=xstart; jj<xstart+width; jj++) {
1333:             idx[loc++] = m*ii + jj;
1334:           }
1335:         }
1336:         ISCreateGeneral(PETSC_COMM_SELF,loc,idx,PETSC_COPY_VALUES,*is_local+loc_outer);
1337:       }
1338:       PetscFree(idx);
1339:       xstart += width;
1340:       loc_outer++;
1341:     }
1342:     ystart += height;
1343:   }
1344:   for (i=0; i<*Nsub; i++) { ISSort((*is)[i]); }
1345:   return(0);
1346: }

1350: /*@C
1351:     PCASMGetLocalSubdomains - Gets the local subdomains (for this processor
1352:     only) for the additive Schwarz preconditioner. 

1354:     Not Collective

1356:     Input Parameter:
1357: .   pc - the preconditioner context

1359:     Output Parameters:
1360: +   n - the number of subdomains for this processor (default value = 1)
1361: .   is - the index sets that define the subdomains for this processor
1362: -   is_local - the index sets that define the local part of the subdomains for this processor (can be PETSC_NULL)
1363:          

1365:     Notes:
1366:     The IS numbering is in the parallel, global numbering of the vector.

1368:     Level: advanced

1370: .keywords: PC, ASM, set, local, subdomains, additive Schwarz

1372: .seealso: PCASMSetTotalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
1373:           PCASMCreateSubdomains2D(), PCASMSetLocalSubdomains(), PCASMGetLocalSubmatrices()
1374: @*/
1375: PetscErrorCode  PCASMGetLocalSubdomains(PC pc,PetscInt *n,IS *is[],IS *is_local[])
1376: {
1377:   PC_ASM         *osm;
1379:   PetscBool      match;

1385:   PetscTypeCompare((PetscObject)pc,PCASM,&match);
1386:   if (!match) {
1387:     if (n)  *n  = 0;
1388:     if (is) *is = PETSC_NULL;
1389:   } else {
1390:     osm = (PC_ASM*)pc->data;
1391:     if (n)  *n  = osm->n_local_true;
1392:     if (is) *is = osm->is;
1393:     if (is_local) *is_local = osm->is_local;
1394:   }
1395:   return(0);
1396: }

1400: /*@C
1401:     PCASMGetLocalSubmatrices - Gets the local submatrices (for this processor
1402:     only) for the additive Schwarz preconditioner. 

1404:     Not Collective

1406:     Input Parameter:
1407: .   pc - the preconditioner context

1409:     Output Parameters:
1410: +   n - the number of matrices for this processor (default value = 1)
1411: -   mat - the matrices
1412:          

1414:     Level: advanced

1416:     Notes: Call after PCSetUp() (or KSPSetUp()) but before PCApply() (or KSPApply()) and before PCSetUpOnBlocks())

1418:            Usually one would use PCSetModifySubmatrices() to change the submatrices in building the preconditioner.

1420: .keywords: PC, ASM, set, local, subdomains, additive Schwarz, block Jacobi

1422: .seealso: PCASMSetTotalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
1423:           PCASMCreateSubdomains2D(), PCASMSetLocalSubdomains(), PCASMGetLocalSubdomains(), PCSetModifySubmatrices()
1424: @*/
1425: PetscErrorCode  PCASMGetLocalSubmatrices(PC pc,PetscInt *n,Mat *mat[])
1426: {
1427:   PC_ASM         *osm;
1429:   PetscBool      match;

1435:   if (!pc->setupcalled) SETERRQ(((PetscObject)pc)->comm,PETSC_ERR_ARG_WRONGSTATE,"Must call after KSPSetUP() or PCSetUp().");
1436:   PetscTypeCompare((PetscObject)pc,PCASM,&match);
1437:   if (!match) {
1438:     if (n)   *n   = 0;
1439:     if (mat) *mat = PETSC_NULL;
1440:   } else {
1441:     osm = (PC_ASM*)pc->data;
1442:     if (n)   *n   = osm->n_local_true;
1443:     if (mat) *mat = osm->pmat;
1444:   }
1445:   return(0);
1446: }