Actual source code: pack.c

 2:  #include packimpl.h

  6: /*@C
  7:     DMCompositeSetCoupling - Sets user provided routines that compute the coupling between the 
  8:       seperate components (DMDA's and arrays) in a DMto build the correct matrix nonzero structure.


 11:     Logically Collective on MPI_Comm

 13:     Input Parameter:
 14: +   dm - the composite object
 15: -   formcouplelocations - routine to set the nonzero locations in the matrix

 17:     Level: advanced

 19:     Notes: See DMSetApplicationContext() and DMGetApplicationContext() for how to get user information into
 20:         this routine

 22: @*/
 23: PetscErrorCode  DMCompositeSetCoupling(DM dm,PetscErrorCode (*FormCoupleLocations)(DM,Mat,PetscInt*,PetscInt*,PetscInt,PetscInt,PetscInt,PetscInt))
 24: {
 25:   DM_Composite *com = (DM_Composite*)dm->data;

 28:   com->FormCoupleLocations = FormCoupleLocations;
 29:   return(0);
 30: }

 34: PetscErrorCode  DMDestroy_Composite(DM dm)
 35: {
 36:   PetscErrorCode         ierr;
 37:   struct DMCompositeLink *next, *prev;
 38:   DM_Composite           *com = (DM_Composite *)dm->data;

 41:   next = com->next;
 42:   while (next) {
 43:     prev = next;
 44:     next = next->next;
 45:     if (prev->type == DMCOMPOSITE_DM) {
 46:       DMDestroy(&prev->dm);
 47:     }
 48:     PetscFree(prev->grstarts);
 49:     PetscFree(prev);
 50:   }
 51:   return(0);
 52: }

 56: PetscErrorCode  DMView_Composite(DM dm,PetscViewer v)
 57: {
 59:   PetscBool      iascii;
 60:   DM_Composite   *com = (DM_Composite *)dm->data;

 63:   PetscTypeCompare((PetscObject)v,PETSCVIEWERASCII,&iascii);
 64:   if (iascii) {
 65:     struct DMCompositeLink *lnk = com->next;
 66:     PetscInt               i;

 68:     PetscViewerASCIIPrintf(v,"DM (%s)\n",((PetscObject)dm)->prefix?((PetscObject)dm)->prefix:"no prefix");
 69:     PetscViewerASCIIPrintf(v,"  contains %d DMs and %d redundant arrays\n",com->nDM,com->nredundant);
 70:     PetscViewerASCIIPushTab(v);
 71:     for (i=0; lnk; lnk=lnk->next,i++) {
 72:       if (lnk->dm) {
 73:         PetscViewerASCIIPrintf(v,"Link %d: DM of type %s\n",i,((PetscObject)lnk->dm)->type_name);
 74:         PetscViewerASCIIPushTab(v);
 75:         DMView(lnk->dm,v);
 76:         PetscViewerASCIIPopTab(v);
 77:       } else {
 78:         PetscViewerASCIIPrintf(v,"Link %d: Redundant array of size %d owned by rank %d\n",i,lnk->nlocal,lnk->rank);
 79:       }
 80:     }
 81:     PetscViewerASCIIPopTab(v);
 82:   }
 83:   return(0);
 84: }

 86: /* --------------------------------------------------------------------------------------*/
 89: PetscErrorCode  DMSetUp_Composite(DM dm)
 90: {
 91:   PetscErrorCode         ierr;
 92:   PetscInt               nprev = 0;
 93:   PetscMPIInt            rank,size;
 94:   DM_Composite           *com = (DM_Composite*)dm->data;
 95:   struct DMCompositeLink *next = com->next;
 96:   PetscLayout            map;

 99:   if (com->setup) SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_ARG_WRONGSTATE,"Packer has already been setup");
100:   PetscLayoutCreate(((PetscObject)dm)->comm,&map);
101:   PetscLayoutSetLocalSize(map,com->n);
102:   PetscLayoutSetSize(map,PETSC_DETERMINE);
103:   PetscLayoutSetBlockSize(map,1);
104:   PetscLayoutSetUp(map);
105:   PetscLayoutGetSize(map,&com->N);
106:   PetscLayoutGetRange(map,&com->rstart,PETSC_NULL);
107:   PetscLayoutDestroy(&map);
108: 
109:   /* now set the rstart for each linked array/vector */
110:   MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
111:   MPI_Comm_size(((PetscObject)dm)->comm,&size);
112:   while (next) {
113:     next->rstart = nprev;
114:     nprev += next->n;
115:     next->grstart = com->rstart + next->rstart;
116:     if (next->type == DMCOMPOSITE_ARRAY) {
117:       MPI_Bcast(&next->grstart,1,MPIU_INT,next->rank,((PetscObject)dm)->comm);
118:     } else {
119:       PetscMalloc(size*sizeof(PetscInt),&next->grstarts);
120:       MPI_Allgather(&next->grstart,1,MPIU_INT,next->grstarts,1,MPIU_INT,((PetscObject)dm)->comm);
121:     }
122:     next = next->next;
123:   }
124:   com->setup = PETSC_TRUE;
125:   return(0);
126: }


131: PetscErrorCode DMCompositeGetAccess_Array(DM dm,struct DMCompositeLink *mine,Vec vec,PetscScalar **array)
132: {
134:   PetscScalar    *varray;
135:   PetscMPIInt    rank;

138:   MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
139:   if (array) {
140:     if (rank == mine->rank) {
141:       VecGetArray(vec,&varray);
142:       *array  = varray + mine->rstart;
143:       VecRestoreArray(vec,&varray);
144:     } else {
145:       *array = 0;
146:     }
147:   }
148:   return(0);
149: }

153: PetscErrorCode DMCompositeGetAccess_DM(DM dm,struct DMCompositeLink *mine,Vec vec,Vec *global)
154: {
156:   PetscScalar    *array;

159:   if (global) {
160:     DMGetGlobalVector(mine->dm,global);
161:     VecGetArray(vec,&array);
162:     VecPlaceArray(*global,array+mine->rstart);
163:     VecRestoreArray(vec,&array);
164:   }
165:   return(0);
166: }

170: PetscErrorCode DMCompositeRestoreAccess_Array(DM dm,struct DMCompositeLink *mine,Vec vec,PetscScalar **array)
171: {
173:   return(0);
174: }

178: PetscErrorCode DMCompositeRestoreAccess_DM(DM dm,struct DMCompositeLink *mine,Vec vec,Vec *global)
179: {

183:   if (global) {
184:     VecResetArray(*global);
185:     DMRestoreGlobalVector(mine->dm,global);
186:   }
187:   return(0);
188: }

192: PetscErrorCode DMCompositeScatter_Array(DM dm,struct DMCompositeLink *mine,Vec vec,PetscScalar *array)
193: {
195:   PetscScalar    *varray;
196:   PetscMPIInt    rank;

199:   MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
200:   if (rank == mine->rank) {
201:     VecGetArray(vec,&varray);
202:     PetscMemcpy(array,varray+mine->rstart,mine->n*sizeof(PetscScalar));
203:     VecRestoreArray(vec,&varray);
204:   }
205:   MPI_Bcast(array,mine->nlocal,MPIU_SCALAR,mine->rank,((PetscObject)dm)->comm);
206:   return(0);
207: }

211: PetscErrorCode DMCompositeScatter_DM(DM dm,struct DMCompositeLink *mine,Vec vec,Vec local)
212: {
214:   PetscScalar    *array;
215:   Vec            global;

218:   DMGetGlobalVector(mine->dm,&global);
219:   VecGetArray(vec,&array);
220:   VecPlaceArray(global,array+mine->rstart);
221:   DMGlobalToLocalBegin(mine->dm,global,INSERT_VALUES,local);
222:   DMGlobalToLocalEnd(mine->dm,global,INSERT_VALUES,local);
223:   VecRestoreArray(vec,&array);
224:   VecResetArray(global);
225:   DMRestoreGlobalVector(mine->dm,&global);
226:   return(0);
227: }

231: PetscErrorCode DMCompositeGather_Array(DM dm,struct DMCompositeLink *mine,Vec vec,InsertMode imode,const PetscScalar *array)
232: {
234:   PetscScalar    *varray;
235:   PetscMPIInt    rank;

238:   MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
239:   if (rank == mine->rank) {
240:     VecGetArray(vec,&varray);
241:     if (varray+mine->rstart == array) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"You need not DMCompositeGather() into objects obtained via DMCompositeGetAccess()");
242:   }
243:   switch (imode) {
244:   case INSERT_VALUES:
245:     if (rank == mine->rank) {
246:       PetscMemcpy(varray+mine->rstart,array,mine->n*sizeof(PetscScalar));
247:     }
248:     break;
249:   case ADD_VALUES: {
250:     PetscInt          i;
251:     void             *source;
252:     PetscScalar       *buffer,*dest;
253:     if (rank == mine->rank) {
254:       dest = &varray[mine->rstart];
255: #if defined(PETSC_HAVE_MPI_IN_PLACE)
256:       buffer = dest;
257:       source = MPI_IN_PLACE;
258: #else
259:       PetscMalloc(mine->nlocal*sizeof(PetscScalar),&buffer);
260:       source = (void *) buffer;
261: #endif
262:       for (i=0; i<mine->nlocal; i++) buffer[i] = varray[mine->rstart+i] + array[i];
263:     } else {
264:       source = (void *) array;
265:       dest   = PETSC_NULL;
266:     }
267:     MPI_Reduce(source,dest,mine->nlocal,MPIU_SCALAR,MPI_SUM,mine->rank,((PetscObject)dm)->comm);
268: #if !defined(PETSC_HAVE_MPI_IN_PLACE)
269:     if (rank == mine->rank) {PetscFree(source);}
270: #endif
271:   } break;
272:   default: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"imode");
273:   }
274:   if (rank == mine->rank) {VecRestoreArray(vec,&varray);}
275:   return(0);
276: }

280: PetscErrorCode DMCompositeGather_DM(DM dm,struct DMCompositeLink *mine,Vec vec,InsertMode imode,Vec local)
281: {
283:   PetscScalar    *array;
284:   Vec            global;

287:   DMGetGlobalVector(mine->dm,&global);
288:   VecGetArray(vec,&array);
289:   VecPlaceArray(global,array+mine->rstart);
290:   DMLocalToGlobalBegin(mine->dm,local,imode,global);
291:   DMLocalToGlobalEnd(mine->dm,local,imode,global);
292:   VecRestoreArray(vec,&array);
293:   VecResetArray(global);
294:   DMRestoreGlobalVector(mine->dm,&global);
295:   return(0);
296: }

298: /* ----------------------------------------------------------------------------------*/

300: #include <stdarg.h>

304: /*@C
305:     DMCompositeGetNumberDM - Get's the number of DM objects in the DMComposite
306:        representation.

308:     Not Collective

310:     Input Parameter:
311: .    dm - the packer object

313:     Output Parameter:
314: .     nDM - the number of DMs

316:     Level: beginner

318: @*/
319: PetscErrorCode  DMCompositeGetNumberDM(DM dm,PetscInt *nDM)
320: {
321:   DM_Composite *com = (DM_Composite*)dm->data;
324:   *nDM = com->nDM;
325:   return(0);
326: }


331: /*@C
332:     DMCompositeGetAccess - Allows one to access the individual packed vectors in their global
333:        representation.

335:     Collective on DMComposite

337:     Input Parameter:
338: +    dm - the packer object
339: .    gvec - the global vector
340: -    ... - the individual sequential or parallel objects (arrays or vectors)

342:     Notes: Use DMCompositeRestoreAccess() to return the vectors when you no longer need them
343:  
344:     Level: advanced

346: @*/
347: PetscErrorCode  DMCompositeGetAccess(DM dm,Vec gvec,...)
348: {
349:   va_list                Argp;
350:   PetscErrorCode         ierr;
351:   struct DMCompositeLink *next;
352:   DM_Composite           *com = (DM_Composite*)dm->data;

357:   next = com->next;
358:   if (!com->setup) {
359:     DMSetUp(dm);
360:   }

362:   /* loop over packed objects, handling one at at time */
363:   va_start(Argp,gvec);
364:   while (next) {
365:     if (next->type == DMCOMPOSITE_ARRAY) {
366:       PetscScalar **array;
367:       array = va_arg(Argp, PetscScalar**);
368:       DMCompositeGetAccess_Array(dm,next,gvec,array);
369:     } else if (next->type == DMCOMPOSITE_DM) {
370:       Vec *vec;
371:       vec  = va_arg(Argp, Vec*);
372:       DMCompositeGetAccess_DM(dm,next,gvec,vec);
373:     } else {
374:       SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
375:     }
376:     next = next->next;
377:   }
378:   va_end(Argp);
379:   return(0);
380: }

384: /*@C
385:     DMCompositeRestoreAccess - Returns the vectors obtained with DMCompositeGetAccess()
386:        representation.

388:     Collective on DMComposite

390:     Input Parameter:
391: +    dm - the packer object
392: .    gvec - the global vector
393: -    ... - the individual sequential or parallel objects (arrays or vectors)
394:  
395:     Level: advanced

397: .seealso  DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
398:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeScatter(),
399:          DMCompositeRestoreAccess(), DMCompositeGetAccess()

401: @*/
402: PetscErrorCode  DMCompositeRestoreAccess(DM dm,Vec gvec,...)
403: {
404:   va_list                Argp;
405:   PetscErrorCode         ierr;
406:   struct DMCompositeLink *next;
407:   DM_Composite           *com = (DM_Composite*)dm->data;

412:   next = com->next;
413:   if (!com->setup) {
414:     DMSetUp(dm);
415:   }

417:   /* loop over packed objects, handling one at at time */
418:   va_start(Argp,gvec);
419:   while (next) {
420:     if (next->type == DMCOMPOSITE_ARRAY) {
421:       PetscScalar **array;
422:       array = va_arg(Argp, PetscScalar**);
423:       DMCompositeRestoreAccess_Array(dm,next,gvec,array);
424:     } else if (next->type == DMCOMPOSITE_DM) {
425:       Vec *vec;
426:       vec  = va_arg(Argp, Vec*);
427:       DMCompositeRestoreAccess_DM(dm,next,gvec,vec);
428:     } else {
429:       SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
430:     }
431:     next = next->next;
432:   }
433:   va_end(Argp);
434:   return(0);
435: }

439: /*@C
440:     DMCompositeScatter - Scatters from a global packed vector into its individual local vectors

442:     Collective on DMComposite

444:     Input Parameter:
445: +    dm - the packer object
446: .    gvec - the global vector
447: -    ... - the individual sequential objects (arrays or vectors), PETSC_NULL for those that are not needed

449:     Level: advanced

451: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
452:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
453:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()

455: @*/
456: PetscErrorCode  DMCompositeScatter(DM dm,Vec gvec,...)
457: {
458:   va_list                Argp;
459:   PetscErrorCode         ierr;
460:   struct DMCompositeLink *next;
461:   PetscInt               cnt;
462:   DM_Composite           *com = (DM_Composite*)dm->data;

467:   if (!com->setup) {
468:     DMSetUp(dm);
469:   }

471:   /* loop over packed objects, handling one at at time */
472:   va_start(Argp,gvec);
473:   for (cnt=3,next=com->next; next; cnt++,next=next->next) {
474:     if (next->type == DMCOMPOSITE_ARRAY) {
475:       PetscScalar *array;
476:       array = va_arg(Argp, PetscScalar*);
479:       if (!array) continue;
480:       DMCompositeScatter_Array(dm,next,gvec,array);
481:     } else if (next->type == DMCOMPOSITE_DM) {
482:       Vec vec;
483:       vec = va_arg(Argp, Vec);
484:       if (!vec) continue;
486:       DMCompositeScatter_DM(dm,next,gvec,vec);
487:     } else {
488:       SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
489:     }
490:   }
491:   va_end(Argp);
492:   return(0);
493: }

497: /*@C
498:     DMCompositeGather - Gathers into a global packed vector from its individual local vectors

500:     Collective on DMComposite

502:     Input Parameter:
503: +    dm - the packer object
504: .    gvec - the global vector
505: -    ... - the individual sequential objects (arrays or vectors), PETSC_NULL for any that are not needed

507:     Level: advanced

509: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
510:          DMCompositeScatter(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
511:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()

513: @*/
514: PetscErrorCode  DMCompositeGather(DM dm,Vec gvec,InsertMode imode,...)
515: {
516:   va_list                Argp;
517:   PetscErrorCode         ierr;
518:   struct DMCompositeLink *next;
519:   DM_Composite           *com = (DM_Composite*)dm->data;
520:   PetscInt               cnt;

525:   if (!com->setup) {
526:     DMSetUp(dm);
527:   }

529:   /* loop over packed objects, handling one at at time */
530:   va_start(Argp,imode);
531:   for (cnt=3,next=com->next; next; cnt++,next=next->next) {
532:     if (next->type == DMCOMPOSITE_ARRAY) {
533:       PetscScalar *array;
534:       array = va_arg(Argp, PetscScalar*);
535:       if (!array) continue;
537:       DMCompositeGather_Array(dm,next,gvec,imode,array);
538:     } else if (next->type == DMCOMPOSITE_DM) {
539:       Vec vec;
540:       vec = va_arg(Argp, Vec);
541:       if (!vec) continue;
543:       DMCompositeGather_DM(dm,next,gvec,imode,vec);
544:     } else {
545:       SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
546:     }
547:   }
548:   va_end(Argp);
549:   return(0);
550: }

554: /*@C
555:     DMCompositeAddArray - adds an "redundant" array to a DMComposite. The array values will 
556:        be stored in part of the array on process orank.

558:     Collective on DMComposite

560:     Input Parameter:
561: +    dm - the packer object
562: .    orank - the process on which the array entries officially live, this number must be
563:              the same on all processes.
564: -    n - the length of the array
565:  
566:     Level: advanced

568: .seealso DMDestroy(), DMCompositeGather(), DMCompositeAddDM(), DMCreateGlobalVector(),
569:          DMCompositeScatter(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
570:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()

572: @*/
573: PetscErrorCode  DMCompositeAddArray(DM dm,PetscMPIInt orank,PetscInt n)
574: {
575:   struct DMCompositeLink *mine,*next;
576:   PetscErrorCode         ierr;
577:   PetscMPIInt            rank;
578:   DM_Composite           *com = (DM_Composite*)dm->data;

582:   next = com->next;
583:   if (com->setup) {
584:     SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_ARG_WRONGSTATE,"Cannot add an array once you have used the DMComposite");
585:   }
586: #if defined(PETSC_USE_DEBUG)
587:   {
588:     PetscMPIInt orankmax;
589:     MPI_Allreduce(&orank,&orankmax,1,MPI_INT,MPI_MAX,((PetscObject)dm)->comm);
590:     if (orank != orankmax) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"orank %d must be equal on all processes, another process has value %d",orank,orankmax);
591:   }
592: #endif

594:   MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
595:   /* create new link */
596:   PetscNew(struct DMCompositeLink,&mine);
597:   mine->nlocal        = n;
598:   mine->n             = (rank == orank) ? n : 0;
599:   mine->rank          = orank;
600:   mine->dm            = PETSC_NULL;
601:   mine->type          = DMCOMPOSITE_ARRAY;
602:   mine->next          = PETSC_NULL;
603:   if (rank == mine->rank) {com->n += n;com->nmine++;}

605:   /* add to end of list */
606:   if (!next) {
607:     com->next = mine;
608:   } else {
609:     while (next->next) next = next->next;
610:     next->next = mine;
611:   }
612:   com->nredundant++;
613:   return(0);
614: }

618: /*@C
619:     DMCompositeAddDM - adds a DM  vector to a DMComposite

621:     Collective on DMComposite

623:     Input Parameter:
624: +    dm - the packer object
625: -    dm - the DM object, if the DM is a da you will need to caste it with a (DM)
626:  
627:     Level: advanced

629: .seealso DMDestroy(), DMCompositeGather(), DMCompositeAddDM(), DMCreateGlobalVector(),
630:          DMCompositeScatter(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
631:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()

633: @*/
634: PetscErrorCode  DMCompositeAddDM(DM dmc,DM dm)
635: {
636:   PetscErrorCode         ierr;
637:   PetscInt               n,nlocal;
638:   struct DMCompositeLink *mine,*next;
639:   Vec                    global,local;
640:   DM_Composite           *com = (DM_Composite*)dmc->data;

645:   next = com->next;
646:   if (com->setup) SETERRQ(((PetscObject)dmc)->comm,PETSC_ERR_ARG_WRONGSTATE,"Cannot add a DM once you have used the DMComposite");

648:   /* create new link */
649:   PetscNew(struct DMCompositeLink,&mine);
650:   PetscObjectReference((PetscObject)dm);
651:   DMGetGlobalVector(dm,&global);
652:   VecGetLocalSize(global,&n);
653:   DMRestoreGlobalVector(dm,&global);
654:   DMGetLocalVector(dm,&local);
655:   VecGetSize(local,&nlocal);
656:   DMRestoreLocalVector(dm,&local);
657:   mine->n      = n;
658:   mine->nlocal = nlocal;
659:   mine->dm     = dm;
660:   mine->type   = DMCOMPOSITE_DM;
661:   mine->next   = PETSC_NULL;
662:   com->n       += n;

664:   /* add to end of list */
665:   if (!next) {
666:     com->next = mine;
667:   } else {
668:     while (next->next) next = next->next;
669:     next->next = mine;
670:   }
671:   com->nDM++;
672:   com->nmine++;
673:   return(0);
674: }

680: PetscErrorCode  VecView_DMComposite(Vec gvec,PetscViewer viewer)
681: {
682:   DM                     dm;
683:   PetscErrorCode         ierr;
684:   struct DMCompositeLink *next;
685:   PetscBool              isdraw;
686:   DM_Composite           *com;

689:   PetscObjectQuery((PetscObject)gvec,"DM",(PetscObject*)&dm);
690:   if (!dm) SETERRQ(((PetscObject)gvec)->comm,PETSC_ERR_ARG_WRONG,"Vector not generated from a DMComposite");
691:   com = (DM_Composite*)dm->data;
692:   next = com->next;

694:   PetscTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw);
695:   if (!isdraw) {
696:     /* do I really want to call this? */
697:     VecView_MPI(gvec,viewer);
698:   } else {
699:     PetscInt cnt = 0;

701:     /* loop over packed objects, handling one at at time */
702:     while (next) {
703:       if (next->type == DMCOMPOSITE_ARRAY) {
704:         PetscScalar *array;
705:         DMCompositeGetAccess_Array(dm,next,gvec,&array);

707:         /*skip it for now */
708:       } else if (next->type == DMCOMPOSITE_DM) {
709:         Vec      vec;
710:         PetscInt bs;

712:         DMCompositeGetAccess_DM(dm,next,gvec,&vec);
713:         VecView(vec,viewer);
714:         VecGetBlockSize(vec,&bs);
715:         DMCompositeRestoreAccess_DM(dm,next,gvec,&vec);
716:         PetscViewerDrawBaseAdd(viewer,bs);
717:         cnt += bs;
718:       } else {
719:         SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
720:       }
721:       next = next->next;
722:     }
723:     PetscViewerDrawBaseAdd(viewer,-cnt);
724:   }
725:   return(0);
726: }


732: PetscErrorCode  DMCreateGlobalVector_Composite(DM dm,Vec *gvec)
733: {
734:   PetscErrorCode         ierr;
735:   DM_Composite           *com = (DM_Composite*)dm->data;

739:   if (!com->setup) {
740:     DMSetUp(dm);
741:   }
742:   VecCreateMPI(((PetscObject)dm)->comm,com->n,com->N,gvec);
743:   PetscObjectCompose((PetscObject)*gvec,"DM",(PetscObject)dm);
744:   VecSetOperation(*gvec,VECOP_VIEW,(void(*)(void))VecView_DMComposite);
745:   return(0);
746: }

750: PetscErrorCode  DMCreateLocalVector_Composite(DM dm,Vec *lvec)
751: {
752:   PetscErrorCode         ierr;
753:   DM_Composite           *com = (DM_Composite*)dm->data;

757:   if (!com->setup) {
758:     DMSetUp(dm);
759:   }
760:   VecCreateSeq(((PetscObject)dm)->comm,com->nghost,lvec);
761:   PetscObjectCompose((PetscObject)*lvec,"DM",(PetscObject)dm);
762:   return(0);
763: }

767: /*@C
768:     DMCompositeGetISLocalToGlobalMappings - gets an ISLocalToGlobalMapping for each DM/array in the DMComposite, maps to the composite global space

770:     Collective on DM

772:     Input Parameter:
773: .    dm - the packer object

775:     Output Parameters:
776: .    ltogs - the individual mappings for each packed vector/array. Note that this includes
777:            all the ghost points that individual ghosted DMDA's may have. Also each process has an
778:            mapping for EACH redundant array (not just the local redundant arrays).

780:     Level: advanced

782:     Notes:
783:        Each entry of ltogs should be destroyed with ISLocalToGlobalMappingDestroy(), the ltogs array should be freed with PetscFree().

785: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
786:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetAccess(), DMCompositeScatter(),
787:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(),DMCompositeGetEntries()

789: @*/
790: PetscErrorCode  DMCompositeGetISLocalToGlobalMappings(DM dm,ISLocalToGlobalMapping **ltogs)
791: {
792:   PetscErrorCode         ierr;
793:   PetscInt               i,*idx,n,cnt;
794:   struct DMCompositeLink *next;
795:   PetscMPIInt            rank;
796:   DM_Composite           *com = (DM_Composite*)dm->data;

800:   PetscMalloc((com->nDM+com->nredundant)*sizeof(ISLocalToGlobalMapping),ltogs);
801:   next = com->next;
802:   MPI_Comm_rank(((PetscObject)dm)->comm,&rank);

804:   /* loop over packed objects, handling one at at time */
805:   cnt = 0;
806:   while (next) {
807:     if (next->type == DMCOMPOSITE_ARRAY) {
808:       PetscMalloc(next->nlocal*sizeof(PetscInt),&idx);
809:       if (rank == next->rank) {
810:         for (i=0; i<next->nlocal; i++) idx[i] = next->grstart + i;
811:       }
812:       MPI_Bcast(idx,next->nlocal,MPIU_INT,next->rank,((PetscObject)dm)->comm);
813:       ISLocalToGlobalMappingCreate(((PetscObject)dm)->comm,next->nlocal,idx,PETSC_OWN_POINTER,&(*ltogs)[cnt]);
814:     } else if (next->type == DMCOMPOSITE_DM) {
815:       ISLocalToGlobalMapping ltog;
816:       PetscMPIInt            size;
817:       const PetscInt         *suboff,*indices;
818:       Vec                    global;

820:       /* Get sub-DM global indices for each local dof */
821:       DMGetLocalToGlobalMapping(next->dm,&ltog);
822:       ISLocalToGlobalMappingGetSize(ltog,&n);
823:       ISLocalToGlobalMappingGetIndices(ltog,&indices);
824:       PetscMalloc(n*sizeof(PetscInt),&idx);

826:       /* Get the offsets for the sub-DM global vector */
827:       DMGetGlobalVector(next->dm,&global);
828:       VecGetOwnershipRanges(global,&suboff);
829:       MPI_Comm_size(((PetscObject)global)->comm,&size);

831:       /* Shift the sub-DM definition of the global space to the composite global space */
832:       for (i=0; i<n; i++) {
833:         PetscInt subi = indices[i],lo = 0,hi = size,t;
834:         /* Binary search to find which rank owns subi */
835:         while (hi-lo > 1) {
836:           t = lo + (hi-lo)/2;
837:           if (suboff[t] > subi) hi = t;
838:           else                  lo = t;
839:         }
840:         idx[i] = subi - suboff[lo] + next->grstarts[lo];
841:       }
842:       ISLocalToGlobalMappingRestoreIndices(ltog,&indices);
843:       ISLocalToGlobalMappingCreate(((PetscObject)dm)->comm,n,idx,PETSC_OWN_POINTER,&(*ltogs)[cnt]);
844:       DMRestoreGlobalVector(next->dm,&global);
845:     } else SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
846:     next = next->next;
847:     cnt++;
848:   }
849:   return(0);
850: }

854: /*@C
855:    DMCompositeGetLocalISs - Gets index sets for each DM/array component of a composite local vector

857:    Not Collective

859:    Input Arguments:
860: . dm - composite DM

862:    Output Arguments:
863: . is - array of serial index sets for each each component of the DMComposite

865:    Level: intermediate

867:    Notes:
868:    At present, a composite local vector does not normally exist.  This function is used to provide index sets for
869:    MatGetLocalSubMatrix().  In the future, the scatters for each entry in the DMComposite may be be merged into a single
870:    scatter to a composite local vector.

872:    To get the composite global indices at all local points (including ghosts), use DMCompositeGetISLocalToGlobalMappings().

874:    To get index sets for pieces of the composite global vector, use DMCompositeGetGlobalISs().

876:    Each returned IS should be destroyed with ISDestroy(), the array should be freed with PetscFree().

878: .seealso: DMCompositeGetGlobalISs(), DMCompositeGetISLocalToGlobalMappings(), MatGetLocalSubMatrix(), MatCreateLocalRef()
879: @*/
880: PetscErrorCode  DMCompositeGetLocalISs(DM dm,IS **is)
881: {
882:   PetscErrorCode         ierr;
883:   DM_Composite           *com = (DM_Composite*)dm->data;
884:   struct DMCompositeLink *link;
885:   PetscInt cnt,start;

890:   PetscMalloc(com->nmine*sizeof(IS),is);
891:   for (cnt=0,start=0,link=com->next; link; start+=link->nlocal,cnt++,link=link->next) {
892:     ISCreateStride(PETSC_COMM_SELF,link->nlocal,start,1,&(*is)[cnt]);
893:     if (link->type == DMCOMPOSITE_DM) {
894:       PetscInt bs;
895:       DMGetBlockSize(link->dm,&bs);
896:       ISSetBlockSize((*is)[cnt],bs);
897:     }
898:   }
899:   return(0);
900: }

904: /*@C
905:     DMCompositeGetGlobalISs - Gets the index sets for each composed object

907:     Collective on DMComposite

909:     Input Parameter:
910: .    dm - the packer object

912:     Output Parameters:
913: .    is - the array of index sets

915:     Level: advanced

917:     Notes:
918:        The is entries should be destroyed with ISDestroy(), the is array should be freed with PetscFree()

920:        These could be used to extract a subset of vector entries for a "multi-physics" preconditioner

922:        Use DMCompositeGetLocalISs() for index sets in the packed local numbering, and
923:        DMCompositeGetISLocalToGlobalMappings() for to map local sub-DM (including ghost) indices to packed global
924:        indices.

926: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
927:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetAccess(), DMCompositeScatter(),
928:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(),DMCompositeGetEntries()

930: @*/

932: PetscErrorCode  DMCompositeGetGlobalISs(DM dm,IS *is[])
933: {
934:   PetscErrorCode         ierr;
935:   PetscInt               cnt = 0,*idx,i;
936:   struct DMCompositeLink *next;
937:   PetscMPIInt            rank;
938:   DM_Composite           *com = (DM_Composite*)dm->data;

942:   PetscMalloc((com->nDM+com->nredundant)*sizeof(IS),is);
943:   next = com->next;
944:   MPI_Comm_rank(((PetscObject)dm)->comm,&rank);

946:   /* loop over packed objects, handling one at at time */
947:   while (next) {
948:     PetscMalloc(next->n*sizeof(PetscInt),&idx);
949:     for (i=0; i<next->n; i++) idx[i] = next->grstart + i;
950:     ISCreateGeneral(((PetscObject)dm)->comm,next->n,idx,PETSC_OWN_POINTER,&(*is)[cnt]);
951:     cnt++;
952:     next = next->next;
953:   }
954:   return(0);
955: }

957: /* -------------------------------------------------------------------------------------*/
960: PetscErrorCode DMCompositeGetLocalVectors_Array(DM dm,struct DMCompositeLink *mine,PetscScalar **array)
961: {
964:   if (array) {
965:     PetscMalloc(mine->nlocal*sizeof(PetscScalar),array);
966:   }
967:   return(0);
968: }

972: PetscErrorCode DMCompositeGetLocalVectors_DM(DM dm,struct DMCompositeLink *mine,Vec *local)
973: {
976:   if (local) {
977:     DMGetLocalVector(mine->dm,local);
978:   }
979:   return(0);
980: }

984: PetscErrorCode DMCompositeRestoreLocalVectors_Array(DM dm,struct DMCompositeLink *mine,PetscScalar **array)
985: {
988:   if (array) {
989:     PetscFree(*array);
990:   }
991:   return(0);
992: }

996: PetscErrorCode DMCompositeRestoreLocalVectors_DM(DM dm,struct DMCompositeLink *mine,Vec *local)
997: {
1000:   if (local) {
1001:     DMRestoreLocalVector(mine->dm,local);
1002:   }
1003:   return(0);
1004: }

1008: /*@C
1009:     DMCompositeGetLocalVectors - Gets local vectors and arrays for each part of a DMComposite.
1010:        Use DMCompositeRestoreLocalVectors() to return them.

1012:     Not Collective

1014:     Input Parameter:
1015: .    dm - the packer object
1016:  
1017:     Output Parameter:
1018: .    ... - the individual sequential objects (arrays or vectors)
1019:  
1020:     Level: advanced

1022: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
1023:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
1024:          DMCompositeRestoreLocalVectors(), DMCompositeScatter(), DMCompositeGetEntries()

1026: @*/
1027: PetscErrorCode  DMCompositeGetLocalVectors(DM dm,...)
1028: {
1029:   va_list                Argp;
1030:   PetscErrorCode         ierr;
1031:   struct DMCompositeLink *next;
1032:   DM_Composite           *com = (DM_Composite*)dm->data;

1036:   next = com->next;
1037:   /* loop over packed objects, handling one at at time */
1038:   va_start(Argp,dm);
1039:   while (next) {
1040:     if (next->type == DMCOMPOSITE_ARRAY) {
1041:       PetscScalar **array;
1042:       array = va_arg(Argp, PetscScalar**);
1043:       DMCompositeGetLocalVectors_Array(dm,next,array);
1044:     } else if (next->type == DMCOMPOSITE_DM) {
1045:       Vec *vec;
1046:       vec = va_arg(Argp, Vec*);
1047:       DMCompositeGetLocalVectors_DM(dm,next,vec);
1048:     } else {
1049:       SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1050:     }
1051:     next = next->next;
1052:   }
1053:   va_end(Argp);
1054:   return(0);
1055: }

1059: /*@C
1060:     DMCompositeRestoreLocalVectors - Restores local vectors and arrays for each part of a DMComposite.

1062:     Not Collective

1064:     Input Parameter:
1065: .    dm - the packer object
1066:  
1067:     Output Parameter:
1068: .    ... - the individual sequential objects (arrays or vectors)
1069:  
1070:     Level: advanced

1072: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
1073:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
1074:          DMCompositeGetLocalVectors(), DMCompositeScatter(), DMCompositeGetEntries()

1076: @*/
1077: PetscErrorCode  DMCompositeRestoreLocalVectors(DM dm,...)
1078: {
1079:   va_list                Argp;
1080:   PetscErrorCode         ierr;
1081:   struct DMCompositeLink *next;
1082:   DM_Composite           *com = (DM_Composite*)dm->data;

1086:   next = com->next;
1087:   /* loop over packed objects, handling one at at time */
1088:   va_start(Argp,dm);
1089:   while (next) {
1090:     if (next->type == DMCOMPOSITE_ARRAY) {
1091:       PetscScalar **array;
1092:       array = va_arg(Argp, PetscScalar**);
1093:       DMCompositeRestoreLocalVectors_Array(dm,next,array);
1094:     } else if (next->type == DMCOMPOSITE_DM) {
1095:       Vec *vec;
1096:       vec = va_arg(Argp, Vec*);
1097:       DMCompositeRestoreLocalVectors_DM(dm,next,vec);
1098:     } else {
1099:       SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1100:     }
1101:     next = next->next;
1102:   }
1103:   va_end(Argp);
1104:   return(0);
1105: }

1107: /* -------------------------------------------------------------------------------------*/
1110: PetscErrorCode DMCompositeGetEntries_Array(DM dm,struct DMCompositeLink *mine,PetscInt *n)
1111: {
1113:   if (n) *n = mine->nlocal;
1114:   return(0);
1115: }

1119: PetscErrorCode DMCompositeGetEntries_DM(DM dmi,struct DMCompositeLink *mine,DM *dm)
1120: {
1122:   if (dm) *dm = mine->dm;
1123:   return(0);
1124: }

1128: /*@C
1129:     DMCompositeGetEntries - Gets the DM, redundant size, etc for each entry in a DMComposite.

1131:     Not Collective

1133:     Input Parameter:
1134: .    dm - the packer object
1135:  
1136:     Output Parameter:
1137: .    ... - the individual entries, DMs or integer sizes)
1138:  
1139:     Level: advanced

1141: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
1142:          DMCompositeGather(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
1143:          DMCompositeRestoreLocalVectors(), DMCompositeGetLocalVectors(),  DMCompositeScatter(),
1144:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors()

1146: @*/
1147: PetscErrorCode  DMCompositeGetEntries(DM dm,...)
1148: {
1149:   va_list                Argp;
1150:   PetscErrorCode         ierr;
1151:   struct DMCompositeLink *next;
1152:   DM_Composite           *com = (DM_Composite*)dm->data;

1156:   next = com->next;
1157:   /* loop over packed objects, handling one at at time */
1158:   va_start(Argp,dm);
1159:   while (next) {
1160:     if (next->type == DMCOMPOSITE_ARRAY) {
1161:       PetscInt *n;
1162:       n = va_arg(Argp, PetscInt*);
1163:       DMCompositeGetEntries_Array(dm,next,n);
1164:     } else if (next->type == DMCOMPOSITE_DM) {
1165:       DM *dmn;
1166:       dmn = va_arg(Argp, DM*);
1167:       DMCompositeGetEntries_DM(dm,next,dmn);
1168:     } else {
1169:       SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1170:     }
1171:     next = next->next;
1172:   }
1173:   va_end(Argp);
1174:   return(0);
1175: }

1179: PetscErrorCode  DMRefine_Composite(DM dmi,MPI_Comm comm,DM *fine)
1180: {
1181:   PetscErrorCode         ierr;
1182:   struct DMCompositeLink *next;
1183:   DM_Composite           *com = (DM_Composite*)dmi->data;
1184:   DM                     dm;

1188:   next = com->next;
1189:   DMCompositeCreate(comm,fine);

1191:   /* loop over packed objects, handling one at at time */
1192:   while (next) {
1193:     if (next->type == DMCOMPOSITE_ARRAY) {
1194:       DMCompositeAddArray(*fine,next->rank,next->nlocal);
1195:     } else if (next->type == DMCOMPOSITE_DM) {
1196:       DMRefine(next->dm,comm,&dm);
1197:       DMCompositeAddDM(*fine,dm);
1198:       PetscObjectDereference((PetscObject)dm);
1199:     } else {
1200:       SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1201:     }
1202:     next = next->next;
1203:   }
1204:   return(0);
1205: }

1209: PetscErrorCode  DMCoarsen_Composite(DM dmi,MPI_Comm comm,DM *fine)
1210: {
1211:   PetscErrorCode         ierr;
1212:   struct DMCompositeLink *next;
1213:   DM_Composite           *com = (DM_Composite*)dmi->data;
1214:   DM                     dm;

1218:   next = com->next;
1219:   DMCompositeCreate(comm,fine);

1221:   /* loop over packed objects, handling one at at time */
1222:   while (next) {
1223:     if (next->type == DMCOMPOSITE_ARRAY) {
1224:       DMCompositeAddArray(*fine,next->rank,next->nlocal);
1225:     } else if (next->type == DMCOMPOSITE_DM) {
1226:       DMCoarsen(next->dm,comm,&dm);
1227:       DMCompositeAddDM(*fine,dm);
1228:       PetscObjectDereference((PetscObject)dm);
1229:     } else {
1230:       SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1231:     }
1232:     next = next->next;
1233:   }
1234:   return(0);
1235: }

1237: struct MatPackLink {
1238:   Mat                A;
1239:   struct MatPackLink *next;
1240: };

1242: struct MatPack {
1243:   DM                 right,left;
1244:   struct MatPackLink *next;
1245: };

1249: PetscErrorCode MatMultBoth_Shell_Pack(Mat A,Vec x,Vec y,PetscBool  add)
1250: {
1251:   struct MatPack         *mpack;
1252:   struct DMCompositeLink *xnext,*ynext;
1253:   struct MatPackLink     *anext;
1254:   PetscScalar            *xarray,*yarray;
1255:   PetscErrorCode         ierr;
1256:   PetscInt               i;
1257:   Vec                    xglobal,yglobal;
1258:   PetscMPIInt            rank;
1259:   DM_Composite           *comright;
1260:   DM_Composite           *comleft;

1263:   MatShellGetContext(A,(void**)&mpack);
1264:   MPI_Comm_rank(((PetscObject)mpack->right)->comm,&rank);
1265:   comright = (DM_Composite*)mpack->right->data;
1266:   comleft = (DM_Composite*)mpack->left->data;
1267:   xnext = comright->next;
1268:   ynext = comleft->next;
1269:   anext = mpack->next;

1271:   while (xnext) {
1272:     if (xnext->type == DMCOMPOSITE_ARRAY) {
1273:       if (rank == xnext->rank) {
1274:         VecGetArray(x,&xarray);
1275:         VecGetArray(y,&yarray);
1276:         if (add) {
1277:           for (i=0; i<xnext->n; i++) {
1278:             yarray[ynext->rstart+i] += xarray[xnext->rstart+i];
1279:           }
1280:         } else {
1281:           PetscMemcpy(yarray+ynext->rstart,xarray+xnext->rstart,xnext->n*sizeof(PetscScalar));
1282:         }
1283:         VecRestoreArray(x,&xarray);
1284:         VecRestoreArray(y,&yarray);
1285:       }
1286:     } else if (xnext->type == DMCOMPOSITE_DM) {
1287:       VecGetArray(x,&xarray);
1288:       VecGetArray(y,&yarray);
1289:       DMGetGlobalVector(xnext->dm,&xglobal);
1290:       DMGetGlobalVector(ynext->dm,&yglobal);
1291:       VecPlaceArray(xglobal,xarray+xnext->rstart);
1292:       VecPlaceArray(yglobal,yarray+ynext->rstart);
1293:       if (add) {
1294:         MatMultAdd(anext->A,xglobal,yglobal,yglobal);
1295:       } else {
1296:         MatMult(anext->A,xglobal,yglobal);
1297:       }
1298:       VecRestoreArray(x,&xarray);
1299:       VecRestoreArray(y,&yarray);
1300:       VecResetArray(xglobal);
1301:       VecResetArray(yglobal);
1302:       DMRestoreGlobalVector(xnext->dm,&xglobal);
1303:       DMRestoreGlobalVector(ynext->dm,&yglobal);
1304:       anext = anext->next;
1305:     } else {
1306:       SETERRQ(((PetscObject)A)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1307:     }
1308:     xnext = xnext->next;
1309:     ynext = ynext->next;
1310:   }
1311:   return(0);
1312: }

1316: PetscErrorCode MatMultAdd_Shell_Pack(Mat A,Vec x,Vec y,Vec z)
1317: {
1320:   if (z != y) SETERRQ(((PetscObject)A)->comm,PETSC_ERR_SUP,"Handles y == z only");
1321:   MatMultBoth_Shell_Pack(A,x,y,PETSC_TRUE);
1322:   return(0);
1323: }

1327: PetscErrorCode MatMult_Shell_Pack(Mat A,Vec x,Vec y)
1328: {
1331:   MatMultBoth_Shell_Pack(A,x,y,PETSC_FALSE);
1332:   return(0);
1333: }

1337: PetscErrorCode MatMultTranspose_Shell_Pack(Mat A,Vec x,Vec y)
1338: {
1339:   struct MatPack         *mpack;
1340:   struct DMCompositeLink *xnext,*ynext;
1341:   struct MatPackLink     *anext;
1342:   PetscScalar            *xarray,*yarray;
1343:   PetscErrorCode         ierr;
1344:   Vec                    xglobal,yglobal;
1345:   PetscMPIInt            rank;
1346:   DM_Composite           *comright;
1347:   DM_Composite           *comleft;

1350:   MatShellGetContext(A,(void**)&mpack);
1351:   MPI_Comm_rank(((PetscObject)mpack->right)->comm,&rank);
1352:   comright = (DM_Composite*)mpack->right->data;
1353:   comleft = (DM_Composite*)mpack->left->data;
1354:   ynext = comright->next;
1355:   xnext = comleft->next;
1356:   anext = mpack->next;

1358:   while (xnext) {
1359:     if (xnext->type == DMCOMPOSITE_ARRAY) {
1360:       if (rank == ynext->rank) {
1361:         VecGetArray(x,&xarray);
1362:         VecGetArray(y,&yarray);
1363:         PetscMemcpy(yarray+ynext->rstart,xarray+xnext->rstart,xnext->n*sizeof(PetscScalar));
1364:         VecRestoreArray(x,&xarray);
1365:         VecRestoreArray(y,&yarray);
1366:       }
1367:     } else if (xnext->type == DMCOMPOSITE_DM) {
1368:       VecGetArray(x,&xarray);
1369:       VecGetArray(y,&yarray);
1370:       DMGetGlobalVector(xnext->dm,&xglobal);
1371:       DMGetGlobalVector(ynext->dm,&yglobal);
1372:       VecPlaceArray(xglobal,xarray+xnext->rstart);
1373:       VecPlaceArray(yglobal,yarray+ynext->rstart);
1374:       MatMultTranspose(anext->A,xglobal,yglobal);
1375:       VecRestoreArray(x,&xarray);
1376:       VecRestoreArray(y,&yarray);
1377:       VecResetArray(xglobal);
1378:       VecResetArray(yglobal);
1379:       DMRestoreGlobalVector(xnext->dm,&xglobal);
1380:       DMRestoreGlobalVector(ynext->dm,&yglobal);
1381:       anext = anext->next;
1382:     } else {
1383:       SETERRQ(((PetscObject)A)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1384:     }
1385:     xnext = xnext->next;
1386:     ynext = ynext->next;
1387:   }
1388:   return(0);
1389: }

1393: PetscErrorCode MatDestroy_Shell_Pack(Mat A)
1394: {
1395:   struct MatPack     *mpack;
1396:   struct MatPackLink *anext,*oldanext;
1397:   PetscErrorCode     ierr;

1400:   MatShellGetContext(A,(void**)&mpack);
1401:   anext = mpack->next;

1403:   while (anext) {
1404:     MatDestroy(&anext->A);
1405:     oldanext = anext;
1406:     anext    = anext->next;
1407:     PetscFree(oldanext);
1408:   }
1409:   PetscFree(mpack);
1410:   PetscObjectChangeTypeName((PetscObject)A,0);
1411:   return(0);
1412: }

1416: PetscErrorCode  DMGetInterpolation_Composite(DM coarse,DM fine,Mat *A,Vec *v)
1417: {
1418:   PetscErrorCode         ierr;
1419:   PetscInt               m,n,M,N;
1420:   struct DMCompositeLink *nextc;
1421:   struct DMCompositeLink *nextf;
1422:   struct MatPackLink     *nextmat,*pnextmat = 0;
1423:   struct MatPack         *mpack;
1424:   Vec                    gcoarse,gfine;
1425:   DM_Composite           *comcoarse = (DM_Composite*)coarse->data;
1426:   DM_Composite           *comfine = (DM_Composite*)fine->data;

1431:   nextc = comcoarse->next;
1432:   nextf = comfine->next;
1433:   /* use global vectors only for determining matrix layout */
1434:   DMCreateGlobalVector(coarse,&gcoarse);
1435:   DMCreateGlobalVector(fine,&gfine);
1436:   VecGetLocalSize(gcoarse,&n);
1437:   VecGetLocalSize(gfine,&m);
1438:   VecGetSize(gcoarse,&N);
1439:   VecGetSize(gfine,&M);
1440:   VecDestroy(&gcoarse);
1441:   VecDestroy(&gfine);

1443:   PetscNew(struct MatPack,&mpack);
1444:   mpack->right = coarse;
1445:   mpack->left  = fine;
1446:   MatCreate(((PetscObject)fine)->comm,A);
1447:   MatSetSizes(*A,m,n,M,N);
1448:   MatSetType(*A,MATSHELL);
1449:   MatShellSetContext(*A,mpack);
1450:   MatShellSetOperation(*A,MATOP_MULT,(void(*)(void))MatMult_Shell_Pack);
1451:   MatShellSetOperation(*A,MATOP_MULT_TRANSPOSE,(void(*)(void))MatMultTranspose_Shell_Pack);
1452:   MatShellSetOperation(*A,MATOP_MULT_ADD,(void(*)(void))MatMultAdd_Shell_Pack);
1453:   MatShellSetOperation(*A,MATOP_DESTROY,(void(*)(void))MatDestroy_Shell_Pack);

1455:   /* loop over packed objects, handling one at at time */
1456:   while (nextc) {
1457:     if (nextc->type != nextf->type) SETERRQ(((PetscObject)fine)->comm,PETSC_ERR_ARG_INCOMP,"Two DM have different layout");

1459:     if (nextc->type == DMCOMPOSITE_ARRAY) {
1460:       ;
1461:     } else if (nextc->type == DMCOMPOSITE_DM) {
1462:       PetscNew(struct MatPackLink,&nextmat);
1463:       nextmat->next = 0;
1464:       if (pnextmat) {
1465:         pnextmat->next = nextmat;
1466:         pnextmat       = nextmat;
1467:       } else {
1468:         pnextmat    = nextmat;
1469:         mpack->next = nextmat;
1470:       }
1471:       DMGetInterpolation(nextc->dm,nextf->dm,&nextmat->A,PETSC_NULL);
1472:     } else {
1473:       SETERRQ(((PetscObject)fine)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1474:     }
1475:     nextc = nextc->next;
1476:     nextf = nextf->next;
1477:   }
1478:   return(0);
1479: }

1483: static PetscErrorCode DMCreateLocalToGlobalMapping_Composite(DM dm)
1484: {
1485:   DM_Composite           *com = (DM_Composite*)dm->data;
1486:   ISLocalToGlobalMapping *ltogs;
1487:   PetscInt               i;
1488:   PetscErrorCode         ierr;

1491:   /* Set the ISLocalToGlobalMapping on the new matrix */
1492:   DMCompositeGetISLocalToGlobalMappings(dm,&ltogs);
1493:   ISLocalToGlobalMappingConcatenate(((PetscObject)dm)->comm,com->nDM+com->nredundant,ltogs,&dm->ltogmap);
1494:   for (i=0; i<com->nDM+com->nredundant; i++) {ISLocalToGlobalMappingDestroy(&ltogs[i]);}
1495:   PetscFree(ltogs);
1496:   return(0);
1497: }


1502: PetscErrorCode  DMGetColoring_Composite(DM dm,ISColoringType ctype,const MatType mtype,ISColoring *coloring)
1503: {
1504:   PetscErrorCode         ierr;
1505:   PetscInt               n,i,cnt;
1506:   ISColoringValue        *colors;
1507:   PetscBool              dense = PETSC_FALSE;
1508:   ISColoringValue        maxcol = 0;
1509:   DM_Composite           *com = (DM_Composite*)dm->data;

1513:   if (ctype == IS_COLORING_GHOSTED) {
1514:     SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Currently you must use -dmmg_iscoloring_type global" );
1515:   } else if (ctype == IS_COLORING_GLOBAL) {
1516:     n = com->n;
1517:   } else SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_ARG_OUTOFRANGE,"Unknown ISColoringType");
1518:   PetscMalloc(n*sizeof(ISColoringValue),&colors); /* freed in ISColoringDestroy() */

1520:   PetscOptionsGetBool(PETSC_NULL,"-dmcomposite_dense_jacobian",&dense,PETSC_NULL);
1521:   if (dense) {
1522:     for (i=0; i<n; i++) {
1523:       colors[i] = (ISColoringValue)(com->rstart + i);
1524:     }
1525:     maxcol = com->N;
1526:   } else {
1527:     struct DMCompositeLink *next = com->next;
1528:     PetscMPIInt            rank;
1529: 
1530:     MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
1531:     cnt  = 0;
1532:     while (next) {
1533:       if (next->type == DMCOMPOSITE_ARRAY) {
1534:         if (rank == next->rank) {  /* each column gets is own color */
1535:           for (i=com->rstart+next->rstart; i<com->rstart+next->rstart+next->n; i++) {
1536:             colors[cnt++] = maxcol++;
1537:           }
1538:         }
1539:         MPI_Bcast(&maxcol,1,MPIU_COLORING_VALUE,next->rank,((PetscObject)dm)->comm);
1540:       } else if (next->type == DMCOMPOSITE_DM) {
1541:         ISColoring     lcoloring;

1543:         DMGetColoring(next->dm,IS_COLORING_GLOBAL,mtype,&lcoloring);
1544:         for (i=0; i<lcoloring->N; i++) {
1545:           colors[cnt++] = maxcol + lcoloring->colors[i];
1546:         }
1547:         maxcol += lcoloring->n;
1548:         ISColoringDestroy(&lcoloring);
1549:       } else SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1550:       next = next->next;
1551:     }
1552:   }
1553:   ISColoringCreate(((PetscObject)dm)->comm,maxcol,n,colors,coloring);
1554:   return(0);
1555: }

1559: PetscErrorCode  DMGlobalToLocalBegin_Composite(DM dm,Vec gvec,InsertMode mode,Vec lvec)
1560: {
1561:   PetscErrorCode         ierr;
1562:   struct DMCompositeLink *next;
1563:   PetscInt               cnt = 3;
1564:   PetscMPIInt            rank;
1565:   PetscScalar            *garray,*larray;
1566:   DM_Composite           *com = (DM_Composite*)dm->data;

1571:   next = com->next;
1572:   if (!com->setup) {
1573:     DMSetUp(dm);
1574:   }
1575:   MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
1576:   VecGetArray(gvec,&garray);
1577:   VecGetArray(lvec,&larray);

1579:   /* loop over packed objects, handling one at at time */
1580:   while (next) {
1581:     if (next->type == DMCOMPOSITE_ARRAY) {
1582:       if (rank == next->rank) {
1583:         PetscMemcpy(larray,garray,next->n*sizeof(PetscScalar));
1584:         garray += next->n;
1585:       }
1586:       /* does not handle ADD_VALUES */
1587:       MPI_Bcast(larray,next->nlocal,MPIU_SCALAR,next->rank,((PetscObject)dm)->comm);
1588:     } else if (next->type == DMCOMPOSITE_DM) {
1589:       Vec      local,global;
1590:       PetscInt N;

1592:       DMGetGlobalVector(next->dm,&global);
1593:       VecGetLocalSize(global,&N);
1594:       VecPlaceArray(global,garray);
1595:       DMGetLocalVector(next->dm,&local);
1596:       VecPlaceArray(local,larray);
1597:       DMGlobalToLocalBegin(next->dm,global,mode,local);
1598:       DMGlobalToLocalEnd(next->dm,global,mode,local);
1599:       VecResetArray(global);
1600:       VecResetArray(local);
1601:       DMRestoreGlobalVector(next->dm,&global);
1602:       DMRestoreGlobalVector(next->dm,&local);
1603:     } else {
1604:       SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1605:     }
1606:     cnt++;
1607:     larray += next->nlocal;
1608:     next    = next->next;
1609:   }

1611:   VecRestoreArray(gvec,PETSC_NULL);
1612:   VecRestoreArray(lvec,PETSC_NULL);
1613:   return(0);
1614: }

1618: PetscErrorCode  DMGlobalToLocalEnd_Composite(DM dm,Vec gvec,InsertMode mode,Vec lvec)
1619: {
1621:   return(0);
1622: }

1627: PetscErrorCode  DMCreate_Composite(DM p)
1628: {
1630:   DM_Composite   *com;

1633:   PetscNewLog(p,DM_Composite,&com);
1634:   p->data = com;
1635:   PetscObjectChangeTypeName((PetscObject)p,"DMComposite");
1636:   com->n            = 0;
1637:   com->next         = PETSC_NULL;
1638:   com->nredundant   = 0;
1639:   com->nDM          = 0;

1641:   p->ops->createglobalvector              = DMCreateGlobalVector_Composite;
1642:   p->ops->createlocalvector               = DMCreateLocalVector_Composite;
1643:   p->ops->createlocaltoglobalmapping      = DMCreateLocalToGlobalMapping_Composite;
1644:   p->ops->createlocaltoglobalmappingblock = 0;
1645:   p->ops->refine                          = DMRefine_Composite;
1646:   p->ops->coarsen                         = DMCoarsen_Composite;
1647:   p->ops->getinterpolation                = DMGetInterpolation_Composite;
1648:   p->ops->getmatrix                       = DMGetMatrix_Composite;
1649:   p->ops->getcoloring                     = DMGetColoring_Composite;
1650:   p->ops->globaltolocalbegin              = DMGlobalToLocalBegin_Composite;
1651:   p->ops->globaltolocalend                = DMGlobalToLocalEnd_Composite;
1652:   p->ops->destroy                         = DMDestroy_Composite;
1653:   p->ops->view                            = DMView_Composite;
1654:   p->ops->setup                           = DMSetUp_Composite;
1655:   return(0);
1656: }

1661: /*@C
1662:     DMCompositeCreate - Creates a vector packer, used to generate "composite"
1663:       vectors made up of several subvectors.

1665:     Collective on MPI_Comm

1667:     Input Parameter:
1668: .   comm - the processors that will share the global vector

1670:     Output Parameters:
1671: .   packer - the packer object

1673:     Level: advanced

1675: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCompositeScatter(),
1676:          DMCompositeGather(), DMCreateGlobalVector(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess()
1677:          DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()

1679: @*/
1680: PetscErrorCode  DMCompositeCreate(MPI_Comm comm,DM *packer)
1681: {

1686:   DMCreate(comm,packer);
1687:   DMSetType(*packer,DMCOMPOSITE);
1688:   return(0);
1689: }