Actual source code: pack.c
2: #include packimpl.h
6: /*@C
7: DMCompositeSetCoupling - Sets user provided routines that compute the coupling between the
8: seperate components (DMDA's and arrays) in a DMto build the correct matrix nonzero structure.
11: Logically Collective on MPI_Comm
13: Input Parameter:
14: + dm - the composite object
15: - formcouplelocations - routine to set the nonzero locations in the matrix
17: Level: advanced
19: Notes: See DMSetApplicationContext() and DMGetApplicationContext() for how to get user information into
20: this routine
22: @*/
23: PetscErrorCode DMCompositeSetCoupling(DM dm,PetscErrorCode (*FormCoupleLocations)(DM,Mat,PetscInt*,PetscInt*,PetscInt,PetscInt,PetscInt,PetscInt))
24: {
25: DM_Composite *com = (DM_Composite*)dm->data;
28: com->FormCoupleLocations = FormCoupleLocations;
29: return(0);
30: }
34: PetscErrorCode DMDestroy_Composite(DM dm)
35: {
36: PetscErrorCode ierr;
37: struct DMCompositeLink *next, *prev;
38: DM_Composite *com = (DM_Composite *)dm->data;
41: next = com->next;
42: while (next) {
43: prev = next;
44: next = next->next;
45: if (prev->type == DMCOMPOSITE_DM) {
46: DMDestroy(&prev->dm);
47: }
48: PetscFree(prev->grstarts);
49: PetscFree(prev);
50: }
51: return(0);
52: }
56: PetscErrorCode DMView_Composite(DM dm,PetscViewer v)
57: {
59: PetscBool iascii;
60: DM_Composite *com = (DM_Composite *)dm->data;
63: PetscTypeCompare((PetscObject)v,PETSCVIEWERASCII,&iascii);
64: if (iascii) {
65: struct DMCompositeLink *lnk = com->next;
66: PetscInt i;
68: PetscViewerASCIIPrintf(v,"DM (%s)\n",((PetscObject)dm)->prefix?((PetscObject)dm)->prefix:"no prefix");
69: PetscViewerASCIIPrintf(v," contains %d DMs and %d redundant arrays\n",com->nDM,com->nredundant);
70: PetscViewerASCIIPushTab(v);
71: for (i=0; lnk; lnk=lnk->next,i++) {
72: if (lnk->dm) {
73: PetscViewerASCIIPrintf(v,"Link %d: DM of type %s\n",i,((PetscObject)lnk->dm)->type_name);
74: PetscViewerASCIIPushTab(v);
75: DMView(lnk->dm,v);
76: PetscViewerASCIIPopTab(v);
77: } else {
78: PetscViewerASCIIPrintf(v,"Link %d: Redundant array of size %d owned by rank %d\n",i,lnk->nlocal,lnk->rank);
79: }
80: }
81: PetscViewerASCIIPopTab(v);
82: }
83: return(0);
84: }
86: /* --------------------------------------------------------------------------------------*/
89: PetscErrorCode DMSetUp_Composite(DM dm)
90: {
91: PetscErrorCode ierr;
92: PetscInt nprev = 0;
93: PetscMPIInt rank,size;
94: DM_Composite *com = (DM_Composite*)dm->data;
95: struct DMCompositeLink *next = com->next;
96: PetscLayout map;
99: if (com->setup) SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_ARG_WRONGSTATE,"Packer has already been setup");
100: PetscLayoutCreate(((PetscObject)dm)->comm,&map);
101: PetscLayoutSetLocalSize(map,com->n);
102: PetscLayoutSetSize(map,PETSC_DETERMINE);
103: PetscLayoutSetBlockSize(map,1);
104: PetscLayoutSetUp(map);
105: PetscLayoutGetSize(map,&com->N);
106: PetscLayoutGetRange(map,&com->rstart,PETSC_NULL);
107: PetscLayoutDestroy(&map);
108:
109: /* now set the rstart for each linked array/vector */
110: MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
111: MPI_Comm_size(((PetscObject)dm)->comm,&size);
112: while (next) {
113: next->rstart = nprev;
114: nprev += next->n;
115: next->grstart = com->rstart + next->rstart;
116: if (next->type == DMCOMPOSITE_ARRAY) {
117: MPI_Bcast(&next->grstart,1,MPIU_INT,next->rank,((PetscObject)dm)->comm);
118: } else {
119: PetscMalloc(size*sizeof(PetscInt),&next->grstarts);
120: MPI_Allgather(&next->grstart,1,MPIU_INT,next->grstarts,1,MPIU_INT,((PetscObject)dm)->comm);
121: }
122: next = next->next;
123: }
124: com->setup = PETSC_TRUE;
125: return(0);
126: }
131: PetscErrorCode DMCompositeGetAccess_Array(DM dm,struct DMCompositeLink *mine,Vec vec,PetscScalar **array)
132: {
134: PetscScalar *varray;
135: PetscMPIInt rank;
138: MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
139: if (array) {
140: if (rank == mine->rank) {
141: VecGetArray(vec,&varray);
142: *array = varray + mine->rstart;
143: VecRestoreArray(vec,&varray);
144: } else {
145: *array = 0;
146: }
147: }
148: return(0);
149: }
153: PetscErrorCode DMCompositeGetAccess_DM(DM dm,struct DMCompositeLink *mine,Vec vec,Vec *global)
154: {
156: PetscScalar *array;
159: if (global) {
160: DMGetGlobalVector(mine->dm,global);
161: VecGetArray(vec,&array);
162: VecPlaceArray(*global,array+mine->rstart);
163: VecRestoreArray(vec,&array);
164: }
165: return(0);
166: }
170: PetscErrorCode DMCompositeRestoreAccess_Array(DM dm,struct DMCompositeLink *mine,Vec vec,PetscScalar **array)
171: {
173: return(0);
174: }
178: PetscErrorCode DMCompositeRestoreAccess_DM(DM dm,struct DMCompositeLink *mine,Vec vec,Vec *global)
179: {
183: if (global) {
184: VecResetArray(*global);
185: DMRestoreGlobalVector(mine->dm,global);
186: }
187: return(0);
188: }
192: PetscErrorCode DMCompositeScatter_Array(DM dm,struct DMCompositeLink *mine,Vec vec,PetscScalar *array)
193: {
195: PetscScalar *varray;
196: PetscMPIInt rank;
199: MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
200: if (rank == mine->rank) {
201: VecGetArray(vec,&varray);
202: PetscMemcpy(array,varray+mine->rstart,mine->n*sizeof(PetscScalar));
203: VecRestoreArray(vec,&varray);
204: }
205: MPI_Bcast(array,mine->nlocal,MPIU_SCALAR,mine->rank,((PetscObject)dm)->comm);
206: return(0);
207: }
211: PetscErrorCode DMCompositeScatter_DM(DM dm,struct DMCompositeLink *mine,Vec vec,Vec local)
212: {
214: PetscScalar *array;
215: Vec global;
218: DMGetGlobalVector(mine->dm,&global);
219: VecGetArray(vec,&array);
220: VecPlaceArray(global,array+mine->rstart);
221: DMGlobalToLocalBegin(mine->dm,global,INSERT_VALUES,local);
222: DMGlobalToLocalEnd(mine->dm,global,INSERT_VALUES,local);
223: VecRestoreArray(vec,&array);
224: VecResetArray(global);
225: DMRestoreGlobalVector(mine->dm,&global);
226: return(0);
227: }
231: PetscErrorCode DMCompositeGather_Array(DM dm,struct DMCompositeLink *mine,Vec vec,InsertMode imode,const PetscScalar *array)
232: {
234: PetscScalar *varray;
235: PetscMPIInt rank;
238: MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
239: if (rank == mine->rank) {
240: VecGetArray(vec,&varray);
241: if (varray+mine->rstart == array) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"You need not DMCompositeGather() into objects obtained via DMCompositeGetAccess()");
242: }
243: switch (imode) {
244: case INSERT_VALUES:
245: if (rank == mine->rank) {
246: PetscMemcpy(varray+mine->rstart,array,mine->n*sizeof(PetscScalar));
247: }
248: break;
249: case ADD_VALUES: {
250: PetscInt i;
251: void *source;
252: PetscScalar *buffer,*dest;
253: if (rank == mine->rank) {
254: dest = &varray[mine->rstart];
255: #if defined(PETSC_HAVE_MPI_IN_PLACE)
256: buffer = dest;
257: source = MPI_IN_PLACE;
258: #else
259: PetscMalloc(mine->nlocal*sizeof(PetscScalar),&buffer);
260: source = (void *) buffer;
261: #endif
262: for (i=0; i<mine->nlocal; i++) buffer[i] = varray[mine->rstart+i] + array[i];
263: } else {
264: source = (void *) array;
265: dest = PETSC_NULL;
266: }
267: MPI_Reduce(source,dest,mine->nlocal,MPIU_SCALAR,MPI_SUM,mine->rank,((PetscObject)dm)->comm);
268: #if !defined(PETSC_HAVE_MPI_IN_PLACE)
269: if (rank == mine->rank) {PetscFree(source);}
270: #endif
271: } break;
272: default: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"imode");
273: }
274: if (rank == mine->rank) {VecRestoreArray(vec,&varray);}
275: return(0);
276: }
280: PetscErrorCode DMCompositeGather_DM(DM dm,struct DMCompositeLink *mine,Vec vec,InsertMode imode,Vec local)
281: {
283: PetscScalar *array;
284: Vec global;
287: DMGetGlobalVector(mine->dm,&global);
288: VecGetArray(vec,&array);
289: VecPlaceArray(global,array+mine->rstart);
290: DMLocalToGlobalBegin(mine->dm,local,imode,global);
291: DMLocalToGlobalEnd(mine->dm,local,imode,global);
292: VecRestoreArray(vec,&array);
293: VecResetArray(global);
294: DMRestoreGlobalVector(mine->dm,&global);
295: return(0);
296: }
298: /* ----------------------------------------------------------------------------------*/
300: #include <stdarg.h>
304: /*@C
305: DMCompositeGetNumberDM - Get's the number of DM objects in the DMComposite
306: representation.
308: Not Collective
310: Input Parameter:
311: . dm - the packer object
313: Output Parameter:
314: . nDM - the number of DMs
316: Level: beginner
318: @*/
319: PetscErrorCode DMCompositeGetNumberDM(DM dm,PetscInt *nDM)
320: {
321: DM_Composite *com = (DM_Composite*)dm->data;
324: *nDM = com->nDM;
325: return(0);
326: }
331: /*@C
332: DMCompositeGetAccess - Allows one to access the individual packed vectors in their global
333: representation.
335: Collective on DMComposite
337: Input Parameter:
338: + dm - the packer object
339: . gvec - the global vector
340: - ... - the individual sequential or parallel objects (arrays or vectors)
342: Notes: Use DMCompositeRestoreAccess() to return the vectors when you no longer need them
343:
344: Level: advanced
346: @*/
347: PetscErrorCode DMCompositeGetAccess(DM dm,Vec gvec,...)
348: {
349: va_list Argp;
350: PetscErrorCode ierr;
351: struct DMCompositeLink *next;
352: DM_Composite *com = (DM_Composite*)dm->data;
357: next = com->next;
358: if (!com->setup) {
359: DMSetUp(dm);
360: }
362: /* loop over packed objects, handling one at at time */
363: va_start(Argp,gvec);
364: while (next) {
365: if (next->type == DMCOMPOSITE_ARRAY) {
366: PetscScalar **array;
367: array = va_arg(Argp, PetscScalar**);
368: DMCompositeGetAccess_Array(dm,next,gvec,array);
369: } else if (next->type == DMCOMPOSITE_DM) {
370: Vec *vec;
371: vec = va_arg(Argp, Vec*);
372: DMCompositeGetAccess_DM(dm,next,gvec,vec);
373: } else {
374: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
375: }
376: next = next->next;
377: }
378: va_end(Argp);
379: return(0);
380: }
384: /*@C
385: DMCompositeRestoreAccess - Returns the vectors obtained with DMCompositeGetAccess()
386: representation.
388: Collective on DMComposite
390: Input Parameter:
391: + dm - the packer object
392: . gvec - the global vector
393: - ... - the individual sequential or parallel objects (arrays or vectors)
394:
395: Level: advanced
397: .seealso DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
398: DMCompositeGather(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeScatter(),
399: DMCompositeRestoreAccess(), DMCompositeGetAccess()
401: @*/
402: PetscErrorCode DMCompositeRestoreAccess(DM dm,Vec gvec,...)
403: {
404: va_list Argp;
405: PetscErrorCode ierr;
406: struct DMCompositeLink *next;
407: DM_Composite *com = (DM_Composite*)dm->data;
412: next = com->next;
413: if (!com->setup) {
414: DMSetUp(dm);
415: }
417: /* loop over packed objects, handling one at at time */
418: va_start(Argp,gvec);
419: while (next) {
420: if (next->type == DMCOMPOSITE_ARRAY) {
421: PetscScalar **array;
422: array = va_arg(Argp, PetscScalar**);
423: DMCompositeRestoreAccess_Array(dm,next,gvec,array);
424: } else if (next->type == DMCOMPOSITE_DM) {
425: Vec *vec;
426: vec = va_arg(Argp, Vec*);
427: DMCompositeRestoreAccess_DM(dm,next,gvec,vec);
428: } else {
429: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
430: }
431: next = next->next;
432: }
433: va_end(Argp);
434: return(0);
435: }
439: /*@C
440: DMCompositeScatter - Scatters from a global packed vector into its individual local vectors
442: Collective on DMComposite
444: Input Parameter:
445: + dm - the packer object
446: . gvec - the global vector
447: - ... - the individual sequential objects (arrays or vectors), PETSC_NULL for those that are not needed
449: Level: advanced
451: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
452: DMCompositeGather(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
453: DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()
455: @*/
456: PetscErrorCode DMCompositeScatter(DM dm,Vec gvec,...)
457: {
458: va_list Argp;
459: PetscErrorCode ierr;
460: struct DMCompositeLink *next;
461: PetscInt cnt;
462: DM_Composite *com = (DM_Composite*)dm->data;
467: if (!com->setup) {
468: DMSetUp(dm);
469: }
471: /* loop over packed objects, handling one at at time */
472: va_start(Argp,gvec);
473: for (cnt=3,next=com->next; next; cnt++,next=next->next) {
474: if (next->type == DMCOMPOSITE_ARRAY) {
475: PetscScalar *array;
476: array = va_arg(Argp, PetscScalar*);
479: if (!array) continue;
480: DMCompositeScatter_Array(dm,next,gvec,array);
481: } else if (next->type == DMCOMPOSITE_DM) {
482: Vec vec;
483: vec = va_arg(Argp, Vec);
484: if (!vec) continue;
486: DMCompositeScatter_DM(dm,next,gvec,vec);
487: } else {
488: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
489: }
490: }
491: va_end(Argp);
492: return(0);
493: }
497: /*@C
498: DMCompositeGather - Gathers into a global packed vector from its individual local vectors
500: Collective on DMComposite
502: Input Parameter:
503: + dm - the packer object
504: . gvec - the global vector
505: - ... - the individual sequential objects (arrays or vectors), PETSC_NULL for any that are not needed
507: Level: advanced
509: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
510: DMCompositeScatter(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
511: DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()
513: @*/
514: PetscErrorCode DMCompositeGather(DM dm,Vec gvec,InsertMode imode,...)
515: {
516: va_list Argp;
517: PetscErrorCode ierr;
518: struct DMCompositeLink *next;
519: DM_Composite *com = (DM_Composite*)dm->data;
520: PetscInt cnt;
525: if (!com->setup) {
526: DMSetUp(dm);
527: }
529: /* loop over packed objects, handling one at at time */
530: va_start(Argp,imode);
531: for (cnt=3,next=com->next; next; cnt++,next=next->next) {
532: if (next->type == DMCOMPOSITE_ARRAY) {
533: PetscScalar *array;
534: array = va_arg(Argp, PetscScalar*);
535: if (!array) continue;
537: DMCompositeGather_Array(dm,next,gvec,imode,array);
538: } else if (next->type == DMCOMPOSITE_DM) {
539: Vec vec;
540: vec = va_arg(Argp, Vec);
541: if (!vec) continue;
543: DMCompositeGather_DM(dm,next,gvec,imode,vec);
544: } else {
545: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
546: }
547: }
548: va_end(Argp);
549: return(0);
550: }
554: /*@C
555: DMCompositeAddArray - adds an "redundant" array to a DMComposite. The array values will
556: be stored in part of the array on process orank.
558: Collective on DMComposite
560: Input Parameter:
561: + dm - the packer object
562: . orank - the process on which the array entries officially live, this number must be
563: the same on all processes.
564: - n - the length of the array
565:
566: Level: advanced
568: .seealso DMDestroy(), DMCompositeGather(), DMCompositeAddDM(), DMCreateGlobalVector(),
569: DMCompositeScatter(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
570: DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()
572: @*/
573: PetscErrorCode DMCompositeAddArray(DM dm,PetscMPIInt orank,PetscInt n)
574: {
575: struct DMCompositeLink *mine,*next;
576: PetscErrorCode ierr;
577: PetscMPIInt rank;
578: DM_Composite *com = (DM_Composite*)dm->data;
582: next = com->next;
583: if (com->setup) {
584: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_ARG_WRONGSTATE,"Cannot add an array once you have used the DMComposite");
585: }
586: #if defined(PETSC_USE_DEBUG)
587: {
588: PetscMPIInt orankmax;
589: MPI_Allreduce(&orank,&orankmax,1,MPI_INT,MPI_MAX,((PetscObject)dm)->comm);
590: if (orank != orankmax) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"orank %d must be equal on all processes, another process has value %d",orank,orankmax);
591: }
592: #endif
594: MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
595: /* create new link */
596: PetscNew(struct DMCompositeLink,&mine);
597: mine->nlocal = n;
598: mine->n = (rank == orank) ? n : 0;
599: mine->rank = orank;
600: mine->dm = PETSC_NULL;
601: mine->type = DMCOMPOSITE_ARRAY;
602: mine->next = PETSC_NULL;
603: if (rank == mine->rank) {com->n += n;com->nmine++;}
605: /* add to end of list */
606: if (!next) {
607: com->next = mine;
608: } else {
609: while (next->next) next = next->next;
610: next->next = mine;
611: }
612: com->nredundant++;
613: return(0);
614: }
618: /*@C
619: DMCompositeAddDM - adds a DM vector to a DMComposite
621: Collective on DMComposite
623: Input Parameter:
624: + dm - the packer object
625: - dm - the DM object, if the DM is a da you will need to caste it with a (DM)
626:
627: Level: advanced
629: .seealso DMDestroy(), DMCompositeGather(), DMCompositeAddDM(), DMCreateGlobalVector(),
630: DMCompositeScatter(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
631: DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()
633: @*/
634: PetscErrorCode DMCompositeAddDM(DM dmc,DM dm)
635: {
636: PetscErrorCode ierr;
637: PetscInt n,nlocal;
638: struct DMCompositeLink *mine,*next;
639: Vec global,local;
640: DM_Composite *com = (DM_Composite*)dmc->data;
645: next = com->next;
646: if (com->setup) SETERRQ(((PetscObject)dmc)->comm,PETSC_ERR_ARG_WRONGSTATE,"Cannot add a DM once you have used the DMComposite");
648: /* create new link */
649: PetscNew(struct DMCompositeLink,&mine);
650: PetscObjectReference((PetscObject)dm);
651: DMGetGlobalVector(dm,&global);
652: VecGetLocalSize(global,&n);
653: DMRestoreGlobalVector(dm,&global);
654: DMGetLocalVector(dm,&local);
655: VecGetSize(local,&nlocal);
656: DMRestoreLocalVector(dm,&local);
657: mine->n = n;
658: mine->nlocal = nlocal;
659: mine->dm = dm;
660: mine->type = DMCOMPOSITE_DM;
661: mine->next = PETSC_NULL;
662: com->n += n;
664: /* add to end of list */
665: if (!next) {
666: com->next = mine;
667: } else {
668: while (next->next) next = next->next;
669: next->next = mine;
670: }
671: com->nDM++;
672: com->nmine++;
673: return(0);
674: }
680: PetscErrorCode VecView_DMComposite(Vec gvec,PetscViewer viewer)
681: {
682: DM dm;
683: PetscErrorCode ierr;
684: struct DMCompositeLink *next;
685: PetscBool isdraw;
686: DM_Composite *com;
689: PetscObjectQuery((PetscObject)gvec,"DM",(PetscObject*)&dm);
690: if (!dm) SETERRQ(((PetscObject)gvec)->comm,PETSC_ERR_ARG_WRONG,"Vector not generated from a DMComposite");
691: com = (DM_Composite*)dm->data;
692: next = com->next;
694: PetscTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw);
695: if (!isdraw) {
696: /* do I really want to call this? */
697: VecView_MPI(gvec,viewer);
698: } else {
699: PetscInt cnt = 0;
701: /* loop over packed objects, handling one at at time */
702: while (next) {
703: if (next->type == DMCOMPOSITE_ARRAY) {
704: PetscScalar *array;
705: DMCompositeGetAccess_Array(dm,next,gvec,&array);
707: /*skip it for now */
708: } else if (next->type == DMCOMPOSITE_DM) {
709: Vec vec;
710: PetscInt bs;
712: DMCompositeGetAccess_DM(dm,next,gvec,&vec);
713: VecView(vec,viewer);
714: VecGetBlockSize(vec,&bs);
715: DMCompositeRestoreAccess_DM(dm,next,gvec,&vec);
716: PetscViewerDrawBaseAdd(viewer,bs);
717: cnt += bs;
718: } else {
719: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
720: }
721: next = next->next;
722: }
723: PetscViewerDrawBaseAdd(viewer,-cnt);
724: }
725: return(0);
726: }
732: PetscErrorCode DMCreateGlobalVector_Composite(DM dm,Vec *gvec)
733: {
734: PetscErrorCode ierr;
735: DM_Composite *com = (DM_Composite*)dm->data;
739: if (!com->setup) {
740: DMSetUp(dm);
741: }
742: VecCreateMPI(((PetscObject)dm)->comm,com->n,com->N,gvec);
743: PetscObjectCompose((PetscObject)*gvec,"DM",(PetscObject)dm);
744: VecSetOperation(*gvec,VECOP_VIEW,(void(*)(void))VecView_DMComposite);
745: return(0);
746: }
750: PetscErrorCode DMCreateLocalVector_Composite(DM dm,Vec *lvec)
751: {
752: PetscErrorCode ierr;
753: DM_Composite *com = (DM_Composite*)dm->data;
757: if (!com->setup) {
758: DMSetUp(dm);
759: }
760: VecCreateSeq(((PetscObject)dm)->comm,com->nghost,lvec);
761: PetscObjectCompose((PetscObject)*lvec,"DM",(PetscObject)dm);
762: return(0);
763: }
767: /*@C
768: DMCompositeGetISLocalToGlobalMappings - gets an ISLocalToGlobalMapping for each DM/array in the DMComposite, maps to the composite global space
770: Collective on DM
772: Input Parameter:
773: . dm - the packer object
775: Output Parameters:
776: . ltogs - the individual mappings for each packed vector/array. Note that this includes
777: all the ghost points that individual ghosted DMDA's may have. Also each process has an
778: mapping for EACH redundant array (not just the local redundant arrays).
780: Level: advanced
782: Notes:
783: Each entry of ltogs should be destroyed with ISLocalToGlobalMappingDestroy(), the ltogs array should be freed with PetscFree().
785: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
786: DMCompositeGather(), DMCompositeCreate(), DMCompositeGetAccess(), DMCompositeScatter(),
787: DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(),DMCompositeGetEntries()
789: @*/
790: PetscErrorCode DMCompositeGetISLocalToGlobalMappings(DM dm,ISLocalToGlobalMapping **ltogs)
791: {
792: PetscErrorCode ierr;
793: PetscInt i,*idx,n,cnt;
794: struct DMCompositeLink *next;
795: PetscMPIInt rank;
796: DM_Composite *com = (DM_Composite*)dm->data;
800: PetscMalloc((com->nDM+com->nredundant)*sizeof(ISLocalToGlobalMapping),ltogs);
801: next = com->next;
802: MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
804: /* loop over packed objects, handling one at at time */
805: cnt = 0;
806: while (next) {
807: if (next->type == DMCOMPOSITE_ARRAY) {
808: PetscMalloc(next->nlocal*sizeof(PetscInt),&idx);
809: if (rank == next->rank) {
810: for (i=0; i<next->nlocal; i++) idx[i] = next->grstart + i;
811: }
812: MPI_Bcast(idx,next->nlocal,MPIU_INT,next->rank,((PetscObject)dm)->comm);
813: ISLocalToGlobalMappingCreate(((PetscObject)dm)->comm,next->nlocal,idx,PETSC_OWN_POINTER,&(*ltogs)[cnt]);
814: } else if (next->type == DMCOMPOSITE_DM) {
815: ISLocalToGlobalMapping ltog;
816: PetscMPIInt size;
817: const PetscInt *suboff,*indices;
818: Vec global;
820: /* Get sub-DM global indices for each local dof */
821: DMGetLocalToGlobalMapping(next->dm,<og);
822: ISLocalToGlobalMappingGetSize(ltog,&n);
823: ISLocalToGlobalMappingGetIndices(ltog,&indices);
824: PetscMalloc(n*sizeof(PetscInt),&idx);
826: /* Get the offsets for the sub-DM global vector */
827: DMGetGlobalVector(next->dm,&global);
828: VecGetOwnershipRanges(global,&suboff);
829: MPI_Comm_size(((PetscObject)global)->comm,&size);
831: /* Shift the sub-DM definition of the global space to the composite global space */
832: for (i=0; i<n; i++) {
833: PetscInt subi = indices[i],lo = 0,hi = size,t;
834: /* Binary search to find which rank owns subi */
835: while (hi-lo > 1) {
836: t = lo + (hi-lo)/2;
837: if (suboff[t] > subi) hi = t;
838: else lo = t;
839: }
840: idx[i] = subi - suboff[lo] + next->grstarts[lo];
841: }
842: ISLocalToGlobalMappingRestoreIndices(ltog,&indices);
843: ISLocalToGlobalMappingCreate(((PetscObject)dm)->comm,n,idx,PETSC_OWN_POINTER,&(*ltogs)[cnt]);
844: DMRestoreGlobalVector(next->dm,&global);
845: } else SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
846: next = next->next;
847: cnt++;
848: }
849: return(0);
850: }
854: /*@C
855: DMCompositeGetLocalISs - Gets index sets for each DM/array component of a composite local vector
857: Not Collective
859: Input Arguments:
860: . dm - composite DM
862: Output Arguments:
863: . is - array of serial index sets for each each component of the DMComposite
865: Level: intermediate
867: Notes:
868: At present, a composite local vector does not normally exist. This function is used to provide index sets for
869: MatGetLocalSubMatrix(). In the future, the scatters for each entry in the DMComposite may be be merged into a single
870: scatter to a composite local vector.
872: To get the composite global indices at all local points (including ghosts), use DMCompositeGetISLocalToGlobalMappings().
874: To get index sets for pieces of the composite global vector, use DMCompositeGetGlobalISs().
876: Each returned IS should be destroyed with ISDestroy(), the array should be freed with PetscFree().
878: .seealso: DMCompositeGetGlobalISs(), DMCompositeGetISLocalToGlobalMappings(), MatGetLocalSubMatrix(), MatCreateLocalRef()
879: @*/
880: PetscErrorCode DMCompositeGetLocalISs(DM dm,IS **is)
881: {
882: PetscErrorCode ierr;
883: DM_Composite *com = (DM_Composite*)dm->data;
884: struct DMCompositeLink *link;
885: PetscInt cnt,start;
890: PetscMalloc(com->nmine*sizeof(IS),is);
891: for (cnt=0,start=0,link=com->next; link; start+=link->nlocal,cnt++,link=link->next) {
892: ISCreateStride(PETSC_COMM_SELF,link->nlocal,start,1,&(*is)[cnt]);
893: if (link->type == DMCOMPOSITE_DM) {
894: PetscInt bs;
895: DMGetBlockSize(link->dm,&bs);
896: ISSetBlockSize((*is)[cnt],bs);
897: }
898: }
899: return(0);
900: }
904: /*@C
905: DMCompositeGetGlobalISs - Gets the index sets for each composed object
907: Collective on DMComposite
909: Input Parameter:
910: . dm - the packer object
912: Output Parameters:
913: . is - the array of index sets
915: Level: advanced
917: Notes:
918: The is entries should be destroyed with ISDestroy(), the is array should be freed with PetscFree()
920: These could be used to extract a subset of vector entries for a "multi-physics" preconditioner
922: Use DMCompositeGetLocalISs() for index sets in the packed local numbering, and
923: DMCompositeGetISLocalToGlobalMappings() for to map local sub-DM (including ghost) indices to packed global
924: indices.
926: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
927: DMCompositeGather(), DMCompositeCreate(), DMCompositeGetAccess(), DMCompositeScatter(),
928: DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(),DMCompositeGetEntries()
930: @*/
932: PetscErrorCode DMCompositeGetGlobalISs(DM dm,IS *is[])
933: {
934: PetscErrorCode ierr;
935: PetscInt cnt = 0,*idx,i;
936: struct DMCompositeLink *next;
937: PetscMPIInt rank;
938: DM_Composite *com = (DM_Composite*)dm->data;
942: PetscMalloc((com->nDM+com->nredundant)*sizeof(IS),is);
943: next = com->next;
944: MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
946: /* loop over packed objects, handling one at at time */
947: while (next) {
948: PetscMalloc(next->n*sizeof(PetscInt),&idx);
949: for (i=0; i<next->n; i++) idx[i] = next->grstart + i;
950: ISCreateGeneral(((PetscObject)dm)->comm,next->n,idx,PETSC_OWN_POINTER,&(*is)[cnt]);
951: cnt++;
952: next = next->next;
953: }
954: return(0);
955: }
957: /* -------------------------------------------------------------------------------------*/
960: PetscErrorCode DMCompositeGetLocalVectors_Array(DM dm,struct DMCompositeLink *mine,PetscScalar **array)
961: {
964: if (array) {
965: PetscMalloc(mine->nlocal*sizeof(PetscScalar),array);
966: }
967: return(0);
968: }
972: PetscErrorCode DMCompositeGetLocalVectors_DM(DM dm,struct DMCompositeLink *mine,Vec *local)
973: {
976: if (local) {
977: DMGetLocalVector(mine->dm,local);
978: }
979: return(0);
980: }
984: PetscErrorCode DMCompositeRestoreLocalVectors_Array(DM dm,struct DMCompositeLink *mine,PetscScalar **array)
985: {
988: if (array) {
989: PetscFree(*array);
990: }
991: return(0);
992: }
996: PetscErrorCode DMCompositeRestoreLocalVectors_DM(DM dm,struct DMCompositeLink *mine,Vec *local)
997: {
1000: if (local) {
1001: DMRestoreLocalVector(mine->dm,local);
1002: }
1003: return(0);
1004: }
1008: /*@C
1009: DMCompositeGetLocalVectors - Gets local vectors and arrays for each part of a DMComposite.
1010: Use DMCompositeRestoreLocalVectors() to return them.
1012: Not Collective
1014: Input Parameter:
1015: . dm - the packer object
1016:
1017: Output Parameter:
1018: . ... - the individual sequential objects (arrays or vectors)
1019:
1020: Level: advanced
1022: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
1023: DMCompositeGather(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
1024: DMCompositeRestoreLocalVectors(), DMCompositeScatter(), DMCompositeGetEntries()
1026: @*/
1027: PetscErrorCode DMCompositeGetLocalVectors(DM dm,...)
1028: {
1029: va_list Argp;
1030: PetscErrorCode ierr;
1031: struct DMCompositeLink *next;
1032: DM_Composite *com = (DM_Composite*)dm->data;
1036: next = com->next;
1037: /* loop over packed objects, handling one at at time */
1038: va_start(Argp,dm);
1039: while (next) {
1040: if (next->type == DMCOMPOSITE_ARRAY) {
1041: PetscScalar **array;
1042: array = va_arg(Argp, PetscScalar**);
1043: DMCompositeGetLocalVectors_Array(dm,next,array);
1044: } else if (next->type == DMCOMPOSITE_DM) {
1045: Vec *vec;
1046: vec = va_arg(Argp, Vec*);
1047: DMCompositeGetLocalVectors_DM(dm,next,vec);
1048: } else {
1049: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1050: }
1051: next = next->next;
1052: }
1053: va_end(Argp);
1054: return(0);
1055: }
1059: /*@C
1060: DMCompositeRestoreLocalVectors - Restores local vectors and arrays for each part of a DMComposite.
1062: Not Collective
1064: Input Parameter:
1065: . dm - the packer object
1066:
1067: Output Parameter:
1068: . ... - the individual sequential objects (arrays or vectors)
1069:
1070: Level: advanced
1072: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
1073: DMCompositeGather(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
1074: DMCompositeGetLocalVectors(), DMCompositeScatter(), DMCompositeGetEntries()
1076: @*/
1077: PetscErrorCode DMCompositeRestoreLocalVectors(DM dm,...)
1078: {
1079: va_list Argp;
1080: PetscErrorCode ierr;
1081: struct DMCompositeLink *next;
1082: DM_Composite *com = (DM_Composite*)dm->data;
1086: next = com->next;
1087: /* loop over packed objects, handling one at at time */
1088: va_start(Argp,dm);
1089: while (next) {
1090: if (next->type == DMCOMPOSITE_ARRAY) {
1091: PetscScalar **array;
1092: array = va_arg(Argp, PetscScalar**);
1093: DMCompositeRestoreLocalVectors_Array(dm,next,array);
1094: } else if (next->type == DMCOMPOSITE_DM) {
1095: Vec *vec;
1096: vec = va_arg(Argp, Vec*);
1097: DMCompositeRestoreLocalVectors_DM(dm,next,vec);
1098: } else {
1099: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1100: }
1101: next = next->next;
1102: }
1103: va_end(Argp);
1104: return(0);
1105: }
1107: /* -------------------------------------------------------------------------------------*/
1110: PetscErrorCode DMCompositeGetEntries_Array(DM dm,struct DMCompositeLink *mine,PetscInt *n)
1111: {
1113: if (n) *n = mine->nlocal;
1114: return(0);
1115: }
1119: PetscErrorCode DMCompositeGetEntries_DM(DM dmi,struct DMCompositeLink *mine,DM *dm)
1120: {
1122: if (dm) *dm = mine->dm;
1123: return(0);
1124: }
1128: /*@C
1129: DMCompositeGetEntries - Gets the DM, redundant size, etc for each entry in a DMComposite.
1131: Not Collective
1133: Input Parameter:
1134: . dm - the packer object
1135:
1136: Output Parameter:
1137: . ... - the individual entries, DMs or integer sizes)
1138:
1139: Level: advanced
1141: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCreateGlobalVector(),
1142: DMCompositeGather(), DMCompositeCreate(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess(),
1143: DMCompositeRestoreLocalVectors(), DMCompositeGetLocalVectors(), DMCompositeScatter(),
1144: DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors()
1146: @*/
1147: PetscErrorCode DMCompositeGetEntries(DM dm,...)
1148: {
1149: va_list Argp;
1150: PetscErrorCode ierr;
1151: struct DMCompositeLink *next;
1152: DM_Composite *com = (DM_Composite*)dm->data;
1156: next = com->next;
1157: /* loop over packed objects, handling one at at time */
1158: va_start(Argp,dm);
1159: while (next) {
1160: if (next->type == DMCOMPOSITE_ARRAY) {
1161: PetscInt *n;
1162: n = va_arg(Argp, PetscInt*);
1163: DMCompositeGetEntries_Array(dm,next,n);
1164: } else if (next->type == DMCOMPOSITE_DM) {
1165: DM *dmn;
1166: dmn = va_arg(Argp, DM*);
1167: DMCompositeGetEntries_DM(dm,next,dmn);
1168: } else {
1169: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1170: }
1171: next = next->next;
1172: }
1173: va_end(Argp);
1174: return(0);
1175: }
1179: PetscErrorCode DMRefine_Composite(DM dmi,MPI_Comm comm,DM *fine)
1180: {
1181: PetscErrorCode ierr;
1182: struct DMCompositeLink *next;
1183: DM_Composite *com = (DM_Composite*)dmi->data;
1184: DM dm;
1188: next = com->next;
1189: DMCompositeCreate(comm,fine);
1191: /* loop over packed objects, handling one at at time */
1192: while (next) {
1193: if (next->type == DMCOMPOSITE_ARRAY) {
1194: DMCompositeAddArray(*fine,next->rank,next->nlocal);
1195: } else if (next->type == DMCOMPOSITE_DM) {
1196: DMRefine(next->dm,comm,&dm);
1197: DMCompositeAddDM(*fine,dm);
1198: PetscObjectDereference((PetscObject)dm);
1199: } else {
1200: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1201: }
1202: next = next->next;
1203: }
1204: return(0);
1205: }
1209: PetscErrorCode DMCoarsen_Composite(DM dmi,MPI_Comm comm,DM *fine)
1210: {
1211: PetscErrorCode ierr;
1212: struct DMCompositeLink *next;
1213: DM_Composite *com = (DM_Composite*)dmi->data;
1214: DM dm;
1218: next = com->next;
1219: DMCompositeCreate(comm,fine);
1221: /* loop over packed objects, handling one at at time */
1222: while (next) {
1223: if (next->type == DMCOMPOSITE_ARRAY) {
1224: DMCompositeAddArray(*fine,next->rank,next->nlocal);
1225: } else if (next->type == DMCOMPOSITE_DM) {
1226: DMCoarsen(next->dm,comm,&dm);
1227: DMCompositeAddDM(*fine,dm);
1228: PetscObjectDereference((PetscObject)dm);
1229: } else {
1230: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1231: }
1232: next = next->next;
1233: }
1234: return(0);
1235: }
1237: struct MatPackLink {
1238: Mat A;
1239: struct MatPackLink *next;
1240: };
1242: struct MatPack {
1243: DM right,left;
1244: struct MatPackLink *next;
1245: };
1249: PetscErrorCode MatMultBoth_Shell_Pack(Mat A,Vec x,Vec y,PetscBool add)
1250: {
1251: struct MatPack *mpack;
1252: struct DMCompositeLink *xnext,*ynext;
1253: struct MatPackLink *anext;
1254: PetscScalar *xarray,*yarray;
1255: PetscErrorCode ierr;
1256: PetscInt i;
1257: Vec xglobal,yglobal;
1258: PetscMPIInt rank;
1259: DM_Composite *comright;
1260: DM_Composite *comleft;
1263: MatShellGetContext(A,(void**)&mpack);
1264: MPI_Comm_rank(((PetscObject)mpack->right)->comm,&rank);
1265: comright = (DM_Composite*)mpack->right->data;
1266: comleft = (DM_Composite*)mpack->left->data;
1267: xnext = comright->next;
1268: ynext = comleft->next;
1269: anext = mpack->next;
1271: while (xnext) {
1272: if (xnext->type == DMCOMPOSITE_ARRAY) {
1273: if (rank == xnext->rank) {
1274: VecGetArray(x,&xarray);
1275: VecGetArray(y,&yarray);
1276: if (add) {
1277: for (i=0; i<xnext->n; i++) {
1278: yarray[ynext->rstart+i] += xarray[xnext->rstart+i];
1279: }
1280: } else {
1281: PetscMemcpy(yarray+ynext->rstart,xarray+xnext->rstart,xnext->n*sizeof(PetscScalar));
1282: }
1283: VecRestoreArray(x,&xarray);
1284: VecRestoreArray(y,&yarray);
1285: }
1286: } else if (xnext->type == DMCOMPOSITE_DM) {
1287: VecGetArray(x,&xarray);
1288: VecGetArray(y,&yarray);
1289: DMGetGlobalVector(xnext->dm,&xglobal);
1290: DMGetGlobalVector(ynext->dm,&yglobal);
1291: VecPlaceArray(xglobal,xarray+xnext->rstart);
1292: VecPlaceArray(yglobal,yarray+ynext->rstart);
1293: if (add) {
1294: MatMultAdd(anext->A,xglobal,yglobal,yglobal);
1295: } else {
1296: MatMult(anext->A,xglobal,yglobal);
1297: }
1298: VecRestoreArray(x,&xarray);
1299: VecRestoreArray(y,&yarray);
1300: VecResetArray(xglobal);
1301: VecResetArray(yglobal);
1302: DMRestoreGlobalVector(xnext->dm,&xglobal);
1303: DMRestoreGlobalVector(ynext->dm,&yglobal);
1304: anext = anext->next;
1305: } else {
1306: SETERRQ(((PetscObject)A)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1307: }
1308: xnext = xnext->next;
1309: ynext = ynext->next;
1310: }
1311: return(0);
1312: }
1316: PetscErrorCode MatMultAdd_Shell_Pack(Mat A,Vec x,Vec y,Vec z)
1317: {
1320: if (z != y) SETERRQ(((PetscObject)A)->comm,PETSC_ERR_SUP,"Handles y == z only");
1321: MatMultBoth_Shell_Pack(A,x,y,PETSC_TRUE);
1322: return(0);
1323: }
1327: PetscErrorCode MatMult_Shell_Pack(Mat A,Vec x,Vec y)
1328: {
1331: MatMultBoth_Shell_Pack(A,x,y,PETSC_FALSE);
1332: return(0);
1333: }
1337: PetscErrorCode MatMultTranspose_Shell_Pack(Mat A,Vec x,Vec y)
1338: {
1339: struct MatPack *mpack;
1340: struct DMCompositeLink *xnext,*ynext;
1341: struct MatPackLink *anext;
1342: PetscScalar *xarray,*yarray;
1343: PetscErrorCode ierr;
1344: Vec xglobal,yglobal;
1345: PetscMPIInt rank;
1346: DM_Composite *comright;
1347: DM_Composite *comleft;
1350: MatShellGetContext(A,(void**)&mpack);
1351: MPI_Comm_rank(((PetscObject)mpack->right)->comm,&rank);
1352: comright = (DM_Composite*)mpack->right->data;
1353: comleft = (DM_Composite*)mpack->left->data;
1354: ynext = comright->next;
1355: xnext = comleft->next;
1356: anext = mpack->next;
1358: while (xnext) {
1359: if (xnext->type == DMCOMPOSITE_ARRAY) {
1360: if (rank == ynext->rank) {
1361: VecGetArray(x,&xarray);
1362: VecGetArray(y,&yarray);
1363: PetscMemcpy(yarray+ynext->rstart,xarray+xnext->rstart,xnext->n*sizeof(PetscScalar));
1364: VecRestoreArray(x,&xarray);
1365: VecRestoreArray(y,&yarray);
1366: }
1367: } else if (xnext->type == DMCOMPOSITE_DM) {
1368: VecGetArray(x,&xarray);
1369: VecGetArray(y,&yarray);
1370: DMGetGlobalVector(xnext->dm,&xglobal);
1371: DMGetGlobalVector(ynext->dm,&yglobal);
1372: VecPlaceArray(xglobal,xarray+xnext->rstart);
1373: VecPlaceArray(yglobal,yarray+ynext->rstart);
1374: MatMultTranspose(anext->A,xglobal,yglobal);
1375: VecRestoreArray(x,&xarray);
1376: VecRestoreArray(y,&yarray);
1377: VecResetArray(xglobal);
1378: VecResetArray(yglobal);
1379: DMRestoreGlobalVector(xnext->dm,&xglobal);
1380: DMRestoreGlobalVector(ynext->dm,&yglobal);
1381: anext = anext->next;
1382: } else {
1383: SETERRQ(((PetscObject)A)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1384: }
1385: xnext = xnext->next;
1386: ynext = ynext->next;
1387: }
1388: return(0);
1389: }
1393: PetscErrorCode MatDestroy_Shell_Pack(Mat A)
1394: {
1395: struct MatPack *mpack;
1396: struct MatPackLink *anext,*oldanext;
1397: PetscErrorCode ierr;
1400: MatShellGetContext(A,(void**)&mpack);
1401: anext = mpack->next;
1403: while (anext) {
1404: MatDestroy(&anext->A);
1405: oldanext = anext;
1406: anext = anext->next;
1407: PetscFree(oldanext);
1408: }
1409: PetscFree(mpack);
1410: PetscObjectChangeTypeName((PetscObject)A,0);
1411: return(0);
1412: }
1416: PetscErrorCode DMGetInterpolation_Composite(DM coarse,DM fine,Mat *A,Vec *v)
1417: {
1418: PetscErrorCode ierr;
1419: PetscInt m,n,M,N;
1420: struct DMCompositeLink *nextc;
1421: struct DMCompositeLink *nextf;
1422: struct MatPackLink *nextmat,*pnextmat = 0;
1423: struct MatPack *mpack;
1424: Vec gcoarse,gfine;
1425: DM_Composite *comcoarse = (DM_Composite*)coarse->data;
1426: DM_Composite *comfine = (DM_Composite*)fine->data;
1431: nextc = comcoarse->next;
1432: nextf = comfine->next;
1433: /* use global vectors only for determining matrix layout */
1434: DMCreateGlobalVector(coarse,&gcoarse);
1435: DMCreateGlobalVector(fine,&gfine);
1436: VecGetLocalSize(gcoarse,&n);
1437: VecGetLocalSize(gfine,&m);
1438: VecGetSize(gcoarse,&N);
1439: VecGetSize(gfine,&M);
1440: VecDestroy(&gcoarse);
1441: VecDestroy(&gfine);
1443: PetscNew(struct MatPack,&mpack);
1444: mpack->right = coarse;
1445: mpack->left = fine;
1446: MatCreate(((PetscObject)fine)->comm,A);
1447: MatSetSizes(*A,m,n,M,N);
1448: MatSetType(*A,MATSHELL);
1449: MatShellSetContext(*A,mpack);
1450: MatShellSetOperation(*A,MATOP_MULT,(void(*)(void))MatMult_Shell_Pack);
1451: MatShellSetOperation(*A,MATOP_MULT_TRANSPOSE,(void(*)(void))MatMultTranspose_Shell_Pack);
1452: MatShellSetOperation(*A,MATOP_MULT_ADD,(void(*)(void))MatMultAdd_Shell_Pack);
1453: MatShellSetOperation(*A,MATOP_DESTROY,(void(*)(void))MatDestroy_Shell_Pack);
1455: /* loop over packed objects, handling one at at time */
1456: while (nextc) {
1457: if (nextc->type != nextf->type) SETERRQ(((PetscObject)fine)->comm,PETSC_ERR_ARG_INCOMP,"Two DM have different layout");
1459: if (nextc->type == DMCOMPOSITE_ARRAY) {
1460: ;
1461: } else if (nextc->type == DMCOMPOSITE_DM) {
1462: PetscNew(struct MatPackLink,&nextmat);
1463: nextmat->next = 0;
1464: if (pnextmat) {
1465: pnextmat->next = nextmat;
1466: pnextmat = nextmat;
1467: } else {
1468: pnextmat = nextmat;
1469: mpack->next = nextmat;
1470: }
1471: DMGetInterpolation(nextc->dm,nextf->dm,&nextmat->A,PETSC_NULL);
1472: } else {
1473: SETERRQ(((PetscObject)fine)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1474: }
1475: nextc = nextc->next;
1476: nextf = nextf->next;
1477: }
1478: return(0);
1479: }
1483: static PetscErrorCode DMCreateLocalToGlobalMapping_Composite(DM dm)
1484: {
1485: DM_Composite *com = (DM_Composite*)dm->data;
1486: ISLocalToGlobalMapping *ltogs;
1487: PetscInt i;
1488: PetscErrorCode ierr;
1491: /* Set the ISLocalToGlobalMapping on the new matrix */
1492: DMCompositeGetISLocalToGlobalMappings(dm,<ogs);
1493: ISLocalToGlobalMappingConcatenate(((PetscObject)dm)->comm,com->nDM+com->nredundant,ltogs,&dm->ltogmap);
1494: for (i=0; i<com->nDM+com->nredundant; i++) {ISLocalToGlobalMappingDestroy(<ogs[i]);}
1495: PetscFree(ltogs);
1496: return(0);
1497: }
1502: PetscErrorCode DMGetColoring_Composite(DM dm,ISColoringType ctype,const MatType mtype,ISColoring *coloring)
1503: {
1504: PetscErrorCode ierr;
1505: PetscInt n,i,cnt;
1506: ISColoringValue *colors;
1507: PetscBool dense = PETSC_FALSE;
1508: ISColoringValue maxcol = 0;
1509: DM_Composite *com = (DM_Composite*)dm->data;
1513: if (ctype == IS_COLORING_GHOSTED) {
1514: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Currently you must use -dmmg_iscoloring_type global" );
1515: } else if (ctype == IS_COLORING_GLOBAL) {
1516: n = com->n;
1517: } else SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_ARG_OUTOFRANGE,"Unknown ISColoringType");
1518: PetscMalloc(n*sizeof(ISColoringValue),&colors); /* freed in ISColoringDestroy() */
1520: PetscOptionsGetBool(PETSC_NULL,"-dmcomposite_dense_jacobian",&dense,PETSC_NULL);
1521: if (dense) {
1522: for (i=0; i<n; i++) {
1523: colors[i] = (ISColoringValue)(com->rstart + i);
1524: }
1525: maxcol = com->N;
1526: } else {
1527: struct DMCompositeLink *next = com->next;
1528: PetscMPIInt rank;
1529:
1530: MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
1531: cnt = 0;
1532: while (next) {
1533: if (next->type == DMCOMPOSITE_ARRAY) {
1534: if (rank == next->rank) { /* each column gets is own color */
1535: for (i=com->rstart+next->rstart; i<com->rstart+next->rstart+next->n; i++) {
1536: colors[cnt++] = maxcol++;
1537: }
1538: }
1539: MPI_Bcast(&maxcol,1,MPIU_COLORING_VALUE,next->rank,((PetscObject)dm)->comm);
1540: } else if (next->type == DMCOMPOSITE_DM) {
1541: ISColoring lcoloring;
1543: DMGetColoring(next->dm,IS_COLORING_GLOBAL,mtype,&lcoloring);
1544: for (i=0; i<lcoloring->N; i++) {
1545: colors[cnt++] = maxcol + lcoloring->colors[i];
1546: }
1547: maxcol += lcoloring->n;
1548: ISColoringDestroy(&lcoloring);
1549: } else SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1550: next = next->next;
1551: }
1552: }
1553: ISColoringCreate(((PetscObject)dm)->comm,maxcol,n,colors,coloring);
1554: return(0);
1555: }
1559: PetscErrorCode DMGlobalToLocalBegin_Composite(DM dm,Vec gvec,InsertMode mode,Vec lvec)
1560: {
1561: PetscErrorCode ierr;
1562: struct DMCompositeLink *next;
1563: PetscInt cnt = 3;
1564: PetscMPIInt rank;
1565: PetscScalar *garray,*larray;
1566: DM_Composite *com = (DM_Composite*)dm->data;
1571: next = com->next;
1572: if (!com->setup) {
1573: DMSetUp(dm);
1574: }
1575: MPI_Comm_rank(((PetscObject)dm)->comm,&rank);
1576: VecGetArray(gvec,&garray);
1577: VecGetArray(lvec,&larray);
1579: /* loop over packed objects, handling one at at time */
1580: while (next) {
1581: if (next->type == DMCOMPOSITE_ARRAY) {
1582: if (rank == next->rank) {
1583: PetscMemcpy(larray,garray,next->n*sizeof(PetscScalar));
1584: garray += next->n;
1585: }
1586: /* does not handle ADD_VALUES */
1587: MPI_Bcast(larray,next->nlocal,MPIU_SCALAR,next->rank,((PetscObject)dm)->comm);
1588: } else if (next->type == DMCOMPOSITE_DM) {
1589: Vec local,global;
1590: PetscInt N;
1592: DMGetGlobalVector(next->dm,&global);
1593: VecGetLocalSize(global,&N);
1594: VecPlaceArray(global,garray);
1595: DMGetLocalVector(next->dm,&local);
1596: VecPlaceArray(local,larray);
1597: DMGlobalToLocalBegin(next->dm,global,mode,local);
1598: DMGlobalToLocalEnd(next->dm,global,mode,local);
1599: VecResetArray(global);
1600: VecResetArray(local);
1601: DMRestoreGlobalVector(next->dm,&global);
1602: DMRestoreGlobalVector(next->dm,&local);
1603: } else {
1604: SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_SUP,"Cannot handle that object type yet");
1605: }
1606: cnt++;
1607: larray += next->nlocal;
1608: next = next->next;
1609: }
1611: VecRestoreArray(gvec,PETSC_NULL);
1612: VecRestoreArray(lvec,PETSC_NULL);
1613: return(0);
1614: }
1618: PetscErrorCode DMGlobalToLocalEnd_Composite(DM dm,Vec gvec,InsertMode mode,Vec lvec)
1619: {
1621: return(0);
1622: }
1627: PetscErrorCode DMCreate_Composite(DM p)
1628: {
1630: DM_Composite *com;
1633: PetscNewLog(p,DM_Composite,&com);
1634: p->data = com;
1635: PetscObjectChangeTypeName((PetscObject)p,"DMComposite");
1636: com->n = 0;
1637: com->next = PETSC_NULL;
1638: com->nredundant = 0;
1639: com->nDM = 0;
1641: p->ops->createglobalvector = DMCreateGlobalVector_Composite;
1642: p->ops->createlocalvector = DMCreateLocalVector_Composite;
1643: p->ops->createlocaltoglobalmapping = DMCreateLocalToGlobalMapping_Composite;
1644: p->ops->createlocaltoglobalmappingblock = 0;
1645: p->ops->refine = DMRefine_Composite;
1646: p->ops->coarsen = DMCoarsen_Composite;
1647: p->ops->getinterpolation = DMGetInterpolation_Composite;
1648: p->ops->getmatrix = DMGetMatrix_Composite;
1649: p->ops->getcoloring = DMGetColoring_Composite;
1650: p->ops->globaltolocalbegin = DMGlobalToLocalBegin_Composite;
1651: p->ops->globaltolocalend = DMGlobalToLocalEnd_Composite;
1652: p->ops->destroy = DMDestroy_Composite;
1653: p->ops->view = DMView_Composite;
1654: p->ops->setup = DMSetUp_Composite;
1655: return(0);
1656: }
1661: /*@C
1662: DMCompositeCreate - Creates a vector packer, used to generate "composite"
1663: vectors made up of several subvectors.
1665: Collective on MPI_Comm
1667: Input Parameter:
1668: . comm - the processors that will share the global vector
1670: Output Parameters:
1671: . packer - the packer object
1673: Level: advanced
1675: .seealso DMDestroy(), DMCompositeAddArray(), DMCompositeAddDM(), DMCompositeScatter(),
1676: DMCompositeGather(), DMCreateGlobalVector(), DMCompositeGetISLocalToGlobalMappings(), DMCompositeGetAccess()
1677: DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(), DMCompositeGetEntries()
1679: @*/
1680: PetscErrorCode DMCompositeCreate(MPI_Comm comm,DM *packer)
1681: {
1686: DMCreate(comm,packer);
1687: DMSetType(*packer,DMCOMPOSITE);
1688: return(0);
1689: }