Actual source code: pinit.c

  2: /*
  3:    This file defines the initialization of PETSc, including PetscInitialize()
  4: */

  6: #include <petscsys.h>        /*I  "petscsys.h"   I*/

  8: #if defined(PETSC_HAVE_CUSP)
  9: #include <cublas.h>
 10: #endif

 12: #if defined(PETSC_USE_LOG)
 14: #endif
 17: /* -----------------------------------------------------------------------------------------*/


 29: /* this is used by the _, __, and ___ macros (see include/petscerror.h) */
 30: PetscErrorCode __g0;

 32: /* user may set this BEFORE calling PetscInitialize() */
 33: MPI_Comm PETSC_COMM_WORLD = MPI_COMM_NULL;

 35: PetscMPIInt Petsc_Counter_keyval   = MPI_KEYVAL_INVALID;
 36: PetscMPIInt Petsc_InnerComm_keyval = MPI_KEYVAL_INVALID;
 37: PetscMPIInt Petsc_OuterComm_keyval = MPI_KEYVAL_INVALID;

 39: /*
 40:      Declare and set all the string names of the PETSc enums
 41: */
 42: const char *PetscBools[]     = {"FALSE","TRUE","PetscBool","PETSC_",0};
 43: const char *PetscCopyModes[] = {"COPY_VALUES","OWN_POINTER","USE_POINTER","PetscCopyMode","PETSC_",0};
 44: const char *PetscDataTypes[] = {"INT","DOUBLE","COMPLEX","LONG","SHORT","FLOAT",
 45:                                 "CHAR","LOGICAL","ENUM","BOOL","LONGDOUBLE","PetscDataType","PETSC_",0};

 47: PetscBool  PetscPreLoadingUsed = PETSC_FALSE;
 48: PetscBool  PetscPreLoadingOn   = PETSC_FALSE;

 50: /*
 51:        Checks the options database for initializations related to the 
 52:     PETSc components
 53: */
 56: PetscErrorCode  PetscOptionsCheckInitial_Components(void)
 57: {
 58:   PetscBool  flg1;

 62:   PetscOptionsHasName(PETSC_NULL,"-help",&flg1);
 63:   if (flg1) {
 64: #if defined (PETSC_USE_LOG)
 65:     MPI_Comm   comm = PETSC_COMM_WORLD;
 66:     (*PetscHelpPrintf)(comm,"------Additional PETSc component options--------\n");
 67:     (*PetscHelpPrintf)(comm," -log_summary_exclude: <vec,mat,pc.ksp,snes>\n");
 68:     (*PetscHelpPrintf)(comm," -info_exclude: <null,vec,mat,pc,ksp,snes,ts>\n");
 69:     (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
 70: #endif
 71:   }
 72:   return(0);
 73: }

 75: #if defined(PETSC_HAVE_MATLAB_ENGINE)

 80: /*
 81:       PetscInitializeMatlab - Calls PetscInitialize() from C/C++ without the pointers to argc and args

 83:    Collective
 84:   
 85:    Level: advanced

 87:     Notes: this is called only by the PETSc MATLAB interface. Even though it might start MPI it sets the flag to 
 88:      indicate that it did NOT start MPI so that the PetscFinalize() does not end MPI, thus allowing PetscInitialize() to 
 89:      be called multiple times from MATLAB without the problem of trying to initialize MPI more than once.

 91:      Turns off PETSc signal handling because that can interact with MATLAB's signal handling causing random crashes.

 93: .seealso: PetscInitialize(), PetscInitializeFortran(), PetscInitializeNoArguments()
 94: */
 95: PetscErrorCode  PetscInitializeMatlab(int argc,char **args,const char *filename,const char *help)
 96: {
 98:   int            myargc = argc;
 99:   char           **myargs = args;

102:   PetscInitialize(&myargc,&myargs,filename,help);
103:   PetscPopSignalHandler();
104:   PetscBeganMPI = PETSC_FALSE;
105:   PetscFunctionReturn(ierr);
106: }

110: /*
111:       PetscInitializedMatlab - Has PETSc been initialized already?

113:    Not Collective
114:   
115:    Level: advanced

117:     Notes: this is called only by the PETSc MATLAB interface.

119: .seealso: PetscInitialize(), PetscInitializeFortran(), PetscInitializeNoArguments()
120: */
121: int  PetscInitializedMatlab(void)
122: {
123:   PetscBool flg;

125:   PetscInitialized(&flg);
126:   if (flg) return 1;
127:   else return 0;
128: }

132: /*
133:       Used by MATLAB interface to get communicator
134: */
135: PetscErrorCode  PetscGetPETSC_COMM_SELFMatlab(MPI_Comm *comm)
136: {
138:   *comm = PETSC_COMM_SELF;
139:   return(0);
140: }
141: #endif

145: /*@C
146:       PetscInitializeNoArguments - Calls PetscInitialize() from C/C++ without
147:         the command line arguments.

149:    Collective
150:   
151:    Level: advanced

153: .seealso: PetscInitialize(), PetscInitializeFortran()
154: @*/
155: PetscErrorCode  PetscInitializeNoArguments(void)
156: {
158:   int            argc = 0;
159:   char           **args = 0;

162:   PetscInitialize(&argc,&args,PETSC_NULL,PETSC_NULL);
163:   PetscFunctionReturn(ierr);
164: }

168: /*@
169:       PetscInitialized - Determine whether PETSc is initialized.
170:   
171: 7   Level: beginner

173: .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran()
174: @*/
175: PetscErrorCode  PetscInitialized(PetscBool  *isInitialized)
176: {
179:   *isInitialized = PetscInitializeCalled;
180:   return(0);
181: }

185: /*@
186:       PetscFinalized - Determine whether PetscFinalize() has been called yet
187:   
188:    Level: developer

190: .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran()
191: @*/
192: PetscErrorCode  PetscFinalized(PetscBool  *isFinalized)
193: {
196:   *isFinalized = PetscFinalizeCalled;
197:   return(0);
198: }


203: /*
204:        This function is the MPI reduction operation used to compute the sum of the 
205:    first half of the datatype and the max of the second half.
206: */
207: MPI_Op PetscMaxSum_Op = 0;

212: void  MPIAPI PetscMaxSum_Local(void *in,void *out,int *cnt,MPI_Datatype *datatype)
213: {
214:   PetscInt *xin = (PetscInt*)in,*xout = (PetscInt*)out,i,count = *cnt;

217:   if (*datatype != MPIU_2INT) {
218:     (*PetscErrorPrintf)("Can only handle MPIU_2INT data types");
219:     MPI_Abort(MPI_COMM_WORLD,1);
220:   }

222:   for (i=0; i<count; i++) {
223:     xout[2*i]    = PetscMax(xout[2*i],xin[2*i]);
224:     xout[2*i+1] += xin[2*i+1];
225:   }
226:   PetscFunctionReturnVoid();
227: }

230: /*
231:     Returns the max of the first entry owned by this processor and the
232: sum of the second entry.

234:     The reason nprocs[2*i] contains lengths nprocs[2*i+1] contains flag of 1 if length is nonzero
235: is so that the PetscMaxSum_Op() can set TWO values, if we passed in only nprocs[i] with lengths
236: there would be no place to store the both needed results.
237: */
240: PetscErrorCode  PetscMaxSum(MPI_Comm comm,const PetscInt nprocs[],PetscInt *max,PetscInt *sum)
241: {
242:   PetscMPIInt    size,rank;
243:   PetscInt       *work;

247:   MPI_Comm_size(comm,&size);
248:   MPI_Comm_rank(comm,&rank);
249:   PetscMalloc(2*size*sizeof(PetscInt),&work);
250:   MPI_Allreduce((void*)nprocs,work,size,MPIU_2INT,PetscMaxSum_Op,comm);
251:   *max   = work[2*rank];
252:   *sum   = work[2*rank+1];
253:   PetscFree(work);
254:   return(0);
255: }

257: /* ----------------------------------------------------------------------------*/
258: MPI_Op  PetscADMax_Op = 0;

263: void  MPIAPI PetscADMax_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
264: {
265:   PetscScalar *xin = (PetscScalar *)in,*xout = (PetscScalar*)out;
266:   PetscInt    i,count = *cnt;

269:   if (*datatype != MPIU_2SCALAR) {
270:     (*PetscErrorPrintf)("Can only handle MPIU_2SCALAR data (i.e. double or complex) types");
271:     MPI_Abort(MPI_COMM_WORLD,1);
272:   }

274:   for (i=0; i<count; i++) {
275:     if (PetscRealPart(xout[2*i]) < PetscRealPart(xin[2*i])) {
276:       xout[2*i]   = xin[2*i];
277:       xout[2*i+1] = xin[2*i+1];
278:     }
279:   }
280:   PetscFunctionReturnVoid();
281: }

284: MPI_Op  PetscADMin_Op = 0;

289: void  MPIAPI PetscADMin_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
290: {
291:   PetscScalar *xin = (PetscScalar *)in,*xout = (PetscScalar*)out;
292:   PetscInt    i,count = *cnt;

295:   if (*datatype != MPIU_2SCALAR) {
296:     (*PetscErrorPrintf)("Can only handle MPIU_2SCALAR data (i.e. double or complex) types");
297:     MPI_Abort(MPI_COMM_WORLD,1);
298:   }

300:   for (i=0; i<count; i++) {
301:     if (PetscRealPart(xout[2*i]) > PetscRealPart(xin[2*i])) {
302:       xout[2*i]   = xin[2*i];
303:       xout[2*i+1] = xin[2*i+1];
304:     }
305:   }
306:   PetscFunctionReturnVoid();
307: }
309: /* ---------------------------------------------------------------------------------------*/

311: #if (defined(PETSC_USE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128)
312: MPI_Op MPIU_SUM = 0;

317: void  PetscSum_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
318: {
319:   PetscInt    i,count = *cnt;

322:   if (*datatype == MPIU_SCALAR) {
323:     PetscScalar *xin = (PetscScalar *)in,*xout = (PetscScalar*)out;
324:     for (i=0; i<count; i++) {
325:       xout[i] += xin[i];
326:     }
327:   } else if (*datatype == MPIU_REAL) {
328:     PetscReal *xin = (PetscReal *)in,*xout = (PetscReal*)out;
329:     for (i=0; i<count; i++) {
330:       xout[i] += xin[i];
331:     }
332:   } else {
333:     (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_SCALAR data (i.e. double or complex) types");
334:     MPI_Abort(MPI_COMM_WORLD,1);
335:   }
336:   PetscFunctionReturnVoid();
337: }
339: #endif

341: #if defined(PETSC_USE_REAL___FLOAT128)
342: MPI_Op MPIU_MAX = 0;
343: MPI_Op MPIU_MIN = 0;

348: void  PetscMax_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
349: {
350:   PetscScalar *xin = (PetscScalar *)in,*xout = (PetscScalar*)out;
351:   PetscInt    i,count = *cnt;

354:   if (*datatype != MPIU_SCALAR) {
355:     (*PetscErrorPrintf)("Can only handle MPIU_SCALAR data (i.e. double or complex) types");
356:     MPI_Abort(MPI_COMM_WORLD,1);
357:   }

359:   for (i=0; i<count; i++) {
360:     xout[i] = PetscMax(xout[i],xin[i]);
361:   }
362:   PetscFunctionReturnVoid();
363: }

369: void  PetscMin_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
370: {
371:   PetscScalar *xin = (PetscScalar *)in,*xout = (PetscScalar*)out;
372:   PetscInt    i,count = *cnt;

375:   if (*datatype != MPIU_SCALAR) {
376:     (*PetscErrorPrintf)("Can only handle MPIU_SCALAR data (i.e. double or complex) types");
377:     MPI_Abort(MPI_COMM_WORLD,1);
378:   }

380:   for (i=0; i<count; i++) {
381:     xout[i] = PetscMin(xout[i],xin[i]);
382:   }
383:   PetscFunctionReturnVoid();
384: }
386: #endif

391: /*
392:    Private routine to delete internal tag/name counter storage when a communicator is freed.

394:    This is called by MPI, not by users. This is called by MPI_Comm_free() when the communicator that has this  data as an attribute is freed.


398: */
399: PetscMPIInt  MPIAPI Petsc_DelCounter(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state)
400: {

404:   PetscInfo1(0,"Deleting counter data in an MPI_Comm %ld\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
405:   PetscFree(count_val);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
406:   PetscFunctionReturn(MPI_SUCCESS);
407: }

413: /*
414:   This does not actually free anything, it simply marks when a reference count to an internal or external MPI_Comm reaches zero and the
415:   the external MPI_Comm drops its reference to the internal or external MPI_Comm

417:   This is called by MPI, not by users. This is called when MPI_Comm_free() is called on the communicator.


421: */
422: PetscMPIInt  MPIAPI Petsc_DelComm(MPI_Comm comm,PetscMPIInt keyval,void *attr_val,void *extra_state)
423: {
424:   PetscErrorCode   ierr;
425:   PetscMPIInt      flg;
426:   MPI_Comm         icomm;
427:   void             *ptr;

430:   MPI_Attr_get(comm,Petsc_InnerComm_keyval,&ptr,&flg);
431:   if (flg) {
432:     /*  Use PetscMemcpy() because casting from pointer to integer of different size is not allowed with some compilers  */
433:     PetscMemcpy(&icomm,&ptr,sizeof(MPI_Comm));
434:     MPI_Attr_get(icomm,Petsc_OuterComm_keyval,&ptr,&flg);
435:     if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected reference to outer comm");
436:     MPI_Attr_delete(icomm,Petsc_OuterComm_keyval);
437:     PetscInfo1(0,"User MPI_Comm m %ld is being freed, removing reference from inner PETSc comm to this outer comm\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
438:   } else {
439:     PetscInfo1(0,"Removing reference to PETSc communicator imbedded in a user MPI_Comm m %ld\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
440:   }
441:   PetscFunctionReturn(MPI_SUCCESS);
442: }

445: #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32)
446: #if !defined(PETSC_WORDS_BIGENDIAN)
452: #endif
453: #endif

455: int  PetscGlobalArgc   = 0;
456: char **PetscGlobalArgs = 0;

460: /*@C
461:    PetscGetArgs - Allows you to access the raw command line arguments anywhere
462:      after PetscInitialize() is called but before PetscFinalize().

464:    Not Collective

466:    Output Parameters:
467: +  argc - count of number of command line arguments
468: -  args - the command line arguments

470:    Level: intermediate

472:    Notes:
473:       This is usually used to pass the command line arguments into other libraries
474:    that are called internally deep in PETSc or the application.

476:       The first argument contains the program name as is normal for C arguments.

478:    Concepts: command line arguments
479:    
480: .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArguments()

482: @*/
483: PetscErrorCode  PetscGetArgs(int *argc,char ***args)
484: {
486:   if (!PetscInitializeCalled && PetscFinalizeCalled) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()");
487:   *argc = PetscGlobalArgc;
488:   *args = PetscGlobalArgs;
489:   return(0);
490: }

494: /*@C
495:    PetscGetArguments - Allows you to access the  command line arguments anywhere
496:      after PetscInitialize() is called but before PetscFinalize().

498:    Not Collective

500:    Output Parameters:
501: .  args - the command line arguments

503:    Level: intermediate

505:    Notes:
506:       This does NOT start with the program name and IS null terminated (final arg is void)

508:    Concepts: command line arguments
509:    
510: .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscFreeArguments()

512: @*/
513: PetscErrorCode  PetscGetArguments(char ***args)
514: {
515:   PetscInt       i,argc = PetscGlobalArgc;

519:   if (!PetscInitializeCalled && PetscFinalizeCalled) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()");
520:   if (!argc) {*args = 0; return(0);}
521:   PetscMalloc(argc*sizeof(char*),args);
522:   for (i=0; i<argc-1; i++) {
523:     PetscStrallocpy(PetscGlobalArgs[i+1],&(*args)[i]);
524:   }
525:   (*args)[argc-1] = 0;
526:   return(0);
527: }

531: /*@C
532:    PetscFreeArguments - Frees the memory obtained with PetscGetArguments()

534:    Not Collective

536:    Output Parameters:
537: .  args - the command line arguments 

539:    Level: intermediate

541:    Concepts: command line arguments
542:    
543: .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscGetArguments()

545: @*/
546: PetscErrorCode  PetscFreeArguments(char **args)
547: {
548:   PetscInt       i = 0;

552:   if (!args) {return(0);}
553:   while (args[i]) {
554:     PetscFree(args[i]);
555:     i++;
556:   }
557:   PetscFree(args);
558:   return(0);
559: }

563: /*@C
564:    PetscInitialize - Initializes the PETSc database and MPI. 
565:    PetscInitialize() calls MPI_Init() if that has yet to be called,
566:    so this routine should always be called near the beginning of 
567:    your program -- usually the very first line! 

569:    Collective on MPI_COMM_WORLD or PETSC_COMM_WORLD if it has been set

571:    Input Parameters:
572: +  argc - count of number of command line arguments
573: .  args - the command line arguments
574: .  file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use PETSC_NULL to not check for
575:           code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files
576: -  help - [optional] Help message to print, use PETSC_NULL for no message

578:    If you wish PETSc code to run ONLY on a subcommunicator of MPI_COMM_WORLD, create that
579:    communicator first and assign it to PETSC_COMM_WORLD BEFORE calling PetscInitialize(). Thus if you are running a 
580:    four process job and two processes will run PETSc and have PetscInitialize() and PetscFinalize() and two process will not,
581:    then do this. If ALL processes in the job are using PetscInitialize() and PetscFinalize() then you don't need to do this, even
582:    if different subcommunicators of the job are doing different things with PETSc.

584:    Options Database Keys:
585: +  -start_in_debugger [noxterm,dbx,xdb,gdb,...] - Starts program in debugger
586: .  -on_error_attach_debugger [noxterm,dbx,xdb,gdb,...] - Starts debugger when error detected
587: .  -on_error_emacs <machinename> causes emacsclient to jump to error file
588: .  -on_error_abort calls abort() when error detected (no traceback)
589: .  -on_error_mpiabort calls MPI_abort() when error detected
590: .  -error_output_stderr prints error messages to stderr instead of the default stdout
591: .  -error_output_none does not print the error messages (but handles errors in the same way as if this was not called)
592: .  -debugger_nodes [node1,node2,...] - Indicates nodes to start in debugger
593: .  -debugger_pause [sleeptime] (in seconds) - Pauses debugger
594: .  -stop_for_debugger - Print message on how to attach debugger manually to 
595:                         process and wait (-debugger_pause) seconds for attachment
596: .  -malloc - Indicates use of PETSc error-checking malloc (on by default for debug version of libraries)
597: .  -malloc no - Indicates not to use error-checking malloc
598: .  -malloc_debug - check for memory corruption at EVERY malloc or free
599: .  -fp_trap - Stops on floating point exceptions (Note that on the
600:               IBM RS6000 this slows code by at least a factor of 10.)
601: .  -no_signal_handler - Indicates not to trap error signals
602: .  -shared_tmp - indicates /tmp directory is shared by all processors
603: .  -not_shared_tmp - each processor has own /tmp
604: .  -tmp - alternative name of /tmp directory
605: .  -get_total_flops - returns total flops done by all processors
606: .  -memory_info - Print memory usage at end of run
607: -  -server <port> - start PETSc webserver (default port is 8080)

609:    Options Database Keys for Profiling:
610:    See the <a href="../../docs/manual.pdf#nameddest=Chapter 11 Profiling">profiling chapter of the users manual</a> for details.
611: +  -log_trace [filename] - Print traces of all PETSc calls
612:         to the screen (useful to determine where a program
613:         hangs without running in the debugger).  See PetscLogTraceBegin().
614: .  -info <optional filename> - Prints verbose information to the screen
615: -  -info_exclude <null,vec,mat,pc,ksp,snes,ts> - Excludes some of the verbose messages

617:    Environmental Variables:
618: +   PETSC_TMP - alternative tmp directory
619: .   PETSC_SHARED_TMP - tmp is shared by all processes
620: .   PETSC_NOT_SHARED_TMP - each process has its own private tmp
621: .   PETSC_VIEWER_SOCKET_PORT - socket number to use for socket viewer
622: -   PETSC_VIEWER_SOCKET_MACHINE - machine to use for socket viewer to connect to


625:    Level: beginner

627:    Notes:
628:    If for some reason you must call MPI_Init() separately, call
629:    it before PetscInitialize().

631:    Fortran Version:
632:    In Fortran this routine has the format
633: $       call PetscInitialize(file,ierr)

635: +   ierr - error return code
636: -  file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use PETSC_NULL_CHARACTER to not check for
637:           code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files
638:            
639:    Important Fortran Note:
640:    In Fortran, you MUST use PETSC_NULL_CHARACTER to indicate a
641:    null character string; you CANNOT just use PETSC_NULL as 
642:    in the C version. See the <a href="../../docs/manual.pdf">users manual</a> for details.

644:    If your main program is C but you call Fortran code that also uses PETSc you need to call PetscInitializeFortran() soon after 
645:    calling PetscInitialize().

647:    Concepts: initializing PETSc
648:    
649: .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscInitializeNoArguments()

651: @*/
652: PetscErrorCode  PetscInitialize(int *argc,char ***args,const char file[],const char help[])
653: {
655:   PetscMPIInt    flag, size;
656:   PetscInt       nodesize;
657:   PetscBool      flg;
658:   char           hostname[256];

661:   if (PetscInitializeCalled) return(0);

663:   /* these must be initialized in a routine, not as a constant declaration*/
664:   PETSC_STDOUT = stdout;
665:   PETSC_STDERR = stderr;

667:   PetscOptionsCreate();

669:   /*
670:      We initialize the program name here (before MPI_Init()) because MPICH has a bug in 
671:      it that it sets args[0] on all processors to be args[0] on the first processor.
672:   */
673:   if (argc && *argc) {
674:     PetscSetProgramName(**args);
675:   } else {
676:     PetscSetProgramName("Unknown Name");
677:   }

679:   MPI_Initialized(&flag);
680:   if (!flag) {
681:     if (PETSC_COMM_WORLD != MPI_COMM_NULL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"You cannot set PETSC_COMM_WORLD if you have not initialized MPI first");
682:     MPI_Init(argc,args);
683:     PetscBeganMPI = PETSC_TRUE;
684:   }
685:   if (argc && args) {
686:     PetscGlobalArgc = *argc;
687:     PetscGlobalArgs = *args;
688:   }
689:   PetscFinalizeCalled   = PETSC_FALSE;

691:   if (PETSC_COMM_WORLD == MPI_COMM_NULL) {
692:     PETSC_COMM_WORLD = MPI_COMM_WORLD;
693:   }
694:   MPI_Errhandler_set(PETSC_COMM_WORLD,MPI_ERRORS_RETURN);

696:   /* Done after init due to a bug in MPICH-GM? */
697:   PetscErrorPrintfInitialize();

699:   MPI_Comm_rank(MPI_COMM_WORLD,&PetscGlobalRank);
700:   MPI_Comm_size(MPI_COMM_WORLD,&PetscGlobalSize);

702: #if defined(PETSC_USE_COMPLEX)
703:   /* 
704:      Initialized the global complex variable; this is because with 
705:      shared libraries the constructors for global variables
706:      are not called; at least on IRIX.
707:   */
708:   {
709: #if defined(PETSC_CLANGUAGE_CXX)
710:     PetscScalar ic(0.0,1.0);
711:     PETSC_i = ic;
712: #else
713:     PETSC_i = I;
714: #endif
715:   }

717: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
718:   MPI_Type_contiguous(2,MPIU_REAL,&MPIU_C_DOUBLE_COMPLEX);
719:   MPI_Type_commit(&MPIU_C_DOUBLE_COMPLEX);
720:   MPI_Type_contiguous(2,MPI_FLOAT,&MPIU_C_COMPLEX);
721:   MPI_Type_commit(&MPIU_C_COMPLEX);
722:   MPI_Op_create(PetscSum_Local,1,&MPIU_SUM);
723: #endif
724: #endif

726:   /*
727:      Create the PETSc MPI reduction operator that sums of the first
728:      half of the entries and maxes the second half.
729:   */
730:   MPI_Op_create(PetscMaxSum_Local,1,&PetscMaxSum_Op);

732: #if defined(PETSC_USE_REAL___FLOAT128)
733:   MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU___FLOAT128);
734:   MPI_Type_commit(&MPIU___FLOAT128);
735:   MPI_Op_create(PetscSum_Local,1,&MPIU_SUM);
736:   MPI_Op_create(PetscMax_Local,1,&MPIU_MAX);
737:   MPI_Op_create(PetscMin_Local,1,&MPIU_MIN);
738: #endif

740:   MPI_Type_contiguous(2,MPIU_SCALAR,&MPIU_2SCALAR);
741:   MPI_Type_commit(&MPIU_2SCALAR);
742:   MPI_Op_create(PetscADMax_Local,1,&PetscADMax_Op);
743:   MPI_Op_create(PetscADMin_Local,1,&PetscADMin_Op);

745:   MPI_Type_contiguous(2,MPIU_INT,&MPIU_2INT);
746:   MPI_Type_commit(&MPIU_2INT);

748:   /*
749:      Attributes to be set on PETSc communicators
750:   */
751:   MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelCounter,&Petsc_Counter_keyval,(void*)0);
752:   MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm,&Petsc_InnerComm_keyval,(void*)0);
753:   MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm,&Petsc_OuterComm_keyval,(void*)0);

755:   /*
756:      Build the options database
757:   */
758:   PetscOptionsInsert(argc,args,file);

760: 
761:   /*
762:      Print main application help message
763:   */
764:   PetscOptionsHasName(PETSC_NULL,"-help",&flg);
765:   if (help && flg) {
766:     PetscPrintf(PETSC_COMM_WORLD,help);
767:   }
768:   PetscOptionsCheckInitial_Private();
769: 
770:   /* SHOULD PUT IN GUARDS: Make sure logging is initialized, even if we do not print it out */
771: #if defined(PETSC_USE_LOG)
772:   PetscLogBegin_Private();
773: #endif

775:   /*
776:      Load the dynamic libraries (on machines that support them), this registers all
777:      the solvers etc. (On non-dynamic machines this initializes the PetscDraw and PetscViewer classes)
778:   */
779:   PetscInitialize_DynamicLibraries();

781:   MPI_Comm_size(PETSC_COMM_WORLD,&size);
782:   PetscInfo1(0,"PETSc successfully started: number of processors = %d\n",size);
783:   PetscGetHostName(hostname,256);
784:   PetscInfo1(0,"Running on machine: %s\n",hostname);

786:   PetscOptionsCheckInitial_Components();
787:   /* Check the options database for options related to the options database itself */
788:   PetscOptionsSetFromOptions();

790: #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32)
791:   /* 
792:       Tell MPI about our own data representation converter, this would/should be used if extern32 is not supported by the MPI

794:       Currently not used because it is not supported by MPICH.
795:   */
796: #if !defined(PETSC_WORDS_BIGENDIAN)
797:   MPI_Register_datarep((char *)"petsc",PetscDataRep_read_conv_fn,PetscDataRep_write_conv_fn,PetscDataRep_extent_fn,PETSC_NULL);
798: #endif  
799: #endif

801:   PetscOptionsGetInt(PETSC_NULL,"-hmpi_spawn_size",&nodesize,&flg);
802:   if (flg) {
803: #if defined(PETSC_HAVE_MPI_COMM_SPAWN)
804:     PetscHMPISpawn((PetscMPIInt) nodesize); /* worker nodes never return from here; they go directly to PetscEnd() */
805: #else
806:     SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"PETSc built without MPI 2 (MPI_Comm_spawn) support, use -hmpi_merge_size instead");
807: #endif
808:   } else {
809:     PetscOptionsGetInt(PETSC_NULL,"-hmpi_merge_size",&nodesize,&flg);
810:     if (flg) {
811:       PetscHMPIMerge((PetscMPIInt) nodesize,PETSC_NULL,PETSC_NULL);
812:       if (PetscHMPIWorker) { /* if worker then never enter user code */
813:         PetscInitializeCalled = PETSC_TRUE;
814:         PetscEnd();
815:       }
816:     }
817:   }

819: #if defined(PETSC_HAVE_CUDA)
820:   cublasInit();
821: #endif

823: #if defined(PETSC_HAVE_AMS)
824:   PetscOptionsHasName(PETSC_NULL,"-ams_publish_objects",&flg);
825:   if (flg) {
826:     PetscAMSPublishAll = PETSC_TRUE;
827:   }
828: #endif

830:   PetscOptionsHasName(PETSC_NULL,"-python",&flg);
831:   if (flg) {
832:     PetscInitializeCalled = PETSC_TRUE;
833:     PetscPythonInitialize(PETSC_NULL,PETSC_NULL);
834:   }

836:   /*
837:       Once we are completedly initialized then we can set this variables
838:   */
839:   PetscInitializeCalled = PETSC_TRUE;
840:   return(0);
841: }


848: /*@C 
849:    PetscFinalize - Checks for options to be called at the conclusion
850:    of the program. MPI_Finalize() is called only if the user had not
851:    called MPI_Init() before calling PetscInitialize().

853:    Collective on PETSC_COMM_WORLD

855:    Options Database Keys:
856: +  -options_table - Calls PetscOptionsView()
857: .  -options_left - Prints unused options that remain in the database
858: .  -objects_left  - Prints list of all objects that have not been freed
859: .  -mpidump - Calls PetscMPIDump()
860: .  -malloc_dump - Calls PetscMallocDump()
861: .  -malloc_info - Prints total memory usage
862: -  -malloc_log - Prints summary of memory usage

864:    Options Database Keys for Profiling:
865:    See the <a href="../../docs/manual.pdf#nameddest=Chapter 11 Profiling">profiling chapter of the users manual</a> for details.
866: +  -log_summary [filename] - Prints summary of flop and timing
867:         information to screen. If the filename is specified the
868:         summary is written to the file.  See PetscLogView().
869: .  -log_summary_python [filename] - Prints data on of flop and timing usage to a file or screen.
870:         See PetscLogPrintSViewPython().
871: .  -log_all [filename] - Logs extensive profiling information
872:         See PetscLogDump(). 
873: .  -log [filename] - Logs basic profiline information  See PetscLogDump().
874: .  -log_sync - Log the synchronization in scatters, inner products
875:         and norms
876: -  -log_mpe [filename] - Creates a logfile viewable by the 
877:       utility Upshot/Nupshot (in MPICH distribution)

879:    Level: beginner

881:    Note:
882:    See PetscInitialize() for more general runtime options.

884: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscEnd()
885: @*/
886: PetscErrorCode  PetscFinalize(void)
887: {
889:   PetscMPIInt    rank;
890:   PetscInt       i,nopt;
891:   PetscBool      flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,objects_left = PETSC_FALSE;
892: #if defined(PETSC_HAVE_AMS)
893:   PetscBool      flg = PETSC_FALSE;
894: #endif
895: #if defined(PETSC_USE_LOG)
896:   char           mname[PETSC_MAX_PATH_LEN];
897: #endif
898: 

901:   if (!PetscInitializeCalled) {
902:     printf("PetscInitialize() must be called before PetscFinalize()\n");
903:     PetscFunctionReturn(PETSC_ERR_ARG_WRONGSTATE);
904:   }
905:   PetscInfo(PETSC_NULL,"PetscFinalize() called\n");

907: #if defined(PETSC_HAVE_AMS)
908:   PetscOptionsGetBool(PETSC_NULL,"-options_gui",&flg,PETSC_NULL);
909:   if (flg) {
910:     PetscOptionsAMSDestroy();
911:   }
912: #endif

914:   PetscHMPIFinalize();
915: #if defined(PETSC_HAVE_PTHREADCLASSES)
916:   if (PetscThreadFinalize) {
917:     /* thread pool case */
918:     (*PetscThreadFinalize)();
919:   }
920:   free(ThreadCoreAffinity);
921: #endif

923:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
924:   PetscOptionsGetBool(PETSC_NULL,"-malloc_info",&flg2,PETSC_NULL);
925:   if (!flg2) {
926:     flg2 = PETSC_FALSE;
927:     PetscOptionsGetBool(PETSC_NULL,"-memory_info",&flg2,PETSC_NULL);
928:   }
929:   if (flg2) {
930:     PetscMemoryShowUsage(PETSC_VIEWER_STDOUT_WORLD,"Summary of Memory Usage in PETSc\n");
931:   }

933: #if defined(PETSC_USE_LOG)
934:   flg1 = PETSC_FALSE;
935:   PetscOptionsGetBool(PETSC_NULL,"-get_total_flops",&flg1,PETSC_NULL);
936:   if (flg1) {
937:     PetscLogDouble flops = 0;
938:     MPI_Reduce(&_TotalFlops,&flops,1,MPI_DOUBLE,MPI_SUM,0,PETSC_COMM_WORLD);
939:     PetscPrintf(PETSC_COMM_WORLD,"Total flops over all processors %g\n",flops);
940:   }
941: #endif


944: #if defined(PETSC_USE_LOG)
945: #if defined(PETSC_HAVE_MPE)
946:   mname[0] = 0;
947:   PetscOptionsGetString(PETSC_NULL,"-log_mpe",mname,PETSC_MAX_PATH_LEN,&flg1);
948:   if (flg1){
949:     if (mname[0]) {PetscLogMPEDump(mname);}
950:     else          {PetscLogMPEDump(0);}
951:   }
952: #endif
953:   mname[0] = 0;
954:   PetscOptionsGetString(PETSC_NULL,"-log_summary",mname,PETSC_MAX_PATH_LEN,&flg1);
955:   if (flg1) {
956:     PetscViewer viewer;
957:     if (mname[0])  {
958:       PetscViewerASCIIOpen(PETSC_COMM_WORLD,mname,&viewer);
959:       PetscLogView(viewer);
960:       PetscViewerDestroy(&viewer);
961:     } else {
962:       viewer = PETSC_VIEWER_STDOUT_WORLD;
963:       PetscLogView(viewer);
964:     }
965:   }
966: 
967:   mname[0] = 0;
968:   PetscOptionsGetString(PETSC_NULL,"-log_summary_python",mname,PETSC_MAX_PATH_LEN,&flg1);
969:   if (flg1) {
970:     PetscViewer viewer;
971:     if (mname[0])  {
972:       PetscViewerASCIIOpen(PETSC_COMM_WORLD,mname,&viewer);
973:       PetscLogViewPython(viewer);
974:       PetscViewerDestroy(&viewer);
975:     } else {
976:       viewer = PETSC_VIEWER_STDOUT_WORLD;
977:       PetscLogViewPython(viewer);
978:     }
979:   }
980: 
981:   PetscOptionsGetString(PETSC_NULL,"-log_detailed",mname,PETSC_MAX_PATH_LEN,&flg1);
982:   if (flg1) {
983:     if (mname[0])  {PetscLogPrintDetailed(PETSC_COMM_WORLD,mname);}
984:     else           {PetscLogPrintDetailed(PETSC_COMM_WORLD,0);}
985:   }
986: 
987:   mname[0] = 0;
988:   PetscOptionsGetString(PETSC_NULL,"-log_all",mname,PETSC_MAX_PATH_LEN,&flg1);
989:   PetscOptionsGetString(PETSC_NULL,"-log",mname,PETSC_MAX_PATH_LEN,&flg2);
990:   if (flg1 || flg2){
991:     if (mname[0]) PetscLogDump(mname);
992:     else          PetscLogDump(0);
993:   }
994: #endif

996: #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD)
997:   if (PetscStackActive) {
998:     PetscStackDestroy();
999:   }
1000: #endif

1002:   flg1 = PETSC_FALSE;
1003:   PetscOptionsGetBool(PETSC_NULL,"-no_signal_handler",&flg1,PETSC_NULL);
1004:   if (!flg1) { PetscPopSignalHandler();}
1005:   flg1 = PETSC_FALSE;
1006:   PetscOptionsGetBool(PETSC_NULL,"-mpidump",&flg1,PETSC_NULL);
1007:   if (flg1) {
1008:     PetscMPIDump(stdout);
1009:   }
1010:   flg1 = PETSC_FALSE;
1011:   flg2 = PETSC_FALSE;
1012:   /* preemptive call to avoid listing this option in options table as unused */
1013:   PetscOptionsHasName(PETSC_NULL,"-malloc_dump",&flg1);
1014:   PetscOptionsGetBool(PETSC_NULL,"-options_table",&flg2,PETSC_NULL);

1016:   if (flg2) {
1017:     PetscOptionsView(PETSC_VIEWER_STDOUT_WORLD);
1018:   }

1020:   /* to prevent PETSc -options_left from warning */
1021:   PetscOptionsHasName(PETSC_NULL,"-nox",&flg1);
1022:   PetscOptionsHasName(PETSC_NULL,"-nox_warning",&flg1);
1023:   PetscOptionsGetBool(PETSC_NULL,"-objects_left",&objects_left,PETSC_NULL);

1025:   if (!PetscHMPIWorker) { /* worker processes skip this because they do not usually process options */
1026:     flg3 = PETSC_FALSE; /* default value is required */
1027:     PetscOptionsGetBool(PETSC_NULL,"-options_left",&flg3,&flg1);
1028:     PetscOptionsAllUsed(&nopt);
1029:     if (flg3) {
1030:       if (!flg2) { /* have not yet printed the options */
1031:         PetscOptionsView(PETSC_VIEWER_STDOUT_WORLD);
1032:       }
1033:       if (!nopt) {
1034:         PetscPrintf(PETSC_COMM_WORLD,"There are no unused options.\n");
1035:       } else if (nopt == 1) {
1036:         PetscPrintf(PETSC_COMM_WORLD,"There is one unused database option. It is:\n");
1037:       } else {
1038:         PetscPrintf(PETSC_COMM_WORLD,"There are %D unused database options. They are:\n",nopt);
1039:       }
1040:     }
1041: #if defined(PETSC_USE_DEBUG)
1042:     if (nopt && !flg3 && !flg1) {
1043:       PetscPrintf(PETSC_COMM_WORLD,"WARNING! There are options you set that were not used!\n");
1044:       PetscPrintf(PETSC_COMM_WORLD,"WARNING! could be spelling mistake, etc!\n");
1045:       PetscOptionsLeft();
1046:     } else if (nopt && flg3) {
1047: #else 
1048:     if (nopt && flg3) {
1049: #endif
1050:       PetscOptionsLeft();
1051:     }
1052:   }

1054:   /*
1055:      Free all objects registered with PetscObjectRegisterDestroy() such as PETSC_VIEWER_XXX_().
1056:   */
1057:   PetscObjectRegisterDestroyAll();

1059:   /* 
1060:        List all objects the user may have forgot to free 
1061:   */
1062:   if (objects_left && PetscObjectsCounts) {
1063:     PetscPrintf(PETSC_COMM_WORLD,"The following objects %D were never freed\n",PetscObjectsCounts);
1064:   }
1065:   for (i=0; i<PetscObjectsMaxCounts; i++) {
1066:     if (PetscObjects[i]) {
1067:       if (objects_left) {
1068:         PetscPrintf(PETSC_COMM_WORLD,"  %s %s %s\n",PetscObjects[i]->class_name,PetscObjects[i]->type_name,PetscObjects[i]->name);
1069:       }
1070:     }
1071:   }
1072:   /* cannot actually destroy the left over objects, but destroy the list */
1073:   PetscObjectsCounts    = 0;
1074:   PetscObjectsMaxCounts = 0;
1075:   PetscFree(PetscObjects);


1078: #if defined(PETSC_USE_LOG)
1079:   PetscLogDestroy();
1080: #endif

1082:   /*
1083:        Free all the registered create functions, such as KSPList, VecList, SNESList, etc
1084:   */
1085:   PetscFListDestroyAll();

1087:   /* 
1088:      Destroy any packages that registered a finalize 
1089:   */
1090:   PetscRegisterFinalizeAll();

1092:   /*
1093:      Destroy all the function registration lists created
1094:   */
1095:   PetscFinalize_DynamicLibraries();

1097:   if (petsc_history) {
1098:     PetscCloseHistoryFile(&petsc_history);
1099:     petsc_history = 0;
1100:   }

1102:   PetscInfoAllow(PETSC_FALSE,PETSC_NULL);

1104:   {
1105:     char fname[PETSC_MAX_PATH_LEN];
1106:     FILE *fd;
1107:     int  err;

1109:     fname[0] = 0;
1110:     PetscOptionsGetString(PETSC_NULL,"-malloc_dump",fname,250,&flg1);
1111:     if (flg1 && fname[0]) {
1112:       char sname[PETSC_MAX_PATH_LEN];

1114:       sprintf(sname,"%s_%d",fname,rank);
1115:       fd   = fopen(sname,"w"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",sname);
1116:       PetscMallocDump(fd);
1117:       err = fclose(fd);
1118:       if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
1119:     } else if (flg1) {
1120:       MPI_Comm local_comm;

1122:       MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);
1123:       PetscSequentialPhaseBegin_Private(local_comm,1);
1124:         PetscMallocDump(stdout);
1125:       PetscSequentialPhaseEnd_Private(local_comm,1);
1126:       MPI_Comm_free(&local_comm);
1127:     }
1128:   }
1129:   {
1130:     char fname[PETSC_MAX_PATH_LEN];
1131:     FILE *fd;
1132: 
1133:     fname[0] = 0;
1134:     PetscOptionsGetString(PETSC_NULL,"-malloc_log",fname,250,&flg1);
1135:     if (flg1 && fname[0]) {
1136:       char sname[PETSC_MAX_PATH_LEN];
1137:       int  err;

1139:       sprintf(sname,"%s_%d",fname,rank);
1140:       fd   = fopen(sname,"w"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",sname);
1141:       PetscMallocDumpLog(fd);
1142:       err = fclose(fd);
1143:       if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
1144:     } else if (flg1) {
1145:       PetscMallocDumpLog(stdout);
1146:     }
1147:   }
1148:   /* Can be destroyed only after all the options are used */
1149:   PetscOptionsDestroy();

1151:   PetscGlobalArgc = 0;
1152:   PetscGlobalArgs = 0;

1154: #if defined(PETSC_USE_REAL___FLOAT128)
1155:   MPI_Type_free(&MPIU___FLOAT128);
1156:   MPI_Op_free(&MPIU_SUM);
1157:   MPI_Op_free(&MPIU_MAX);
1158:   MPI_Op_free(&MPIU_MIN);
1159: #endif

1161: #if defined(PETSC_USE_COMPLEX)
1162: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
1163:   MPI_Op_free(&MPIU_SUM);
1164:   MPI_Type_free(&MPIU_C_DOUBLE_COMPLEX);
1165:   MPI_Type_free(&MPIU_C_COMPLEX);
1166: #endif
1167: #endif
1168:   MPI_Type_free(&MPIU_2SCALAR);
1169:   MPI_Type_free(&MPIU_2INT);
1170:   MPI_Op_free(&PetscMaxSum_Op);
1171:   MPI_Op_free(&PetscADMax_Op);
1172:   MPI_Op_free(&PetscADMin_Op);

1174:   /* 
1175:      Destroy any known inner MPI_Comm's and attributes pointing to them
1176:      Note this will not destroy any new communicators the user has created.

1178:      If all PETSc objects were not destroyed those left over objects will have hanging references to 
1179:      the MPI_Comms that were freed; but that is ok because those PETSc objects will never be used again
1180:  */
1181:   {
1182:     PetscCommCounter *counter;
1183:     PetscMPIInt      flg;
1184:     MPI_Comm         icomm;
1185:     void             *ptr;
1186:     MPI_Attr_get(PETSC_COMM_SELF,Petsc_InnerComm_keyval,&ptr,&flg);
1187:     if (flg) {
1188:       /*  Use PetscMemcpy() because casting from pointer to integer of different size is not allowed with some compilers  */
1189:       PetscMemcpy(&icomm,&ptr,sizeof(MPI_Comm));
1190:       MPI_Attr_get(icomm,Petsc_Counter_keyval,&counter,&flg);
1191:       if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory");

1193:       MPI_Attr_delete(PETSC_COMM_SELF,Petsc_InnerComm_keyval);
1194:       MPI_Attr_delete(icomm,Petsc_Counter_keyval);
1195:       MPI_Comm_free(&icomm);
1196:     }
1197:     MPI_Attr_get(PETSC_COMM_WORLD,Petsc_InnerComm_keyval,&ptr,&flg);
1198:     if (flg) {
1199:       /*  Use PetscMemcpy() because casting from pointer to integer of different size is not allowed with some compilers  */
1200:       PetscMemcpy(&icomm,&ptr,sizeof(MPI_Comm));
1201:       MPI_Attr_get(icomm,Petsc_Counter_keyval,&counter,&flg);
1202:       if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory");

1204:       MPI_Attr_delete(PETSC_COMM_WORLD,Petsc_InnerComm_keyval);
1205:       MPI_Attr_delete(icomm,Petsc_Counter_keyval);
1206:       MPI_Comm_free(&icomm);
1207:     }
1208:   }

1210:   MPI_Keyval_free(&Petsc_Counter_keyval);
1211:   MPI_Keyval_free(&Petsc_InnerComm_keyval);
1212:   MPI_Keyval_free(&Petsc_OuterComm_keyval);

1214:   PetscInfo(0,"PETSc successfully ended!\n");
1215:   if (PetscBeganMPI) {
1216: #if defined(PETSC_HAVE_MPI_FINALIZED)
1217:     PetscMPIInt flag;
1218:     MPI_Finalized(&flag);
1219:     if (flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"MPI_Finalize() has already been called, even though MPI_Init() was called by PetscInitialize()");
1220: #endif
1221:     MPI_Finalize();
1222:   }

1224:   if (PETSC_ZOPEFD){
1225:     if (PETSC_ZOPEFD != PETSC_STDOUT) fprintf(PETSC_ZOPEFD, "<<<end>>>");
1226:     else fprintf(PETSC_STDOUT, "<<<end>>>");
1227:   }

1229: #if defined(PETSC_HAVE_CUDA)
1230:   cublasShutdown();
1231: #endif
1232: /*

1234:      Note: In certain cases PETSC_COMM_WORLD is never MPI_Comm_free()ed because 
1235:    the communicator has some outstanding requests on it. Specifically if the 
1236:    flag PETSC_HAVE_BROKEN_REQUEST_FREE is set (for IBM MPI implementation). See 
1237:    src/vec/utils/vpscat.c. Due to this the memory allocated in PetscCommDuplicate()
1238:    is never freed as it should be. Thus one may obtain messages of the form
1239:    [ 1] 8 bytes PetscCommDuplicate() line 645 in src/sys/mpiu.c indicating the
1240:    memory was not freed.

1242: */
1243:   PetscMallocClear();
1244:   PetscInitializeCalled = PETSC_FALSE;
1245:   PetscFinalizeCalled   = PETSC_TRUE;
1246:   PetscFunctionReturn(ierr);
1247: }