Actual source code: pinit.c
2: /*
3: This file defines the initialization of PETSc, including PetscInitialize()
4: */
6: #include <petscsys.h> /*I "petscsys.h" I*/
8: #if defined(PETSC_HAVE_CUSP)
9: #include <cublas.h>
10: #endif
12: #if defined(PETSC_USE_LOG)
14: #endif
17: /* -----------------------------------------------------------------------------------------*/
29: /* this is used by the _, __, and ___ macros (see include/petscerror.h) */
30: PetscErrorCode __g0;
32: /* user may set this BEFORE calling PetscInitialize() */
33: MPI_Comm PETSC_COMM_WORLD = MPI_COMM_NULL;
35: PetscMPIInt Petsc_Counter_keyval = MPI_KEYVAL_INVALID;
36: PetscMPIInt Petsc_InnerComm_keyval = MPI_KEYVAL_INVALID;
37: PetscMPIInt Petsc_OuterComm_keyval = MPI_KEYVAL_INVALID;
39: /*
40: Declare and set all the string names of the PETSc enums
41: */
42: const char *PetscBools[] = {"FALSE","TRUE","PetscBool","PETSC_",0};
43: const char *PetscCopyModes[] = {"COPY_VALUES","OWN_POINTER","USE_POINTER","PetscCopyMode","PETSC_",0};
44: const char *PetscDataTypes[] = {"INT","DOUBLE","COMPLEX","LONG","SHORT","FLOAT",
45: "CHAR","LOGICAL","ENUM","BOOL","LONGDOUBLE","PetscDataType","PETSC_",0};
47: PetscBool PetscPreLoadingUsed = PETSC_FALSE;
48: PetscBool PetscPreLoadingOn = PETSC_FALSE;
50: /*
51: Checks the options database for initializations related to the
52: PETSc components
53: */
56: PetscErrorCode PetscOptionsCheckInitial_Components(void)
57: {
58: PetscBool flg1;
62: PetscOptionsHasName(PETSC_NULL,"-help",&flg1);
63: if (flg1) {
64: #if defined (PETSC_USE_LOG)
65: MPI_Comm comm = PETSC_COMM_WORLD;
66: (*PetscHelpPrintf)(comm,"------Additional PETSc component options--------\n");
67: (*PetscHelpPrintf)(comm," -log_summary_exclude: <vec,mat,pc.ksp,snes>\n");
68: (*PetscHelpPrintf)(comm," -info_exclude: <null,vec,mat,pc,ksp,snes,ts>\n");
69: (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
70: #endif
71: }
72: return(0);
73: }
75: #if defined(PETSC_HAVE_MATLAB_ENGINE)
80: /*
81: PetscInitializeMatlab - Calls PetscInitialize() from C/C++ without the pointers to argc and args
83: Collective
84:
85: Level: advanced
87: Notes: this is called only by the PETSc MATLAB interface. Even though it might start MPI it sets the flag to
88: indicate that it did NOT start MPI so that the PetscFinalize() does not end MPI, thus allowing PetscInitialize() to
89: be called multiple times from MATLAB without the problem of trying to initialize MPI more than once.
91: Turns off PETSc signal handling because that can interact with MATLAB's signal handling causing random crashes.
93: .seealso: PetscInitialize(), PetscInitializeFortran(), PetscInitializeNoArguments()
94: */
95: PetscErrorCode PetscInitializeMatlab(int argc,char **args,const char *filename,const char *help)
96: {
98: int myargc = argc;
99: char **myargs = args;
102: PetscInitialize(&myargc,&myargs,filename,help);
103: PetscPopSignalHandler();
104: PetscBeganMPI = PETSC_FALSE;
105: PetscFunctionReturn(ierr);
106: }
110: /*
111: PetscInitializedMatlab - Has PETSc been initialized already?
113: Not Collective
114:
115: Level: advanced
117: Notes: this is called only by the PETSc MATLAB interface.
119: .seealso: PetscInitialize(), PetscInitializeFortran(), PetscInitializeNoArguments()
120: */
121: int PetscInitializedMatlab(void)
122: {
123: PetscBool flg;
125: PetscInitialized(&flg);
126: if (flg) return 1;
127: else return 0;
128: }
132: /*
133: Used by MATLAB interface to get communicator
134: */
135: PetscErrorCode PetscGetPETSC_COMM_SELFMatlab(MPI_Comm *comm)
136: {
138: *comm = PETSC_COMM_SELF;
139: return(0);
140: }
141: #endif
145: /*@C
146: PetscInitializeNoArguments - Calls PetscInitialize() from C/C++ without
147: the command line arguments.
149: Collective
150:
151: Level: advanced
153: .seealso: PetscInitialize(), PetscInitializeFortran()
154: @*/
155: PetscErrorCode PetscInitializeNoArguments(void)
156: {
158: int argc = 0;
159: char **args = 0;
162: PetscInitialize(&argc,&args,PETSC_NULL,PETSC_NULL);
163: PetscFunctionReturn(ierr);
164: }
168: /*@
169: PetscInitialized - Determine whether PETSc is initialized.
170:
171: 7 Level: beginner
173: .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran()
174: @*/
175: PetscErrorCode PetscInitialized(PetscBool *isInitialized)
176: {
179: *isInitialized = PetscInitializeCalled;
180: return(0);
181: }
185: /*@
186: PetscFinalized - Determine whether PetscFinalize() has been called yet
187:
188: Level: developer
190: .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran()
191: @*/
192: PetscErrorCode PetscFinalized(PetscBool *isFinalized)
193: {
196: *isFinalized = PetscFinalizeCalled;
197: return(0);
198: }
203: /*
204: This function is the MPI reduction operation used to compute the sum of the
205: first half of the datatype and the max of the second half.
206: */
207: MPI_Op PetscMaxSum_Op = 0;
212: void MPIAPI PetscMaxSum_Local(void *in,void *out,int *cnt,MPI_Datatype *datatype)
213: {
214: PetscInt *xin = (PetscInt*)in,*xout = (PetscInt*)out,i,count = *cnt;
217: if (*datatype != MPIU_2INT) {
218: (*PetscErrorPrintf)("Can only handle MPIU_2INT data types");
219: MPI_Abort(MPI_COMM_WORLD,1);
220: }
222: for (i=0; i<count; i++) {
223: xout[2*i] = PetscMax(xout[2*i],xin[2*i]);
224: xout[2*i+1] += xin[2*i+1];
225: }
226: PetscFunctionReturnVoid();
227: }
230: /*
231: Returns the max of the first entry owned by this processor and the
232: sum of the second entry.
234: The reason nprocs[2*i] contains lengths nprocs[2*i+1] contains flag of 1 if length is nonzero
235: is so that the PetscMaxSum_Op() can set TWO values, if we passed in only nprocs[i] with lengths
236: there would be no place to store the both needed results.
237: */
240: PetscErrorCode PetscMaxSum(MPI_Comm comm,const PetscInt nprocs[],PetscInt *max,PetscInt *sum)
241: {
242: PetscMPIInt size,rank;
243: PetscInt *work;
247: MPI_Comm_size(comm,&size);
248: MPI_Comm_rank(comm,&rank);
249: PetscMalloc(2*size*sizeof(PetscInt),&work);
250: MPI_Allreduce((void*)nprocs,work,size,MPIU_2INT,PetscMaxSum_Op,comm);
251: *max = work[2*rank];
252: *sum = work[2*rank+1];
253: PetscFree(work);
254: return(0);
255: }
257: /* ----------------------------------------------------------------------------*/
258: MPI_Op PetscADMax_Op = 0;
263: void MPIAPI PetscADMax_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
264: {
265: PetscScalar *xin = (PetscScalar *)in,*xout = (PetscScalar*)out;
266: PetscInt i,count = *cnt;
269: if (*datatype != MPIU_2SCALAR) {
270: (*PetscErrorPrintf)("Can only handle MPIU_2SCALAR data (i.e. double or complex) types");
271: MPI_Abort(MPI_COMM_WORLD,1);
272: }
274: for (i=0; i<count; i++) {
275: if (PetscRealPart(xout[2*i]) < PetscRealPart(xin[2*i])) {
276: xout[2*i] = xin[2*i];
277: xout[2*i+1] = xin[2*i+1];
278: }
279: }
280: PetscFunctionReturnVoid();
281: }
284: MPI_Op PetscADMin_Op = 0;
289: void MPIAPI PetscADMin_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
290: {
291: PetscScalar *xin = (PetscScalar *)in,*xout = (PetscScalar*)out;
292: PetscInt i,count = *cnt;
295: if (*datatype != MPIU_2SCALAR) {
296: (*PetscErrorPrintf)("Can only handle MPIU_2SCALAR data (i.e. double or complex) types");
297: MPI_Abort(MPI_COMM_WORLD,1);
298: }
300: for (i=0; i<count; i++) {
301: if (PetscRealPart(xout[2*i]) > PetscRealPart(xin[2*i])) {
302: xout[2*i] = xin[2*i];
303: xout[2*i+1] = xin[2*i+1];
304: }
305: }
306: PetscFunctionReturnVoid();
307: }
309: /* ---------------------------------------------------------------------------------------*/
311: #if (defined(PETSC_USE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128)
312: MPI_Op MPIU_SUM = 0;
317: void PetscSum_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
318: {
319: PetscInt i,count = *cnt;
322: if (*datatype == MPIU_SCALAR) {
323: PetscScalar *xin = (PetscScalar *)in,*xout = (PetscScalar*)out;
324: for (i=0; i<count; i++) {
325: xout[i] += xin[i];
326: }
327: } else if (*datatype == MPIU_REAL) {
328: PetscReal *xin = (PetscReal *)in,*xout = (PetscReal*)out;
329: for (i=0; i<count; i++) {
330: xout[i] += xin[i];
331: }
332: } else {
333: (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_SCALAR data (i.e. double or complex) types");
334: MPI_Abort(MPI_COMM_WORLD,1);
335: }
336: PetscFunctionReturnVoid();
337: }
339: #endif
341: #if defined(PETSC_USE_REAL___FLOAT128)
342: MPI_Op MPIU_MAX = 0;
343: MPI_Op MPIU_MIN = 0;
348: void PetscMax_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
349: {
350: PetscScalar *xin = (PetscScalar *)in,*xout = (PetscScalar*)out;
351: PetscInt i,count = *cnt;
354: if (*datatype != MPIU_SCALAR) {
355: (*PetscErrorPrintf)("Can only handle MPIU_SCALAR data (i.e. double or complex) types");
356: MPI_Abort(MPI_COMM_WORLD,1);
357: }
359: for (i=0; i<count; i++) {
360: xout[i] = PetscMax(xout[i],xin[i]);
361: }
362: PetscFunctionReturnVoid();
363: }
369: void PetscMin_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
370: {
371: PetscScalar *xin = (PetscScalar *)in,*xout = (PetscScalar*)out;
372: PetscInt i,count = *cnt;
375: if (*datatype != MPIU_SCALAR) {
376: (*PetscErrorPrintf)("Can only handle MPIU_SCALAR data (i.e. double or complex) types");
377: MPI_Abort(MPI_COMM_WORLD,1);
378: }
380: for (i=0; i<count; i++) {
381: xout[i] = PetscMin(xout[i],xin[i]);
382: }
383: PetscFunctionReturnVoid();
384: }
386: #endif
391: /*
392: Private routine to delete internal tag/name counter storage when a communicator is freed.
394: This is called by MPI, not by users. This is called by MPI_Comm_free() when the communicator that has this data as an attribute is freed.
398: */
399: PetscMPIInt MPIAPI Petsc_DelCounter(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state)
400: {
404: PetscInfo1(0,"Deleting counter data in an MPI_Comm %ld\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
405: PetscFree(count_val);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
406: PetscFunctionReturn(MPI_SUCCESS);
407: }
413: /*
414: This does not actually free anything, it simply marks when a reference count to an internal or external MPI_Comm reaches zero and the
415: the external MPI_Comm drops its reference to the internal or external MPI_Comm
417: This is called by MPI, not by users. This is called when MPI_Comm_free() is called on the communicator.
421: */
422: PetscMPIInt MPIAPI Petsc_DelComm(MPI_Comm comm,PetscMPIInt keyval,void *attr_val,void *extra_state)
423: {
424: PetscErrorCode ierr;
425: PetscMPIInt flg;
426: MPI_Comm icomm;
427: void *ptr;
430: MPI_Attr_get(comm,Petsc_InnerComm_keyval,&ptr,&flg);
431: if (flg) {
432: /* Use PetscMemcpy() because casting from pointer to integer of different size is not allowed with some compilers */
433: PetscMemcpy(&icomm,&ptr,sizeof(MPI_Comm));
434: MPI_Attr_get(icomm,Petsc_OuterComm_keyval,&ptr,&flg);
435: if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected reference to outer comm");
436: MPI_Attr_delete(icomm,Petsc_OuterComm_keyval);
437: PetscInfo1(0,"User MPI_Comm m %ld is being freed, removing reference from inner PETSc comm to this outer comm\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
438: } else {
439: PetscInfo1(0,"Removing reference to PETSc communicator imbedded in a user MPI_Comm m %ld\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
440: }
441: PetscFunctionReturn(MPI_SUCCESS);
442: }
445: #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32)
446: #if !defined(PETSC_WORDS_BIGENDIAN)
452: #endif
453: #endif
455: int PetscGlobalArgc = 0;
456: char **PetscGlobalArgs = 0;
460: /*@C
461: PetscGetArgs - Allows you to access the raw command line arguments anywhere
462: after PetscInitialize() is called but before PetscFinalize().
464: Not Collective
466: Output Parameters:
467: + argc - count of number of command line arguments
468: - args - the command line arguments
470: Level: intermediate
472: Notes:
473: This is usually used to pass the command line arguments into other libraries
474: that are called internally deep in PETSc or the application.
476: The first argument contains the program name as is normal for C arguments.
478: Concepts: command line arguments
479:
480: .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArguments()
482: @*/
483: PetscErrorCode PetscGetArgs(int *argc,char ***args)
484: {
486: if (!PetscInitializeCalled && PetscFinalizeCalled) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()");
487: *argc = PetscGlobalArgc;
488: *args = PetscGlobalArgs;
489: return(0);
490: }
494: /*@C
495: PetscGetArguments - Allows you to access the command line arguments anywhere
496: after PetscInitialize() is called but before PetscFinalize().
498: Not Collective
500: Output Parameters:
501: . args - the command line arguments
503: Level: intermediate
505: Notes:
506: This does NOT start with the program name and IS null terminated (final arg is void)
508: Concepts: command line arguments
509:
510: .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscFreeArguments()
512: @*/
513: PetscErrorCode PetscGetArguments(char ***args)
514: {
515: PetscInt i,argc = PetscGlobalArgc;
519: if (!PetscInitializeCalled && PetscFinalizeCalled) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()");
520: if (!argc) {*args = 0; return(0);}
521: PetscMalloc(argc*sizeof(char*),args);
522: for (i=0; i<argc-1; i++) {
523: PetscStrallocpy(PetscGlobalArgs[i+1],&(*args)[i]);
524: }
525: (*args)[argc-1] = 0;
526: return(0);
527: }
531: /*@C
532: PetscFreeArguments - Frees the memory obtained with PetscGetArguments()
534: Not Collective
536: Output Parameters:
537: . args - the command line arguments
539: Level: intermediate
541: Concepts: command line arguments
542:
543: .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscGetArguments()
545: @*/
546: PetscErrorCode PetscFreeArguments(char **args)
547: {
548: PetscInt i = 0;
552: if (!args) {return(0);}
553: while (args[i]) {
554: PetscFree(args[i]);
555: i++;
556: }
557: PetscFree(args);
558: return(0);
559: }
563: /*@C
564: PetscInitialize - Initializes the PETSc database and MPI.
565: PetscInitialize() calls MPI_Init() if that has yet to be called,
566: so this routine should always be called near the beginning of
567: your program -- usually the very first line!
569: Collective on MPI_COMM_WORLD or PETSC_COMM_WORLD if it has been set
571: Input Parameters:
572: + argc - count of number of command line arguments
573: . args - the command line arguments
574: . file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use PETSC_NULL to not check for
575: code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files
576: - help - [optional] Help message to print, use PETSC_NULL for no message
578: If you wish PETSc code to run ONLY on a subcommunicator of MPI_COMM_WORLD, create that
579: communicator first and assign it to PETSC_COMM_WORLD BEFORE calling PetscInitialize(). Thus if you are running a
580: four process job and two processes will run PETSc and have PetscInitialize() and PetscFinalize() and two process will not,
581: then do this. If ALL processes in the job are using PetscInitialize() and PetscFinalize() then you don't need to do this, even
582: if different subcommunicators of the job are doing different things with PETSc.
584: Options Database Keys:
585: + -start_in_debugger [noxterm,dbx,xdb,gdb,...] - Starts program in debugger
586: . -on_error_attach_debugger [noxterm,dbx,xdb,gdb,...] - Starts debugger when error detected
587: . -on_error_emacs <machinename> causes emacsclient to jump to error file
588: . -on_error_abort calls abort() when error detected (no traceback)
589: . -on_error_mpiabort calls MPI_abort() when error detected
590: . -error_output_stderr prints error messages to stderr instead of the default stdout
591: . -error_output_none does not print the error messages (but handles errors in the same way as if this was not called)
592: . -debugger_nodes [node1,node2,...] - Indicates nodes to start in debugger
593: . -debugger_pause [sleeptime] (in seconds) - Pauses debugger
594: . -stop_for_debugger - Print message on how to attach debugger manually to
595: process and wait (-debugger_pause) seconds for attachment
596: . -malloc - Indicates use of PETSc error-checking malloc (on by default for debug version of libraries)
597: . -malloc no - Indicates not to use error-checking malloc
598: . -malloc_debug - check for memory corruption at EVERY malloc or free
599: . -fp_trap - Stops on floating point exceptions (Note that on the
600: IBM RS6000 this slows code by at least a factor of 10.)
601: . -no_signal_handler - Indicates not to trap error signals
602: . -shared_tmp - indicates /tmp directory is shared by all processors
603: . -not_shared_tmp - each processor has own /tmp
604: . -tmp - alternative name of /tmp directory
605: . -get_total_flops - returns total flops done by all processors
606: . -memory_info - Print memory usage at end of run
607: - -server <port> - start PETSc webserver (default port is 8080)
609: Options Database Keys for Profiling:
610: See the <a href="../../docs/manual.pdf#nameddest=Chapter 11 Profiling">profiling chapter of the users manual</a> for details.
611: + -log_trace [filename] - Print traces of all PETSc calls
612: to the screen (useful to determine where a program
613: hangs without running in the debugger). See PetscLogTraceBegin().
614: . -info <optional filename> - Prints verbose information to the screen
615: - -info_exclude <null,vec,mat,pc,ksp,snes,ts> - Excludes some of the verbose messages
617: Environmental Variables:
618: + PETSC_TMP - alternative tmp directory
619: . PETSC_SHARED_TMP - tmp is shared by all processes
620: . PETSC_NOT_SHARED_TMP - each process has its own private tmp
621: . PETSC_VIEWER_SOCKET_PORT - socket number to use for socket viewer
622: - PETSC_VIEWER_SOCKET_MACHINE - machine to use for socket viewer to connect to
625: Level: beginner
627: Notes:
628: If for some reason you must call MPI_Init() separately, call
629: it before PetscInitialize().
631: Fortran Version:
632: In Fortran this routine has the format
633: $ call PetscInitialize(file,ierr)
635: + ierr - error return code
636: - file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use PETSC_NULL_CHARACTER to not check for
637: code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files
638:
639: Important Fortran Note:
640: In Fortran, you MUST use PETSC_NULL_CHARACTER to indicate a
641: null character string; you CANNOT just use PETSC_NULL as
642: in the C version. See the <a href="../../docs/manual.pdf">users manual</a> for details.
644: If your main program is C but you call Fortran code that also uses PETSc you need to call PetscInitializeFortran() soon after
645: calling PetscInitialize().
647: Concepts: initializing PETSc
648:
649: .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscInitializeNoArguments()
651: @*/
652: PetscErrorCode PetscInitialize(int *argc,char ***args,const char file[],const char help[])
653: {
655: PetscMPIInt flag, size;
656: PetscInt nodesize;
657: PetscBool flg;
658: char hostname[256];
661: if (PetscInitializeCalled) return(0);
663: /* these must be initialized in a routine, not as a constant declaration*/
664: PETSC_STDOUT = stdout;
665: PETSC_STDERR = stderr;
667: PetscOptionsCreate();
669: /*
670: We initialize the program name here (before MPI_Init()) because MPICH has a bug in
671: it that it sets args[0] on all processors to be args[0] on the first processor.
672: */
673: if (argc && *argc) {
674: PetscSetProgramName(**args);
675: } else {
676: PetscSetProgramName("Unknown Name");
677: }
679: MPI_Initialized(&flag);
680: if (!flag) {
681: if (PETSC_COMM_WORLD != MPI_COMM_NULL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"You cannot set PETSC_COMM_WORLD if you have not initialized MPI first");
682: MPI_Init(argc,args);
683: PetscBeganMPI = PETSC_TRUE;
684: }
685: if (argc && args) {
686: PetscGlobalArgc = *argc;
687: PetscGlobalArgs = *args;
688: }
689: PetscFinalizeCalled = PETSC_FALSE;
691: if (PETSC_COMM_WORLD == MPI_COMM_NULL) {
692: PETSC_COMM_WORLD = MPI_COMM_WORLD;
693: }
694: MPI_Errhandler_set(PETSC_COMM_WORLD,MPI_ERRORS_RETURN);
696: /* Done after init due to a bug in MPICH-GM? */
697: PetscErrorPrintfInitialize();
699: MPI_Comm_rank(MPI_COMM_WORLD,&PetscGlobalRank);
700: MPI_Comm_size(MPI_COMM_WORLD,&PetscGlobalSize);
702: #if defined(PETSC_USE_COMPLEX)
703: /*
704: Initialized the global complex variable; this is because with
705: shared libraries the constructors for global variables
706: are not called; at least on IRIX.
707: */
708: {
709: #if defined(PETSC_CLANGUAGE_CXX)
710: PetscScalar ic(0.0,1.0);
711: PETSC_i = ic;
712: #else
713: PETSC_i = I;
714: #endif
715: }
717: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
718: MPI_Type_contiguous(2,MPIU_REAL,&MPIU_C_DOUBLE_COMPLEX);
719: MPI_Type_commit(&MPIU_C_DOUBLE_COMPLEX);
720: MPI_Type_contiguous(2,MPI_FLOAT,&MPIU_C_COMPLEX);
721: MPI_Type_commit(&MPIU_C_COMPLEX);
722: MPI_Op_create(PetscSum_Local,1,&MPIU_SUM);
723: #endif
724: #endif
726: /*
727: Create the PETSc MPI reduction operator that sums of the first
728: half of the entries and maxes the second half.
729: */
730: MPI_Op_create(PetscMaxSum_Local,1,&PetscMaxSum_Op);
732: #if defined(PETSC_USE_REAL___FLOAT128)
733: MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU___FLOAT128);
734: MPI_Type_commit(&MPIU___FLOAT128);
735: MPI_Op_create(PetscSum_Local,1,&MPIU_SUM);
736: MPI_Op_create(PetscMax_Local,1,&MPIU_MAX);
737: MPI_Op_create(PetscMin_Local,1,&MPIU_MIN);
738: #endif
740: MPI_Type_contiguous(2,MPIU_SCALAR,&MPIU_2SCALAR);
741: MPI_Type_commit(&MPIU_2SCALAR);
742: MPI_Op_create(PetscADMax_Local,1,&PetscADMax_Op);
743: MPI_Op_create(PetscADMin_Local,1,&PetscADMin_Op);
745: MPI_Type_contiguous(2,MPIU_INT,&MPIU_2INT);
746: MPI_Type_commit(&MPIU_2INT);
748: /*
749: Attributes to be set on PETSc communicators
750: */
751: MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelCounter,&Petsc_Counter_keyval,(void*)0);
752: MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm,&Petsc_InnerComm_keyval,(void*)0);
753: MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm,&Petsc_OuterComm_keyval,(void*)0);
755: /*
756: Build the options database
757: */
758: PetscOptionsInsert(argc,args,file);
760:
761: /*
762: Print main application help message
763: */
764: PetscOptionsHasName(PETSC_NULL,"-help",&flg);
765: if (help && flg) {
766: PetscPrintf(PETSC_COMM_WORLD,help);
767: }
768: PetscOptionsCheckInitial_Private();
769:
770: /* SHOULD PUT IN GUARDS: Make sure logging is initialized, even if we do not print it out */
771: #if defined(PETSC_USE_LOG)
772: PetscLogBegin_Private();
773: #endif
775: /*
776: Load the dynamic libraries (on machines that support them), this registers all
777: the solvers etc. (On non-dynamic machines this initializes the PetscDraw and PetscViewer classes)
778: */
779: PetscInitialize_DynamicLibraries();
781: MPI_Comm_size(PETSC_COMM_WORLD,&size);
782: PetscInfo1(0,"PETSc successfully started: number of processors = %d\n",size);
783: PetscGetHostName(hostname,256);
784: PetscInfo1(0,"Running on machine: %s\n",hostname);
786: PetscOptionsCheckInitial_Components();
787: /* Check the options database for options related to the options database itself */
788: PetscOptionsSetFromOptions();
790: #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32)
791: /*
792: Tell MPI about our own data representation converter, this would/should be used if extern32 is not supported by the MPI
794: Currently not used because it is not supported by MPICH.
795: */
796: #if !defined(PETSC_WORDS_BIGENDIAN)
797: MPI_Register_datarep((char *)"petsc",PetscDataRep_read_conv_fn,PetscDataRep_write_conv_fn,PetscDataRep_extent_fn,PETSC_NULL);
798: #endif
799: #endif
801: PetscOptionsGetInt(PETSC_NULL,"-hmpi_spawn_size",&nodesize,&flg);
802: if (flg) {
803: #if defined(PETSC_HAVE_MPI_COMM_SPAWN)
804: PetscHMPISpawn((PetscMPIInt) nodesize); /* worker nodes never return from here; they go directly to PetscEnd() */
805: #else
806: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"PETSc built without MPI 2 (MPI_Comm_spawn) support, use -hmpi_merge_size instead");
807: #endif
808: } else {
809: PetscOptionsGetInt(PETSC_NULL,"-hmpi_merge_size",&nodesize,&flg);
810: if (flg) {
811: PetscHMPIMerge((PetscMPIInt) nodesize,PETSC_NULL,PETSC_NULL);
812: if (PetscHMPIWorker) { /* if worker then never enter user code */
813: PetscInitializeCalled = PETSC_TRUE;
814: PetscEnd();
815: }
816: }
817: }
819: #if defined(PETSC_HAVE_CUDA)
820: cublasInit();
821: #endif
823: #if defined(PETSC_HAVE_AMS)
824: PetscOptionsHasName(PETSC_NULL,"-ams_publish_objects",&flg);
825: if (flg) {
826: PetscAMSPublishAll = PETSC_TRUE;
827: }
828: #endif
830: PetscOptionsHasName(PETSC_NULL,"-python",&flg);
831: if (flg) {
832: PetscInitializeCalled = PETSC_TRUE;
833: PetscPythonInitialize(PETSC_NULL,PETSC_NULL);
834: }
836: /*
837: Once we are completedly initialized then we can set this variables
838: */
839: PetscInitializeCalled = PETSC_TRUE;
840: return(0);
841: }
848: /*@C
849: PetscFinalize - Checks for options to be called at the conclusion
850: of the program. MPI_Finalize() is called only if the user had not
851: called MPI_Init() before calling PetscInitialize().
853: Collective on PETSC_COMM_WORLD
855: Options Database Keys:
856: + -options_table - Calls PetscOptionsView()
857: . -options_left - Prints unused options that remain in the database
858: . -objects_left - Prints list of all objects that have not been freed
859: . -mpidump - Calls PetscMPIDump()
860: . -malloc_dump - Calls PetscMallocDump()
861: . -malloc_info - Prints total memory usage
862: - -malloc_log - Prints summary of memory usage
864: Options Database Keys for Profiling:
865: See the <a href="../../docs/manual.pdf#nameddest=Chapter 11 Profiling">profiling chapter of the users manual</a> for details.
866: + -log_summary [filename] - Prints summary of flop and timing
867: information to screen. If the filename is specified the
868: summary is written to the file. See PetscLogView().
869: . -log_summary_python [filename] - Prints data on of flop and timing usage to a file or screen.
870: See PetscLogPrintSViewPython().
871: . -log_all [filename] - Logs extensive profiling information
872: See PetscLogDump().
873: . -log [filename] - Logs basic profiline information See PetscLogDump().
874: . -log_sync - Log the synchronization in scatters, inner products
875: and norms
876: - -log_mpe [filename] - Creates a logfile viewable by the
877: utility Upshot/Nupshot (in MPICH distribution)
879: Level: beginner
881: Note:
882: See PetscInitialize() for more general runtime options.
884: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscEnd()
885: @*/
886: PetscErrorCode PetscFinalize(void)
887: {
889: PetscMPIInt rank;
890: PetscInt i,nopt;
891: PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,objects_left = PETSC_FALSE;
892: #if defined(PETSC_HAVE_AMS)
893: PetscBool flg = PETSC_FALSE;
894: #endif
895: #if defined(PETSC_USE_LOG)
896: char mname[PETSC_MAX_PATH_LEN];
897: #endif
898:
901: if (!PetscInitializeCalled) {
902: printf("PetscInitialize() must be called before PetscFinalize()\n");
903: PetscFunctionReturn(PETSC_ERR_ARG_WRONGSTATE);
904: }
905: PetscInfo(PETSC_NULL,"PetscFinalize() called\n");
907: #if defined(PETSC_HAVE_AMS)
908: PetscOptionsGetBool(PETSC_NULL,"-options_gui",&flg,PETSC_NULL);
909: if (flg) {
910: PetscOptionsAMSDestroy();
911: }
912: #endif
914: PetscHMPIFinalize();
915: #if defined(PETSC_HAVE_PTHREADCLASSES)
916: if (PetscThreadFinalize) {
917: /* thread pool case */
918: (*PetscThreadFinalize)();
919: }
920: free(ThreadCoreAffinity);
921: #endif
923: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
924: PetscOptionsGetBool(PETSC_NULL,"-malloc_info",&flg2,PETSC_NULL);
925: if (!flg2) {
926: flg2 = PETSC_FALSE;
927: PetscOptionsGetBool(PETSC_NULL,"-memory_info",&flg2,PETSC_NULL);
928: }
929: if (flg2) {
930: PetscMemoryShowUsage(PETSC_VIEWER_STDOUT_WORLD,"Summary of Memory Usage in PETSc\n");
931: }
933: #if defined(PETSC_USE_LOG)
934: flg1 = PETSC_FALSE;
935: PetscOptionsGetBool(PETSC_NULL,"-get_total_flops",&flg1,PETSC_NULL);
936: if (flg1) {
937: PetscLogDouble flops = 0;
938: MPI_Reduce(&_TotalFlops,&flops,1,MPI_DOUBLE,MPI_SUM,0,PETSC_COMM_WORLD);
939: PetscPrintf(PETSC_COMM_WORLD,"Total flops over all processors %g\n",flops);
940: }
941: #endif
944: #if defined(PETSC_USE_LOG)
945: #if defined(PETSC_HAVE_MPE)
946: mname[0] = 0;
947: PetscOptionsGetString(PETSC_NULL,"-log_mpe",mname,PETSC_MAX_PATH_LEN,&flg1);
948: if (flg1){
949: if (mname[0]) {PetscLogMPEDump(mname);}
950: else {PetscLogMPEDump(0);}
951: }
952: #endif
953: mname[0] = 0;
954: PetscOptionsGetString(PETSC_NULL,"-log_summary",mname,PETSC_MAX_PATH_LEN,&flg1);
955: if (flg1) {
956: PetscViewer viewer;
957: if (mname[0]) {
958: PetscViewerASCIIOpen(PETSC_COMM_WORLD,mname,&viewer);
959: PetscLogView(viewer);
960: PetscViewerDestroy(&viewer);
961: } else {
962: viewer = PETSC_VIEWER_STDOUT_WORLD;
963: PetscLogView(viewer);
964: }
965: }
966:
967: mname[0] = 0;
968: PetscOptionsGetString(PETSC_NULL,"-log_summary_python",mname,PETSC_MAX_PATH_LEN,&flg1);
969: if (flg1) {
970: PetscViewer viewer;
971: if (mname[0]) {
972: PetscViewerASCIIOpen(PETSC_COMM_WORLD,mname,&viewer);
973: PetscLogViewPython(viewer);
974: PetscViewerDestroy(&viewer);
975: } else {
976: viewer = PETSC_VIEWER_STDOUT_WORLD;
977: PetscLogViewPython(viewer);
978: }
979: }
980:
981: PetscOptionsGetString(PETSC_NULL,"-log_detailed",mname,PETSC_MAX_PATH_LEN,&flg1);
982: if (flg1) {
983: if (mname[0]) {PetscLogPrintDetailed(PETSC_COMM_WORLD,mname);}
984: else {PetscLogPrintDetailed(PETSC_COMM_WORLD,0);}
985: }
986:
987: mname[0] = 0;
988: PetscOptionsGetString(PETSC_NULL,"-log_all",mname,PETSC_MAX_PATH_LEN,&flg1);
989: PetscOptionsGetString(PETSC_NULL,"-log",mname,PETSC_MAX_PATH_LEN,&flg2);
990: if (flg1 || flg2){
991: if (mname[0]) PetscLogDump(mname);
992: else PetscLogDump(0);
993: }
994: #endif
996: #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD)
997: if (PetscStackActive) {
998: PetscStackDestroy();
999: }
1000: #endif
1002: flg1 = PETSC_FALSE;
1003: PetscOptionsGetBool(PETSC_NULL,"-no_signal_handler",&flg1,PETSC_NULL);
1004: if (!flg1) { PetscPopSignalHandler();}
1005: flg1 = PETSC_FALSE;
1006: PetscOptionsGetBool(PETSC_NULL,"-mpidump",&flg1,PETSC_NULL);
1007: if (flg1) {
1008: PetscMPIDump(stdout);
1009: }
1010: flg1 = PETSC_FALSE;
1011: flg2 = PETSC_FALSE;
1012: /* preemptive call to avoid listing this option in options table as unused */
1013: PetscOptionsHasName(PETSC_NULL,"-malloc_dump",&flg1);
1014: PetscOptionsGetBool(PETSC_NULL,"-options_table",&flg2,PETSC_NULL);
1016: if (flg2) {
1017: PetscOptionsView(PETSC_VIEWER_STDOUT_WORLD);
1018: }
1020: /* to prevent PETSc -options_left from warning */
1021: PetscOptionsHasName(PETSC_NULL,"-nox",&flg1);
1022: PetscOptionsHasName(PETSC_NULL,"-nox_warning",&flg1);
1023: PetscOptionsGetBool(PETSC_NULL,"-objects_left",&objects_left,PETSC_NULL);
1025: if (!PetscHMPIWorker) { /* worker processes skip this because they do not usually process options */
1026: flg3 = PETSC_FALSE; /* default value is required */
1027: PetscOptionsGetBool(PETSC_NULL,"-options_left",&flg3,&flg1);
1028: PetscOptionsAllUsed(&nopt);
1029: if (flg3) {
1030: if (!flg2) { /* have not yet printed the options */
1031: PetscOptionsView(PETSC_VIEWER_STDOUT_WORLD);
1032: }
1033: if (!nopt) {
1034: PetscPrintf(PETSC_COMM_WORLD,"There are no unused options.\n");
1035: } else if (nopt == 1) {
1036: PetscPrintf(PETSC_COMM_WORLD,"There is one unused database option. It is:\n");
1037: } else {
1038: PetscPrintf(PETSC_COMM_WORLD,"There are %D unused database options. They are:\n",nopt);
1039: }
1040: }
1041: #if defined(PETSC_USE_DEBUG)
1042: if (nopt && !flg3 && !flg1) {
1043: PetscPrintf(PETSC_COMM_WORLD,"WARNING! There are options you set that were not used!\n");
1044: PetscPrintf(PETSC_COMM_WORLD,"WARNING! could be spelling mistake, etc!\n");
1045: PetscOptionsLeft();
1046: } else if (nopt && flg3) {
1047: #else
1048: if (nopt && flg3) {
1049: #endif
1050: PetscOptionsLeft();
1051: }
1052: }
1054: /*
1055: Free all objects registered with PetscObjectRegisterDestroy() such as PETSC_VIEWER_XXX_().
1056: */
1057: PetscObjectRegisterDestroyAll();
1059: /*
1060: List all objects the user may have forgot to free
1061: */
1062: if (objects_left && PetscObjectsCounts) {
1063: PetscPrintf(PETSC_COMM_WORLD,"The following objects %D were never freed\n",PetscObjectsCounts);
1064: }
1065: for (i=0; i<PetscObjectsMaxCounts; i++) {
1066: if (PetscObjects[i]) {
1067: if (objects_left) {
1068: PetscPrintf(PETSC_COMM_WORLD," %s %s %s\n",PetscObjects[i]->class_name,PetscObjects[i]->type_name,PetscObjects[i]->name);
1069: }
1070: }
1071: }
1072: /* cannot actually destroy the left over objects, but destroy the list */
1073: PetscObjectsCounts = 0;
1074: PetscObjectsMaxCounts = 0;
1075: PetscFree(PetscObjects);
1078: #if defined(PETSC_USE_LOG)
1079: PetscLogDestroy();
1080: #endif
1082: /*
1083: Free all the registered create functions, such as KSPList, VecList, SNESList, etc
1084: */
1085: PetscFListDestroyAll();
1087: /*
1088: Destroy any packages that registered a finalize
1089: */
1090: PetscRegisterFinalizeAll();
1092: /*
1093: Destroy all the function registration lists created
1094: */
1095: PetscFinalize_DynamicLibraries();
1097: if (petsc_history) {
1098: PetscCloseHistoryFile(&petsc_history);
1099: petsc_history = 0;
1100: }
1102: PetscInfoAllow(PETSC_FALSE,PETSC_NULL);
1104: {
1105: char fname[PETSC_MAX_PATH_LEN];
1106: FILE *fd;
1107: int err;
1109: fname[0] = 0;
1110: PetscOptionsGetString(PETSC_NULL,"-malloc_dump",fname,250,&flg1);
1111: if (flg1 && fname[0]) {
1112: char sname[PETSC_MAX_PATH_LEN];
1114: sprintf(sname,"%s_%d",fname,rank);
1115: fd = fopen(sname,"w"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",sname);
1116: PetscMallocDump(fd);
1117: err = fclose(fd);
1118: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
1119: } else if (flg1) {
1120: MPI_Comm local_comm;
1122: MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);
1123: PetscSequentialPhaseBegin_Private(local_comm,1);
1124: PetscMallocDump(stdout);
1125: PetscSequentialPhaseEnd_Private(local_comm,1);
1126: MPI_Comm_free(&local_comm);
1127: }
1128: }
1129: {
1130: char fname[PETSC_MAX_PATH_LEN];
1131: FILE *fd;
1132:
1133: fname[0] = 0;
1134: PetscOptionsGetString(PETSC_NULL,"-malloc_log",fname,250,&flg1);
1135: if (flg1 && fname[0]) {
1136: char sname[PETSC_MAX_PATH_LEN];
1137: int err;
1139: sprintf(sname,"%s_%d",fname,rank);
1140: fd = fopen(sname,"w"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",sname);
1141: PetscMallocDumpLog(fd);
1142: err = fclose(fd);
1143: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
1144: } else if (flg1) {
1145: PetscMallocDumpLog(stdout);
1146: }
1147: }
1148: /* Can be destroyed only after all the options are used */
1149: PetscOptionsDestroy();
1151: PetscGlobalArgc = 0;
1152: PetscGlobalArgs = 0;
1154: #if defined(PETSC_USE_REAL___FLOAT128)
1155: MPI_Type_free(&MPIU___FLOAT128);
1156: MPI_Op_free(&MPIU_SUM);
1157: MPI_Op_free(&MPIU_MAX);
1158: MPI_Op_free(&MPIU_MIN);
1159: #endif
1161: #if defined(PETSC_USE_COMPLEX)
1162: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
1163: MPI_Op_free(&MPIU_SUM);
1164: MPI_Type_free(&MPIU_C_DOUBLE_COMPLEX);
1165: MPI_Type_free(&MPIU_C_COMPLEX);
1166: #endif
1167: #endif
1168: MPI_Type_free(&MPIU_2SCALAR);
1169: MPI_Type_free(&MPIU_2INT);
1170: MPI_Op_free(&PetscMaxSum_Op);
1171: MPI_Op_free(&PetscADMax_Op);
1172: MPI_Op_free(&PetscADMin_Op);
1174: /*
1175: Destroy any known inner MPI_Comm's and attributes pointing to them
1176: Note this will not destroy any new communicators the user has created.
1178: If all PETSc objects were not destroyed those left over objects will have hanging references to
1179: the MPI_Comms that were freed; but that is ok because those PETSc objects will never be used again
1180: */
1181: {
1182: PetscCommCounter *counter;
1183: PetscMPIInt flg;
1184: MPI_Comm icomm;
1185: void *ptr;
1186: MPI_Attr_get(PETSC_COMM_SELF,Petsc_InnerComm_keyval,&ptr,&flg);
1187: if (flg) {
1188: /* Use PetscMemcpy() because casting from pointer to integer of different size is not allowed with some compilers */
1189: PetscMemcpy(&icomm,&ptr,sizeof(MPI_Comm));
1190: MPI_Attr_get(icomm,Petsc_Counter_keyval,&counter,&flg);
1191: if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory");
1193: MPI_Attr_delete(PETSC_COMM_SELF,Petsc_InnerComm_keyval);
1194: MPI_Attr_delete(icomm,Petsc_Counter_keyval);
1195: MPI_Comm_free(&icomm);
1196: }
1197: MPI_Attr_get(PETSC_COMM_WORLD,Petsc_InnerComm_keyval,&ptr,&flg);
1198: if (flg) {
1199: /* Use PetscMemcpy() because casting from pointer to integer of different size is not allowed with some compilers */
1200: PetscMemcpy(&icomm,&ptr,sizeof(MPI_Comm));
1201: MPI_Attr_get(icomm,Petsc_Counter_keyval,&counter,&flg);
1202: if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory");
1204: MPI_Attr_delete(PETSC_COMM_WORLD,Petsc_InnerComm_keyval);
1205: MPI_Attr_delete(icomm,Petsc_Counter_keyval);
1206: MPI_Comm_free(&icomm);
1207: }
1208: }
1210: MPI_Keyval_free(&Petsc_Counter_keyval);
1211: MPI_Keyval_free(&Petsc_InnerComm_keyval);
1212: MPI_Keyval_free(&Petsc_OuterComm_keyval);
1214: PetscInfo(0,"PETSc successfully ended!\n");
1215: if (PetscBeganMPI) {
1216: #if defined(PETSC_HAVE_MPI_FINALIZED)
1217: PetscMPIInt flag;
1218: MPI_Finalized(&flag);
1219: if (flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"MPI_Finalize() has already been called, even though MPI_Init() was called by PetscInitialize()");
1220: #endif
1221: MPI_Finalize();
1222: }
1224: if (PETSC_ZOPEFD){
1225: if (PETSC_ZOPEFD != PETSC_STDOUT) fprintf(PETSC_ZOPEFD, "<<<end>>>");
1226: else fprintf(PETSC_STDOUT, "<<<end>>>");
1227: }
1229: #if defined(PETSC_HAVE_CUDA)
1230: cublasShutdown();
1231: #endif
1232: /*
1234: Note: In certain cases PETSC_COMM_WORLD is never MPI_Comm_free()ed because
1235: the communicator has some outstanding requests on it. Specifically if the
1236: flag PETSC_HAVE_BROKEN_REQUEST_FREE is set (for IBM MPI implementation). See
1237: src/vec/utils/vpscat.c. Due to this the memory allocated in PetscCommDuplicate()
1238: is never freed as it should be. Thus one may obtain messages of the form
1239: [ 1] 8 bytes PetscCommDuplicate() line 645 in src/sys/mpiu.c indicating the
1240: memory was not freed.
1242: */
1243: PetscMallocClear();
1244: PetscInitializeCalled = PETSC_FALSE;
1245: PetscFinalizeCalled = PETSC_TRUE;
1246: PetscFunctionReturn(ierr);
1247: }