Actual source code: init.c

  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot know 
  6:   what malloc is being used until it has already processed the input.
  7: */

  9: #include <petscsys.h>        /*I  "petscsys.h"   I*/

 11: #define PETSC_PTHREADCLASSES_DEBUG 0

 13: #ifndef _GNU_SOURCE
 14: #define _GNU_SOURCE
 15: #endif
 16: #if defined(PETSC_HAVE_SCHED_H)
 17: #ifndef __USE_GNU
 19: #endif
 20: #include <sched.h>
 21: #endif
 22: #if defined(PETSC_HAVE_PTHREAD_H)
 23: #include <pthread.h>
 24: #endif

 26: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 27: #include <sys/sysinfo.h>
 28: #endif
 29: #if defined(PETSC_HAVE_UNISTD_H)
 30: #include <unistd.h>
 31: #endif
 32: #if defined(PETSC_HAVE_STDLIB_H)
 33: #include <stdlib.h>
 34: #endif
 35: #if defined(PETSC_HAVE_MALLOC_H)
 36: #include <malloc.h>
 37: #endif
 38: #if defined(PETSC_HAVE_VALGRIND)
 39: #include <valgrind/valgrind.h>
 40: #endif

 42: /* ------------------------Nasty global variables -------------------------------*/
 43: /*
 44:      Indicates if PETSc started up MPI, or it was 
 45:    already started before PETSc was initialized.
 46: */
 47: PetscBool    PetscBeganMPI         = PETSC_FALSE;
 48: PetscBool    PetscInitializeCalled = PETSC_FALSE;
 49: PetscBool    PetscFinalizeCalled   = PETSC_FALSE;
 50: PetscBool    PetscUseThreadPool    = PETSC_FALSE;
 51: PetscBool    PetscThreadGo         = PETSC_TRUE;
 52: PetscMPIInt  PetscGlobalRank = -1;
 53: PetscMPIInt  PetscGlobalSize = -1;

 55: #if defined(PETSC_HAVE_PTHREADCLASSES)
 56: PetscMPIInt  PetscMaxThreads = 2;
 57: pthread_t*   PetscThreadPoint;
 58: #define PETSC_HAVE_PTHREAD_BARRIER
 59: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
 60: pthread_barrier_t* BarrPoint;   /* used by 'true' thread pool */
 61: #endif
 62: PetscErrorCode ithreaderr = 0;
 63: int*         pVal;

 65: #define CACHE_LINE_SIZE 64  /* used by 'chain', 'main','tree' thread pools */
 66: int* ThreadCoreAffinity;

 68: typedef enum {JobInitiated,ThreadsWorking,JobCompleted} estat;  /* used by 'chain','tree' thread pool */

 70: typedef struct {
 71:   pthread_mutex_t** mutexarray;
 72:   pthread_cond_t**  cond1array;
 73:   pthread_cond_t** cond2array;
 74:   void* (*pfunc)(void*);
 75:   void** pdata;
 76:   PetscBool startJob;
 77:   estat eJobStat;
 78:   PetscBool** arrThreadStarted;
 79:   PetscBool** arrThreadReady;
 80: } sjob_tree;
 81: sjob_tree job_tree;
 82: typedef struct {
 83:   pthread_mutex_t** mutexarray;
 84:   pthread_cond_t**  cond1array;
 85:   pthread_cond_t** cond2array;
 86:   void* (*pfunc)(void*);
 87:   void** pdata;
 88:   PetscBool** arrThreadReady;
 89: } sjob_main;
 90: sjob_main job_main;
 91: typedef struct {
 92:   pthread_mutex_t** mutexarray;
 93:   pthread_cond_t**  cond1array;
 94:   pthread_cond_t** cond2array;
 95:   void* (*pfunc)(void*);
 96:   void** pdata;
 97:   PetscBool startJob;
 98:   estat eJobStat;
 99:   PetscBool** arrThreadStarted;
100:   PetscBool** arrThreadReady;
101: } sjob_chain;
102: sjob_chain job_chain;
103: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
104: typedef struct {
105:   pthread_mutex_t mutex;
106:   pthread_cond_t cond;
107:   void* (*pfunc)(void*);
108:   void** pdata;
109:   pthread_barrier_t* pbarr;
110:   int iNumJobThreads;
111:   int iNumReadyThreads;
112:   PetscBool startJob;
113: } sjob_true;
114: sjob_true job_true = {PTHREAD_MUTEX_INITIALIZER,PTHREAD_COND_INITIALIZER,NULL,NULL,NULL,0,0,PETSC_FALSE};
115: #endif

117: pthread_cond_t  main_cond  = PTHREAD_COND_INITIALIZER;  /* used by 'true', 'chain','tree' thread pools */
118: char* arrmutex; /* used by 'chain','main','tree' thread pools */
119: char* arrcond1; /* used by 'chain','main','tree' thread pools */
120: char* arrcond2; /* used by 'chain','main','tree' thread pools */
121: char* arrstart; /* used by 'chain','main','tree' thread pools */
122: char* arrready; /* used by 'chain','main','tree' thread pools */

124: /* Function Pointers */
125: void*          (*PetscThreadFunc)(void*) = NULL;
126: void*          (*PetscThreadInitialize)(PetscInt) = NULL;
127: PetscErrorCode (*PetscThreadFinalize)(void) = NULL;
128: void           (*MainWait)(void) = NULL;
129: PetscErrorCode (*MainJob)(void* (*pFunc)(void*),void**,PetscInt) = NULL;
130: /**** Tree Thread Pool Functions ****/
131: void*          PetscThreadFunc_Tree(void*);
132: void*          PetscThreadInitialize_Tree(PetscInt);
133: PetscErrorCode PetscThreadFinalize_Tree(void);
134: void           MainWait_Tree(void);
135: PetscErrorCode MainJob_Tree(void* (*pFunc)(void*),void**,PetscInt);
136: /**** Main Thread Pool Functions ****/
137: void*          PetscThreadFunc_Main(void*);
138: void*          PetscThreadInitialize_Main(PetscInt);
139: PetscErrorCode PetscThreadFinalize_Main(void);
140: void           MainWait_Main(void);
141: PetscErrorCode MainJob_Main(void* (*pFunc)(void*),void**,PetscInt);
142: /**** Chain Thread Pool Functions ****/
143: void*          PetscThreadFunc_Chain(void*);
144: void*          PetscThreadInitialize_Chain(PetscInt);
145: PetscErrorCode PetscThreadFinalize_Chain(void);
146: void           MainWait_Chain(void);
147: PetscErrorCode MainJob_Chain(void* (*pFunc)(void*),void**,PetscInt);
148: /**** True Thread Pool Functions ****/
149: void*          PetscThreadFunc_True(void*);
150: void*          PetscThreadInitialize_True(PetscInt);
151: PetscErrorCode PetscThreadFinalize_True(void);
152: void           MainWait_True(void);
153: PetscErrorCode MainJob_True(void* (*pFunc)(void*),void**,PetscInt);
154: /**** NO Thread Pool Function  ****/
155: PetscErrorCode MainJob_Spawn(void* (*pFunc)(void*),void**,PetscInt);
156: /****  ****/
157: void* FuncFinish(void*);
158: void* PetscThreadRun(MPI_Comm Comm,void* (*pFunc)(void*),int,pthread_t*,void**);
159: void* PetscThreadStop(MPI_Comm Comm,int,pthread_t*);
160: #endif

162: #if defined(PETSC_USE_COMPLEX)
163: #if defined(PETSC_COMPLEX_INSTANTIATE)
164: template <> class std::complex<double>; /* instantiate complex template class */
165: #endif
166: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
167: MPI_Datatype   MPIU_C_DOUBLE_COMPLEX;
168: MPI_Datatype   MPIU_C_COMPLEX;
169: #endif
170: PetscScalar    PETSC_i;
171: #else
172: PetscScalar    PETSC_i = 0.0;
173: #endif
174: #if defined(PETSC_USE_REAL___FLOAT128)
175: MPI_Datatype   MPIU___FLOAT128 = 0;
176: #endif
177: MPI_Datatype   MPIU_2SCALAR = 0;
178: MPI_Datatype   MPIU_2INT = 0;

180: /*
181:      These are needed by petscbt.h
182: */
183: #include <petscbt.h>
184: char      _BT_mask = ' ';
185: char      _BT_c = ' ';
186: PetscInt  _BT_idx  = 0;

188: /*
189:        Function that is called to display all error messages
190: */
191: PetscErrorCode  (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
192: PetscErrorCode  (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
193: #if defined(PETSC_HAVE_MATLAB_ENGINE)
194: PetscErrorCode  (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintf_Matlab;
195: #else
196: PetscErrorCode  (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
197: #endif
198: /*
199:   This is needed to turn on/off cusp synchronization */
200: PetscBool   synchronizeCUSP = PETSC_FALSE;

202: /* ------------------------------------------------------------------------------*/
203: /* 
204:    Optional file where all PETSc output from various prints is saved
205: */
206: FILE *petsc_history = PETSC_NULL;

210: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
211: {
213:   PetscMPIInt    rank,size;
214:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
215:   char           version[256];

218:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
219:   if (!rank) {
220:     char        arch[10];
221:     int         err;
222:     PetscViewer viewer;

224:     PetscGetArchType(arch,10);
225:     PetscGetDate(date,64);
226:     PetscGetVersion(version,256);
227:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
228:     if (filename) {
229:       PetscFixFilename(filename,fname);
230:     } else {
231:       PetscGetHomeDirectory(pfile,240);
232:       PetscStrcat(pfile,"/.petschistory");
233:       PetscFixFilename(pfile,fname);
234:     }

236:     *fd = fopen(fname,"a"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);
237:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
238:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
239:     PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
240:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
241:     PetscViewerASCIIOpenWithFILE(PETSC_COMM_WORLD,*fd,&viewer);
242:     PetscOptionsView(viewer);
243:     PetscViewerDestroy(&viewer);
244:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
245:     err = fflush(*fd);
246:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
247:   }
248:   return(0);
249: }

253: PetscErrorCode  PetscCloseHistoryFile(FILE **fd)
254: {
256:   PetscMPIInt    rank;
257:   char           date[64];
258:   int            err;

261:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
262:   if (!rank) {
263:     PetscGetDate(date,64);
264:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
265:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
266:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
267:     err = fflush(*fd);
268:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
269:     err = fclose(*fd);
270:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
271:   }
272:   return(0);
273: }

275: /* ------------------------------------------------------------------------------*/

277: /* 
278:    This is ugly and probably belongs somewhere else, but I want to 
279:   be able to put a true MPI abort error handler with command line args.

281:     This is so MPI errors in the debugger will leave all the stack 
282:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
283:   in the debugger hence we call abort() instead of MPI_Abort().
284: */

288: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag)
289: {
291:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
292:   abort();
293: }

297: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag)
298: {

302:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
303:   PetscAttachDebugger();
304:   if (ierr) { /* hopeless so get out */
305:     MPI_Abort(*comm,*flag);
306:   }
307: }

311: /*@C 
312:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 
313:      wishes a clean exit somewhere deep in the program.

315:    Collective on PETSC_COMM_WORLD

317:    Options Database Keys are the same as for PetscFinalize()

319:    Level: advanced

321:    Note:
322:    See PetscInitialize() for more general runtime options.

324: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
325: @*/
326: PetscErrorCode  PetscEnd(void)
327: {
329:   PetscFinalize();
330:   exit(0);
331:   return 0;
332: }

334: PetscBool    PetscOptionsPublish = PETSC_FALSE;
337: static char       emacsmachinename[256];

339: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0;
340: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = 0;

344: /*@C 
345:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
346:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
347:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

349:    Input Parameter:
350: +  help - the help function (may be PETSC_NULL)
351: -  version - the version function (may be PETSC_NULL)

353:    Level: developer

355:    Concepts: package help message

357: @*/
358: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
359: {
361:   PetscExternalHelpFunction    = help;
362:   PetscExternalVersionFunction = version;
363:   return(0);
364: }

368: PetscErrorCode  PetscOptionsCheckInitial_Private(void)
369: {
370:   char           string[64],mname[PETSC_MAX_PATH_LEN],*f;
371:   MPI_Comm       comm = PETSC_COMM_WORLD;
372:   PetscBool      flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flg4 = PETSC_FALSE,flag,flgz,flgzout;
374:   PetscReal      si;
375:   int            i;
376:   PetscMPIInt    rank;
377:   char           version[256];

380:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);

382:   /*
383:       Setup the memory management; support for tracing malloc() usage 
384:   */
385:   PetscOptionsHasName(PETSC_NULL,"-malloc_log",&flg3);
386: #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD)
387:   PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg1,&flg2);
388:   if ((!flg2 || flg1) && !petscsetmallocvisited) {
389: #if defined(PETSC_HAVE_VALGRIND)
390:     if (flg2 || !(RUNNING_ON_VALGRIND)) {
391:       /* turn off default -malloc if valgrind is being used */
392: #endif
393:       PetscSetUseTrMalloc_Private();
394: #if defined(PETSC_HAVE_VALGRIND)
395:     }
396: #endif
397:   }
398: #else
399:   PetscOptionsGetBool(PETSC_NULL,"-malloc_dump",&flg1,PETSC_NULL);
400:   PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg2,PETSC_NULL);
401:   if (flg1 || flg2 || flg3) {PetscSetUseTrMalloc_Private();}
402: #endif
403:   if (flg3) {
404:     PetscMallocSetDumpLog();
405:   }
406:   flg1 = PETSC_FALSE;
407:   PetscOptionsGetBool(PETSC_NULL,"-malloc_debug",&flg1,PETSC_NULL);
408:   if (flg1) {
409:     PetscSetUseTrMalloc_Private();
410:     PetscMallocDebug(PETSC_TRUE);
411:   }

413:   flg1 = PETSC_FALSE;
414:   PetscOptionsGetBool(PETSC_NULL,"-malloc_info",&flg1,PETSC_NULL);
415:   if (!flg1) {
416:     flg1 = PETSC_FALSE;
417:     PetscOptionsGetBool(PETSC_NULL,"-memory_info",&flg1,PETSC_NULL);
418:   }
419:   if (flg1) {
420:     PetscMemorySetGetMaximumUsage();
421:   }

423:   /*
424:       Set the display variable for graphics
425:   */
426:   PetscSetDisplay();


429:   /*
430:       Print the PETSc version information
431:   */
432:   PetscOptionsHasName(PETSC_NULL,"-v",&flg1);
433:   PetscOptionsHasName(PETSC_NULL,"-version",&flg2);
434:   PetscOptionsHasName(PETSC_NULL,"-help",&flg3);
435:   if (flg1 || flg2 || flg3){

437:     /*
438:        Print "higher-level" package version message 
439:     */
440:     if (PetscExternalVersionFunction) {
441:       (*PetscExternalVersionFunction)(comm);
442:     }

444:     PetscGetVersion(version,256);
445:     (*PetscHelpPrintf)(comm,"--------------------------------------------\
446: ------------------------------\n");
447:     (*PetscHelpPrintf)(comm,"%s\n",version);
448:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
449:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
450:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
451:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
452:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
453:     (*PetscHelpPrintf)(comm,"--------------------------------------------\
454: ------------------------------\n");
455:   }

457:   /*
458:        Print "higher-level" package help message 
459:   */
460:   if (flg3){
461:     if (PetscExternalHelpFunction) {
462:       (*PetscExternalHelpFunction)(comm);
463:     }
464:   }

466:   /*
467:       Setup the error handling
468:   */
469:   flg1 = PETSC_FALSE;
470:   PetscOptionsGetBool(PETSC_NULL,"-on_error_abort",&flg1,PETSC_NULL);
471:   if (flg1) { PetscPushErrorHandler(PetscAbortErrorHandler,0);}
472:   flg1 = PETSC_FALSE;
473:   PetscOptionsGetBool(PETSC_NULL,"-on_error_mpiabort",&flg1,PETSC_NULL);
474:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);}
475:   flg1 = PETSC_FALSE;
476:   PetscOptionsGetBool(PETSC_NULL,"-mpi_return_on_error",&flg1,PETSC_NULL);
477:   if (flg1) {
478:     MPI_Errhandler_set(comm,MPI_ERRORS_RETURN);
479:   }
480:   flg1 = PETSC_FALSE;
481:   PetscOptionsGetBool(PETSC_NULL,"-no_signal_handler",&flg1,PETSC_NULL);
482:   if (!flg1) {PetscPushSignalHandler(PetscDefaultSignalHandler,(void*)0);}
483:   flg1 = PETSC_FALSE;
484:   PetscOptionsGetBool(PETSC_NULL,"-fp_trap",&flg1,PETSC_NULL);
485:   if (flg1) {PetscSetFPTrap(PETSC_FP_TRAP_ON);}

487:   /*
488:       Setup debugger information
489:   */
490:   PetscSetDefaultDebugger();
491:   PetscOptionsGetString(PETSC_NULL,"-on_error_attach_debugger",string,64,&flg1);
492:   if (flg1) {
493:     MPI_Errhandler err_handler;

495:     PetscSetDebuggerFromString(string);
496:     MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_DebuggerOnError,&err_handler);
497:     MPI_Errhandler_set(comm,err_handler);
498:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);
499:   }
500:   PetscOptionsGetString(PETSC_NULL,"-debug_terminal",string,64,&flg1);
501:   if (flg1) { PetscSetDebugTerminal(string); }
502:   PetscOptionsGetString(PETSC_NULL,"-start_in_debugger",string,64,&flg1);
503:   PetscOptionsGetString(PETSC_NULL,"-stop_for_debugger",string,64,&flg2);
504:   if (flg1 || flg2) {
505:     PetscMPIInt    size;
506:     PetscInt       lsize,*nodes;
507:     MPI_Errhandler err_handler;
508:     /*
509:        we have to make sure that all processors have opened 
510:        connections to all other processors, otherwise once the 
511:        debugger has stated it is likely to receive a SIGUSR1
512:        and kill the program. 
513:     */
514:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
515:     if (size > 2) {
516:       PetscMPIInt dummy = 0;
517:       MPI_Status  status;
518:       for (i=0; i<size; i++) {
519:         if (rank != i) {
520:           MPI_Send(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD);
521:         }
522:       }
523:       for (i=0; i<size; i++) {
524:         if (rank != i) {
525:           MPI_Recv(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD,&status);
526:         }
527:       }
528:     }
529:     /* check if this processor node should be in debugger */
530:     PetscMalloc(size*sizeof(PetscInt),&nodes);
531:     lsize = size;
532:     PetscOptionsGetIntArray(PETSC_NULL,"-debugger_nodes",nodes,&lsize,&flag);
533:     if (flag) {
534:       for (i=0; i<lsize; i++) {
535:         if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
536:       }
537:     }
538:     if (!flag) {
539:       PetscSetDebuggerFromString(string);
540:       PetscPushErrorHandler(PetscAbortErrorHandler,0);
541:       if (flg1) {
542:         PetscAttachDebugger();
543:       } else {
544:         PetscStopForDebugger();
545:       }
546:       MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_AbortOnError,&err_handler);
547:       MPI_Errhandler_set(comm,err_handler);
548:     }
549:     PetscFree(nodes);
550:   }

552:   PetscOptionsGetString(PETSC_NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
553:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

555: #if defined(PETSC_USE_SOCKET_VIEWER)
556:   /*
557:     Activates new sockets for zope if needed
558:   */
559:   PetscOptionsHasName(PETSC_NULL,"-zope", &flgz);
560:   PetscOptionsHasName(PETSC_NULL,"-nostdout", &flgzout);
561:   if (flgz){
562:     int  sockfd;
563:     char hostname[256];
564:     char username[256];
565:     int  remoteport = 9999;

567:     PetscOptionsGetString(PETSC_NULL, "-zope", hostname, 256, &flgz);
568:     if (!hostname[0]){
569:       PetscGetHostName(hostname,256);
570:     }
571:     PetscOpenSocket(hostname, remoteport, &sockfd);
572:     PetscGetUserName(username, 256);
573:     PETSC_ZOPEFD = fdopen(sockfd, "w");
574:     if (flgzout){
575:       PETSC_STDOUT = PETSC_ZOPEFD;
576:       fprintf(PETSC_STDOUT, "<<<user>>> %s\n",username);
577:       fprintf(PETSC_STDOUT, "<<<start>>>");
578:     } else {
579:       fprintf(PETSC_ZOPEFD, "<<<user>>> %s\n",username);
580:       fprintf(PETSC_ZOPEFD, "<<<start>>>");
581:     }
582:   }
583: #endif
584: #if defined(PETSC_USE_SERVER)
585:   PetscOptionsHasName(PETSC_NULL,"-server", &flgz);
586:   if (flgz){
587:     PetscInt port = PETSC_DECIDE;
588:     PetscOptionsGetInt(PETSC_NULL,"-server",&port,PETSC_NULL);
589:     PetscWebServe(PETSC_COMM_WORLD,(int)port);
590:   }
591: #endif

593:   /*
594:         Setup profiling and logging
595:   */
596: #if defined (PETSC_USE_INFO)
597:   {
598:     char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0;
599:     PetscOptionsGetString(PETSC_NULL,"-info",logname,250,&flg1);
600:     if (flg1 && logname[0]) {
601:       PetscInfoAllow(PETSC_TRUE,logname);
602:     } else if (flg1) {
603:       PetscInfoAllow(PETSC_TRUE,PETSC_NULL);
604:     }
605:   }
606: #endif
607: #if defined(PETSC_USE_LOG)
608:   mname[0] = 0;
609:   PetscOptionsGetString(PETSC_NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
610:   if (flg1) {
611:     if (mname[0]) {
612:       PetscOpenHistoryFile(mname,&petsc_history);
613:     } else {
614:       PetscOpenHistoryFile(0,&petsc_history);
615:     }
616:   }
617: #if defined(PETSC_HAVE_MPE)
618:   flg1 = PETSC_FALSE;
619:   PetscOptionsHasName(PETSC_NULL,"-log_mpe",&flg1);
620:   if (flg1) PetscLogMPEBegin();
621: #endif
622:   flg1 = PETSC_FALSE;
623:   flg2 = PETSC_FALSE;
624:   flg3 = PETSC_FALSE;
625:   PetscOptionsGetBool(PETSC_NULL,"-log_all",&flg1,PETSC_NULL);
626:   PetscOptionsGetBool(PETSC_NULL,"-log",&flg2,PETSC_NULL);
627:   PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);
628:   PetscOptionsHasName(PETSC_NULL,"-log_summary_python",&flg4);
629:   if (flg1)                      {  PetscLogAllBegin(); }
630:   else if (flg2 || flg3 || flg4) {  PetscLogBegin();}
631: 
632:   PetscOptionsGetString(PETSC_NULL,"-log_trace",mname,250,&flg1);
633:   if (flg1) {
634:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
635:     FILE *file;
636:     if (mname[0]) {
637:       sprintf(name,"%s.%d",mname,rank);
638:       PetscFixFilename(name,fname);
639:       file = fopen(fname,"w");
640:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
641:     } else {
642:       file = PETSC_STDOUT;
643:     }
644:     PetscLogTraceBegin(file);
645:   }
646: #endif

648:   /*
649:       Setup building of stack frames for all function calls
650:   */
651: #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD)
652:   PetscStackCreate();
653: #endif

655:   PetscOptionsGetBool(PETSC_NULL,"-options_gui",&PetscOptionsPublish,PETSC_NULL);

657: #if defined(PETSC_HAVE_PTHREADCLASSES)
658:   /*
659:       Determine whether user specified maximum number of threads
660:    */
661:   PetscOptionsGetInt(PETSC_NULL,"-thread_max",&PetscMaxThreads,PETSC_NULL);

663:   PetscOptionsHasName(PETSC_NULL,"-main",&flg1);
664:   if(flg1) {
665:     cpu_set_t mset;
666:     int icorr,ncorr = get_nprocs();
667:     PetscOptionsGetInt(PETSC_NULL,"-main",&icorr,PETSC_NULL);
668:     CPU_ZERO(&mset);
669:     CPU_SET(icorr%ncorr,&mset);
670:     sched_setaffinity(0,sizeof(cpu_set_t),&mset);
671:   }

673:   PetscInt N_CORES = get_nprocs();
674:   ThreadCoreAffinity = (int*)malloc(N_CORES*sizeof(int));
675:   char tstr[9];
676:   char tbuf[2];
677:   strcpy(tstr,"-thread");
678:   for(i=0;i<PetscMaxThreads;i++) {
679:     ThreadCoreAffinity[i] = i;
680:     sprintf(tbuf,"%d",i);
681:     strcat(tstr,tbuf);
682:     PetscOptionsHasName(PETSC_NULL,tstr,&flg1);
683:     if(flg1) {
684:       PetscOptionsGetInt(PETSC_NULL,tstr,&ThreadCoreAffinity[i],PETSC_NULL);
685:       ThreadCoreAffinity[i] = ThreadCoreAffinity[i]%N_CORES; /* check on the user */
686:     }
687:     tstr[7] = '\0';
688:   }

690:   /*
691:       Determine whether to use thread pool
692:    */
693:   PetscOptionsHasName(PETSC_NULL,"-use_thread_pool",&flg1);
694:   if (flg1) {
695:     PetscUseThreadPool = PETSC_TRUE;
696:     /* get the thread pool type */
697:     PetscInt ipool = 0;
698:     const char *choices[4] = {"true","tree","main","chain"};

700:     PetscOptionsGetEList(PETSC_NULL,"-use_thread_pool",choices,4,&ipool,PETSC_NULL);
701:     switch(ipool) {
702:     case 1:
703:       PetscThreadFunc       = &PetscThreadFunc_Tree;
704:       PetscThreadInitialize = &PetscThreadInitialize_Tree;
705:       PetscThreadFinalize   = &PetscThreadFinalize_Tree;
706:       MainWait              = &MainWait_Tree;
707:       MainJob               = &MainJob_Tree;
708:       PetscInfo(PETSC_NULL,"Using tree thread pool\n");
709:       break;
710:     case 2:
711:       PetscThreadFunc       = &PetscThreadFunc_Main;
712:       PetscThreadInitialize = &PetscThreadInitialize_Main;
713:       PetscThreadFinalize   = &PetscThreadFinalize_Main;
714:       MainWait              = &MainWait_Main;
715:       MainJob               = &MainJob_Main;
716:       PetscInfo(PETSC_NULL,"Using main thread pool\n");
717:       break;
718: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
719:     case 3:
720: #else
721:     default:
722: #endif
723:       PetscThreadFunc       = &PetscThreadFunc_Chain;
724:       PetscThreadInitialize = &PetscThreadInitialize_Chain;
725:       PetscThreadFinalize   = &PetscThreadFinalize_Chain;
726:       MainWait              = &MainWait_Chain;
727:       MainJob               = &MainJob_Chain;
728:       PetscInfo(PETSC_NULL,"Using chain thread pool\n");
729:       break;
730: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
731:     default:
732:       PetscThreadFunc       = &PetscThreadFunc_True;
733:       PetscThreadInitialize = &PetscThreadInitialize_True;
734:       PetscThreadFinalize   = &PetscThreadFinalize_True;
735:       MainWait              = &MainWait_True;
736:       MainJob               = &MainJob_True;
737:       PetscInfo(PETSC_NULL,"Using true thread pool\n");
738:       break;
739: #endif
740:     }
741:     PetscThreadInitialize(PetscMaxThreads);
742:   } else {
743:     /* need to define these in the case on 'no threads' or 'thread create/destroy' */
744:     /* could take any of the above versions */
745:     MainJob               = &MainJob_Spawn;
746:   }
747: #endif
748:   /*
749:        Print basic help message
750:   */
751:   PetscOptionsHasName(PETSC_NULL,"-help",&flg1);
752:   if (flg1) {
753:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
754:     (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
755:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
756:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
757:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
758:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
759:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
760:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
761:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
762:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
763:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
764:     (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
765:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
766:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
767:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
768:     (*PetscHelpPrintf)(comm," -display display: Location where graphics and debuggers are displayed\n");
769:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
770:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
771:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
772:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
773:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
774:     (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");
775:     (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");
776:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
777:     (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");
778:     (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");
779:     (*PetscHelpPrintf)(comm," -options_table: dump list of options inputted\n");
780:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
781:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
782:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
783:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
784:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
785:     (*PetscHelpPrintf)(comm," -memory_info: print memory usage at end of run\n");
786:     (*PetscHelpPrintf)(comm," -server <port>: Run PETSc webserver (default port is 8080) see PetscWebServe()\n");
787: #if defined(PETSC_USE_LOG)
788:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
789:     (*PetscHelpPrintf)(comm," -log[_all _summary _summary_python]: logging objects and events\n");
790:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
791: #if defined(PETSC_HAVE_MPE)
792:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through upshot\n");
793: #endif
794:     (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");
795: #endif
796:     (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
797:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
798:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
799:     (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
800:   }

802:   PetscOptionsGetReal(PETSC_NULL,"-petsc_sleep",&si,&flg1);
803:   if (flg1) {
804:     PetscSleep(si);
805:   }

807:   PetscOptionsGetString(PETSC_NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);
808:   PetscStrstr(mname,"null",&f);
809:   if (f) {
810:     PetscInfoDeactivateClass(PETSC_NULL);
811:   }

813: #if defined(PETSC_HAVE_CUSP)
814:   PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);
815:   if (flg3) flg1 = PETSC_TRUE;
816:   else flg1 = PETSC_FALSE;
817:   PetscOptionsGetBool(PETSC_NULL,"-cusp_synchronize",&flg1,PETSC_NULL);
818:   if (flg1) synchronizeCUSP = PETSC_TRUE;
819: #endif

821:   return(0);
822: }

824: #if defined(PETSC_HAVE_PTHREADCLASSES)

826: /**** 'Tree' Thread Pool Functions ****/
827: void* PetscThreadFunc_Tree(void* arg) {
828:   PetscErrorCode iterr;
829:   int icorr,ierr;
830:   int* pId = (int*)arg;
831:   int ThreadId = *pId,Mary = 2,i,SubWorker;
832:   PetscBool PeeOn;
833:   cpu_set_t mset;
834:   if (PETSC_PTHREADCLASSES_DEBUG) printf("Thread %d In Tree Thread Function\n",ThreadId);
835:   icorr = ThreadCoreAffinity[ThreadId];
836:   CPU_ZERO(&mset);
837:   CPU_SET(icorr,&mset);
838:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

840:   if((Mary*ThreadId+1)>(PetscMaxThreads-1)) {
841:     PeeOn = PETSC_TRUE;
842:   }
843:   else {
844:     PeeOn = PETSC_FALSE;
845:   }
846:   if(PeeOn==PETSC_FALSE) {
847:     /* check your subordinates, wait for them to be ready */
848:     for(i=1;i<=Mary;i++) {
849:       SubWorker = Mary*ThreadId+i;
850:       if(SubWorker<PetscMaxThreads) {
851:         pthread_mutex_lock(job_tree.mutexarray[SubWorker]);
852:         while(*(job_tree.arrThreadReady[SubWorker])==PETSC_FALSE) {
853:           /* upon entry, automically releases the lock and blocks
854:            upon return, has the lock */
855:           pthread_cond_wait(job_tree.cond1array[SubWorker],job_tree.mutexarray[SubWorker]);
856:         }
857:         pthread_mutex_unlock(job_tree.mutexarray[SubWorker]);
858:       }
859:     }
860:     /* your subordinates are now ready */
861:   }
862:   pthread_mutex_lock(job_tree.mutexarray[ThreadId]);
863:   /* update your ready status */
864:   *(job_tree.arrThreadReady[ThreadId]) = PETSC_TRUE;
865:   if(ThreadId==0) {
866:     job_tree.eJobStat = JobCompleted;
867:     /* ignal main */
868:     pthread_cond_signal(&main_cond);
869:   }
870:   else {
871:     /* tell your boss that you're ready to work */
872:     pthread_cond_signal(job_tree.cond1array[ThreadId]);
873:   }
874:   /* the while loop needs to have an exit
875:   the 'main' thread can terminate all the threads by performing a broadcast
876:    and calling FuncFinish */
877:   while(PetscThreadGo) {
878:     /*need to check the condition to ensure we don't have to wait
879:       waiting when you don't have to causes problems
880:      also need to check the condition to ensure proper handling of spurious wakeups */
881:     while(*(job_tree.arrThreadReady[ThreadId])==PETSC_TRUE) {
882:       /* upon entry, automically releases the lock and blocks
883:        upon return, has the lock */
884:         pthread_cond_wait(job_tree.cond2array[ThreadId],job_tree.mutexarray[ThreadId]);
885:         *(job_tree.arrThreadStarted[ThreadId]) = PETSC_TRUE;
886:         *(job_tree.arrThreadReady[ThreadId])   = PETSC_FALSE;
887:     }
888:     if(ThreadId==0) {
889:       job_tree.startJob = PETSC_FALSE;
890:       job_tree.eJobStat = ThreadsWorking;
891:     }
892:     pthread_mutex_unlock(job_tree.mutexarray[ThreadId]);
893:     if(PeeOn==PETSC_FALSE) {
894:       /* tell your subordinates it's time to get to work */
895:       for(i=1; i<=Mary; i++) {
896:         SubWorker = Mary*ThreadId+i;
897:         if(SubWorker<PetscMaxThreads) {
898:           pthread_cond_signal(job_tree.cond2array[SubWorker]);
899:         }
900:       }
901:     }
902:     /* do your job */
903:     if(job_tree.pdata==NULL) {
904:       iterr = (PetscErrorCode)(long int)job_tree.pfunc(job_tree.pdata);
905:     }
906:     else {
907:       iterr = (PetscErrorCode)(long int)job_tree.pfunc(job_tree.pdata[ThreadId]);
908:     }
909:     if(iterr!=0) {
910:       ithreaderr = 1;
911:     }
912:     if(PetscThreadGo) {
913:       /* reset job, get ready for more */
914:       if(PeeOn==PETSC_FALSE) {
915:         /* check your subordinates, waiting for them to be ready
916:          how do you know for a fact that a given subordinate has actually started? */
917:         for(i=1;i<=Mary;i++) {
918:           SubWorker = Mary*ThreadId+i;
919:           if(SubWorker<PetscMaxThreads) {
920:             pthread_mutex_lock(job_tree.mutexarray[SubWorker]);
921:             while(*(job_tree.arrThreadReady[SubWorker])==PETSC_FALSE||*(job_tree.arrThreadStarted[SubWorker])==PETSC_FALSE) {
922:               /* upon entry, automically releases the lock and blocks
923:                upon return, has the lock */
924:               pthread_cond_wait(job_tree.cond1array[SubWorker],job_tree.mutexarray[SubWorker]);
925:             }
926:             pthread_mutex_unlock(job_tree.mutexarray[SubWorker]);
927:           }
928:         }
929:         /* your subordinates are now ready */
930:       }
931:       pthread_mutex_lock(job_tree.mutexarray[ThreadId]);
932:       *(job_tree.arrThreadReady[ThreadId]) = PETSC_TRUE;
933:       if(ThreadId==0) {
934:         job_tree.eJobStat = JobCompleted; /* oot thread: last thread to complete, guaranteed! */
935:         /* root thread signals 'main' */
936:         pthread_cond_signal(&main_cond);
937:       }
938:       else {
939:         /* signal your boss before you go to sleep */
940:         pthread_cond_signal(job_tree.cond1array[ThreadId]);
941:       }
942:     }
943:   }
944:   return NULL;
945: }

949: void* PetscThreadInitialize_Tree(PetscInt N) {
950:   PetscInt i,ierr;
951:   int status;

953:   if(PetscUseThreadPool) {
954:     size_t Val1 = (size_t)CACHE_LINE_SIZE;
955:     size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
956:     arrmutex = (char*)memalign(Val1,Val2);
957:     arrcond1 = (char*)memalign(Val1,Val2);
958:     arrcond2 = (char*)memalign(Val1,Val2);
959:     arrstart = (char*)memalign(Val1,Val2);
960:     arrready = (char*)memalign(Val1,Val2);
961:     job_tree.mutexarray       = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
962:     job_tree.cond1array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
963:     job_tree.cond2array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
964:     job_tree.arrThreadStarted = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
965:     job_tree.arrThreadReady   = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
966:     /* initialize job structure */
967:     for(i=0; i<PetscMaxThreads; i++) {
968:       job_tree.mutexarray[i]        = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
969:       job_tree.cond1array[i]        = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
970:       job_tree.cond2array[i]        = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
971:       job_tree.arrThreadStarted[i]  = (PetscBool*)(arrstart+CACHE_LINE_SIZE*i);
972:       job_tree.arrThreadReady[i]    = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
973:     }
974:     for(i=0; i<PetscMaxThreads; i++) {
975:       pthread_mutex_init(job_tree.mutexarray[i],NULL);
976:       pthread_cond_init(job_tree.cond1array[i],NULL);
977:       pthread_cond_init(job_tree.cond2array[i],NULL);
978:       *(job_tree.arrThreadStarted[i])  = PETSC_FALSE;
979:       *(job_tree.arrThreadReady[i])    = PETSC_FALSE;
980:     }
981:     job_tree.pfunc = NULL;
982:     job_tree.pdata = (void**)malloc(N*sizeof(void*));
983:     job_tree.startJob = PETSC_FALSE;
984:     job_tree.eJobStat = JobInitiated;
985:     pVal = (int*)malloc(N*sizeof(int));
986:     /* allocate memory in the heap for the thread structure */
987:     PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
988:     /* create threads */
989:     for(i=0; i<N; i++) {
990:       pVal[i] = i;
991:       status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
992:       /* should check status */
993:     }
994:   }
995:   return NULL;
996: }

1000: PetscErrorCode PetscThreadFinalize_Tree() {
1001:   int i,ierr;
1002:   void* jstatus;


1006:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1007:   /* join the threads */
1008:   for(i=0; i<PetscMaxThreads; i++) {
1009:     pthread_join(PetscThreadPoint[i],&jstatus);
1010:     /* do error checking*/
1011:   }
1012:   free(PetscThreadPoint);
1013:   free(arrmutex);
1014:   free(arrcond1);
1015:   free(arrcond2);
1016:   free(arrstart);
1017:   free(arrready);
1018:   free(job_tree.pdata);
1019:   free(pVal);

1021:   return(0);
1022: }

1026: void MainWait_Tree() {
1027:   int ierr;
1028:   pthread_mutex_lock(job_tree.mutexarray[0]);
1029:   while(job_tree.eJobStat<JobCompleted||job_tree.startJob==PETSC_TRUE) {
1030:     pthread_cond_wait(&main_cond,job_tree.mutexarray[0]);
1031:   }
1032:   pthread_mutex_unlock(job_tree.mutexarray[0]);
1033: }

1037: PetscErrorCode MainJob_Tree(void* (*pFunc)(void*),void** data,PetscInt n) {
1038:   int i,ierr;
1039:   PetscErrorCode ijoberr = 0;

1041:   MainWait();
1042:   job_tree.pfunc = pFunc;
1043:   job_tree.pdata = data;
1044:   job_tree.startJob = PETSC_TRUE;
1045:   for(i=0; i<PetscMaxThreads; i++) {
1046:     *(job_tree.arrThreadStarted[i]) = PETSC_FALSE;
1047:   }
1048:   job_tree.eJobStat = JobInitiated;
1049:   pthread_cond_signal(job_tree.cond2array[0]);
1050:   if(pFunc!=FuncFinish) {
1051:     MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1052:   }

1054:   if(ithreaderr) {
1055:     ijoberr = ithreaderr;
1056:   }
1057:   return ijoberr;
1058: }
1059: /****  ****/

1061: /**** 'Main' Thread Pool Functions ****/
1062: void* PetscThreadFunc_Main(void* arg) {
1063:   PetscErrorCode iterr;
1064:   int icorr,ierr;
1065:   int* pId = (int*)arg;
1066:   int ThreadId = *pId;
1067:   cpu_set_t mset;
1068:   if (PETSC_PTHREADCLASSES_DEBUG) printf("Thread %d In Main Thread Function\n",ThreadId);
1069:   icorr = ThreadCoreAffinity[ThreadId];
1070:   CPU_ZERO(&mset);
1071:   CPU_SET(icorr,&mset);
1072:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

1074:   pthread_mutex_lock(job_main.mutexarray[ThreadId]);
1075:   /* update your ready status */
1076:   *(job_main.arrThreadReady[ThreadId]) = PETSC_TRUE;
1077:   /* tell the BOSS that you're ready to work before you go to sleep */
1078:   pthread_cond_signal(job_main.cond1array[ThreadId]);

1080:   /* the while loop needs to have an exit
1081:      the 'main' thread can terminate all the threads by performing a broadcast
1082:      and calling FuncFinish */
1083:   while(PetscThreadGo) {
1084:     /* need to check the condition to ensure we don't have to wait
1085:        waiting when you don't have to causes problems
1086:      also need to check the condition to ensure proper handling of spurious wakeups */
1087:     while(*(job_main.arrThreadReady[ThreadId])==PETSC_TRUE) {
1088:       /* upon entry, atomically releases the lock and blocks
1089:        upon return, has the lock */
1090:         pthread_cond_wait(job_main.cond2array[ThreadId],job_main.mutexarray[ThreadId]);
1091:         /* (job_main.arrThreadReady[ThreadId])   = PETSC_FALSE; */
1092:     }
1093:     pthread_mutex_unlock(job_main.mutexarray[ThreadId]);
1094:     if(job_main.pdata==NULL) {
1095:       iterr = (PetscErrorCode)(long int)job_main.pfunc(job_main.pdata);
1096:     }
1097:     else {
1098:       iterr = (PetscErrorCode)(long int)job_main.pfunc(job_main.pdata[ThreadId]);
1099:     }
1100:     if(iterr!=0) {
1101:       ithreaderr = 1;
1102:     }
1103:     if(PetscThreadGo) {
1104:       /* reset job, get ready for more */
1105:       pthread_mutex_lock(job_main.mutexarray[ThreadId]);
1106:       *(job_main.arrThreadReady[ThreadId]) = PETSC_TRUE;
1107:       /* tell the BOSS that you're ready to work before you go to sleep */
1108:       pthread_cond_signal(job_main.cond1array[ThreadId]);
1109:     }
1110:   }
1111:   return NULL;
1112: }

1116: void* PetscThreadInitialize_Main(PetscInt N) {
1117:   PetscInt i,ierr;
1118:   int status;

1120:   if(PetscUseThreadPool) {
1121:     size_t Val1 = (size_t)CACHE_LINE_SIZE;
1122:     size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
1123:     arrmutex = (char*)memalign(Val1,Val2);
1124:     arrcond1 = (char*)memalign(Val1,Val2);
1125:     arrcond2 = (char*)memalign(Val1,Val2);
1126:     arrstart = (char*)memalign(Val1,Val2);
1127:     arrready = (char*)memalign(Val1,Val2);
1128:     job_main.mutexarray       = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
1129:     job_main.cond1array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1130:     job_main.cond2array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1131:     job_main.arrThreadReady   = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1132:     /* initialize job structure */
1133:     for(i=0; i<PetscMaxThreads; i++) {
1134:       job_main.mutexarray[i]        = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
1135:       job_main.cond1array[i]        = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
1136:       job_main.cond2array[i]        = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
1137:       job_main.arrThreadReady[i]    = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
1138:     }
1139:     for(i=0; i<PetscMaxThreads; i++) {
1140:       pthread_mutex_init(job_main.mutexarray[i],NULL);
1141:       pthread_cond_init(job_main.cond1array[i],NULL);
1142:       pthread_cond_init(job_main.cond2array[i],NULL);
1143:       *(job_main.arrThreadReady[i])    = PETSC_FALSE;
1144:     }
1145:     job_main.pfunc = NULL;
1146:     job_main.pdata = (void**)malloc(N*sizeof(void*));
1147:     pVal = (int*)malloc(N*sizeof(int));
1148:     /* allocate memory in the heap for the thread structure */
1149:     PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1150:     /* create threads */
1151:     for(i=0; i<N; i++) {
1152:       pVal[i] = i;
1153:       status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1154:       /* error check */
1155:     }
1156:   }
1157:   else {
1158:   }
1159:   return NULL;
1160: }

1164: PetscErrorCode PetscThreadFinalize_Main() {
1165:   int i,ierr;
1166:   void* jstatus;


1170:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1171:   /* join the threads */
1172:   for(i=0; i<PetscMaxThreads; i++) {
1173:     pthread_join(PetscThreadPoint[i],&jstatus);
1174:   }
1175:   free(PetscThreadPoint);
1176:   free(arrmutex);
1177:   free(arrcond1);
1178:   free(arrcond2);
1179:   free(arrstart);
1180:   free(arrready);
1181:   free(job_main.pdata);
1182:   free(pVal);

1184:   return(0);
1185: }

1189: void MainWait_Main() {
1190:   int i,ierr;
1191:   for(i=0; i<PetscMaxThreads; i++) {
1192:     pthread_mutex_lock(job_main.mutexarray[i]);
1193:     while(*(job_main.arrThreadReady[i])==PETSC_FALSE) {
1194:       pthread_cond_wait(job_main.cond1array[i],job_main.mutexarray[i]);
1195:     }
1196:     pthread_mutex_unlock(job_main.mutexarray[i]);
1197:   }
1198: }

1202: PetscErrorCode MainJob_Main(void* (*pFunc)(void*),void** data,PetscInt n) {
1203:   int i,ierr;
1204:   PetscErrorCode ijoberr = 0;

1206:   MainWait(); /* you know everyone is waiting to be signalled! */
1207:   job_main.pfunc = pFunc;
1208:   job_main.pdata = data;
1209:   for(i=0; i<PetscMaxThreads; i++) {
1210:     *(job_main.arrThreadReady[i]) = PETSC_FALSE; /* why do this?  suppose you get into MainWait first */
1211:   }
1212:   /* tell the threads to go to work */
1213:   for(i=0; i<PetscMaxThreads; i++) {
1214:     pthread_cond_signal(job_main.cond2array[i]);
1215:   }
1216:   if(pFunc!=FuncFinish) {
1217:     MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1218:   }

1220:   if(ithreaderr) {
1221:     ijoberr = ithreaderr;
1222:   }
1223:   return ijoberr;
1224: }
1225: /****  ****/

1227: /**** Chain Thread Functions ****/
1228: void* PetscThreadFunc_Chain(void* arg) {
1229:   PetscErrorCode iterr;
1230:   int icorr,ierr;
1231:   int* pId = (int*)arg;
1232:   int ThreadId = *pId;
1233:   int SubWorker = ThreadId + 1;
1234:   PetscBool PeeOn;
1235:   cpu_set_t mset;
1236:   if (PETSC_PTHREADCLASSES_DEBUG) printf("Thread %d In Chain Thread Function\n",ThreadId);
1237:   icorr = ThreadCoreAffinity[ThreadId];
1238:   CPU_ZERO(&mset);
1239:   CPU_SET(icorr,&mset);
1240:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

1242:   if(ThreadId==(PetscMaxThreads-1)) {
1243:     PeeOn = PETSC_TRUE;
1244:   }
1245:   else {
1246:     PeeOn = PETSC_FALSE;
1247:   }
1248:   if(PeeOn==PETSC_FALSE) {
1249:     /* check your subordinate, wait for him to be ready */
1250:     pthread_mutex_lock(job_chain.mutexarray[SubWorker]);
1251:     while(*(job_chain.arrThreadReady[SubWorker])==PETSC_FALSE) {
1252:       /* upon entry, automically releases the lock and blocks
1253:        upon return, has the lock */
1254:       pthread_cond_wait(job_chain.cond1array[SubWorker],job_chain.mutexarray[SubWorker]);
1255:     }
1256:     pthread_mutex_unlock(job_chain.mutexarray[SubWorker]);
1257:     /* your subordinate is now ready*/
1258:   }
1259:   pthread_mutex_lock(job_chain.mutexarray[ThreadId]);
1260:   /* update your ready status */
1261:   *(job_chain.arrThreadReady[ThreadId]) = PETSC_TRUE;
1262:   if(ThreadId==0) {
1263:     job_chain.eJobStat = JobCompleted;
1264:     /* signal main */
1265:     pthread_cond_signal(&main_cond);
1266:   }
1267:   else {
1268:     /* tell your boss that you're ready to work */
1269:     pthread_cond_signal(job_chain.cond1array[ThreadId]);
1270:   }
1271:   /*  the while loop needs to have an exit
1272:      the 'main' thread can terminate all the threads by performing a broadcast
1273:    and calling FuncFinish */
1274:   while(PetscThreadGo) {
1275:     /* need to check the condition to ensure we don't have to wait
1276:        waiting when you don't have to causes problems
1277:      also need to check the condition to ensure proper handling of spurious wakeups */
1278:     while(*(job_chain.arrThreadReady[ThreadId])==PETSC_TRUE) {
1279:       /*upon entry, automically releases the lock and blocks
1280:        upon return, has the lock */
1281:         pthread_cond_wait(job_chain.cond2array[ThreadId],job_chain.mutexarray[ThreadId]);
1282:         *(job_chain.arrThreadStarted[ThreadId]) = PETSC_TRUE;
1283:         *(job_chain.arrThreadReady[ThreadId])   = PETSC_FALSE;
1284:     }
1285:     if(ThreadId==0) {
1286:       job_chain.startJob = PETSC_FALSE;
1287:       job_chain.eJobStat = ThreadsWorking;
1288:     }
1289:     pthread_mutex_unlock(job_chain.mutexarray[ThreadId]);
1290:     if(PeeOn==PETSC_FALSE) {
1291:       /* tell your subworker it's time to get to work */
1292:       pthread_cond_signal(job_chain.cond2array[SubWorker]);
1293:     }
1294:     /* do your job */
1295:     if(job_chain.pdata==NULL) {
1296:       iterr = (PetscErrorCode)(long int)job_chain.pfunc(job_chain.pdata);
1297:     }
1298:     else {
1299:       iterr = (PetscErrorCode)(long int)job_chain.pfunc(job_chain.pdata[ThreadId]);
1300:     }
1301:     if(iterr!=0) {
1302:       ithreaderr = 1;
1303:     }
1304:     if(PetscThreadGo) {
1305:       /* reset job, get ready for more */
1306:       if(PeeOn==PETSC_FALSE) {
1307:         /* check your subordinate, wait for him to be ready
1308:          how do you know for a fact that your subordinate has actually started? */
1309:         pthread_mutex_lock(job_chain.mutexarray[SubWorker]);
1310:         while(*(job_chain.arrThreadReady[SubWorker])==PETSC_FALSE||*(job_chain.arrThreadStarted[SubWorker])==PETSC_FALSE) {
1311:           /* upon entry, automically releases the lock and blocks
1312:            upon return, has the lock */
1313:           pthread_cond_wait(job_chain.cond1array[SubWorker],job_chain.mutexarray[SubWorker]);
1314:         }
1315:         pthread_mutex_unlock(job_chain.mutexarray[SubWorker]);
1316:         /* your subordinate is now ready */
1317:       }
1318:       pthread_mutex_lock(job_chain.mutexarray[ThreadId]);
1319:       *(job_chain.arrThreadReady[ThreadId]) = PETSC_TRUE;
1320:       if(ThreadId==0) {
1321:         job_chain.eJobStat = JobCompleted; /* foreman: last thread to complete, guaranteed! */
1322:         /* root thread (foreman) signals 'main' */
1323:         pthread_cond_signal(&main_cond);
1324:       }
1325:       else {
1326:         /* signal your boss before you go to sleep */
1327:         pthread_cond_signal(job_chain.cond1array[ThreadId]);
1328:       }
1329:     }
1330:   }
1331:   return NULL;
1332: }

1336: void* PetscThreadInitialize_Chain(PetscInt N) {
1337:   PetscInt i,ierr;
1338:   int status;

1340:   if(PetscUseThreadPool) {
1341:     size_t Val1 = (size_t)CACHE_LINE_SIZE;
1342:     size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
1343:     arrmutex = (char*)memalign(Val1,Val2);
1344:     arrcond1 = (char*)memalign(Val1,Val2);
1345:     arrcond2 = (char*)memalign(Val1,Val2);
1346:     arrstart = (char*)memalign(Val1,Val2);
1347:     arrready = (char*)memalign(Val1,Val2);
1348:     job_chain.mutexarray       = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
1349:     job_chain.cond1array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1350:     job_chain.cond2array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1351:     job_chain.arrThreadStarted = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1352:     job_chain.arrThreadReady   = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1353:     /* initialize job structure */
1354:     for(i=0; i<PetscMaxThreads; i++) {
1355:       job_chain.mutexarray[i]        = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
1356:       job_chain.cond1array[i]        = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
1357:       job_chain.cond2array[i]        = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
1358:       job_chain.arrThreadStarted[i]  = (PetscBool*)(arrstart+CACHE_LINE_SIZE*i);
1359:       job_chain.arrThreadReady[i]    = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
1360:     }
1361:     for(i=0; i<PetscMaxThreads; i++) {
1362:       pthread_mutex_init(job_chain.mutexarray[i],NULL);
1363:       pthread_cond_init(job_chain.cond1array[i],NULL);
1364:       pthread_cond_init(job_chain.cond2array[i],NULL);
1365:       *(job_chain.arrThreadStarted[i])  = PETSC_FALSE;
1366:       *(job_chain.arrThreadReady[i])    = PETSC_FALSE;
1367:     }
1368:     job_chain.pfunc = NULL;
1369:     job_chain.pdata = (void**)malloc(N*sizeof(void*));
1370:     job_chain.startJob = PETSC_FALSE;
1371:     job_chain.eJobStat = JobInitiated;
1372:     pVal = (int*)malloc(N*sizeof(int));
1373:     /* allocate memory in the heap for the thread structure */
1374:     PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1375:     /* create threads */
1376:     for(i=0; i<N; i++) {
1377:       pVal[i] = i;
1378:       status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1379:       /* should check error */
1380:     }
1381:   }
1382:   else {
1383:   }
1384:   return NULL;
1385: }


1390: PetscErrorCode PetscThreadFinalize_Chain() {
1391:   int i,ierr;
1392:   void* jstatus;


1396:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1397:   /* join the threads */
1398:   for(i=0; i<PetscMaxThreads; i++) {
1399:     pthread_join(PetscThreadPoint[i],&jstatus);
1400:     /* should check error */
1401:   }
1402:   free(PetscThreadPoint);
1403:   free(arrmutex);
1404:   free(arrcond1);
1405:   free(arrcond2);
1406:   free(arrstart);
1407:   free(arrready);
1408:   free(job_chain.pdata);
1409:   free(pVal);

1411:   return(0);
1412: }

1416: void MainWait_Chain() {
1417:   int ierr;
1418:   pthread_mutex_lock(job_chain.mutexarray[0]);
1419:   while(job_chain.eJobStat<JobCompleted||job_chain.startJob==PETSC_TRUE) {
1420:     pthread_cond_wait(&main_cond,job_chain.mutexarray[0]);
1421:   }
1422:   pthread_mutex_unlock(job_chain.mutexarray[0]);
1423: }

1427: PetscErrorCode MainJob_Chain(void* (*pFunc)(void*),void** data,PetscInt n) {
1428:   int i,ierr;
1429:   PetscErrorCode ijoberr = 0;

1431:   MainWait();
1432:   job_chain.pfunc = pFunc;
1433:   job_chain.pdata = data;
1434:   job_chain.startJob = PETSC_TRUE;
1435:   for(i=0; i<PetscMaxThreads; i++) {
1436:     *(job_chain.arrThreadStarted[i]) = PETSC_FALSE;
1437:   }
1438:   job_chain.eJobStat = JobInitiated;
1439:   pthread_cond_signal(job_chain.cond2array[0]);
1440:   if(pFunc!=FuncFinish) {
1441:     MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1442:   }

1444:   if(ithreaderr) {
1445:     ijoberr = ithreaderr;
1446:   }
1447:   return ijoberr;
1448: }
1449: /****  ****/

1451: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
1452: /**** True Thread Functions ****/
1453: void* PetscThreadFunc_True(void* arg) {
1454:   int icorr,ierr,iVal;
1455:   int* pId = (int*)arg;
1456:   int ThreadId = *pId;
1457:   PetscErrorCode iterr;
1458:   cpu_set_t mset;
1459:   if (PETSC_PTHREADCLASSES_DEBUG) printf("Thread %d In True Pool Thread Function\n",ThreadId);
1460:   icorr = ThreadCoreAffinity[ThreadId];
1461:   CPU_ZERO(&mset);
1462:   CPU_SET(icorr,&mset);
1463:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

1465:   pthread_mutex_lock(&job_true.mutex);
1466:   job_true.iNumReadyThreads++;
1467:   if(job_true.iNumReadyThreads==PetscMaxThreads) {
1468:     pthread_cond_signal(&main_cond);
1469:   }
1470:   /*the while loop needs to have an exit
1471:     the 'main' thread can terminate all the threads by performing a broadcast
1472:    and calling FuncFinish */
1473:   while(PetscThreadGo) {
1474:     /*need to check the condition to ensure we don't have to wait
1475:       waiting when you don't have to causes problems
1476:      also need to wait if another thread sneaks in and messes with the predicate */
1477:     while(job_true.startJob==PETSC_FALSE&&job_true.iNumJobThreads==0) {
1478:       /* upon entry, automically releases the lock and blocks
1479:        upon return, has the lock */
1480:       if (PETSC_PTHREADCLASSES_DEBUG) printf("Thread %d Going to Sleep!\n",ThreadId);
1481:       pthread_cond_wait(&job_true.cond,&job_true.mutex);
1482:     }
1483:     job_true.startJob = PETSC_FALSE;
1484:     job_true.iNumJobThreads--;
1485:     job_true.iNumReadyThreads--;
1486:     iVal = PetscMaxThreads-job_true.iNumReadyThreads-1;
1487:     pthread_mutex_unlock(&job_true.mutex);
1488:     if(job_true.pdata==NULL) {
1489:       iterr = (PetscErrorCode)(long int)job_true.pfunc(job_true.pdata);
1490:     }
1491:     else {
1492:       iterr = (PetscErrorCode)(long int)job_true.pfunc(job_true.pdata[iVal]);
1493:     }
1494:     if(iterr!=0) {
1495:       ithreaderr = 1;
1496:     }
1497:     if (PETSC_PTHREADCLASSES_DEBUG) printf("Thread %d Finished Job\n",ThreadId);
1498:     /* the barrier is necessary BECAUSE: look at job_true.iNumReadyThreads
1499:       what happens if a thread finishes before they all start? BAD!
1500:      what happens if a thread finishes before any else start? BAD! */
1501:     pthread_barrier_wait(job_true.pbarr); /* ensures all threads are finished */
1502:     /* reset job */
1503:     if(PetscThreadGo) {
1504:       pthread_mutex_lock(&job_true.mutex);
1505:       job_true.iNumReadyThreads++;
1506:       if(job_true.iNumReadyThreads==PetscMaxThreads) {
1507:         /* signal the 'main' thread that the job is done! (only done once) */
1508:         pthread_cond_signal(&main_cond);
1509:       }
1510:     }
1511:   }
1512:   return NULL;
1513: }

1517: void* PetscThreadInitialize_True(PetscInt N) {
1518:   PetscInt i;
1519:   int status;

1521:   pVal = (int*)malloc(N*sizeof(int));
1522:   /* allocate memory in the heap for the thread structure */
1523:   PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1524:   BarrPoint = (pthread_barrier_t*)malloc((N+1)*sizeof(pthread_barrier_t)); /* BarrPoint[0] makes no sense, don't use it! */
1525:   job_true.pdata = (void**)malloc(N*sizeof(void*));
1526:   for(i=0; i<N; i++) {
1527:     pVal[i] = i;
1528:     status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1529:     /* error check to ensure proper thread creation */
1530:     status = pthread_barrier_init(&BarrPoint[i+1],NULL,i+1);
1531:     /* should check error */
1532:   }
1533:   if (PETSC_PTHREADCLASSES_DEBUG) printf("Finished True Thread Pool Initialization\n");
1534:   return NULL;
1535: }


1540: PetscErrorCode PetscThreadFinalize_True() {
1541:   int i,ierr;
1542:   void* jstatus;


1546:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1547:   /* join the threads */
1548:   for(i=0; i<PetscMaxThreads; i++) {
1549:     pthread_join(PetscThreadPoint[i],&jstatus);
1550:   }
1551:   free(BarrPoint);
1552:   free(PetscThreadPoint);

1554:   return(0);
1555: }

1559: void MainWait_True() {
1560:   int ierr;
1561:   pthread_mutex_lock(&job_true.mutex);
1562:   while(job_true.iNumReadyThreads<PetscMaxThreads||job_true.startJob==PETSC_TRUE) {
1563:     pthread_cond_wait(&main_cond,&job_true.mutex);
1564:   }
1565:   pthread_mutex_unlock(&job_true.mutex);
1566: }

1570: PetscErrorCode MainJob_True(void* (*pFunc)(void*),void** data,PetscInt n) {
1571:   int ierr;
1572:   PetscErrorCode ijoberr = 0;

1574:   MainWait();
1575:   job_true.pfunc = pFunc;
1576:   job_true.pdata = data;
1577:   job_true.pbarr = &BarrPoint[n];
1578:   job_true.iNumJobThreads = n;
1579:   job_true.startJob = PETSC_TRUE;
1580:   pthread_cond_broadcast(&job_true.cond);
1581:   if(pFunc!=FuncFinish) {
1582:     MainWait(); /* why wait after? guarantees that job gets done */
1583:   }

1585:   if(ithreaderr) {
1586:     ijoberr = ithreaderr;
1587:   }
1588:   return ijoberr;
1589: }
1590: /**** NO THREAD POOL FUNCTION ****/
1593: PetscErrorCode MainJob_Spawn(void* (*pFunc)(void*),void** data,PetscInt n) {
1594:   PetscErrorCode ijoberr = 0;

1596:   pthread_t* apThread = (pthread_t*)malloc(n*sizeof(pthread_t));
1597:   PetscThreadPoint = apThread; /* point to same place */
1598:   PetscThreadRun(MPI_COMM_WORLD,pFunc,n,apThread,data);
1599:   PetscThreadStop(MPI_COMM_WORLD,n,apThread); /* ensures that all threads are finished with the job */
1600:   free(apThread);

1602:   return ijoberr;
1603: }
1604: /****  ****/
1605: #endif

1607: void* FuncFinish(void* arg) {
1608:   PetscThreadGo = PETSC_FALSE;
1609:   return(0);
1610: }

1612: #endif