49 static void __kmp_enable_tasking( kmp_task_team_t *task_team, kmp_info_t *this_thr );
50 static void __kmp_alloc_task_deque( kmp_info_t *thread, kmp_thread_data_t *thread_data );
51 static int __kmp_realloc_task_threads_data( kmp_info_t *thread, kmp_task_team_t *task_team );
54 # define __kmp_static_delay( arg )
58 __kmp_static_delay(
int arg )
61 # if KMP_ARCH_X86_64 && KMP_OS_LINUX
62 KMP_ASSERT( arg != 0 );
64 KMP_ASSERT( arg >= 0 );
70 __kmp_static_yield(
int arg )
75 #ifdef BUILD_TIED_TASK_STACK
87 __kmp_trace_task_stack( kmp_int32 gtid, kmp_thread_data_t *thread_data,
int threshold,
char *location )
89 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks;
90 kmp_taskdata_t **stack_top = task_stack -> ts_top;
91 kmp_int32 entries = task_stack -> ts_entries;
92 kmp_taskdata_t *tied_task;
94 KA_TRACE(threshold, (
"__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "
95 "first_block = %p, stack_top = %p \n",
96 location, gtid, entries, task_stack->ts_first_block, stack_top ) );
98 KMP_DEBUG_ASSERT( stack_top != NULL );
99 KMP_DEBUG_ASSERT( entries > 0 );
101 while ( entries != 0 )
103 KMP_DEBUG_ASSERT( stack_top != & task_stack->ts_first_block.sb_block[0] );
105 if ( entries & TASK_STACK_INDEX_MASK == 0 )
107 kmp_stack_block_t *stack_block = (kmp_stack_block_t *) (stack_top) ;
109 stack_block = stack_block -> sb_prev;
110 stack_top = & stack_block -> sb_block[TASK_STACK_BLOCK_SIZE];
117 tied_task = * stack_top;
119 KMP_DEBUG_ASSERT( tied_task != NULL );
120 KMP_DEBUG_ASSERT( tied_task -> td_flags.tasktype == TASK_TIED );
122 KA_TRACE(threshold, (
"__kmp_trace_task_stack(%s): gtid=%d, entry=%d, "
123 "stack_top=%p, tied_task=%p\n",
124 location, gtid, entries, stack_top, tied_task ) );
126 KMP_DEBUG_ASSERT( stack_top == & task_stack->ts_first_block.sb_block[0] );
128 KA_TRACE(threshold, (
"__kmp_trace_task_stack(exit): location = %s, gtid = %d\n",
141 __kmp_init_task_stack( kmp_int32 gtid, kmp_thread_data_t *thread_data )
143 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks;
144 kmp_stack_block_t *first_block;
147 first_block = & task_stack -> ts_first_block;
148 task_stack -> ts_top = (kmp_taskdata_t **) first_block;
149 memset( (
void *) first_block,
'\0', TASK_STACK_BLOCK_SIZE *
sizeof(kmp_taskdata_t *));
152 task_stack -> ts_entries = TASK_STACK_EMPTY;
153 first_block -> sb_next = NULL;
154 first_block -> sb_prev = NULL;
165 __kmp_free_task_stack( kmp_int32 gtid, kmp_thread_data_t *thread_data )
167 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks;
168 kmp_stack_block_t *stack_block = & task_stack -> ts_first_block;
170 KMP_DEBUG_ASSERT( task_stack -> ts_entries == TASK_STACK_EMPTY );
172 while ( stack_block != NULL ) {
173 kmp_stack_block_t *next_block = (stack_block) ? stack_block -> sb_next : NULL;
175 stack_block -> sb_next = NULL;
176 stack_block -> sb_prev = NULL;
177 if (stack_block != & task_stack -> ts_first_block) {
178 __kmp_thread_free( thread, stack_block );
180 stack_block = next_block;
183 task_stack -> ts_entries = 0;
184 task_stack -> ts_top = NULL;
197 __kmp_push_task_stack( kmp_int32 gtid, kmp_info_t *thread, kmp_taskdata_t * tied_task )
200 kmp_thread_data_t *thread_data = & thread -> th.th_task_team ->
201 tt.tt_threads_data[ __kmp_tid_from_gtid( gtid ) ];
202 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks ;
204 if ( tied_task->td_flags.team_serial || tied_task->td_flags.tasking_ser ) {
208 KMP_DEBUG_ASSERT( tied_task -> td_flags.tasktype == TASK_TIED );
209 KMP_DEBUG_ASSERT( task_stack -> ts_top != NULL );
211 KA_TRACE(20, (
"__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n",
212 gtid, thread, tied_task ) );
214 * (task_stack -> ts_top) = tied_task;
217 task_stack -> ts_top++;
218 task_stack -> ts_entries++;
220 if ( task_stack -> ts_entries & TASK_STACK_INDEX_MASK == 0 )
223 kmp_stack_block_t *stack_block =
224 (kmp_stack_block_t *) (task_stack -> ts_top - TASK_STACK_BLOCK_SIZE);
227 if ( stack_block -> sb_next != NULL )
229 task_stack -> ts_top = & stack_block -> sb_next -> sb_block[0];
233 kmp_stack_block_t *new_block = (kmp_stack_block_t *)
234 __kmp_thread_calloc(thread,
sizeof(kmp_stack_block_t));
236 task_stack -> ts_top = & new_block -> sb_block[0];
237 stack_block -> sb_next = new_block;
238 new_block -> sb_prev = stack_block;
239 new_block -> sb_next = NULL;
241 KA_TRACE(30, (
"__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n",
242 gtid, tied_task, new_block ) );
245 KA_TRACE(20, (
"__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n", gtid, tied_task ) );
258 __kmp_pop_task_stack( kmp_int32 gtid, kmp_info_t *thread, kmp_taskdata_t *ending_task )
261 kmp_thread_data_t *thread_data = & thread -> th.th_task_team -> tt_threads_data[ __kmp_tid_from_gtid( gtid ) ];
262 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks ;
263 kmp_taskdata_t *tied_task;
265 if ( ending_task->td_flags.team_serial || ending_task->td_flags.tasking_ser ) {
269 KMP_DEBUG_ASSERT( task_stack -> ts_top != NULL );
270 KMP_DEBUG_ASSERT( task_stack -> ts_entries > 0 );
272 KA_TRACE(20, (
"__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n", gtid, thread ) );
275 if ( task_stack -> ts_entries & TASK_STACK_INDEX_MASK == 0 )
277 kmp_stack_block_t *stack_block =
278 (kmp_stack_block_t *) (task_stack -> ts_top) ;
280 stack_block = stack_block -> sb_prev;
281 task_stack -> ts_top = & stack_block -> sb_block[TASK_STACK_BLOCK_SIZE];
285 task_stack -> ts_top--;
286 task_stack -> ts_entries--;
288 tied_task = * (task_stack -> ts_top );
290 KMP_DEBUG_ASSERT( tied_task != NULL );
291 KMP_DEBUG_ASSERT( tied_task -> td_flags.tasktype == TASK_TIED );
292 KMP_DEBUG_ASSERT( tied_task == ending_task );
294 KA_TRACE(20, (
"__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n", gtid, tied_task ) );
303 __kmp_push_task(kmp_int32 gtid, kmp_task_t * task )
305 kmp_info_t * thread = __kmp_threads[ gtid ];
306 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
307 kmp_task_team_t * task_team = thread->th.th_task_team;
308 kmp_int32 tid = __kmp_tid_from_gtid( gtid );
309 kmp_thread_data_t * thread_data;
311 KA_TRACE(20, (
"__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata ) );
314 if ( taskdata->td_flags.task_serial ) {
315 KA_TRACE(20, (
"__kmp_push_task: T#%d team serialized; returning TASK_NOT_PUSHED for task %p\n",
317 return TASK_NOT_PUSHED;
321 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
322 if ( ! KMP_TASKING_ENABLED( task_team, thread->th.th_task_state ) ) {
323 __kmp_enable_tasking( task_team, thread );
325 KMP_DEBUG_ASSERT( TCR_4(task_team -> tt.tt_found_tasks) == TRUE );
326 KMP_DEBUG_ASSERT( TCR_PTR(task_team -> tt.tt_threads_data) != NULL );
329 thread_data = & task_team -> tt.tt_threads_data[ tid ];
332 if (thread_data -> td.td_deque == NULL ) {
333 __kmp_alloc_task_deque( thread, thread_data );
337 if ( TCR_4(thread_data -> td.td_deque_ntasks) >= TASK_DEQUE_SIZE )
339 KA_TRACE(20, (
"__kmp_push_task: T#%d deque is full; returning TASK_NOT_PUSHED for task %p\n",
341 return TASK_NOT_PUSHED;
345 __kmp_acquire_bootstrap_lock( & thread_data -> td.td_deque_lock );
348 KMP_DEBUG_ASSERT( TCR_4(thread_data -> td.td_deque_ntasks) < TASK_DEQUE_SIZE );
350 thread_data -> td.td_deque[ thread_data -> td.td_deque_tail ] = taskdata;
352 thread_data -> td.td_deque_tail = ( thread_data -> td.td_deque_tail + 1 ) & TASK_DEQUE_MASK;
353 TCW_4(thread_data -> td.td_deque_ntasks, TCR_4(thread_data -> td.td_deque_ntasks) + 1);
355 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
357 KA_TRACE(20, (
"__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "
358 "task=%p ntasks=%d head=%u tail=%u\n",
359 gtid, taskdata, thread_data->td.td_deque_ntasks,
360 thread_data->td.td_deque_tail, thread_data->td.td_deque_head) );
362 return TASK_SUCCESSFULLY_PUSHED;
371 __kmp_pop_current_task_from_thread( kmp_info_t *this_thr )
373 KF_TRACE( 10, (
"__kmp_pop_current_task_from_thread(enter): T#%d this_thread=%p, curtask=%p, "
374 "curtask_parent=%p\n",
375 0, this_thr, this_thr -> th.th_current_task,
376 this_thr -> th.th_current_task -> td_parent ) );
378 this_thr -> th.th_current_task = this_thr -> th.th_current_task -> td_parent;
380 KF_TRACE( 10, (
"__kmp_pop_current_task_from_thread(exit): T#%d this_thread=%p, curtask=%p, "
381 "curtask_parent=%p\n",
382 0, this_thr, this_thr -> th.th_current_task,
383 this_thr -> th.th_current_task -> td_parent ) );
394 __kmp_push_current_task_to_thread( kmp_info_t *this_thr, kmp_team_t *team,
int tid )
397 KF_TRACE( 10, (
"__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p curtask=%p "
399 tid, this_thr, this_thr->th.th_current_task,
400 team->t.t_implicit_task_taskdata[tid].td_parent ) );
402 KMP_DEBUG_ASSERT (this_thr != NULL);
405 if( this_thr->th.th_current_task != & team -> t.t_implicit_task_taskdata[ 0 ] ) {
406 team -> t.t_implicit_task_taskdata[ 0 ].td_parent = this_thr->th.th_current_task;
407 this_thr->th.th_current_task = & team -> t.t_implicit_task_taskdata[ 0 ];
410 team -> t.t_implicit_task_taskdata[ tid ].td_parent = team -> t.t_implicit_task_taskdata[ 0 ].td_parent;
411 this_thr->th.th_current_task = & team -> t.t_implicit_task_taskdata[ tid ];
414 KF_TRACE( 10, (
"__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p curtask=%p "
416 tid, this_thr, this_thr->th.th_current_task,
417 team->t.t_implicit_task_taskdata[tid].td_parent ) );
428 __kmp_task_start( kmp_int32 gtid, kmp_task_t * task, kmp_taskdata_t * current_task )
430 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
431 kmp_info_t * thread = __kmp_threads[ gtid ];
433 KA_TRACE(10, (
"__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n",
434 gtid, taskdata, current_task) );
436 KMP_DEBUG_ASSERT( taskdata -> td_flags.tasktype == TASK_EXPLICIT );
441 current_task -> td_flags.executing = 0;
444 #ifdef BUILD_TIED_TASK_STACK
445 if ( taskdata -> td_flags.tiedness == TASK_TIED )
447 __kmp_push_task_stack( gtid, thread, taskdata );
452 thread -> th.th_current_task = taskdata;
454 KMP_DEBUG_ASSERT( taskdata -> td_flags.started == 0 );
455 KMP_DEBUG_ASSERT( taskdata -> td_flags.executing == 0 );
456 taskdata -> td_flags.started = 1;
457 taskdata -> td_flags.executing = 1;
458 KMP_DEBUG_ASSERT( taskdata -> td_flags.complete == 0 );
459 KMP_DEBUG_ASSERT( taskdata -> td_flags.freed == 0 );
466 KA_TRACE(10, (
"__kmp_task_start(exit): T#%d task=%p\n",
480 __kmpc_omp_task_begin_if0(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * task )
482 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
483 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
485 KA_TRACE(10, (
"__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p current_task=%p\n",
486 gtid, loc_ref, taskdata, current_task ) );
488 taskdata -> td_flags.task_serial = 1;
489 __kmp_task_start( gtid, task, current_task );
491 KA_TRACE(10, (
"__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n",
492 gtid, loc_ref, taskdata ) );
503 __kmpc_omp_task_begin(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * task )
505 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
507 KA_TRACE(10, (
"__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n",
508 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task ) );
510 __kmp_task_start( gtid, task, current_task );
512 KA_TRACE(10, (
"__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n",
513 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
517 #endif // TASK_UNUSED
527 __kmp_free_task( kmp_int32 gtid, kmp_taskdata_t * taskdata, kmp_info_t * thread )
529 KA_TRACE(30, (
"__kmp_free_task: T#%d freeing data from task %p\n",
533 KMP_DEBUG_ASSERT( taskdata->td_flags.tasktype == TASK_EXPLICIT );
534 KMP_DEBUG_ASSERT( taskdata->td_flags.executing == 0 );
535 KMP_DEBUG_ASSERT( taskdata->td_flags.complete == 1 );
536 KMP_DEBUG_ASSERT( taskdata->td_flags.freed == 0 );
537 KMP_DEBUG_ASSERT( TCR_4(taskdata->td_allocated_child_tasks) == 0 || taskdata->td_flags.task_serial == 1);
538 KMP_DEBUG_ASSERT( TCR_4(taskdata->td_incomplete_child_tasks) == 0 );
540 taskdata->td_flags.freed = 1;
543 __kmp_fast_free( thread, taskdata );
545 __kmp_thread_free( thread, taskdata );
548 KA_TRACE(20, (
"__kmp_free_task: T#%d freed task %p\n",
560 __kmp_free_task_and_ancestors( kmp_int32 gtid, kmp_taskdata_t * taskdata, kmp_info_t * thread )
562 kmp_int32 children = 0;
563 kmp_int32 team_or_tasking_serialized = taskdata -> td_flags.team_serial || taskdata -> td_flags.tasking_ser;
565 KMP_DEBUG_ASSERT( taskdata -> td_flags.tasktype == TASK_EXPLICIT );
567 if ( !team_or_tasking_serialized ) {
568 children = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata -> td_allocated_child_tasks) ) - 1;
569 KMP_DEBUG_ASSERT( children >= 0 );
573 while ( children == 0 )
575 kmp_taskdata_t * parent_taskdata = taskdata -> td_parent;
577 KA_TRACE(20, (
"__kmp_free_task_and_ancestors(enter): T#%d task %p complete "
578 "and freeing itself\n", gtid, taskdata) );
581 __kmp_free_task( gtid, taskdata, thread );
583 taskdata = parent_taskdata;
587 if ( team_or_tasking_serialized || taskdata -> td_flags.tasktype == TASK_IMPLICIT )
590 if ( !team_or_tasking_serialized ) {
592 children = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata -> td_allocated_child_tasks) ) - 1;
593 KMP_DEBUG_ASSERT( children >= 0 );
597 KA_TRACE(20, (
"__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
598 "not freeing it yet\n", gtid, taskdata, children) );
608 __kmp_task_finish( kmp_int32 gtid, kmp_task_t *task, kmp_taskdata_t *resumed_task )
610 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
611 kmp_info_t * thread = __kmp_threads[ gtid ];
612 kmp_int32 children = 0;
614 KA_TRACE(10, (
"__kmp_task_finish(enter): T#%d finishing task %p and resuming task %p\n",
615 gtid, taskdata, resumed_task) );
617 KMP_DEBUG_ASSERT( taskdata -> td_flags.tasktype == TASK_EXPLICIT );
620 #ifdef BUILD_TIED_TASK_STACK
621 if ( taskdata -> td_flags.tiedness == TASK_TIED )
623 __kmp_pop_task_stack( gtid, thread, taskdata );
627 KMP_DEBUG_ASSERT( taskdata -> td_flags.executing == 1 );
628 KMP_DEBUG_ASSERT( taskdata -> td_flags.complete == 0 );
629 taskdata -> td_flags.executing = 0;
630 taskdata -> td_flags.complete = 1;
631 KMP_DEBUG_ASSERT( taskdata -> td_flags.started == 1 );
632 KMP_DEBUG_ASSERT( taskdata -> td_flags.freed == 0 );
635 if ( !( taskdata -> td_flags.team_serial || taskdata -> td_flags.tasking_ser ) ) {
637 children = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata -> td_parent -> td_incomplete_child_tasks) ) - 1;
638 KMP_DEBUG_ASSERT( children >= 0 );
640 if ( taskdata->td_taskgroup )
641 KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata->td_taskgroup->count) );
642 __kmp_release_deps(gtid,taskdata);
646 KA_TRACE(20, (
"__kmp_task_finish: T#%d finished task %p, %d incomplete children\n",
647 gtid, taskdata, children) );
651 KMP_DEBUG_ASSERT( (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) ==
652 taskdata->td_flags.task_serial);
653 if ( taskdata->td_flags.task_serial )
655 if (resumed_task == NULL) {
656 resumed_task = taskdata->td_parent;
660 KMP_DEBUG_ASSERT( resumed_task == taskdata->td_parent );
664 KMP_DEBUG_ASSERT( resumed_task != NULL );
668 __kmp_free_task_and_ancestors(gtid, taskdata, thread);
670 __kmp_threads[ gtid ] -> th.th_current_task = resumed_task;
674 resumed_task->td_flags.executing = 1;
676 KA_TRACE(10, (
"__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n",
677 gtid, taskdata, resumed_task) );
689 __kmpc_omp_task_complete_if0(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task )
691 KA_TRACE(10, (
"__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n",
692 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
694 __kmp_task_finish( gtid, task, NULL );
696 KA_TRACE(10, (
"__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n",
697 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
708 __kmpc_omp_task_complete(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task )
710 KA_TRACE(10, (
"__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n",
711 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
713 __kmp_task_finish( gtid, task, NULL );
715 KA_TRACE(10, (
"__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n",
716 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
719 #endif // TASK_UNUSED
734 __kmp_init_implicit_task(
ident_t *loc_ref, kmp_info_t *this_thr, kmp_team_t *team,
int tid,
int set_curr_task )
736 kmp_taskdata_t * task = & team->t.t_implicit_task_taskdata[ tid ];
738 KF_TRACE(10, (
"__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n",
739 tid, team, task, set_curr_task ?
"TRUE" :
"FALSE" ) );
741 task->td_task_id = KMP_GEN_TASK_ID();
742 task->td_team = team;
744 task->td_ident = loc_ref;
745 task->td_taskwait_ident = NULL;
746 task->td_taskwait_counter = 0;
747 task->td_taskwait_thread = 0;
749 task->td_flags.tiedness = TASK_TIED;
750 task->td_flags.tasktype = TASK_IMPLICIT;
752 task->td_flags.task_serial = 1;
753 task->td_flags.tasking_ser = ( __kmp_tasking_mode == tskm_immediate_exec );
754 task->td_flags.team_serial = ( team->t.t_serialized ) ? 1 : 0;
756 task->td_flags.started = 1;
757 task->td_flags.executing = 1;
758 task->td_flags.complete = 0;
759 task->td_flags.freed = 0;
762 task->td_dephash = NULL;
763 task->td_depnode = NULL;
767 task->td_incomplete_child_tasks = 0;
768 task->td_allocated_child_tasks = 0;
770 task->td_taskgroup = NULL;
772 __kmp_push_current_task_to_thread( this_thr, team, tid );
774 KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0);
775 KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0);
778 KF_TRACE(10, (
"__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n",
785 __kmp_round_up_to_val(
size_t size,
size_t val ) {
786 if ( size & ( val - 1 ) ) {
787 size &= ~ ( val - 1 );
788 if ( size <= KMP_SIZE_T_MAX - val ) {
809 __kmp_task_alloc(
ident_t *loc_ref, kmp_int32 gtid, kmp_tasking_flags_t *flags,
810 size_t sizeof_kmp_task_t,
size_t sizeof_shareds,
811 kmp_routine_entry_t task_entry )
814 kmp_taskdata_t *taskdata;
815 kmp_info_t *thread = __kmp_threads[ gtid ];
816 kmp_team_t *team = thread->th.th_team;
817 kmp_taskdata_t *parent_task = thread->th.th_current_task;
818 size_t shareds_offset;
820 KA_TRACE(10, (
"__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
821 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
822 gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t,
823 sizeof_shareds, task_entry) );
825 if ( parent_task->td_flags.final ) {
826 if (flags->merged_if0) {
833 shareds_offset =
sizeof( kmp_taskdata_t ) + sizeof_kmp_task_t;
834 shareds_offset = __kmp_round_up_to_val( shareds_offset,
sizeof(
void * ));
837 KA_TRACE(30, (
"__kmp_task_alloc: T#%d First malloc size: %ld\n",
838 gtid, shareds_offset) );
839 KA_TRACE(30, (
"__kmp_task_alloc: T#%d Second malloc size: %ld\n",
840 gtid, sizeof_shareds) );
844 taskdata = (kmp_taskdata_t *) __kmp_fast_allocate( thread, shareds_offset + sizeof_shareds );
846 taskdata = (kmp_taskdata_t *) __kmp_thread_malloc( thread, shareds_offset + sizeof_shareds );
849 task = KMP_TASKDATA_TO_TASK(taskdata);
853 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)taskdata) & (
sizeof(
double)-1) ) == 0 );
854 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)task) & (
sizeof(
double)-1) ) == 0 );
856 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)taskdata) & (
sizeof(_Quad)-1) ) == 0 );
857 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)task) & (
sizeof(_Quad)-1) ) == 0 );
859 if (sizeof_shareds > 0) {
861 task->shareds = & ((
char *) taskdata)[ shareds_offset ];
863 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)task->shareds) & (
sizeof(
void *)-1) ) == 0 );
865 task->shareds = NULL;
867 task->routine = task_entry;
870 taskdata->td_task_id = KMP_GEN_TASK_ID();
871 taskdata->td_team = team;
872 taskdata->td_alloc_thread = thread;
873 taskdata->td_parent = parent_task;
874 taskdata->td_level = parent_task->td_level + 1;
875 taskdata->td_ident = loc_ref;
876 taskdata->td_taskwait_ident = NULL;
877 taskdata->td_taskwait_counter = 0;
878 taskdata->td_taskwait_thread = 0;
879 KMP_DEBUG_ASSERT( taskdata->td_parent != NULL );
880 copy_icvs( &taskdata->td_icvs, &taskdata->td_parent->td_icvs );
882 taskdata->td_flags.tiedness = flags->tiedness;
883 taskdata->td_flags.final = flags->final;
884 taskdata->td_flags.merged_if0 = flags->merged_if0;
885 taskdata->td_flags.tasktype = TASK_EXPLICIT;
888 taskdata->td_flags.tasking_ser = ( __kmp_tasking_mode == tskm_immediate_exec );
891 taskdata->td_flags.team_serial = ( team->t.t_serialized ) ? 1 : 0;
896 taskdata->td_flags.task_serial = ( taskdata->td_flags.final
897 || taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser );
899 taskdata->td_flags.started = 0;
900 taskdata->td_flags.executing = 0;
901 taskdata->td_flags.complete = 0;
902 taskdata->td_flags.freed = 0;
904 taskdata->td_flags.native = flags->native;
906 taskdata->td_incomplete_child_tasks = 0;
907 taskdata->td_allocated_child_tasks = 1;
909 taskdata->td_taskgroup = parent_task->td_taskgroup;
910 taskdata->td_dephash = NULL;
911 taskdata->td_depnode = NULL;
914 if ( !( taskdata -> td_flags.team_serial || taskdata -> td_flags.tasking_ser ) ) {
915 KMP_TEST_THEN_INC32( (kmp_int32 *)(& parent_task->td_incomplete_child_tasks) );
917 if ( parent_task->td_taskgroup )
918 KMP_TEST_THEN_INC32( (kmp_int32 *)(& parent_task->td_taskgroup->count) );
921 if ( taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT ) {
922 KMP_TEST_THEN_INC32( (kmp_int32 *)(& taskdata->td_parent->td_allocated_child_tasks) );
926 KA_TRACE(20, (
"__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",
927 gtid, taskdata, taskdata->td_parent) );
934 __kmpc_omp_task_alloc(
ident_t *loc_ref, kmp_int32 gtid, kmp_int32 flags,
935 size_t sizeof_kmp_task_t,
size_t sizeof_shareds,
936 kmp_routine_entry_t task_entry )
939 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *) & flags;
941 input_flags->native = FALSE;
944 KA_TRACE(10, (
"__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s) "
945 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
946 gtid, loc_ref, input_flags->tiedness ?
"tied " :
"untied",
947 sizeof_kmp_task_t, sizeof_shareds, task_entry) );
949 retval = __kmp_task_alloc( loc_ref, gtid, input_flags, sizeof_kmp_task_t,
950 sizeof_shareds, task_entry );
952 KA_TRACE(20, (
"__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval) );
965 __kmp_invoke_task( kmp_int32 gtid, kmp_task_t *task, kmp_taskdata_t * current_task )
967 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
968 KA_TRACE(30, (
"__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n",
969 gtid, taskdata, current_task) );
971 __kmp_task_start( gtid, task, current_task );
977 #ifdef KMP_GOMP_COMPAT
978 if (taskdata->td_flags.native) {
979 ((void (*)(
void *))(*(task->routine)))(task->shareds);
984 (*(task->routine))(gtid, task);
987 __kmp_task_finish( gtid, task, current_task );
989 KA_TRACE(30, (
"__kmp_inovke_task(exit): T#%d completed task %p, resuming task %p\n",
990 gtid, taskdata, current_task) );
1005 __kmpc_omp_task_parts(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * new_task)
1007 kmp_taskdata_t * new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1009 KA_TRACE(10, (
"__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n",
1010 gtid, loc_ref, new_taskdata ) );
1015 if ( __kmp_push_task( gtid, new_task ) == TASK_NOT_PUSHED )
1017 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
1018 new_taskdata->td_flags.task_serial = 1;
1019 __kmp_invoke_task( gtid, new_task, current_task );
1022 KA_TRACE(10, (
"__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
1023 "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", gtid, loc_ref,
1026 return TASK_CURRENT_NOT_QUEUED;
1041 __kmpc_omp_task(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * new_task)
1043 kmp_taskdata_t * new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1046 KA_TRACE(10, (
"__kmpc_omp_task(enter): T#%d loc=%p task=%p\n",
1047 gtid, loc_ref, new_taskdata ) );
1052 if ( __kmp_push_task( gtid, new_task ) == TASK_NOT_PUSHED )
1054 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
1055 new_taskdata -> td_flags.task_serial = 1;
1056 __kmp_invoke_task( gtid, new_task, current_task );
1059 KA_TRACE(10, (
"__kmpc_omp_task(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
1060 gtid, loc_ref, new_taskdata ) );
1062 return TASK_CURRENT_NOT_QUEUED;
1070 __kmpc_omp_taskwait(
ident_t *loc_ref, kmp_int32 gtid )
1072 kmp_taskdata_t * taskdata;
1073 kmp_info_t * thread;
1074 int thread_finished = FALSE;
1076 KA_TRACE(10, (
"__kmpc_omp_taskwait(enter): T#%d loc=%p\n",
1079 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
1082 thread = __kmp_threads[ gtid ];
1083 taskdata = thread -> th.th_current_task;
1087 taskdata->td_taskwait_counter += 1;
1088 taskdata->td_taskwait_ident = loc_ref;
1089 taskdata->td_taskwait_thread = gtid + 1;
1092 void * itt_sync_obj = __kmp_itt_taskwait_object( gtid );
1093 if ( itt_sync_obj != NULL )
1094 __kmp_itt_taskwait_starting( gtid, itt_sync_obj );
1097 if ( ! taskdata->td_flags.team_serial ) {
1099 while ( TCR_4(taskdata -> td_incomplete_child_tasks) != 0 ) {
1100 __kmp_execute_tasks( thread, gtid, &(taskdata->td_incomplete_child_tasks),
1101 0, FALSE, &thread_finished,
1105 __kmp_task_stealing_constraint );
1109 if ( itt_sync_obj != NULL )
1110 __kmp_itt_taskwait_finished( gtid, itt_sync_obj );
1114 taskdata->td_taskwait_thread = - taskdata->td_taskwait_thread;
1117 KA_TRACE(10, (
"__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
1118 "returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata) );
1120 return TASK_CURRENT_NOT_QUEUED;
1128 __kmpc_omp_taskyield(
ident_t *loc_ref, kmp_int32 gtid,
int end_part )
1130 kmp_taskdata_t * taskdata;
1131 kmp_info_t * thread;
1132 int thread_finished = FALSE;
1134 KA_TRACE(10, (
"__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n",
1135 gtid, loc_ref, end_part) );
1137 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
1140 thread = __kmp_threads[ gtid ];
1141 taskdata = thread -> th.th_current_task;
1146 taskdata->td_taskwait_counter += 1;
1147 taskdata->td_taskwait_ident = loc_ref;
1148 taskdata->td_taskwait_thread = gtid + 1;
1151 void * itt_sync_obj = __kmp_itt_taskwait_object( gtid );
1152 if ( itt_sync_obj != NULL )
1153 __kmp_itt_taskwait_starting( gtid, itt_sync_obj );
1155 if ( ! taskdata->td_flags.team_serial ) {
1156 __kmp_execute_tasks( thread, gtid, NULL, 0, FALSE, &thread_finished,
1160 __kmp_task_stealing_constraint );
1164 if ( itt_sync_obj != NULL )
1165 __kmp_itt_taskwait_finished( gtid, itt_sync_obj );
1169 taskdata->td_taskwait_thread = - taskdata->td_taskwait_thread;
1172 KA_TRACE(10, (
"__kmpc_omp_taskyield(exit): T#%d task %p resuming, "
1173 "returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata) );
1175 return TASK_CURRENT_NOT_QUEUED;
1184 __kmpc_taskgroup(
ident* loc,
int gtid )
1186 kmp_info_t * thread = __kmp_threads[ gtid ];
1187 kmp_taskdata_t * taskdata = thread->th.th_current_task;
1188 kmp_taskgroup_t * tg_new =
1189 (kmp_taskgroup_t *)__kmp_thread_malloc( thread,
sizeof( kmp_taskgroup_t ) );
1190 KA_TRACE(10, (
"__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new) );
1192 tg_new->parent = taskdata->td_taskgroup;
1193 taskdata->td_taskgroup = tg_new;
1202 __kmpc_end_taskgroup(
ident* loc,
int gtid )
1204 kmp_info_t * thread = __kmp_threads[ gtid ];
1205 kmp_taskdata_t * taskdata = thread->th.th_current_task;
1206 kmp_taskgroup_t * taskgroup = taskdata->td_taskgroup;
1207 int thread_finished = FALSE;
1209 KA_TRACE(10, (
"__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc) );
1210 KMP_DEBUG_ASSERT( taskgroup != NULL );
1212 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
1215 void * itt_sync_obj = __kmp_itt_taskwait_object( gtid );
1216 if ( itt_sync_obj != NULL )
1217 __kmp_itt_taskwait_starting( gtid, itt_sync_obj );
1220 if ( ! taskdata->td_flags.team_serial ) {
1221 while ( TCR_4(taskgroup->count) != 0 ) {
1222 __kmp_execute_tasks( thread, gtid, &(taskgroup->count),
1223 0, FALSE, &thread_finished,
1227 __kmp_task_stealing_constraint );
1232 if ( itt_sync_obj != NULL )
1233 __kmp_itt_taskwait_finished( gtid, itt_sync_obj );
1236 KMP_DEBUG_ASSERT( taskgroup->count == 0 );
1239 taskdata->td_taskgroup = taskgroup->parent;
1240 __kmp_thread_free( thread, taskgroup );
1242 KA_TRACE(10, (
"__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n", gtid, taskdata) );
1251 __kmp_remove_my_task( kmp_info_t * thread, kmp_int32 gtid, kmp_task_team_t *task_team,
1252 kmp_int32 is_constrained )
1255 kmp_taskdata_t * taskdata;
1256 kmp_thread_data_t *thread_data;
1259 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
1260 KMP_DEBUG_ASSERT( task_team -> tt.tt_threads_data != NULL );
1262 thread_data = & task_team -> tt.tt_threads_data[ __kmp_tid_from_gtid( gtid ) ];
1264 KA_TRACE(10, (
"__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n",
1265 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1266 thread_data->td.td_deque_tail) );
1268 if (TCR_4(thread_data -> td.td_deque_ntasks) == 0) {
1269 KA_TRACE(10, (
"__kmp_remove_my_task(exit #1): T#%d No tasks to remove: ntasks=%d head=%u tail=%u\n",
1270 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1271 thread_data->td.td_deque_tail) );
1275 __kmp_acquire_bootstrap_lock( & thread_data -> td.td_deque_lock );
1277 if (TCR_4(thread_data -> td.td_deque_ntasks) == 0) {
1278 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
1279 KA_TRACE(10, (
"__kmp_remove_my_task(exit #2): T#%d No tasks to remove: ntasks=%d head=%u tail=%u\n",
1280 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1281 thread_data->td.td_deque_tail) );
1285 tail = ( thread_data -> td.td_deque_tail - 1 ) & TASK_DEQUE_MASK;
1286 taskdata = thread_data -> td.td_deque[ tail ];
1288 if (is_constrained) {
1291 kmp_taskdata_t * current = thread->th.th_current_task;
1292 kmp_int32 level = current->td_level;
1293 kmp_taskdata_t * parent = taskdata->td_parent;
1294 while ( parent != current && parent->td_level > level ) {
1295 parent = parent->td_parent;
1296 KMP_DEBUG_ASSERT(parent != NULL);
1298 if ( parent != current ) {
1300 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
1301 KA_TRACE(10, (
"__kmp_remove_my_task(exit #2): T#%d No tasks to remove: ntasks=%d head=%u tail=%u\n",
1302 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1303 thread_data->td.td_deque_tail) );
1308 thread_data -> td.td_deque_tail = tail;
1309 TCW_4(thread_data -> td.td_deque_ntasks, thread_data -> td.td_deque_ntasks - 1);
1311 __kmp_release_bootstrap_lock( & thread_data->td.td_deque_lock );
1313 KA_TRACE(10, (
"__kmp_remove_my_task(exit #2): T#%d task %p removed: ntasks=%d head=%u tail=%u\n",
1314 gtid, taskdata, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1315 thread_data->td.td_deque_tail) );
1317 task = KMP_TASKDATA_TO_TASK( taskdata );
1328 __kmp_steal_task( kmp_info_t *victim, kmp_int32 gtid, kmp_task_team_t *task_team,
1329 volatile kmp_uint32 *unfinished_threads,
int *thread_finished,
1330 kmp_int32 is_constrained )
1333 kmp_taskdata_t * taskdata;
1334 kmp_thread_data_t *victim_td, *threads_data;
1335 kmp_int32 victim_tid, thread_tid;
1337 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
1339 threads_data = task_team -> tt.tt_threads_data;
1340 KMP_DEBUG_ASSERT( threads_data != NULL );
1342 victim_tid = victim->th.th_info.ds.ds_tid;
1343 victim_td = & threads_data[ victim_tid ];
1345 KA_TRACE(10, (
"__kmp_steal_task(enter): T#%d try to steal from T#%d: task_team=%p ntasks=%d "
1346 "head=%u tail=%u\n",
1347 gtid, __kmp_gtid_from_thread( victim ), task_team, victim_td->td.td_deque_ntasks,
1348 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1350 if ( (TCR_4(victim_td -> td.td_deque_ntasks) == 0) ||
1351 (TCR_PTR(victim->th.th_task_team) != task_team))
1353 KA_TRACE(10, (
"__kmp_steal_task(exit #1): T#%d could not steal from T#%d: task_team=%p "
1354 "ntasks=%d head=%u tail=%u\n",
1355 gtid, __kmp_gtid_from_thread( victim ), task_team, victim_td->td.td_deque_ntasks,
1356 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1360 __kmp_acquire_bootstrap_lock( & victim_td -> td.td_deque_lock );
1363 if ( (TCR_4(victim_td -> td.td_deque_ntasks) == 0) ||
1364 (TCR_PTR(victim->th.th_task_team) != task_team))
1366 __kmp_release_bootstrap_lock( & victim_td -> td.td_deque_lock );
1367 KA_TRACE(10, (
"__kmp_steal_task(exit #2): T#%d could not steal from T#%d: task_team=%p "
1368 "ntasks=%d head=%u tail=%u\n",
1369 gtid, __kmp_gtid_from_thread( victim ), task_team, victim_td->td.td_deque_ntasks,
1370 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1374 KMP_DEBUG_ASSERT( victim_td -> td.td_deque != NULL );
1376 if ( !is_constrained ) {
1377 taskdata = victim_td -> td.td_deque[ victim_td -> td.td_deque_head ];
1379 victim_td -> td.td_deque_head = ( victim_td -> td.td_deque_head + 1 ) & TASK_DEQUE_MASK;
1382 kmp_int32 tail = ( victim_td -> td.td_deque_tail - 1 ) & TASK_DEQUE_MASK;
1383 taskdata = victim_td -> td.td_deque[ tail ];
1386 kmp_taskdata_t * current = __kmp_threads[ gtid ]->th.th_current_task;
1387 kmp_int32 level = current->td_level;
1388 kmp_taskdata_t * parent = taskdata->td_parent;
1389 while ( parent != current && parent->td_level > level ) {
1390 parent = parent->td_parent;
1391 KMP_DEBUG_ASSERT(parent != NULL);
1393 if ( parent != current ) {
1395 __kmp_release_bootstrap_lock( & victim_td -> td.td_deque_lock );
1396 KA_TRACE(10, (
"__kmp_steal_task(exit #2): T#%d could not steal from T#%d: task_team=%p "
1397 "ntasks=%d head=%u tail=%u\n",
1398 gtid, __kmp_gtid_from_thread( threads_data[victim_tid].td.td_thr ),
1399 task_team, victim_td->td.td_deque_ntasks,
1400 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1403 victim_td -> td.td_deque_tail = tail;
1405 if (*thread_finished) {
1409 kmp_uint32 count = KMP_TEST_THEN_INC32( (kmp_int32 *)unfinished_threads );
1411 KA_TRACE(20, (
"__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n",
1412 gtid, count + 1, task_team) );
1414 *thread_finished = FALSE;
1416 TCW_4(victim_td -> td.td_deque_ntasks, TCR_4(victim_td -> td.td_deque_ntasks) - 1);
1418 __kmp_release_bootstrap_lock( & victim_td -> td.td_deque_lock );
1420 KA_TRACE(10, (
"__kmp_steal_task(exit #3): T#%d stole task %p from T#d: task_team=%p "
1421 "ntasks=%d head=%u tail=%u\n",
1422 gtid, taskdata, __kmp_gtid_from_thread( victim ), task_team,
1423 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,
1424 victim_td->td.td_deque_tail) );
1426 task = KMP_TASKDATA_TO_TASK( taskdata );
1442 __kmp_execute_tasks( kmp_info_t *thread,
1444 volatile kmp_uint *spinner,
1447 int *thread_finished,
1449 void * itt_sync_obj,
1451 kmp_int32 is_constrained )
1453 kmp_task_team_t * task_team;
1455 kmp_thread_data_t * threads_data;
1457 kmp_taskdata_t * current_task = thread -> th.th_current_task;
1458 volatile kmp_uint32 * unfinished_threads;
1459 kmp_int32 nthreads, last_stolen, k, tid;
1461 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
1462 KMP_DEBUG_ASSERT( thread == __kmp_threads[ gtid ] );
1464 task_team = thread -> th.th_task_team;
1465 KMP_DEBUG_ASSERT( task_team != NULL );
1467 KA_TRACE(15, (
"__kmp_execute_tasks(enter): T#%d final_spin=%d *thread_finished=%d\n",
1468 gtid, final_spin, *thread_finished) );
1470 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team -> tt.tt_threads_data);
1471 KMP_DEBUG_ASSERT( threads_data != NULL );
1473 nthreads = task_team -> tt.tt_nproc;
1474 unfinished_threads = &(task_team -> tt.tt_unfinished_threads);
1475 KMP_DEBUG_ASSERT( nthreads > 1 );
1476 KMP_DEBUG_ASSERT( TCR_4((
int)*unfinished_threads) >= 0 );
1480 while (( task = __kmp_remove_my_task( thread, gtid, task_team, is_constrained )) != NULL ) {
1481 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1482 if ( __itt_sync_create_ptr || KMP_ITT_DEBUG ) {
1483 if ( itt_sync_obj == NULL ) {
1485 itt_sync_obj = __kmp_itt_barrier_object( gtid, bs_forkjoin_barrier );
1487 __kmp_itt_task_starting( itt_sync_obj );
1490 __kmp_invoke_task( gtid, task, current_task );
1492 if ( itt_sync_obj != NULL )
1493 __kmp_itt_task_finished( itt_sync_obj );
1501 if ((spinner == NULL) || ((!final_spin) && (TCR_4(*spinner) == checker))) {
1502 KA_TRACE(15, (
"__kmp_execute_tasks(exit #1): T#%d spin condition satisfied\n", gtid) );
1505 KMP_YIELD( __kmp_library == library_throughput );
1514 if (! *thread_finished) {
1515 kmp_uint32 count = KMP_TEST_THEN_DEC32( (kmp_int32 *)unfinished_threads ) - 1;
1516 KA_TRACE(20, (
"__kmp_execute_tasks(dec #1): T#%d dec unfinished_threads to %d task_team=%p\n",
1517 gtid, count, task_team) );
1518 *thread_finished = TRUE;
1526 if ((spinner != NULL) && (TCR_4(*spinner) == checker)) {
1527 KA_TRACE(15, (
"__kmp_execute_tasks(exit #2): T#%d spin condition satisfied\n", gtid) );
1533 tid = thread -> th.th_info.ds.ds_tid;
1534 last_stolen = threads_data[ tid ].td.td_deque_last_stolen;
1536 if (last_stolen != -1) {
1537 kmp_info_t *other_thread = threads_data[last_stolen].td.td_thr;
1539 while ((task = __kmp_steal_task( other_thread, gtid, task_team, unfinished_threads,
1540 thread_finished, is_constrained )) != NULL)
1542 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1543 if ( __itt_sync_create_ptr || KMP_ITT_DEBUG ) {
1544 if ( itt_sync_obj == NULL ) {
1546 itt_sync_obj = __kmp_itt_barrier_object( gtid, bs_forkjoin_barrier );
1548 __kmp_itt_task_starting( itt_sync_obj );
1551 __kmp_invoke_task( gtid, task, current_task );
1553 if ( itt_sync_obj != NULL )
1554 __kmp_itt_task_finished( itt_sync_obj );
1558 if ((spinner == NULL) || ((!final_spin) && (TCR_4(*spinner) == checker))) {
1559 KA_TRACE(15, (
"__kmp_execute_tasks(exit #3): T#%d spin condition satisfied\n",
1564 KMP_YIELD( __kmp_library == library_throughput );
1567 if (TCR_4(threads_data[ tid ].td.td_deque_ntasks) != 0) {
1568 KA_TRACE(20, (
"__kmp_execute_tasks: T#%d stolen task spawned other tasks, restart\n",
1575 threads_data[ tid ].td.td_deque_last_stolen = -1;
1583 if (! *thread_finished) {
1584 kmp_uint32 count = KMP_TEST_THEN_DEC32( (kmp_int32 *)unfinished_threads ) - 1;
1585 KA_TRACE(20, (
"__kmp_execute_tasks(dec #2): T#%d dec unfinished_threads to %d "
1586 "task_team=%p\n", gtid, count, task_team) );
1587 *thread_finished = TRUE;
1596 if ((spinner != NULL) && (TCR_4(*spinner) == checker)) {
1597 KA_TRACE(15, (
"__kmp_execute_tasks(exit #4): T#%d spin condition satisfied\n",
1610 k = __kmp_get_random( thread ) % (nthreads - 1);
1611 if ( k >= thread -> th.th_info.ds.ds_tid ) {
1615 kmp_info_t *other_thread = threads_data[k].td.td_thr;
1625 if ( ( __kmp_tasking_mode == tskm_task_teams ) &&
1626 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) &&
1627 (TCR_PTR(other_thread->th.th_sleep_loc) != NULL))
1629 __kmp_resume( __kmp_gtid_from_thread( other_thread ), NULL );
1641 while ((task = __kmp_steal_task( other_thread, gtid, task_team, unfinished_threads,
1642 thread_finished, is_constrained )) != NULL)
1644 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1645 if ( __itt_sync_create_ptr || KMP_ITT_DEBUG ) {
1646 if ( itt_sync_obj == NULL ) {
1648 itt_sync_obj = __kmp_itt_barrier_object( gtid, bs_forkjoin_barrier );
1650 __kmp_itt_task_starting( itt_sync_obj );
1653 __kmp_invoke_task( gtid, task, current_task );
1655 if ( itt_sync_obj != NULL )
1656 __kmp_itt_task_finished( itt_sync_obj );
1661 threads_data[ tid ].td.td_deque_last_stolen = k;
1666 if ((spinner == NULL) || ((!final_spin) && (TCR_4(*spinner) == checker))) {
1667 KA_TRACE(15, (
"__kmp_execute_tasks(exit #5): T#%d spin condition satisfied\n",
1671 KMP_YIELD( __kmp_library == library_throughput );
1675 if (TCR_4(threads_data[ tid ].td.td_deque_ntasks) != 0) {
1676 KA_TRACE(20, (
"__kmp_execute_tasks: T#%d stolen task spawned other tasks, restart\n",
1691 if (! *thread_finished) {
1692 kmp_uint32 count = KMP_TEST_THEN_DEC32( (kmp_int32 *)unfinished_threads ) - 1;
1693 KA_TRACE(20, (
"__kmp_execute_tasks(dec #3): T#%d dec unfinished_threads to %d; "
1695 gtid, count, task_team) );
1696 *thread_finished = TRUE;
1705 if ((spinner != NULL) && (TCR_4(*spinner) == checker)) {
1706 KA_TRACE(15, (
"__kmp_execute_tasks(exit #6): T#%d spin condition satisfied\n",
1713 KA_TRACE(15, (
"__kmp_execute_tasks(exit #7): T#%d can't find work\n", gtid) );
1724 __kmp_enable_tasking( kmp_task_team_t *task_team, kmp_info_t *this_thr )
1726 kmp_team_t *team = this_thr->th.th_team;
1727 kmp_thread_data_t *threads_data;
1728 int nthreads, i, is_init_thread;
1730 KA_TRACE( 10, (
"__kmp_enable_tasking(enter): T#%d\n",
1731 __kmp_gtid_from_thread( this_thr ) ) );
1733 KMP_DEBUG_ASSERT(task_team != NULL);
1734 KMP_DEBUG_ASSERT(team != NULL);
1736 nthreads = task_team->tt.tt_nproc;
1737 KMP_DEBUG_ASSERT(nthreads > 0);
1738 KMP_DEBUG_ASSERT(nthreads == team->t.t_nproc);
1741 is_init_thread = __kmp_realloc_task_threads_data( this_thr, task_team );
1743 if (!is_init_thread) {
1745 KA_TRACE( 20, (
"__kmp_enable_tasking(exit): T#%d: threads array already set up.\n",
1746 __kmp_gtid_from_thread( this_thr ) ) );
1749 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team -> tt.tt_threads_data);
1750 KMP_DEBUG_ASSERT( threads_data != NULL );
1752 if ( ( __kmp_tasking_mode == tskm_task_teams ) &&
1753 ( __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ) )
1758 for (i = 0; i < nthreads; i++) {
1759 volatile kmp_uint *sleep_loc;
1760 kmp_info_t *thread = threads_data[i].td.td_thr;
1762 if (i == this_thr->th.th_info.ds.ds_tid) {
1772 if ( ( sleep_loc = (
volatile kmp_uint *)
1773 TCR_PTR( thread -> th.th_sleep_loc) ) != NULL )
1775 KF_TRACE( 50, (
"__kmp_enable_tasking: T#%d waking up thread T#%d\n",
1776 __kmp_gtid_from_thread( this_thr ),
1777 __kmp_gtid_from_thread( thread ) ) );
1778 __kmp_resume( __kmp_gtid_from_thread( thread ), sleep_loc );
1781 KF_TRACE( 50, (
"__kmp_enable_tasking: T#%d don't wake up thread T#%d\n",
1782 __kmp_gtid_from_thread( this_thr ),
1783 __kmp_gtid_from_thread( thread ) ) );
1788 KA_TRACE( 10, (
"__kmp_enable_tasking(exit): T#%d\n",
1789 __kmp_gtid_from_thread( this_thr ) ) );
1828 static kmp_task_team_t *__kmp_free_task_teams = NULL;
1830 static kmp_bootstrap_lock_t __kmp_task_team_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_task_team_lock );
1842 __kmp_alloc_task_deque( kmp_info_t *thread, kmp_thread_data_t *thread_data )
1844 __kmp_init_bootstrap_lock( & thread_data -> td.td_deque_lock );
1845 KMP_DEBUG_ASSERT( thread_data -> td.td_deque == NULL );
1848 thread_data -> td.td_deque_last_stolen = -1;
1850 KMP_DEBUG_ASSERT( TCR_4(thread_data -> td.td_deque_ntasks) == 0 );
1851 KMP_DEBUG_ASSERT( thread_data -> td.td_deque_head == 0 );
1852 KMP_DEBUG_ASSERT( thread_data -> td.td_deque_tail == 0 );
1854 KE_TRACE( 10, (
"__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n",
1855 __kmp_gtid_from_thread( thread ), TASK_DEQUE_SIZE, thread_data ) );
1859 thread_data -> td.td_deque = (kmp_taskdata_t **)
1860 __kmp_allocate( TASK_DEQUE_SIZE *
sizeof(kmp_taskdata_t *));
1870 __kmp_free_task_deque( kmp_thread_data_t *thread_data )
1872 __kmp_acquire_bootstrap_lock( & thread_data -> td.td_deque_lock );
1874 if ( thread_data -> td.td_deque != NULL ) {
1875 TCW_4(thread_data -> td.td_deque_ntasks, 0);
1876 __kmp_free( thread_data -> td.td_deque );
1877 thread_data -> td.td_deque = NULL;
1879 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
1881 #ifdef BUILD_TIED_TASK_STACK
1883 if ( thread_data -> td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY ) {
1884 __kmp_free_task_stack( __kmp_thread_from_gtid( gtid ), thread_data );
1886 #endif // BUILD_TIED_TASK_STACK
1900 __kmp_realloc_task_threads_data( kmp_info_t *thread, kmp_task_team_t *task_team )
1902 kmp_thread_data_t ** threads_data_p;
1903 kmp_int32 nthreads, maxthreads;
1904 int is_init_thread = FALSE;
1906 if ( TCR_4(task_team -> tt.tt_found_tasks) ) {
1911 threads_data_p = & task_team -> tt.tt_threads_data;
1912 nthreads = task_team -> tt.tt_nproc;
1913 maxthreads = task_team -> tt.tt_max_threads;
1917 __kmp_acquire_bootstrap_lock( & task_team -> tt.tt_threads_lock );
1919 if ( ! TCR_4(task_team -> tt.tt_found_tasks) ) {
1921 kmp_team_t *team = thread -> th.th_team;
1924 is_init_thread = TRUE;
1925 if ( maxthreads < nthreads ) {
1927 if ( *threads_data_p != NULL ) {
1928 kmp_thread_data_t *old_data = *threads_data_p;
1929 kmp_thread_data_t *new_data = NULL;
1931 KE_TRACE( 10, (
"__kmp_realloc_task_threads_data: T#%d reallocating "
1932 "threads data for task_team %p, new_size = %d, old_size = %d\n",
1933 __kmp_gtid_from_thread( thread ), task_team,
1934 nthreads, maxthreads ) );
1939 new_data = (kmp_thread_data_t *)
1940 __kmp_allocate( nthreads *
sizeof(kmp_thread_data_t) );
1942 memcpy( (
void *) new_data, (
void *) old_data,
1943 maxthreads *
sizeof(kmp_taskdata_t *) );
1945 #ifdef BUILD_TIED_TASK_STACK
1947 for (i = maxthreads; i < nthreads; i++) {
1948 kmp_thread_data_t *thread_data = & (*threads_data_p)[i];
1949 __kmp_init_task_stack( __kmp_gtid_from_thread( thread ), thread_data );
1951 #endif // BUILD_TIED_TASK_STACK
1953 (*threads_data_p) = new_data;
1954 __kmp_free( old_data );
1957 KE_TRACE( 10, (
"__kmp_realloc_task_threads_data: T#%d allocating "
1958 "threads data for task_team %p, size = %d\n",
1959 __kmp_gtid_from_thread( thread ), task_team, nthreads ) );
1963 *threads_data_p = (kmp_thread_data_t *)
1964 __kmp_allocate( nthreads *
sizeof(kmp_thread_data_t) );
1965 #ifdef BUILD_TIED_TASK_STACK
1967 for (i = 0; i < nthreads; i++) {
1968 kmp_thread_data_t *thread_data = & (*threads_data_p)[i];
1969 __kmp_init_task_stack( __kmp_gtid_from_thread( thread ), thread_data );
1971 #endif // BUILD_TIED_TASK_STACK
1973 task_team -> tt.tt_max_threads = nthreads;
1977 KMP_DEBUG_ASSERT( *threads_data_p != NULL );
1981 for (i = 0; i < nthreads; i++) {
1982 kmp_thread_data_t *thread_data = & (*threads_data_p)[i];
1983 thread_data -> td.td_thr = team -> t.t_threads[i];
1985 if ( thread_data -> td.td_deque_last_stolen >= nthreads) {
1989 thread_data -> td.td_deque_last_stolen = -1;
1994 TCW_SYNC_4(task_team -> tt.tt_found_tasks, TRUE);
1997 __kmp_release_bootstrap_lock( & task_team -> tt.tt_threads_lock );
1998 return is_init_thread;
2008 __kmp_free_task_threads_data( kmp_task_team_t *task_team )
2010 __kmp_acquire_bootstrap_lock( & task_team -> tt.tt_threads_lock );
2011 if ( task_team -> tt.tt_threads_data != NULL ) {
2013 for (i = 0; i < task_team->tt.tt_max_threads; i++ ) {
2014 __kmp_free_task_deque( & task_team -> tt.tt_threads_data[i] );
2016 __kmp_free( task_team -> tt.tt_threads_data );
2017 task_team -> tt.tt_threads_data = NULL;
2019 __kmp_release_bootstrap_lock( & task_team -> tt.tt_threads_lock );
2028 static kmp_task_team_t *
2029 __kmp_allocate_task_team( kmp_info_t *thread, kmp_team_t *team )
2031 kmp_task_team_t *task_team = NULL;
2034 KA_TRACE( 20, (
"__kmp_allocate_task_team: T#%d entering; team = %p\n",
2035 (thread ? __kmp_gtid_from_thread( thread ) : -1), team ) );
2037 if (TCR_PTR(__kmp_free_task_teams) != NULL) {
2039 __kmp_acquire_bootstrap_lock( &__kmp_task_team_lock );
2040 if (__kmp_free_task_teams != NULL) {
2041 task_team = __kmp_free_task_teams;
2042 TCW_PTR(__kmp_free_task_teams, task_team -> tt.tt_next);
2043 task_team -> tt.tt_next = NULL;
2045 __kmp_release_bootstrap_lock( &__kmp_task_team_lock );
2048 if (task_team == NULL) {
2049 KE_TRACE( 10, (
"__kmp_allocate_task_team: T#%d allocating "
2050 "task team for team %p\n",
2051 __kmp_gtid_from_thread( thread ), team ) );
2055 task_team = (kmp_task_team_t *) __kmp_allocate(
sizeof(kmp_task_team_t) );
2056 __kmp_init_bootstrap_lock( & task_team -> tt.tt_threads_lock );
2062 TCW_4(task_team -> tt.tt_found_tasks, FALSE);
2063 task_team -> tt.tt_nproc = nthreads = team->t.t_nproc;
2065 task_team -> tt.tt_state = 0;
2066 TCW_4( task_team -> tt.tt_unfinished_threads, nthreads );
2067 TCW_4( task_team -> tt.tt_active, TRUE );
2068 TCW_4( task_team -> tt.tt_ref_ct, nthreads - 1);
2070 KA_TRACE( 20, (
"__kmp_allocate_task_team: T#%d exiting; task_team = %p\n",
2071 (thread ? __kmp_gtid_from_thread( thread ) : -1), task_team ) );
2083 __kmp_free_task_team( kmp_info_t *thread, kmp_task_team_t *task_team )
2085 KA_TRACE( 20, (
"__kmp_free_task_team: T#%d task_team = %p\n",
2086 thread ? __kmp_gtid_from_thread( thread ) : -1, task_team ) );
2088 KMP_DEBUG_ASSERT( TCR_4(task_team -> tt.tt_ref_ct) == 0 );
2091 __kmp_acquire_bootstrap_lock( & __kmp_task_team_lock );
2093 KMP_DEBUG_ASSERT( task_team -> tt.tt_next == NULL );
2094 task_team -> tt.tt_next = __kmp_free_task_teams;
2095 TCW_4(task_team -> tt.tt_found_tasks, FALSE);
2096 TCW_PTR(__kmp_free_task_teams, task_team);
2098 __kmp_release_bootstrap_lock( & __kmp_task_team_lock );
2109 __kmp_reap_task_teams(
void )
2111 kmp_task_team_t *task_team;
2113 if ( TCR_PTR(__kmp_free_task_teams) != NULL ) {
2115 __kmp_acquire_bootstrap_lock( &__kmp_task_team_lock );
2116 while ( ( task_team = __kmp_free_task_teams ) != NULL ) {
2117 __kmp_free_task_teams = task_team -> tt.tt_next;
2118 task_team -> tt.tt_next = NULL;
2121 if ( task_team -> tt.tt_threads_data != NULL ) {
2122 __kmp_free_task_threads_data( task_team );
2124 __kmp_free( task_team );
2126 __kmp_release_bootstrap_lock( &__kmp_task_team_lock );
2138 __kmp_unref_task_team( kmp_task_team_t *task_team, kmp_info_t *thread )
2142 ref_ct = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& task_team->tt.tt_ref_ct) ) - 1;
2144 KA_TRACE( 20, (
"__kmp_unref_task_team: T#%d task_team = %p ref_ct = %d\n",
2145 __kmp_gtid_from_thread( thread ), task_team, ref_ct ) );
2148 if ( ref_ct == 0 ) {
2149 __kmp_free_task_team( thread, task_team );
2152 TCW_PTR( *((
volatile kmp_task_team_t **)(&thread->th.th_task_team)), NULL );
2162 __kmp_wait_to_unref_task_teams(
void)
2168 KMP_INIT_YIELD( spins );
2178 for (thread = (kmp_info_t *)__kmp_thread_pool;
2180 thread = thread->th.th_next_pool)
2182 volatile kmp_uint *sleep_loc;
2186 if ( TCR_PTR(thread->th.th_task_team) == NULL ) {
2187 KA_TRACE( 10, (
"__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n",
2188 __kmp_gtid_from_thread( thread ) ) );
2193 if (!__kmp_is_thread_alive(thread, &exit_val)) {
2194 if (TCR_PTR(thread->th.th_task_team) != NULL) {
2195 __kmp_unref_task_team( thread->th.th_task_team, thread );
2203 KA_TRACE( 10, (
"__kmp_wait_to_unref_task_team: Waiting for T#%d to unreference task_team\n",
2204 __kmp_gtid_from_thread( thread ) ) );
2206 if ( __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ) {
2208 if ( ( sleep_loc = (
volatile kmp_uint *) TCR_PTR( thread->th.th_sleep_loc) ) != NULL ) {
2209 KA_TRACE( 10, (
"__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n",
2210 __kmp_gtid_from_thread( thread ), __kmp_gtid_from_thread( thread ) ) );
2211 __kmp_resume( __kmp_gtid_from_thread( thread ), sleep_loc );
2222 KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc );
2223 KMP_YIELD_SPIN( spins );
2236 __kmp_task_team_setup( kmp_info_t *this_thr, kmp_team_t *team )
2238 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
2240 if ( ( team->t.t_task_team == NULL ) && ( team->t.t_nproc > 1 ) ) {
2248 team->t.t_task_team = __kmp_allocate_task_team( this_thr, team );
2249 KA_TRACE( 20, (
"__kmp_task_team_setup: Master T#%d created new "
2250 "task_team %p for team %d\n",
2251 __kmp_gtid_from_thread( this_thr ), team->t.t_task_team,
2252 ((team != NULL) ? team->t.t_id : -1)) );
2260 if ( team->t.t_task_team != NULL ) {
2263 team->t.t_task_team->tt.tt_state = 1 - this_thr->th.th_task_state;
2274 __kmp_task_team_sync( kmp_info_t *this_thr, kmp_team_t *team )
2276 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
2280 if ( this_thr->th.th_task_team != NULL ) {
2281 if ( ! TCR_SYNC_4( this_thr->th.th_task_team->tt.tt_active ) ) {
2282 KMP_DEBUG_ASSERT( ! KMP_MASTER_TID( __kmp_tid_from_gtid( __kmp_gtid_from_thread( this_thr ) ) ) );
2283 __kmp_unref_task_team( this_thr->th.th_task_team, this_thr );
2288 KMP_DEBUG_ASSERT( this_thr->th.th_task_team == team->t.t_task_team );
2296 TCW_PTR(this_thr->th.th_task_team, team->t.t_task_team);
2297 if ( this_thr->th.th_task_team != NULL ) {
2304 this_thr->th.th_task_state = 1 - this_thr->th.th_task_state;
2305 KMP_DEBUG_ASSERT( this_thr->th.th_task_state == TCR_4(team->t.t_task_team->tt.tt_state) );
2307 KA_TRACE( 20, (
"__kmp_task_team_sync: Thread T#%d task team assigned pointer (%p) from Team #%d task team\n",
2308 __kmp_gtid_from_thread( this_thr ), &this_thr->th.th_task_team,
2309 this_thr->th.th_task_team, ((team != NULL) ? (team->t.t_id) : -1) ) );
2319 __kmp_task_team_wait( kmp_info_t *this_thr,
2322 ,
void * itt_sync_obj
2326 kmp_task_team_t *task_team = team->t.t_task_team;
2328 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
2329 KMP_DEBUG_ASSERT( task_team == this_thr->th.th_task_team );
2331 if ( ( task_team != NULL ) && KMP_TASKING_ENABLED( task_team, this_thr->th.th_task_state ) ) {
2332 KA_TRACE( 20, (
"__kmp_task_team_wait: Master T#%d waiting for all tasks: task_team = %p\n",
2333 __kmp_gtid_from_thread( this_thr ), task_team ) );
2341 __kmp_wait_sleep( this_thr, &task_team->tt.tt_unfinished_threads, 0, TRUE
2353 KA_TRACE( 20, (
"__kmp_task_team_wait: Master T#%d deactivating task_team %p\n",
2354 __kmp_gtid_from_thread( this_thr ), task_team ) );
2355 KMP_DEBUG_ASSERT( task_team->tt.tt_nproc > 1 );
2356 TCW_SYNC_4( task_team->tt.tt_active, FALSE );
2359 TCW_PTR(this_thr->th.th_task_team, NULL);
2360 team->t.t_task_team = NULL;
2374 __kmp_tasking_barrier( kmp_team_t *team, kmp_info_t *thread,
int gtid )
2376 volatile kmp_uint32 *spin = &team->t.t_task_team->tt.tt_unfinished_threads;
2378 KMP_DEBUG_ASSERT( __kmp_tasking_mode == tskm_extra_barrier );
2381 KMP_FSYNC_SPIN_INIT( spin, (kmp_uint32*) NULL );
2383 while (! __kmp_execute_tasks( thread, gtid, spin, 0, TRUE, &flag, NULL ) ) {
2386 KMP_FSYNC_SPIN_PREPARE( spin );
2389 if( TCR_4(__kmp_global.g.g_done) ) {
2390 if( __kmp_global.g.g_abort )
2391 __kmp_abort_thread( );
2397 KMP_FSYNC_SPIN_ACQUIRED( (
void*) spin );
2401 #endif // OMP_30_ENABLED