40 #include "kmp_wait_release.h"
49 static void __kmp_enable_tasking( kmp_task_team_t *task_team, kmp_info_t *this_thr );
50 static void __kmp_alloc_task_deque( kmp_info_t *thread, kmp_thread_data_t *thread_data );
51 static int __kmp_realloc_task_threads_data( kmp_info_t *thread, kmp_task_team_t *task_team );
53 static inline void __kmp_null_resume_wrapper(
int gtid,
volatile void *flag) {
54 switch (((kmp_flag_64 *)flag)->get_type()) {
55 case flag32: __kmp_resume_32(gtid, NULL);
break;
56 case flag64: __kmp_resume_64(gtid, NULL);
break;
57 case flag_oncore: __kmp_resume_oncore(gtid, NULL);
break;
61 #ifdef BUILD_TIED_TASK_STACK
73 __kmp_trace_task_stack( kmp_int32 gtid, kmp_thread_data_t *thread_data,
int threshold,
char *location )
75 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks;
76 kmp_taskdata_t **stack_top = task_stack -> ts_top;
77 kmp_int32 entries = task_stack -> ts_entries;
78 kmp_taskdata_t *tied_task;
80 KA_TRACE(threshold, (
"__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "
81 "first_block = %p, stack_top = %p \n",
82 location, gtid, entries, task_stack->ts_first_block, stack_top ) );
84 KMP_DEBUG_ASSERT( stack_top != NULL );
85 KMP_DEBUG_ASSERT( entries > 0 );
87 while ( entries != 0 )
89 KMP_DEBUG_ASSERT( stack_top != & task_stack->ts_first_block.sb_block[0] );
91 if ( entries & TASK_STACK_INDEX_MASK == 0 )
93 kmp_stack_block_t *stack_block = (kmp_stack_block_t *) (stack_top) ;
95 stack_block = stack_block -> sb_prev;
96 stack_top = & stack_block -> sb_block[TASK_STACK_BLOCK_SIZE];
103 tied_task = * stack_top;
105 KMP_DEBUG_ASSERT( tied_task != NULL );
106 KMP_DEBUG_ASSERT( tied_task -> td_flags.tasktype == TASK_TIED );
108 KA_TRACE(threshold, (
"__kmp_trace_task_stack(%s): gtid=%d, entry=%d, "
109 "stack_top=%p, tied_task=%p\n",
110 location, gtid, entries, stack_top, tied_task ) );
112 KMP_DEBUG_ASSERT( stack_top == & task_stack->ts_first_block.sb_block[0] );
114 KA_TRACE(threshold, (
"__kmp_trace_task_stack(exit): location = %s, gtid = %d\n",
127 __kmp_init_task_stack( kmp_int32 gtid, kmp_thread_data_t *thread_data )
129 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks;
130 kmp_stack_block_t *first_block;
133 first_block = & task_stack -> ts_first_block;
134 task_stack -> ts_top = (kmp_taskdata_t **) first_block;
135 memset( (
void *) first_block,
'\0', TASK_STACK_BLOCK_SIZE *
sizeof(kmp_taskdata_t *));
138 task_stack -> ts_entries = TASK_STACK_EMPTY;
139 first_block -> sb_next = NULL;
140 first_block -> sb_prev = NULL;
151 __kmp_free_task_stack( kmp_int32 gtid, kmp_thread_data_t *thread_data )
153 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks;
154 kmp_stack_block_t *stack_block = & task_stack -> ts_first_block;
156 KMP_DEBUG_ASSERT( task_stack -> ts_entries == TASK_STACK_EMPTY );
158 while ( stack_block != NULL ) {
159 kmp_stack_block_t *next_block = (stack_block) ? stack_block -> sb_next : NULL;
161 stack_block -> sb_next = NULL;
162 stack_block -> sb_prev = NULL;
163 if (stack_block != & task_stack -> ts_first_block) {
164 __kmp_thread_free( thread, stack_block );
166 stack_block = next_block;
169 task_stack -> ts_entries = 0;
170 task_stack -> ts_top = NULL;
183 __kmp_push_task_stack( kmp_int32 gtid, kmp_info_t *thread, kmp_taskdata_t * tied_task )
186 kmp_thread_data_t *thread_data = & thread -> th.th_task_team ->
187 tt.tt_threads_data[ __kmp_tid_from_gtid( gtid ) ];
188 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks ;
190 if ( tied_task->td_flags.team_serial || tied_task->td_flags.tasking_ser ) {
194 KMP_DEBUG_ASSERT( tied_task -> td_flags.tasktype == TASK_TIED );
195 KMP_DEBUG_ASSERT( task_stack -> ts_top != NULL );
197 KA_TRACE(20, (
"__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n",
198 gtid, thread, tied_task ) );
200 * (task_stack -> ts_top) = tied_task;
203 task_stack -> ts_top++;
204 task_stack -> ts_entries++;
206 if ( task_stack -> ts_entries & TASK_STACK_INDEX_MASK == 0 )
209 kmp_stack_block_t *stack_block =
210 (kmp_stack_block_t *) (task_stack -> ts_top - TASK_STACK_BLOCK_SIZE);
213 if ( stack_block -> sb_next != NULL )
215 task_stack -> ts_top = & stack_block -> sb_next -> sb_block[0];
219 kmp_stack_block_t *new_block = (kmp_stack_block_t *)
220 __kmp_thread_calloc(thread,
sizeof(kmp_stack_block_t));
222 task_stack -> ts_top = & new_block -> sb_block[0];
223 stack_block -> sb_next = new_block;
224 new_block -> sb_prev = stack_block;
225 new_block -> sb_next = NULL;
227 KA_TRACE(30, (
"__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n",
228 gtid, tied_task, new_block ) );
231 KA_TRACE(20, (
"__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n", gtid, tied_task ) );
244 __kmp_pop_task_stack( kmp_int32 gtid, kmp_info_t *thread, kmp_taskdata_t *ending_task )
247 kmp_thread_data_t *thread_data = & thread -> th.th_task_team -> tt_threads_data[ __kmp_tid_from_gtid( gtid ) ];
248 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks ;
249 kmp_taskdata_t *tied_task;
251 if ( ending_task->td_flags.team_serial || ending_task->td_flags.tasking_ser ) {
255 KMP_DEBUG_ASSERT( task_stack -> ts_top != NULL );
256 KMP_DEBUG_ASSERT( task_stack -> ts_entries > 0 );
258 KA_TRACE(20, (
"__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n", gtid, thread ) );
261 if ( task_stack -> ts_entries & TASK_STACK_INDEX_MASK == 0 )
263 kmp_stack_block_t *stack_block =
264 (kmp_stack_block_t *) (task_stack -> ts_top) ;
266 stack_block = stack_block -> sb_prev;
267 task_stack -> ts_top = & stack_block -> sb_block[TASK_STACK_BLOCK_SIZE];
271 task_stack -> ts_top--;
272 task_stack -> ts_entries--;
274 tied_task = * (task_stack -> ts_top );
276 KMP_DEBUG_ASSERT( tied_task != NULL );
277 KMP_DEBUG_ASSERT( tied_task -> td_flags.tasktype == TASK_TIED );
278 KMP_DEBUG_ASSERT( tied_task == ending_task );
280 KA_TRACE(20, (
"__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n", gtid, tied_task ) );
289 __kmp_push_task(kmp_int32 gtid, kmp_task_t * task )
291 kmp_info_t * thread = __kmp_threads[ gtid ];
292 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
293 kmp_task_team_t * task_team = thread->th.th_task_team;
294 kmp_int32 tid = __kmp_tid_from_gtid( gtid );
295 kmp_thread_data_t * thread_data;
297 KA_TRACE(20, (
"__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata ) );
300 if ( taskdata->td_flags.task_serial ) {
301 KA_TRACE(20, (
"__kmp_push_task: T#%d team serialized; returning TASK_NOT_PUSHED for task %p\n",
303 return TASK_NOT_PUSHED;
307 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
308 if ( ! KMP_TASKING_ENABLED( task_team, thread->th.th_task_state ) ) {
309 __kmp_enable_tasking( task_team, thread );
311 KMP_DEBUG_ASSERT( TCR_4(task_team -> tt.tt_found_tasks) == TRUE );
312 KMP_DEBUG_ASSERT( TCR_PTR(task_team -> tt.tt_threads_data) != NULL );
315 thread_data = & task_team -> tt.tt_threads_data[ tid ];
318 if (thread_data -> td.td_deque == NULL ) {
319 __kmp_alloc_task_deque( thread, thread_data );
323 if ( TCR_4(thread_data -> td.td_deque_ntasks) >= TASK_DEQUE_SIZE )
325 KA_TRACE(20, (
"__kmp_push_task: T#%d deque is full; returning TASK_NOT_PUSHED for task %p\n",
327 return TASK_NOT_PUSHED;
331 __kmp_acquire_bootstrap_lock( & thread_data -> td.td_deque_lock );
334 KMP_DEBUG_ASSERT( TCR_4(thread_data -> td.td_deque_ntasks) < TASK_DEQUE_SIZE );
336 thread_data -> td.td_deque[ thread_data -> td.td_deque_tail ] = taskdata;
338 thread_data -> td.td_deque_tail = ( thread_data -> td.td_deque_tail + 1 ) & TASK_DEQUE_MASK;
339 TCW_4(thread_data -> td.td_deque_ntasks, TCR_4(thread_data -> td.td_deque_ntasks) + 1);
341 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
343 KA_TRACE(20, (
"__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "
344 "task=%p ntasks=%d head=%u tail=%u\n",
345 gtid, taskdata, thread_data->td.td_deque_ntasks,
346 thread_data->td.td_deque_tail, thread_data->td.td_deque_head) );
348 return TASK_SUCCESSFULLY_PUSHED;
357 __kmp_pop_current_task_from_thread( kmp_info_t *this_thr )
359 KF_TRACE( 10, (
"__kmp_pop_current_task_from_thread(enter): T#%d this_thread=%p, curtask=%p, "
360 "curtask_parent=%p\n",
361 0, this_thr, this_thr -> th.th_current_task,
362 this_thr -> th.th_current_task -> td_parent ) );
364 this_thr -> th.th_current_task = this_thr -> th.th_current_task -> td_parent;
366 KF_TRACE( 10, (
"__kmp_pop_current_task_from_thread(exit): T#%d this_thread=%p, curtask=%p, "
367 "curtask_parent=%p\n",
368 0, this_thr, this_thr -> th.th_current_task,
369 this_thr -> th.th_current_task -> td_parent ) );
380 __kmp_push_current_task_to_thread( kmp_info_t *this_thr, kmp_team_t *team,
int tid )
383 KF_TRACE( 10, (
"__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p curtask=%p "
385 tid, this_thr, this_thr->th.th_current_task,
386 team->t.t_implicit_task_taskdata[tid].td_parent ) );
388 KMP_DEBUG_ASSERT (this_thr != NULL);
391 if( this_thr->th.th_current_task != & team -> t.t_implicit_task_taskdata[ 0 ] ) {
392 team -> t.t_implicit_task_taskdata[ 0 ].td_parent = this_thr->th.th_current_task;
393 this_thr->th.th_current_task = & team -> t.t_implicit_task_taskdata[ 0 ];
396 team -> t.t_implicit_task_taskdata[ tid ].td_parent = team -> t.t_implicit_task_taskdata[ 0 ].td_parent;
397 this_thr->th.th_current_task = & team -> t.t_implicit_task_taskdata[ tid ];
400 KF_TRACE( 10, (
"__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p curtask=%p "
402 tid, this_thr, this_thr->th.th_current_task,
403 team->t.t_implicit_task_taskdata[tid].td_parent ) );
414 __kmp_task_start( kmp_int32 gtid, kmp_task_t * task, kmp_taskdata_t * current_task )
416 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
417 kmp_info_t * thread = __kmp_threads[ gtid ];
419 KA_TRACE(10, (
"__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n",
420 gtid, taskdata, current_task) );
422 KMP_DEBUG_ASSERT( taskdata -> td_flags.tasktype == TASK_EXPLICIT );
427 current_task -> td_flags.executing = 0;
430 #ifdef BUILD_TIED_TASK_STACK
431 if ( taskdata -> td_flags.tiedness == TASK_TIED )
433 __kmp_push_task_stack( gtid, thread, taskdata );
438 thread -> th.th_current_task = taskdata;
440 KMP_DEBUG_ASSERT( taskdata -> td_flags.started == 0 );
441 KMP_DEBUG_ASSERT( taskdata -> td_flags.executing == 0 );
442 taskdata -> td_flags.started = 1;
443 taskdata -> td_flags.executing = 1;
444 KMP_DEBUG_ASSERT( taskdata -> td_flags.complete == 0 );
445 KMP_DEBUG_ASSERT( taskdata -> td_flags.freed == 0 );
452 KA_TRACE(10, (
"__kmp_task_start(exit): T#%d task=%p\n",
466 __kmpc_omp_task_begin_if0(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * task )
468 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
469 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
471 KA_TRACE(10, (
"__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p current_task=%p\n",
472 gtid, loc_ref, taskdata, current_task ) );
474 taskdata -> td_flags.task_serial = 1;
475 __kmp_task_start( gtid, task, current_task );
477 KA_TRACE(10, (
"__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n",
478 gtid, loc_ref, taskdata ) );
489 __kmpc_omp_task_begin(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * task )
491 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
493 KA_TRACE(10, (
"__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n",
494 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task ) );
496 __kmp_task_start( gtid, task, current_task );
498 KA_TRACE(10, (
"__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n",
499 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
503 #endif // TASK_UNUSED
513 __kmp_free_task( kmp_int32 gtid, kmp_taskdata_t * taskdata, kmp_info_t * thread )
515 KA_TRACE(30, (
"__kmp_free_task: T#%d freeing data from task %p\n",
519 KMP_DEBUG_ASSERT( taskdata->td_flags.tasktype == TASK_EXPLICIT );
520 KMP_DEBUG_ASSERT( taskdata->td_flags.executing == 0 );
521 KMP_DEBUG_ASSERT( taskdata->td_flags.complete == 1 );
522 KMP_DEBUG_ASSERT( taskdata->td_flags.freed == 0 );
523 KMP_DEBUG_ASSERT( TCR_4(taskdata->td_allocated_child_tasks) == 0 || taskdata->td_flags.task_serial == 1);
524 KMP_DEBUG_ASSERT( TCR_4(taskdata->td_incomplete_child_tasks) == 0 );
526 taskdata->td_flags.freed = 1;
529 __kmp_fast_free( thread, taskdata );
531 __kmp_thread_free( thread, taskdata );
534 KA_TRACE(20, (
"__kmp_free_task: T#%d freed task %p\n",
546 __kmp_free_task_and_ancestors( kmp_int32 gtid, kmp_taskdata_t * taskdata, kmp_info_t * thread )
548 kmp_int32 children = 0;
549 kmp_int32 team_or_tasking_serialized = taskdata -> td_flags.team_serial || taskdata -> td_flags.tasking_ser;
551 KMP_DEBUG_ASSERT( taskdata -> td_flags.tasktype == TASK_EXPLICIT );
553 if ( !team_or_tasking_serialized ) {
554 children = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata -> td_allocated_child_tasks) ) - 1;
555 KMP_DEBUG_ASSERT( children >= 0 );
559 while ( children == 0 )
561 kmp_taskdata_t * parent_taskdata = taskdata -> td_parent;
563 KA_TRACE(20, (
"__kmp_free_task_and_ancestors(enter): T#%d task %p complete "
564 "and freeing itself\n", gtid, taskdata) );
567 __kmp_free_task( gtid, taskdata, thread );
569 taskdata = parent_taskdata;
573 if ( team_or_tasking_serialized || taskdata -> td_flags.tasktype == TASK_IMPLICIT )
576 if ( !team_or_tasking_serialized ) {
578 children = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata -> td_allocated_child_tasks) ) - 1;
579 KMP_DEBUG_ASSERT( children >= 0 );
583 KA_TRACE(20, (
"__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
584 "not freeing it yet\n", gtid, taskdata, children) );
594 __kmp_task_finish( kmp_int32 gtid, kmp_task_t *task, kmp_taskdata_t *resumed_task )
596 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
597 kmp_info_t * thread = __kmp_threads[ gtid ];
598 kmp_int32 children = 0;
600 KA_TRACE(10, (
"__kmp_task_finish(enter): T#%d finishing task %p and resuming task %p\n",
601 gtid, taskdata, resumed_task) );
603 KMP_DEBUG_ASSERT( taskdata -> td_flags.tasktype == TASK_EXPLICIT );
606 #ifdef BUILD_TIED_TASK_STACK
607 if ( taskdata -> td_flags.tiedness == TASK_TIED )
609 __kmp_pop_task_stack( gtid, thread, taskdata );
613 KMP_DEBUG_ASSERT( taskdata -> td_flags.complete == 0 );
614 taskdata -> td_flags.complete = 1;
615 KMP_DEBUG_ASSERT( taskdata -> td_flags.started == 1 );
616 KMP_DEBUG_ASSERT( taskdata -> td_flags.freed == 0 );
619 if ( !( taskdata -> td_flags.team_serial || taskdata -> td_flags.tasking_ser ) ) {
621 children = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata -> td_parent -> td_incomplete_child_tasks) ) - 1;
622 KMP_DEBUG_ASSERT( children >= 0 );
624 if ( taskdata->td_taskgroup )
625 KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata->td_taskgroup->count) );
626 __kmp_release_deps(gtid,taskdata);
633 KMP_DEBUG_ASSERT( taskdata -> td_flags.executing == 1 );
634 taskdata -> td_flags.executing = 0;
636 KA_TRACE(20, (
"__kmp_task_finish: T#%d finished task %p, %d incomplete children\n",
637 gtid, taskdata, children) );
647 if (taskdata->td_flags.destructors_thunk) {
648 kmp_routine_entry_t destr_thunk = task->destructors;
649 KMP_ASSERT(destr_thunk);
650 destr_thunk(gtid, task);
652 #endif // OMP_40_ENABLED
656 KMP_DEBUG_ASSERT( (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) ==
657 taskdata->td_flags.task_serial);
658 if ( taskdata->td_flags.task_serial )
660 if (resumed_task == NULL) {
661 resumed_task = taskdata->td_parent;
665 KMP_DEBUG_ASSERT( resumed_task == taskdata->td_parent );
669 KMP_DEBUG_ASSERT( resumed_task != NULL );
673 __kmp_free_task_and_ancestors(gtid, taskdata, thread);
675 __kmp_threads[ gtid ] -> th.th_current_task = resumed_task;
679 resumed_task->td_flags.executing = 1;
681 KA_TRACE(10, (
"__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n",
682 gtid, taskdata, resumed_task) );
694 __kmpc_omp_task_complete_if0(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task )
696 KA_TRACE(10, (
"__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n",
697 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
699 __kmp_task_finish( gtid, task, NULL );
701 KA_TRACE(10, (
"__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n",
702 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
713 __kmpc_omp_task_complete(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task )
715 KA_TRACE(10, (
"__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n",
716 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
718 __kmp_task_finish( gtid, task, NULL );
720 KA_TRACE(10, (
"__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n",
721 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
724 #endif // TASK_UNUSED
739 __kmp_init_implicit_task(
ident_t *loc_ref, kmp_info_t *this_thr, kmp_team_t *team,
int tid,
int set_curr_task )
741 kmp_taskdata_t * task = & team->t.t_implicit_task_taskdata[ tid ];
743 KF_TRACE(10, (
"__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n",
744 tid, team, task, set_curr_task ?
"TRUE" :
"FALSE" ) );
746 task->td_task_id = KMP_GEN_TASK_ID();
747 task->td_team = team;
749 task->td_ident = loc_ref;
750 task->td_taskwait_ident = NULL;
751 task->td_taskwait_counter = 0;
752 task->td_taskwait_thread = 0;
754 task->td_flags.tiedness = TASK_TIED;
755 task->td_flags.tasktype = TASK_IMPLICIT;
757 task->td_flags.task_serial = 1;
758 task->td_flags.tasking_ser = ( __kmp_tasking_mode == tskm_immediate_exec );
759 task->td_flags.team_serial = ( team->t.t_serialized ) ? 1 : 0;
761 task->td_flags.started = 1;
762 task->td_flags.executing = 1;
763 task->td_flags.complete = 0;
764 task->td_flags.freed = 0;
767 task->td_dephash = NULL;
768 task->td_depnode = NULL;
772 task->td_incomplete_child_tasks = 0;
773 task->td_allocated_child_tasks = 0;
775 task->td_taskgroup = NULL;
777 __kmp_push_current_task_to_thread( this_thr, team, tid );
779 KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0);
780 KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0);
783 KF_TRACE(10, (
"__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n",
790 __kmp_round_up_to_val(
size_t size,
size_t val ) {
791 if ( size & ( val - 1 ) ) {
792 size &= ~ ( val - 1 );
793 if ( size <= KMP_SIZE_T_MAX - val ) {
814 __kmp_task_alloc(
ident_t *loc_ref, kmp_int32 gtid, kmp_tasking_flags_t *flags,
815 size_t sizeof_kmp_task_t,
size_t sizeof_shareds,
816 kmp_routine_entry_t task_entry )
819 kmp_taskdata_t *taskdata;
820 kmp_info_t *thread = __kmp_threads[ gtid ];
821 kmp_team_t *team = thread->th.th_team;
822 kmp_taskdata_t *parent_task = thread->th.th_current_task;
823 size_t shareds_offset;
825 KA_TRACE(10, (
"__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
826 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
827 gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t,
828 sizeof_shareds, task_entry) );
830 if ( parent_task->td_flags.final ) {
831 if (flags->merged_if0) {
838 shareds_offset =
sizeof( kmp_taskdata_t ) + sizeof_kmp_task_t;
839 shareds_offset = __kmp_round_up_to_val( shareds_offset,
sizeof(
void * ));
842 KA_TRACE(30, (
"__kmp_task_alloc: T#%d First malloc size: %ld\n",
843 gtid, shareds_offset) );
844 KA_TRACE(30, (
"__kmp_task_alloc: T#%d Second malloc size: %ld\n",
845 gtid, sizeof_shareds) );
849 taskdata = (kmp_taskdata_t *) __kmp_fast_allocate( thread, shareds_offset + sizeof_shareds );
851 taskdata = (kmp_taskdata_t *) __kmp_thread_malloc( thread, shareds_offset + sizeof_shareds );
854 task = KMP_TASKDATA_TO_TASK(taskdata);
858 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)taskdata) & (
sizeof(
double)-1) ) == 0 );
859 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)task) & (
sizeof(
double)-1) ) == 0 );
861 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)taskdata) & (
sizeof(_Quad)-1) ) == 0 );
862 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)task) & (
sizeof(_Quad)-1) ) == 0 );
864 if (sizeof_shareds > 0) {
866 task->shareds = & ((
char *) taskdata)[ shareds_offset ];
868 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)task->shareds) & (
sizeof(
void *)-1) ) == 0 );
870 task->shareds = NULL;
872 task->routine = task_entry;
875 taskdata->td_task_id = KMP_GEN_TASK_ID();
876 taskdata->td_team = team;
877 taskdata->td_alloc_thread = thread;
878 taskdata->td_parent = parent_task;
879 taskdata->td_level = parent_task->td_level + 1;
880 taskdata->td_ident = loc_ref;
881 taskdata->td_taskwait_ident = NULL;
882 taskdata->td_taskwait_counter = 0;
883 taskdata->td_taskwait_thread = 0;
884 KMP_DEBUG_ASSERT( taskdata->td_parent != NULL );
885 copy_icvs( &taskdata->td_icvs, &taskdata->td_parent->td_icvs );
887 taskdata->td_flags.tiedness = flags->tiedness;
888 taskdata->td_flags.final = flags->final;
889 taskdata->td_flags.merged_if0 = flags->merged_if0;
891 taskdata->td_flags.destructors_thunk = flags->destructors_thunk;
892 #endif // OMP_40_ENABLED
893 taskdata->td_flags.tasktype = TASK_EXPLICIT;
896 taskdata->td_flags.tasking_ser = ( __kmp_tasking_mode == tskm_immediate_exec );
899 taskdata->td_flags.team_serial = ( team->t.t_serialized ) ? 1 : 0;
904 taskdata->td_flags.task_serial = ( taskdata->td_flags.final
905 || taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser );
907 taskdata->td_flags.started = 0;
908 taskdata->td_flags.executing = 0;
909 taskdata->td_flags.complete = 0;
910 taskdata->td_flags.freed = 0;
912 taskdata->td_flags.native = flags->native;
914 taskdata->td_incomplete_child_tasks = 0;
915 taskdata->td_allocated_child_tasks = 1;
917 taskdata->td_taskgroup = parent_task->td_taskgroup;
918 taskdata->td_dephash = NULL;
919 taskdata->td_depnode = NULL;
922 if ( !( taskdata -> td_flags.team_serial || taskdata -> td_flags.tasking_ser ) ) {
923 KMP_TEST_THEN_INC32( (kmp_int32 *)(& parent_task->td_incomplete_child_tasks) );
925 if ( parent_task->td_taskgroup )
926 KMP_TEST_THEN_INC32( (kmp_int32 *)(& parent_task->td_taskgroup->count) );
929 if ( taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT ) {
930 KMP_TEST_THEN_INC32( (kmp_int32 *)(& taskdata->td_parent->td_allocated_child_tasks) );
934 KA_TRACE(20, (
"__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",
935 gtid, taskdata, taskdata->td_parent) );
942 __kmpc_omp_task_alloc(
ident_t *loc_ref, kmp_int32 gtid, kmp_int32 flags,
943 size_t sizeof_kmp_task_t,
size_t sizeof_shareds,
944 kmp_routine_entry_t task_entry )
947 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *) & flags;
949 input_flags->native = FALSE;
952 KA_TRACE(10, (
"__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s) "
953 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
954 gtid, loc_ref, input_flags->tiedness ?
"tied " :
"untied",
955 sizeof_kmp_task_t, sizeof_shareds, task_entry) );
957 retval = __kmp_task_alloc( loc_ref, gtid, input_flags, sizeof_kmp_task_t,
958 sizeof_shareds, task_entry );
960 KA_TRACE(20, (
"__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval) );
973 __kmp_invoke_task( kmp_int32 gtid, kmp_task_t *task, kmp_taskdata_t * current_task )
975 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
979 KA_TRACE(30, (
"__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n",
980 gtid, taskdata, current_task) );
982 __kmp_task_start( gtid, task, current_task );
988 if (__kmp_omp_cancellation) {
989 kmp_info_t *this_thr = __kmp_threads [ gtid ];
990 kmp_team_t * this_team = this_thr->th.th_team;
991 kmp_taskgroup_t * taskgroup = taskdata->td_taskgroup;
992 if ((taskgroup && taskgroup->cancel_request) || (this_team->t.t_cancel_request == cancel_parallel)) {
1003 #endif // OMP_40_ENABLED
1004 #ifdef KMP_GOMP_COMPAT
1005 if (taskdata->td_flags.native) {
1006 ((void (*)(
void *))(*(task->routine)))(task->shareds);
1011 (*(task->routine))(gtid, task);
1015 #endif // OMP_40_ENABLED
1017 __kmp_task_finish( gtid, task, current_task );
1019 KA_TRACE(30, (
"__kmp_inovke_task(exit): T#%d completed task %p, resuming task %p\n",
1020 gtid, taskdata, current_task) );
1035 __kmpc_omp_task_parts(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * new_task)
1037 kmp_taskdata_t * new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1039 KA_TRACE(10, (
"__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n",
1040 gtid, loc_ref, new_taskdata ) );
1045 if ( __kmp_push_task( gtid, new_task ) == TASK_NOT_PUSHED )
1047 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
1048 new_taskdata->td_flags.task_serial = 1;
1049 __kmp_invoke_task( gtid, new_task, current_task );
1052 KA_TRACE(10, (
"__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
1053 "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", gtid, loc_ref,
1056 return TASK_CURRENT_NOT_QUEUED;
1069 __kmp_omp_task( kmp_int32 gtid, kmp_task_t * new_task,
bool serialize_immediate )
1071 kmp_taskdata_t * new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1076 if ( __kmp_push_task( gtid, new_task ) == TASK_NOT_PUSHED )
1078 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
1079 if ( serialize_immediate )
1080 new_taskdata -> td_flags.task_serial = 1;
1081 __kmp_invoke_task( gtid, new_task, current_task );
1085 return TASK_CURRENT_NOT_QUEUED;
1100 __kmpc_omp_task(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * new_task)
1102 kmp_taskdata_t * new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1105 KA_TRACE(10, (
"__kmpc_omp_task(enter): T#%d loc=%p task=%p\n",
1106 gtid, loc_ref, new_taskdata ) );
1108 res = __kmp_omp_task(gtid,new_task,
true);
1110 KA_TRACE(10, (
"__kmpc_omp_task(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
1111 gtid, loc_ref, new_taskdata ) );
1119 __kmpc_omp_taskwait(
ident_t *loc_ref, kmp_int32 gtid )
1121 kmp_taskdata_t * taskdata;
1122 kmp_info_t * thread;
1123 int thread_finished = FALSE;
1125 KA_TRACE(10, (
"__kmpc_omp_taskwait(enter): T#%d loc=%p\n",
1128 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
1131 thread = __kmp_threads[ gtid ];
1132 taskdata = thread -> th.th_current_task;
1136 taskdata->td_taskwait_counter += 1;
1137 taskdata->td_taskwait_ident = loc_ref;
1138 taskdata->td_taskwait_thread = gtid + 1;
1141 void * itt_sync_obj = __kmp_itt_taskwait_object( gtid );
1142 if ( itt_sync_obj != NULL )
1143 __kmp_itt_taskwait_starting( gtid, itt_sync_obj );
1146 if ( ! taskdata->td_flags.team_serial ) {
1148 kmp_flag_32 flag(&(taskdata->td_incomplete_child_tasks), 0U);
1149 while ( TCR_4(taskdata -> td_incomplete_child_tasks) != 0 ) {
1150 flag.execute_tasks(thread, gtid, FALSE, &thread_finished
1151 USE_ITT_BUILD_ARG(itt_sync_obj), __kmp_task_stealing_constraint );
1155 if ( itt_sync_obj != NULL )
1156 __kmp_itt_taskwait_finished( gtid, itt_sync_obj );
1160 taskdata->td_taskwait_thread = - taskdata->td_taskwait_thread;
1163 KA_TRACE(10, (
"__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
1164 "returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata) );
1166 return TASK_CURRENT_NOT_QUEUED;
1174 __kmpc_omp_taskyield(
ident_t *loc_ref, kmp_int32 gtid,
int end_part )
1176 kmp_taskdata_t * taskdata;
1177 kmp_info_t * thread;
1178 int thread_finished = FALSE;
1180 KA_TRACE(10, (
"__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n",
1181 gtid, loc_ref, end_part) );
1183 if ( __kmp_tasking_mode != tskm_immediate_exec && __kmp_init_parallel ) {
1186 thread = __kmp_threads[ gtid ];
1187 taskdata = thread -> th.th_current_task;
1192 taskdata->td_taskwait_counter += 1;
1193 taskdata->td_taskwait_ident = loc_ref;
1194 taskdata->td_taskwait_thread = gtid + 1;
1197 void * itt_sync_obj = __kmp_itt_taskwait_object( gtid );
1198 if ( itt_sync_obj != NULL )
1199 __kmp_itt_taskwait_starting( gtid, itt_sync_obj );
1201 if ( ! taskdata->td_flags.team_serial ) {
1202 kmp_task_team_t * task_team = thread->th.th_task_team;
1203 if (task_team != NULL) {
1204 if (KMP_TASKING_ENABLED(task_team, thread->th.th_task_state)) {
1205 __kmp_execute_tasks_32( thread, gtid, NULL, FALSE, &thread_finished
1206 USE_ITT_BUILD_ARG(itt_sync_obj), __kmp_task_stealing_constraint );
1211 if ( itt_sync_obj != NULL )
1212 __kmp_itt_taskwait_finished( gtid, itt_sync_obj );
1216 taskdata->td_taskwait_thread = - taskdata->td_taskwait_thread;
1219 KA_TRACE(10, (
"__kmpc_omp_taskyield(exit): T#%d task %p resuming, "
1220 "returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata) );
1222 return TASK_CURRENT_NOT_QUEUED;
1231 __kmpc_taskgroup(
ident_t* loc,
int gtid )
1233 kmp_info_t * thread = __kmp_threads[ gtid ];
1234 kmp_taskdata_t * taskdata = thread->th.th_current_task;
1235 kmp_taskgroup_t * tg_new =
1236 (kmp_taskgroup_t *)__kmp_thread_malloc( thread,
sizeof( kmp_taskgroup_t ) );
1237 KA_TRACE(10, (
"__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new) );
1239 tg_new->cancel_request = cancel_noreq;
1240 tg_new->parent = taskdata->td_taskgroup;
1241 taskdata->td_taskgroup = tg_new;
1250 __kmpc_end_taskgroup(
ident_t* loc,
int gtid )
1252 kmp_info_t * thread = __kmp_threads[ gtid ];
1253 kmp_taskdata_t * taskdata = thread->th.th_current_task;
1254 kmp_taskgroup_t * taskgroup = taskdata->td_taskgroup;
1255 int thread_finished = FALSE;
1257 KA_TRACE(10, (
"__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc) );
1258 KMP_DEBUG_ASSERT( taskgroup != NULL );
1260 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
1263 void * itt_sync_obj = __kmp_itt_taskwait_object( gtid );
1264 if ( itt_sync_obj != NULL )
1265 __kmp_itt_taskwait_starting( gtid, itt_sync_obj );
1268 if ( ! taskdata->td_flags.team_serial ) {
1269 kmp_flag_32 flag(&(taskgroup->count), 0U);
1270 while ( TCR_4(taskgroup->count) != 0 ) {
1271 flag.execute_tasks(thread, gtid, FALSE, &thread_finished
1272 USE_ITT_BUILD_ARG(itt_sync_obj), __kmp_task_stealing_constraint );
1277 if ( itt_sync_obj != NULL )
1278 __kmp_itt_taskwait_finished( gtid, itt_sync_obj );
1281 KMP_DEBUG_ASSERT( taskgroup->count == 0 );
1284 taskdata->td_taskgroup = taskgroup->parent;
1285 __kmp_thread_free( thread, taskgroup );
1287 KA_TRACE(10, (
"__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n", gtid, taskdata) );
1296 __kmp_remove_my_task( kmp_info_t * thread, kmp_int32 gtid, kmp_task_team_t *task_team,
1297 kmp_int32 is_constrained )
1300 kmp_taskdata_t * taskdata;
1301 kmp_thread_data_t *thread_data;
1304 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
1305 KMP_DEBUG_ASSERT( task_team -> tt.tt_threads_data != NULL );
1307 thread_data = & task_team -> tt.tt_threads_data[ __kmp_tid_from_gtid( gtid ) ];
1309 KA_TRACE(10, (
"__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n",
1310 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1311 thread_data->td.td_deque_tail) );
1313 if (TCR_4(thread_data -> td.td_deque_ntasks) == 0) {
1314 KA_TRACE(10, (
"__kmp_remove_my_task(exit #1): T#%d No tasks to remove: ntasks=%d head=%u tail=%u\n",
1315 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1316 thread_data->td.td_deque_tail) );
1320 __kmp_acquire_bootstrap_lock( & thread_data -> td.td_deque_lock );
1322 if (TCR_4(thread_data -> td.td_deque_ntasks) == 0) {
1323 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
1324 KA_TRACE(10, (
"__kmp_remove_my_task(exit #2): T#%d No tasks to remove: ntasks=%d head=%u tail=%u\n",
1325 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1326 thread_data->td.td_deque_tail) );
1330 tail = ( thread_data -> td.td_deque_tail - 1 ) & TASK_DEQUE_MASK;
1331 taskdata = thread_data -> td.td_deque[ tail ];
1333 if (is_constrained) {
1336 kmp_taskdata_t * current = thread->th.th_current_task;
1337 kmp_int32 level = current->td_level;
1338 kmp_taskdata_t * parent = taskdata->td_parent;
1339 while ( parent != current && parent->td_level > level ) {
1340 parent = parent->td_parent;
1341 KMP_DEBUG_ASSERT(parent != NULL);
1343 if ( parent != current ) {
1345 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
1346 KA_TRACE(10, (
"__kmp_remove_my_task(exit #2): T#%d No tasks to remove: ntasks=%d head=%u tail=%u\n",
1347 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1348 thread_data->td.td_deque_tail) );
1353 thread_data -> td.td_deque_tail = tail;
1354 TCW_4(thread_data -> td.td_deque_ntasks, thread_data -> td.td_deque_ntasks - 1);
1356 __kmp_release_bootstrap_lock( & thread_data->td.td_deque_lock );
1358 KA_TRACE(10, (
"__kmp_remove_my_task(exit #2): T#%d task %p removed: ntasks=%d head=%u tail=%u\n",
1359 gtid, taskdata, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1360 thread_data->td.td_deque_tail) );
1362 task = KMP_TASKDATA_TO_TASK( taskdata );
1373 __kmp_steal_task( kmp_info_t *victim, kmp_int32 gtid, kmp_task_team_t *task_team,
1374 volatile kmp_uint32 *unfinished_threads,
int *thread_finished,
1375 kmp_int32 is_constrained )
1378 kmp_taskdata_t * taskdata;
1379 kmp_thread_data_t *victim_td, *threads_data;
1380 kmp_int32 victim_tid, thread_tid;
1382 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
1384 threads_data = task_team -> tt.tt_threads_data;
1385 KMP_DEBUG_ASSERT( threads_data != NULL );
1387 victim_tid = victim->th.th_info.ds.ds_tid;
1388 victim_td = & threads_data[ victim_tid ];
1390 KA_TRACE(10, (
"__kmp_steal_task(enter): T#%d try to steal from T#%d: task_team=%p ntasks=%d "
1391 "head=%u tail=%u\n",
1392 gtid, __kmp_gtid_from_thread( victim ), task_team, victim_td->td.td_deque_ntasks,
1393 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1395 if ( (TCR_4(victim_td -> td.td_deque_ntasks) == 0) ||
1396 (TCR_PTR(victim->th.th_task_team) != task_team))
1398 KA_TRACE(10, (
"__kmp_steal_task(exit #1): T#%d could not steal from T#%d: task_team=%p "
1399 "ntasks=%d head=%u tail=%u\n",
1400 gtid, __kmp_gtid_from_thread( victim ), task_team, victim_td->td.td_deque_ntasks,
1401 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1405 __kmp_acquire_bootstrap_lock( & victim_td -> td.td_deque_lock );
1408 if ( (TCR_4(victim_td -> td.td_deque_ntasks) == 0) ||
1409 (TCR_PTR(victim->th.th_task_team) != task_team))
1411 __kmp_release_bootstrap_lock( & victim_td -> td.td_deque_lock );
1412 KA_TRACE(10, (
"__kmp_steal_task(exit #2): T#%d could not steal from T#%d: task_team=%p "
1413 "ntasks=%d head=%u tail=%u\n",
1414 gtid, __kmp_gtid_from_thread( victim ), task_team, victim_td->td.td_deque_ntasks,
1415 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1419 KMP_DEBUG_ASSERT( victim_td -> td.td_deque != NULL );
1421 if ( !is_constrained ) {
1422 taskdata = victim_td -> td.td_deque[ victim_td -> td.td_deque_head ];
1424 victim_td -> td.td_deque_head = ( victim_td -> td.td_deque_head + 1 ) & TASK_DEQUE_MASK;
1427 kmp_int32 tail = ( victim_td -> td.td_deque_tail - 1 ) & TASK_DEQUE_MASK;
1428 taskdata = victim_td -> td.td_deque[ tail ];
1431 kmp_taskdata_t * current = __kmp_threads[ gtid ]->th.th_current_task;
1432 kmp_int32 level = current->td_level;
1433 kmp_taskdata_t * parent = taskdata->td_parent;
1434 while ( parent != current && parent->td_level > level ) {
1435 parent = parent->td_parent;
1436 KMP_DEBUG_ASSERT(parent != NULL);
1438 if ( parent != current ) {
1440 __kmp_release_bootstrap_lock( & victim_td -> td.td_deque_lock );
1441 KA_TRACE(10, (
"__kmp_steal_task(exit #2): T#%d could not steal from T#%d: task_team=%p "
1442 "ntasks=%d head=%u tail=%u\n",
1443 gtid, __kmp_gtid_from_thread( threads_data[victim_tid].td.td_thr ),
1444 task_team, victim_td->td.td_deque_ntasks,
1445 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1448 victim_td -> td.td_deque_tail = tail;
1450 if (*thread_finished) {
1454 kmp_uint32 count = KMP_TEST_THEN_INC32( (kmp_int32 *)unfinished_threads );
1456 KA_TRACE(20, (
"__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n",
1457 gtid, count + 1, task_team) );
1459 *thread_finished = FALSE;
1461 TCW_4(victim_td -> td.td_deque_ntasks, TCR_4(victim_td -> td.td_deque_ntasks) - 1);
1463 __kmp_release_bootstrap_lock( & victim_td -> td.td_deque_lock );
1465 KA_TRACE(10, (
"__kmp_steal_task(exit #3): T#%d stole task %p from T#%d: task_team=%p "
1466 "ntasks=%d head=%u tail=%u\n",
1467 gtid, taskdata, __kmp_gtid_from_thread( victim ), task_team,
1468 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,
1469 victim_td->td.td_deque_tail) );
1471 task = KMP_TASKDATA_TO_TASK( taskdata );
1486 static inline int __kmp_execute_tasks_template(kmp_info_t *thread, kmp_int32 gtid, C *flag,
int final_spin,
1487 int *thread_finished
1488 USE_ITT_BUILD_ARG(
void * itt_sync_obj), kmp_int32 is_constrained)
1490 kmp_task_team_t * task_team;
1492 kmp_thread_data_t * threads_data;
1494 kmp_taskdata_t * current_task = thread -> th.th_current_task;
1495 volatile kmp_uint32 * unfinished_threads;
1496 kmp_int32 nthreads, last_stolen, k, tid;
1498 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
1499 KMP_DEBUG_ASSERT( thread == __kmp_threads[ gtid ] );
1501 task_team = thread -> th.th_task_team;
1502 KMP_DEBUG_ASSERT( task_team != NULL );
1504 KA_TRACE(15, (
"__kmp_execute_tasks_template(enter): T#%d final_spin=%d *thread_finished=%d\n",
1505 gtid, final_spin, *thread_finished) );
1507 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team -> tt.tt_threads_data);
1508 KMP_DEBUG_ASSERT( threads_data != NULL );
1510 nthreads = task_team -> tt.tt_nproc;
1511 unfinished_threads = &(task_team -> tt.tt_unfinished_threads);
1512 KMP_DEBUG_ASSERT( nthreads > 1 );
1513 KMP_DEBUG_ASSERT( TCR_4((
int)*unfinished_threads) >= 0 );
1517 while (( task = __kmp_remove_my_task( thread, gtid, task_team, is_constrained )) != NULL ) {
1518 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1519 if ( __itt_sync_create_ptr || KMP_ITT_DEBUG ) {
1520 if ( itt_sync_obj == NULL ) {
1522 itt_sync_obj = __kmp_itt_barrier_object( gtid, bs_forkjoin_barrier );
1524 __kmp_itt_task_starting( itt_sync_obj );
1527 __kmp_invoke_task( gtid, task, current_task );
1529 if ( itt_sync_obj != NULL )
1530 __kmp_itt_task_finished( itt_sync_obj );
1538 if (flag == NULL || (!final_spin && flag->done_check())) {
1539 KA_TRACE(15, (
"__kmp_execute_tasks_template(exit #1): T#%d spin condition satisfied\n", gtid) );
1542 KMP_YIELD( __kmp_library == library_throughput );
1551 if (! *thread_finished) {
1552 kmp_uint32 count = KMP_TEST_THEN_DEC32( (kmp_int32 *)unfinished_threads ) - 1;
1553 KA_TRACE(20, (
"__kmp_execute_tasks_template(dec #1): T#%d dec unfinished_threads to %d task_team=%p\n",
1554 gtid, count, task_team) );
1555 *thread_finished = TRUE;
1563 if (flag != NULL && flag->done_check()) {
1564 KA_TRACE(15, (
"__kmp_execute_tasks_template(exit #2): T#%d spin condition satisfied\n", gtid) );
1570 tid = thread -> th.th_info.ds.ds_tid;
1571 last_stolen = threads_data[ tid ].td.td_deque_last_stolen;
1573 if (last_stolen != -1) {
1574 kmp_info_t *other_thread = threads_data[last_stolen].td.td_thr;
1576 while ((task = __kmp_steal_task( other_thread, gtid, task_team, unfinished_threads,
1577 thread_finished, is_constrained )) != NULL)
1579 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1580 if ( __itt_sync_create_ptr || KMP_ITT_DEBUG ) {
1581 if ( itt_sync_obj == NULL ) {
1583 itt_sync_obj = __kmp_itt_barrier_object( gtid, bs_forkjoin_barrier );
1585 __kmp_itt_task_starting( itt_sync_obj );
1588 __kmp_invoke_task( gtid, task, current_task );
1590 if ( itt_sync_obj != NULL )
1591 __kmp_itt_task_finished( itt_sync_obj );
1595 if (flag == NULL || (!final_spin && flag->done_check())) {
1596 KA_TRACE(15, (
"__kmp_execute_tasks_template(exit #3): T#%d spin condition satisfied\n",
1601 KMP_YIELD( __kmp_library == library_throughput );
1604 if (TCR_4(threads_data[ tid ].td.td_deque_ntasks) != 0) {
1605 KA_TRACE(20, (
"__kmp_execute_tasks_template: T#%d stolen task spawned other tasks, restart\n",
1612 threads_data[ tid ].td.td_deque_last_stolen = -1;
1620 if (! *thread_finished) {
1621 kmp_uint32 count = KMP_TEST_THEN_DEC32( (kmp_int32 *)unfinished_threads ) - 1;
1622 KA_TRACE(20, (
"__kmp_execute_tasks_template(dec #2): T#%d dec unfinished_threads to %d "
1623 "task_team=%p\n", gtid, count, task_team) );
1624 *thread_finished = TRUE;
1633 if (flag != NULL && flag->done_check()) {
1634 KA_TRACE(15, (
"__kmp_execute_tasks_template(exit #4): T#%d spin condition satisfied\n",
1647 k = __kmp_get_random( thread ) % (nthreads - 1);
1648 if ( k >= thread -> th.th_info.ds.ds_tid ) {
1652 kmp_info_t *other_thread = threads_data[k].td.td_thr;
1662 if ( ( __kmp_tasking_mode == tskm_task_teams ) &&
1663 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) &&
1664 (TCR_PTR(other_thread->th.th_sleep_loc) != NULL))
1666 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(other_thread), other_thread->th.th_sleep_loc);
1677 while ((task = __kmp_steal_task( other_thread, gtid, task_team, unfinished_threads,
1678 thread_finished, is_constrained )) != NULL)
1680 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1681 if ( __itt_sync_create_ptr || KMP_ITT_DEBUG ) {
1682 if ( itt_sync_obj == NULL ) {
1684 itt_sync_obj = __kmp_itt_barrier_object( gtid, bs_forkjoin_barrier );
1686 __kmp_itt_task_starting( itt_sync_obj );
1689 __kmp_invoke_task( gtid, task, current_task );
1691 if ( itt_sync_obj != NULL )
1692 __kmp_itt_task_finished( itt_sync_obj );
1697 threads_data[ tid ].td.td_deque_last_stolen = k;
1702 if (flag == NULL || (!final_spin && flag->done_check())) {
1703 KA_TRACE(15, (
"__kmp_execute_tasks_template(exit #5): T#%d spin condition satisfied\n",
1707 KMP_YIELD( __kmp_library == library_throughput );
1711 if (TCR_4(threads_data[ tid ].td.td_deque_ntasks) != 0) {
1712 KA_TRACE(20, (
"__kmp_execute_tasks_template: T#%d stolen task spawned other tasks, restart\n",
1727 if (! *thread_finished) {
1728 kmp_uint32 count = KMP_TEST_THEN_DEC32( (kmp_int32 *)unfinished_threads ) - 1;
1729 KA_TRACE(20, (
"__kmp_execute_tasks_template(dec #3): T#%d dec unfinished_threads to %d; "
1731 gtid, count, task_team) );
1732 *thread_finished = TRUE;
1741 if (flag != NULL && flag->done_check()) {
1742 KA_TRACE(15, (
"__kmp_execute_tasks_template(exit #6): T#%d spin condition satisfied\n", gtid) );
1748 KA_TRACE(15, (
"__kmp_execute_tasks_template(exit #7): T#%d can't find work\n", gtid) );
1752 int __kmp_execute_tasks_32(kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32 *flag,
int final_spin,
1753 int *thread_finished
1754 USE_ITT_BUILD_ARG(
void * itt_sync_obj), kmp_int32 is_constrained)
1756 return __kmp_execute_tasks_template(thread, gtid, flag, final_spin, thread_finished
1757 USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
1760 int __kmp_execute_tasks_64(kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64 *flag,
int final_spin,
1761 int *thread_finished
1762 USE_ITT_BUILD_ARG(
void * itt_sync_obj), kmp_int32 is_constrained)
1764 return __kmp_execute_tasks_template(thread, gtid, flag, final_spin, thread_finished
1765 USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
1768 int __kmp_execute_tasks_oncore(kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag,
int final_spin,
1769 int *thread_finished
1770 USE_ITT_BUILD_ARG(
void * itt_sync_obj), kmp_int32 is_constrained)
1772 return __kmp_execute_tasks_template(thread, gtid, flag, final_spin, thread_finished
1773 USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
1784 __kmp_enable_tasking( kmp_task_team_t *task_team, kmp_info_t *this_thr )
1786 kmp_team_t *team = this_thr->th.th_team;
1787 kmp_thread_data_t *threads_data;
1788 int nthreads, i, is_init_thread;
1790 KA_TRACE( 10, (
"__kmp_enable_tasking(enter): T#%d\n",
1791 __kmp_gtid_from_thread( this_thr ) ) );
1793 KMP_DEBUG_ASSERT(task_team != NULL);
1794 KMP_DEBUG_ASSERT(team != NULL);
1796 nthreads = task_team->tt.tt_nproc;
1797 KMP_DEBUG_ASSERT(nthreads > 0);
1798 KMP_DEBUG_ASSERT(nthreads == team->t.t_nproc);
1801 is_init_thread = __kmp_realloc_task_threads_data( this_thr, task_team );
1803 if (!is_init_thread) {
1805 KA_TRACE( 20, (
"__kmp_enable_tasking(exit): T#%d: threads array already set up.\n",
1806 __kmp_gtid_from_thread( this_thr ) ) );
1809 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team -> tt.tt_threads_data);
1810 KMP_DEBUG_ASSERT( threads_data != NULL );
1812 if ( ( __kmp_tasking_mode == tskm_task_teams ) &&
1813 ( __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ) )
1818 for (i = 0; i < nthreads; i++) {
1819 volatile void *sleep_loc;
1820 kmp_info_t *thread = threads_data[i].td.td_thr;
1822 if (i == this_thr->th.th_info.ds.ds_tid) {
1832 if ( ( sleep_loc = TCR_PTR( thread -> th.th_sleep_loc) ) != NULL )
1834 KF_TRACE( 50, (
"__kmp_enable_tasking: T#%d waking up thread T#%d\n",
1835 __kmp_gtid_from_thread( this_thr ),
1836 __kmp_gtid_from_thread( thread ) ) );
1837 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc);
1840 KF_TRACE( 50, (
"__kmp_enable_tasking: T#%d don't wake up thread T#%d\n",
1841 __kmp_gtid_from_thread( this_thr ),
1842 __kmp_gtid_from_thread( thread ) ) );
1847 KA_TRACE( 10, (
"__kmp_enable_tasking(exit): T#%d\n",
1848 __kmp_gtid_from_thread( this_thr ) ) );
1888 static kmp_task_team_t *__kmp_free_task_teams = NULL;
1890 static kmp_bootstrap_lock_t __kmp_task_team_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_task_team_lock );
1902 __kmp_alloc_task_deque( kmp_info_t *thread, kmp_thread_data_t *thread_data )
1904 __kmp_init_bootstrap_lock( & thread_data -> td.td_deque_lock );
1905 KMP_DEBUG_ASSERT( thread_data -> td.td_deque == NULL );
1908 thread_data -> td.td_deque_last_stolen = -1;
1910 KMP_DEBUG_ASSERT( TCR_4(thread_data -> td.td_deque_ntasks) == 0 );
1911 KMP_DEBUG_ASSERT( thread_data -> td.td_deque_head == 0 );
1912 KMP_DEBUG_ASSERT( thread_data -> td.td_deque_tail == 0 );
1914 KE_TRACE( 10, (
"__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n",
1915 __kmp_gtid_from_thread( thread ), TASK_DEQUE_SIZE, thread_data ) );
1919 thread_data -> td.td_deque = (kmp_taskdata_t **)
1920 __kmp_allocate( TASK_DEQUE_SIZE *
sizeof(kmp_taskdata_t *));
1930 __kmp_free_task_deque( kmp_thread_data_t *thread_data )
1932 __kmp_acquire_bootstrap_lock( & thread_data -> td.td_deque_lock );
1934 if ( thread_data -> td.td_deque != NULL ) {
1935 TCW_4(thread_data -> td.td_deque_ntasks, 0);
1936 __kmp_free( thread_data -> td.td_deque );
1937 thread_data -> td.td_deque = NULL;
1939 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
1941 #ifdef BUILD_TIED_TASK_STACK
1943 if ( thread_data -> td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY ) {
1944 __kmp_free_task_stack( __kmp_thread_from_gtid( gtid ), thread_data );
1946 #endif // BUILD_TIED_TASK_STACK
1960 __kmp_realloc_task_threads_data( kmp_info_t *thread, kmp_task_team_t *task_team )
1962 kmp_thread_data_t ** threads_data_p;
1963 kmp_int32 nthreads, maxthreads;
1964 int is_init_thread = FALSE;
1966 if ( TCR_4(task_team -> tt.tt_found_tasks) ) {
1971 threads_data_p = & task_team -> tt.tt_threads_data;
1972 nthreads = task_team -> tt.tt_nproc;
1973 maxthreads = task_team -> tt.tt_max_threads;
1977 __kmp_acquire_bootstrap_lock( & task_team -> tt.tt_threads_lock );
1979 if ( ! TCR_4(task_team -> tt.tt_found_tasks) ) {
1981 kmp_team_t *team = thread -> th.th_team;
1984 is_init_thread = TRUE;
1985 if ( maxthreads < nthreads ) {
1987 if ( *threads_data_p != NULL ) {
1988 kmp_thread_data_t *old_data = *threads_data_p;
1989 kmp_thread_data_t *new_data = NULL;
1991 KE_TRACE( 10, (
"__kmp_realloc_task_threads_data: T#%d reallocating "
1992 "threads data for task_team %p, new_size = %d, old_size = %d\n",
1993 __kmp_gtid_from_thread( thread ), task_team,
1994 nthreads, maxthreads ) );
1999 new_data = (kmp_thread_data_t *)
2000 __kmp_allocate( nthreads *
sizeof(kmp_thread_data_t) );
2002 memcpy( (
void *) new_data, (
void *) old_data,
2003 maxthreads *
sizeof(kmp_taskdata_t *) );
2005 #ifdef BUILD_TIED_TASK_STACK
2007 for (i = maxthreads; i < nthreads; i++) {
2008 kmp_thread_data_t *thread_data = & (*threads_data_p)[i];
2009 __kmp_init_task_stack( __kmp_gtid_from_thread( thread ), thread_data );
2011 #endif // BUILD_TIED_TASK_STACK
2013 (*threads_data_p) = new_data;
2014 __kmp_free( old_data );
2017 KE_TRACE( 10, (
"__kmp_realloc_task_threads_data: T#%d allocating "
2018 "threads data for task_team %p, size = %d\n",
2019 __kmp_gtid_from_thread( thread ), task_team, nthreads ) );
2023 *threads_data_p = (kmp_thread_data_t *)
2024 __kmp_allocate( nthreads *
sizeof(kmp_thread_data_t) );
2025 #ifdef BUILD_TIED_TASK_STACK
2027 for (i = 0; i < nthreads; i++) {
2028 kmp_thread_data_t *thread_data = & (*threads_data_p)[i];
2029 __kmp_init_task_stack( __kmp_gtid_from_thread( thread ), thread_data );
2031 #endif // BUILD_TIED_TASK_STACK
2033 task_team -> tt.tt_max_threads = nthreads;
2037 KMP_DEBUG_ASSERT( *threads_data_p != NULL );
2041 for (i = 0; i < nthreads; i++) {
2042 kmp_thread_data_t *thread_data = & (*threads_data_p)[i];
2043 thread_data -> td.td_thr = team -> t.t_threads[i];
2045 if ( thread_data -> td.td_deque_last_stolen >= nthreads) {
2049 thread_data -> td.td_deque_last_stolen = -1;
2054 TCW_SYNC_4(task_team -> tt.tt_found_tasks, TRUE);
2057 __kmp_release_bootstrap_lock( & task_team -> tt.tt_threads_lock );
2058 return is_init_thread;
2068 __kmp_free_task_threads_data( kmp_task_team_t *task_team )
2070 __kmp_acquire_bootstrap_lock( & task_team -> tt.tt_threads_lock );
2071 if ( task_team -> tt.tt_threads_data != NULL ) {
2073 for (i = 0; i < task_team->tt.tt_max_threads; i++ ) {
2074 __kmp_free_task_deque( & task_team -> tt.tt_threads_data[i] );
2076 __kmp_free( task_team -> tt.tt_threads_data );
2077 task_team -> tt.tt_threads_data = NULL;
2079 __kmp_release_bootstrap_lock( & task_team -> tt.tt_threads_lock );
2088 static kmp_task_team_t *
2089 __kmp_allocate_task_team( kmp_info_t *thread, kmp_team_t *team )
2091 kmp_task_team_t *task_team = NULL;
2094 KA_TRACE( 20, (
"__kmp_allocate_task_team: T#%d entering; team = %p\n",
2095 (thread ? __kmp_gtid_from_thread( thread ) : -1), team ) );
2097 if (TCR_PTR(__kmp_free_task_teams) != NULL) {
2099 __kmp_acquire_bootstrap_lock( &__kmp_task_team_lock );
2100 if (__kmp_free_task_teams != NULL) {
2101 task_team = __kmp_free_task_teams;
2102 TCW_PTR(__kmp_free_task_teams, task_team -> tt.tt_next);
2103 task_team -> tt.tt_next = NULL;
2105 __kmp_release_bootstrap_lock( &__kmp_task_team_lock );
2108 if (task_team == NULL) {
2109 KE_TRACE( 10, (
"__kmp_allocate_task_team: T#%d allocating "
2110 "task team for team %p\n",
2111 __kmp_gtid_from_thread( thread ), team ) );
2115 task_team = (kmp_task_team_t *) __kmp_allocate(
sizeof(kmp_task_team_t) );
2116 __kmp_init_bootstrap_lock( & task_team -> tt.tt_threads_lock );
2122 TCW_4(task_team -> tt.tt_found_tasks, FALSE);
2123 task_team -> tt.tt_nproc = nthreads = team->t.t_nproc;
2125 task_team -> tt.tt_state = 0;
2126 TCW_4( task_team -> tt.tt_unfinished_threads, nthreads );
2127 TCW_4( task_team -> tt.tt_active, TRUE );
2128 TCW_4( task_team -> tt.tt_ref_ct, nthreads - 1);
2130 KA_TRACE( 20, (
"__kmp_allocate_task_team: T#%d exiting; task_team = %p\n",
2131 (thread ? __kmp_gtid_from_thread( thread ) : -1), task_team ) );
2143 __kmp_free_task_team( kmp_info_t *thread, kmp_task_team_t *task_team )
2145 KA_TRACE( 20, (
"__kmp_free_task_team: T#%d task_team = %p\n",
2146 thread ? __kmp_gtid_from_thread( thread ) : -1, task_team ) );
2148 KMP_DEBUG_ASSERT( TCR_4(task_team -> tt.tt_ref_ct) == 0 );
2151 __kmp_acquire_bootstrap_lock( & __kmp_task_team_lock );
2153 KMP_DEBUG_ASSERT( task_team -> tt.tt_next == NULL );
2154 task_team -> tt.tt_next = __kmp_free_task_teams;
2155 TCW_4(task_team -> tt.tt_found_tasks, FALSE);
2156 TCW_PTR(__kmp_free_task_teams, task_team);
2158 __kmp_release_bootstrap_lock( & __kmp_task_team_lock );
2169 __kmp_reap_task_teams(
void )
2171 kmp_task_team_t *task_team;
2173 if ( TCR_PTR(__kmp_free_task_teams) != NULL ) {
2175 __kmp_acquire_bootstrap_lock( &__kmp_task_team_lock );
2176 while ( ( task_team = __kmp_free_task_teams ) != NULL ) {
2177 __kmp_free_task_teams = task_team -> tt.tt_next;
2178 task_team -> tt.tt_next = NULL;
2181 if ( task_team -> tt.tt_threads_data != NULL ) {
2182 __kmp_free_task_threads_data( task_team );
2184 __kmp_free( task_team );
2186 __kmp_release_bootstrap_lock( &__kmp_task_team_lock );
2198 __kmp_unref_task_team( kmp_task_team_t *task_team, kmp_info_t *thread )
2202 ref_ct = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& task_team->tt.tt_ref_ct) ) - 1;
2204 KA_TRACE( 20, (
"__kmp_unref_task_team: T#%d task_team = %p ref_ct = %d\n",
2205 __kmp_gtid_from_thread( thread ), task_team, ref_ct ) );
2208 if ( ref_ct == 0 ) {
2209 __kmp_free_task_team( thread, task_team );
2212 TCW_PTR( *((
volatile kmp_task_team_t **)(&thread->th.th_task_team)), NULL );
2222 __kmp_wait_to_unref_task_teams(
void)
2228 KMP_INIT_YIELD( spins );
2238 for (thread = (kmp_info_t *)__kmp_thread_pool;
2240 thread = thread->th.th_next_pool)
2245 if ( TCR_PTR(thread->th.th_task_team) == NULL ) {
2246 KA_TRACE( 10, (
"__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n",
2247 __kmp_gtid_from_thread( thread ) ) );
2252 if (!__kmp_is_thread_alive(thread, &exit_val)) {
2253 if (TCR_PTR(thread->th.th_task_team) != NULL) {
2254 __kmp_unref_task_team( thread->th.th_task_team, thread );
2262 KA_TRACE( 10, (
"__kmp_wait_to_unref_task_team: Waiting for T#%d to unreference task_team\n",
2263 __kmp_gtid_from_thread( thread ) ) );
2265 if ( __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ) {
2266 volatile void *sleep_loc;
2268 if ( ( sleep_loc = TCR_PTR( thread->th.th_sleep_loc) ) != NULL ) {
2269 KA_TRACE( 10, (
"__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n",
2270 __kmp_gtid_from_thread( thread ), __kmp_gtid_from_thread( thread ) ) );
2271 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc);
2282 KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc );
2283 KMP_YIELD_SPIN( spins );
2296 __kmp_task_team_setup( kmp_info_t *this_thr, kmp_team_t *team )
2298 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
2300 if ( ( team->t.t_task_team == NULL ) && ( team->t.t_nproc > 1 ) ) {
2308 team->t.t_task_team = __kmp_allocate_task_team( this_thr, team );
2309 KA_TRACE( 20, (
"__kmp_task_team_setup: Master T#%d created new "
2310 "task_team %p for team %d\n",
2311 __kmp_gtid_from_thread( this_thr ), team->t.t_task_team,
2312 ((team != NULL) ? team->t.t_id : -1)) );
2320 if ( team->t.t_task_team != NULL ) {
2323 team->t.t_task_team->tt.tt_state = 1 - this_thr->th.th_task_state;
2334 __kmp_task_team_sync( kmp_info_t *this_thr, kmp_team_t *team )
2336 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
2340 if ( this_thr->th.th_task_team != NULL ) {
2341 if ( ! TCR_SYNC_4( this_thr->th.th_task_team->tt.tt_active ) ) {
2342 KMP_DEBUG_ASSERT( ! KMP_MASTER_TID( __kmp_tid_from_gtid( __kmp_gtid_from_thread( this_thr ) ) ) );
2343 __kmp_unref_task_team( this_thr->th.th_task_team, this_thr );
2348 KMP_DEBUG_ASSERT( this_thr->th.th_task_team == team->t.t_task_team );
2356 TCW_PTR(this_thr->th.th_task_team, team->t.t_task_team);
2357 if ( this_thr->th.th_task_team != NULL ) {
2364 this_thr->th.th_task_state = 1 - this_thr->th.th_task_state;
2365 KMP_DEBUG_ASSERT( this_thr->th.th_task_state == TCR_4(team->t.t_task_team->tt.tt_state) );
2367 KA_TRACE( 20, (
"__kmp_task_team_sync: Thread T#%d task team assigned pointer (%p) from Team #%d task team\n",
2368 __kmp_gtid_from_thread( this_thr ), &this_thr->th.th_task_team,
2369 this_thr->th.th_task_team, ((team != NULL) ? (team->t.t_id) : -1) ) );
2379 __kmp_task_team_wait( kmp_info_t *this_thr,
2381 USE_ITT_BUILD_ARG(
void * itt_sync_obj)
2384 kmp_task_team_t *task_team = team->t.t_task_team;
2386 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
2387 KMP_DEBUG_ASSERT( task_team == this_thr->th.th_task_team );
2389 if ( ( task_team != NULL ) && KMP_TASKING_ENABLED( task_team, this_thr->th.th_task_state ) ) {
2390 KA_TRACE( 20, (
"__kmp_task_team_wait: Master T#%d waiting for all tasks: task_team = %p\n",
2391 __kmp_gtid_from_thread( this_thr ), task_team ) );
2399 kmp_flag_32 flag(&task_team->tt.tt_unfinished_threads, 0U);
2400 flag.wait(this_thr, TRUE
2401 USE_ITT_BUILD_ARG(itt_sync_obj));
2409 KA_TRACE( 20, (
"__kmp_task_team_wait: Master T#%d deactivating task_team %p\n",
2410 __kmp_gtid_from_thread( this_thr ), task_team ) );
2411 KMP_DEBUG_ASSERT( task_team->tt.tt_nproc > 1 );
2412 TCW_SYNC_4( task_team->tt.tt_active, FALSE );
2415 TCW_PTR(this_thr->th.th_task_team, NULL);
2416 team->t.t_task_team = NULL;
2430 __kmp_tasking_barrier( kmp_team_t *team, kmp_info_t *thread,
int gtid )
2432 volatile kmp_uint32 *spin = &team->t.t_task_team->tt.tt_unfinished_threads;
2434 KMP_DEBUG_ASSERT( __kmp_tasking_mode == tskm_extra_barrier );
2437 KMP_FSYNC_SPIN_INIT( spin, (kmp_uint32*) NULL );
2439 kmp_flag_32 spin_flag(spin, 0U);
2440 while (! spin_flag.execute_tasks(thread, gtid, TRUE, &flag
2441 USE_ITT_BUILD_ARG(NULL), 0 ) ) {
2444 KMP_FSYNC_SPIN_PREPARE( spin );
2447 if( TCR_4(__kmp_global.g.g_done) ) {
2448 if( __kmp_global.g.g_abort )
2449 __kmp_abort_thread( );
2455 KMP_FSYNC_SPIN_ACQUIRED( (
void*) spin );