41 #include "kmp_error.h"
43 #define MAX_MESSAGE 512
64 if (__kmp_ignore_mppbeg() == FALSE) {
65 __kmp_internal_begin();
67 KC_TRACE( 10, (
"__kmpc_begin: called\n" ) );
85 if (__kmp_ignore_mppend() == FALSE) {
86 KC_TRACE( 10, (
"__kmpc_end: called\n" ) );
87 KA_TRACE( 30, (
"__kmpc_end\n" ));
89 __kmp_internal_end_thread( -1 );
115 kmp_int32 gtid = __kmp_entry_gtid();
117 KC_TRACE( 10, (
"__kmpc_global_thread_num: T#%d\n", gtid ) );
138 KC_TRACE( 10, (
"__kmpc_global_num_threads: num_threads = %d\n", __kmp_nth ) );
140 return TCR_4(__kmp_nth);
152 KC_TRACE( 10, (
"__kmpc_bound_thread_num: called\n" ) );
153 return __kmp_tid_from_gtid( __kmp_entry_gtid() );
164 KC_TRACE( 10, (
"__kmpc_bound_num_threads: called\n" ) );
166 return __kmp_entry_thread() -> th.th_team -> t.t_nproc;
188 if (__kmp_par_range == 0) {
195 semi2 = strchr(semi2,
';');
199 semi2 = strchr(semi2 + 1,
';');
203 if (__kmp_par_range_filename[0]) {
204 const char *name = semi2 - 1;
205 while ((name > loc->
psource) && (*name !=
'/') && (*name !=
';')) {
208 if ((*name ==
'/') || (*name ==
';')) {
211 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
212 return __kmp_par_range < 0;
215 semi3 = strchr(semi2 + 1,
';');
216 if (__kmp_par_range_routine[0]) {
217 if ((semi3 != NULL) && (semi3 > semi2)
218 && (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
219 return __kmp_par_range < 0;
222 if (sscanf(semi3 + 1,
"%d", &line_no) == 1) {
223 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
224 return __kmp_par_range > 0;
226 return __kmp_par_range < 0;
242 return __kmp_entry_thread() -> th.th_root -> r.r_active;
257 KA_TRACE( 20, (
"__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
258 global_tid, num_threads ) );
260 __kmp_push_num_threads( loc, global_tid, num_threads );
264 __kmpc_pop_num_threads(
ident_t *loc, kmp_int32 global_tid )
266 KA_TRACE( 20, (
"__kmpc_pop_num_threads: enter\n" ) );
275 __kmpc_push_proc_bind(
ident_t *loc, kmp_int32 global_tid, kmp_int32 proc_bind )
277 KA_TRACE( 20, (
"__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n",
278 global_tid, proc_bind ) );
280 __kmp_push_proc_bind( loc, global_tid, (kmp_proc_bind_t)proc_bind );
298 int gtid = __kmp_entry_gtid();
302 va_start( ap, microtask );
304 __kmp_fork_call( loc, gtid, TRUE,
306 VOLATILE_CAST(microtask_t) microtask,
307 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
309 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM) && KMP_OS_LINUX
315 __kmp_join_call( loc, gtid );
335 KA_TRACE( 20, (
"__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
336 global_tid, num_teams, num_threads ) );
338 __kmp_push_num_teams( loc, global_tid, num_teams, num_threads );
353 int gtid = __kmp_entry_gtid();
354 kmp_info_t *this_thr = __kmp_threads[ gtid ];
356 va_start( ap, microtask );
359 this_thr->th.th_team_microtask = microtask;
360 this_thr->th.th_teams_level = this_thr->th.th_team->t.t_level;
363 if ( this_thr->th.th_set_nth_teams == 0 ) {
364 __kmp_push_num_teams( loc, gtid, 0, 0 );
366 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
367 KMP_DEBUG_ASSERT(this_thr->th.th_set_nth_teams >= 1);
369 __kmp_fork_call( loc, gtid, TRUE,
371 VOLATILE_CAST(microtask_t) __kmp_teams_master,
372 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
373 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM) && KMP_OS_LINUX
379 __kmp_join_call( loc, gtid );
380 this_thr->th.th_team_microtask = NULL;
381 this_thr->th.th_teams_level = 0;
395 __kmpc_invoke_task_func(
int gtid )
397 return __kmp_invoke_task_func( gtid );
415 kmp_info_t *this_thr;
416 kmp_team_t *serial_team;
418 KC_TRACE( 10, (
"__kmpc_serialized_parallel: called by T#%d\n", global_tid ) );
425 if( ! TCR_4( __kmp_init_parallel ) )
426 __kmp_parallel_initialize();
428 this_thr = __kmp_threads[ global_tid ];
429 serial_team = this_thr -> th.th_serial_team;
432 KMP_DEBUG_ASSERT( serial_team );
436 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
437 KMP_DEBUG_ASSERT( this_thr -> th.th_task_team == this_thr -> th.th_team -> t.t_task_team );
438 KMP_DEBUG_ASSERT( serial_team -> t.t_task_team == NULL );
439 KA_TRACE( 20, (
"__kmpc_serialized_parallel: T#%d pushing task_team %p / team %p, new task_team = NULL\n",
440 global_tid, this_thr -> th.th_task_team, this_thr -> th.th_team ) );
441 this_thr -> th.th_task_team = NULL;
443 #endif // OMP_30_ENABLED
446 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
447 if ( this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false ) {
448 proc_bind = proc_bind_false;
450 else if ( proc_bind == proc_bind_default ) {
455 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
460 this_thr->th.th_set_proc_bind = proc_bind_default;
463 if( this_thr -> th.th_team != serial_team ) {
466 int level = this_thr->th.th_team->t.t_level;
468 if( serial_team -> t.t_serialized ) {
471 kmp_team_t *new_team;
472 int tid = this_thr->th.th_info.ds.ds_tid;
474 __kmp_acquire_bootstrap_lock( &__kmp_forkjoin_lock );
476 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1,
481 & this_thr->th.th_current_task->td_icvs,
483 this_thr->th.th_team->t.t_set_nproc[tid],
484 this_thr->th.th_team->t.t_set_dynamic[tid],
485 this_thr->th.th_team->t.t_set_nested[tid],
486 this_thr->th.th_team->t.t_set_blocktime[tid],
487 this_thr->th.th_team->t.t_set_bt_intervals[tid],
488 this_thr->th.th_team->t.t_set_bt_set[tid],
491 __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
492 KMP_ASSERT( new_team );
495 new_team -> t.t_threads[0] = this_thr;
496 new_team -> t.t_parent = this_thr -> th.th_team;
497 serial_team = new_team;
498 this_thr -> th.th_serial_team = serial_team;
500 KF_TRACE( 10, (
"__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
501 global_tid, serial_team ) );
508 KF_TRACE( 10, (
"__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
509 global_tid, serial_team ) );
513 KMP_DEBUG_ASSERT( serial_team->t.t_threads );
514 KMP_DEBUG_ASSERT( serial_team->t.t_threads[0] == this_thr );
515 KMP_DEBUG_ASSERT( this_thr->th.th_team != serial_team );
516 serial_team -> t.t_ident = loc;
517 serial_team -> t.t_serialized = 1;
518 serial_team -> t.t_nproc = 1;
519 serial_team -> t.t_parent = this_thr->th.th_team;
521 serial_team -> t.t_sched = this_thr->th.th_team->t.t_sched;
522 #endif // OMP_30_ENABLED
523 this_thr -> th.th_team = serial_team;
524 serial_team -> t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
527 KF_TRACE( 10, (
"__kmpc_serialized_parallel: T#d curtask=%p\n",
528 global_tid, this_thr->th.th_current_task ) );
529 KMP_ASSERT( this_thr->th.th_current_task->td_flags.executing == 1 );
530 this_thr->th.th_current_task->td_flags.executing = 0;
532 __kmp_push_current_task_to_thread( this_thr, serial_team, 0 );
537 & this_thr->th.th_current_task->td_icvs,
538 & this_thr->th.th_current_task->td_parent->td_icvs );
541 if ( __kmp_nested_nth.used && ( level + 1 < __kmp_nested_nth.used ) ) {
542 this_thr->th.th_current_task->td_icvs.nproc = __kmp_nested_nth.nth[ level + 1 ];
546 if ( __kmp_nested_proc_bind.used && ( level + 1 < __kmp_nested_proc_bind.used ) ) {
547 this_thr->th.th_current_task->td_icvs.proc_bind
548 = __kmp_nested_proc_bind.bind_types[ level + 1 ];
553 serial_team -> t.t_set_nproc[0] = serial_team->t.t_parent->
554 t.t_set_nproc[serial_team->
556 serial_team -> t.t_set_dynamic[0] = serial_team->t.t_parent->
557 t.t_set_dynamic[serial_team->
559 serial_team -> t.t_set_nested[0] = serial_team->t.t_parent->
560 t.t_set_nested[serial_team->
562 serial_team -> t.t_set_blocktime[0] = serial_team->t.t_parent->
563 t.t_set_blocktime[serial_team->
565 serial_team -> t.t_set_bt_intervals[0] = serial_team->t.t_parent->
566 t.t_set_bt_intervals[serial_team->
568 serial_team -> t.t_set_bt_set[0] = serial_team->t.t_parent->
569 t.t_set_bt_set[serial_team->
571 #endif // OMP_30_ENABLED
572 this_thr -> th.th_info.ds.ds_tid = 0;
575 this_thr -> th.th_team_nproc = 1;
576 this_thr -> th.th_team_master = this_thr;
577 this_thr -> th.th_team_serialized = 1;
580 serial_team -> t.t_level = serial_team -> t.t_parent -> t.t_level + 1;
581 serial_team -> t.t_active_level = serial_team -> t.t_parent -> t.t_active_level;
582 #endif // OMP_30_ENABLED
584 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
585 if ( __kmp_inherit_fp_control ) {
586 __kmp_store_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word );
587 __kmp_store_mxcsr( &serial_team->t.t_mxcsr );
588 serial_team->t.t_mxcsr &= KMP_X86_MXCSR_MASK;
589 serial_team->t.t_fp_control_saved = TRUE;
591 serial_team->t.t_fp_control_saved = FALSE;
595 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
596 if ( !serial_team->t.t_dispatch->th_disp_buffer ) {
597 serial_team->t.t_dispatch->th_disp_buffer = (dispatch_private_info_t *)
598 __kmp_allocate(
sizeof( dispatch_private_info_t ) );
600 this_thr -> th.th_dispatch = serial_team->t.t_dispatch;
607 KMP_DEBUG_ASSERT( this_thr->th.th_team == serial_team );
608 KMP_DEBUG_ASSERT( serial_team -> t.t_threads );
609 KMP_DEBUG_ASSERT( serial_team -> t.t_threads[0] == this_thr );
610 ++ serial_team -> t.t_serialized;
611 this_thr -> th.th_team_serialized = serial_team -> t.t_serialized;
615 int level = this_thr->th.th_team->t.t_level;
617 if ( __kmp_nested_nth.used && ( level + 1 < __kmp_nested_nth.used ) ) {
618 this_thr->th.th_current_task->td_icvs.nproc = __kmp_nested_nth.nth[ level + 1 ];
620 serial_team -> t.t_level++;
621 KF_TRACE( 10, (
"__kmpc_serialized_parallel: T#%d increasing nesting level of serial team %p to %d\n",
622 global_tid, serial_team, serial_team -> t.t_level ) );
624 KF_TRACE( 10, (
"__kmpc_serialized_parallel: T#%d reusing team %p for nested serialized parallel region\n",
625 global_tid, serial_team ) );
626 #endif // OMP_30_ENABLED
629 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
631 dispatch_private_info_t * disp_buffer = (dispatch_private_info_t *)
632 __kmp_allocate(
sizeof( dispatch_private_info_t ) );
633 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
634 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
636 this_thr -> th.th_dispatch = serial_team->t.t_dispatch;
641 if ( __kmp_env_consistency_check )
642 __kmp_push_parallel( global_tid, NULL );
645 #if USE_ITT_BUILD && OMP_30_ENABLED
647 if ( ( __itt_frame_begin_v3_ptr && __kmp_forkjoin_frames && ! __kmp_forkjoin_frames_mode ) || KMP_ITT_DEBUG )
649 __kmp_itt_region_forking( global_tid, 1 );
651 if( ( __kmp_forkjoin_frames_mode == 1 || __kmp_forkjoin_frames_mode == 3 ) && __itt_frame_submit_v3_ptr && __itt_get_timestamp_ptr )
654 if( this_thr->th.th_team->t.t_level == 1 ) {
655 this_thr->th.th_frame_time_serialized = __itt_get_timestamp();
673 kmp_internal_control_t *top;
674 kmp_info_t *this_thr;
675 kmp_team_t *serial_team;
677 KC_TRACE( 10, (
"__kmpc_end_serialized_parallel: called by T#%d\n", global_tid ) );
685 if( ! TCR_4( __kmp_init_parallel ) )
686 __kmp_parallel_initialize();
688 this_thr = __kmp_threads[ global_tid ];
689 serial_team = this_thr->th.th_serial_team;
692 KMP_DEBUG_ASSERT( serial_team );
693 KMP_ASSERT( serial_team -> t.t_serialized );
694 KMP_DEBUG_ASSERT( this_thr -> th.th_team == serial_team );
695 KMP_DEBUG_ASSERT( serial_team != this_thr->th.th_root->r.r_root_team );
696 KMP_DEBUG_ASSERT( serial_team -> t.t_threads );
697 KMP_DEBUG_ASSERT( serial_team -> t.t_threads[0] == this_thr );
700 top = serial_team -> t.t_control_stack_top;
701 if ( top && top -> serial_nesting_level == serial_team -> t.t_serialized ) {
704 &serial_team -> t.t_threads[0] -> th.th_current_task -> td_icvs,
707 serial_team -> t.t_set_nproc[0] = top -> nproc;
708 serial_team -> t.t_set_dynamic[0] = top -> dynamic;
709 serial_team -> t.t_set_nested[0] = top -> nested;
710 serial_team -> t.t_set_blocktime[0] = top -> blocktime;
711 serial_team -> t.t_set_bt_intervals[0] = top -> bt_intervals;
712 serial_team -> t.t_set_bt_set[0] = top -> bt_set;
713 #endif // OMP_30_ENABLED
714 serial_team -> t.t_control_stack_top = top -> next;
720 serial_team -> t.t_level--;
721 #endif // OMP_30_ENABLED
724 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
726 dispatch_private_info_t * disp_buffer = serial_team->t.t_dispatch->th_disp_buffer;
727 serial_team->t.t_dispatch->th_disp_buffer =
728 serial_team->t.t_dispatch->th_disp_buffer->next;
729 __kmp_free( disp_buffer );
732 -- serial_team -> t.t_serialized;
733 if ( serial_team -> t.t_serialized == 0 ) {
737 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
738 if ( __kmp_inherit_fp_control && serial_team->t.t_fp_control_saved ) {
739 __kmp_clear_x87_fpu_status_word();
740 __kmp_load_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word );
741 __kmp_load_mxcsr( &serial_team->t.t_mxcsr );
745 this_thr -> th.th_team = serial_team -> t.t_parent;
746 this_thr -> th.th_info.ds.ds_tid = serial_team -> t.t_master_tid;
749 this_thr -> th.th_team_nproc = serial_team -> t.t_parent -> t.t_nproc;
750 this_thr -> th.th_team_master = serial_team -> t.t_parent -> t.t_threads[0];
751 this_thr -> th.th_team_serialized = this_thr -> th.th_team -> t.t_serialized;
754 this_thr -> th.th_dispatch = & this_thr -> th.th_team ->
755 t.t_dispatch[ serial_team -> t.t_master_tid ];
758 __kmp_pop_current_task_from_thread( this_thr );
760 KMP_ASSERT( this_thr -> th.th_current_task -> td_flags.executing == 0 );
761 this_thr -> th.th_current_task -> td_flags.executing = 1;
763 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
768 if ( ( this_thr -> th.th_task_team = this_thr -> th.th_team -> t.t_task_team ) != NULL ) {
769 this_thr -> th.th_task_state = this_thr -> th.th_task_team -> tt.tt_state;
771 KA_TRACE( 20, (
"__kmpc_end_serialized_parallel: T#%d restoring task_team %p / team %p\n",
772 global_tid, this_thr -> th.th_task_team, this_thr -> th.th_team ) );
774 #endif // OMP_30_ENABLED
780 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
781 KA_TRACE( 20, (
"__kmpc_end_serialized_parallel: T#%d decreasing nesting depth of serial team %p to %d\n",
782 global_tid, serial_team, serial_team -> t.t_serialized ) );
784 #endif // OMP_30_ENABLED
789 #if USE_ITT_BUILD && OMP_30_ENABLED
791 if ( ( __itt_frame_end_v3_ptr && __kmp_forkjoin_frames && ! __kmp_forkjoin_frames_mode ) || KMP_ITT_DEBUG )
793 this_thr->th.th_ident = loc;
794 __kmp_itt_region_joined( global_tid, 1 );
796 if( ( __kmp_forkjoin_frames_mode == 1 || __kmp_forkjoin_frames_mode == 3 ) && __itt_frame_submit_v3_ptr ) {
797 if( this_thr->th.th_team->t.t_level == 0 ) {
798 __kmp_itt_frame_submit( global_tid, this_thr->th.th_frame_time_serialized, __itt_timestamp_none, 0, loc );
803 if ( __kmp_env_consistency_check )
804 __kmp_pop_parallel( global_tid, NULL );
822 KC_TRACE( 10, (
"__kmpc_flush: called\n" ) );
831 #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
844 if ( ! __kmp_cpuinfo.initialized ) {
845 __kmp_query_cpuid( & __kmp_cpuinfo );
847 if ( ! __kmp_cpuinfo.sse2 ) {
853 __sync_synchronize();
854 #endif // KMP_COMPILER_ICC
860 #error Unknown or unsupported architecture
862 #endif // OMP_30_ENABLED
880 int explicit_barrier_flag;
881 KC_TRACE( 10, (
"__kmpc_barrier: called T#%d\n", global_tid ) );
883 if (! TCR_4(__kmp_init_parallel))
884 __kmp_parallel_initialize();
886 if ( __kmp_env_consistency_check ) {
888 KMP_WARNING( ConstructIdentInvalid );
891 __kmp_check_barrier( global_tid, ct_barrier, loc );
894 __kmp_threads[ global_tid ]->th.th_ident = loc;
902 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
917 KC_TRACE( 10, (
"__kmpc_master: called T#%d\n", global_tid ) );
919 if( ! TCR_4( __kmp_init_parallel ) )
920 __kmp_parallel_initialize();
922 if( KMP_MASTER_GTID( global_tid ))
925 if ( __kmp_env_consistency_check ) {
927 __kmp_push_sync( global_tid, ct_master, loc, NULL );
929 __kmp_check_sync( global_tid, ct_master, loc, NULL );
946 KC_TRACE( 10, (
"__kmpc_end_master: called T#%d\n", global_tid ) );
948 KMP_DEBUG_ASSERT( KMP_MASTER_GTID( global_tid ));
950 if ( __kmp_env_consistency_check ) {
952 KMP_WARNING( ThreadIdentInvalid );
954 if( KMP_MASTER_GTID( global_tid ))
955 __kmp_pop_sync( global_tid, ct_master, loc );
971 KMP_DEBUG_ASSERT( __kmp_init_serial );
973 KC_TRACE( 10, (
"__kmpc_ordered: called T#%d\n", gtid ));
975 if (! TCR_4(__kmp_init_parallel))
976 __kmp_parallel_initialize();
979 __kmp_itt_ordered_prep( gtid );
983 th = __kmp_threads[ gtid ];
985 if ( th -> th.th_dispatch -> th_deo_fcn != 0 )
986 (*th->th.th_dispatch->th_deo_fcn)( & gtid, & cid, loc );
988 __kmp_parallel_deo( & gtid, & cid, loc );
991 __kmp_itt_ordered_start( gtid );
1008 KC_TRACE( 10, (
"__kmpc_end_ordered: called T#%d\n", gtid ) );
1011 __kmp_itt_ordered_end( gtid );
1015 th = __kmp_threads[ gtid ];
1017 if ( th -> th.th_dispatch -> th_dxo_fcn != 0 )
1018 (*th->th.th_dispatch->th_dxo_fcn)( & gtid, & cid, loc );
1020 __kmp_parallel_dxo( & gtid, & cid, loc );
1024 __kmp_static_yield(
int arg ) {
1028 static kmp_user_lock_p
1029 __kmp_get_critical_section_ptr( kmp_critical_name * crit,
ident_t const * loc, kmp_int32 gtid )
1031 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1037 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
1039 if ( lck == NULL ) {
1044 lck = __kmp_user_lock_allocate( &idx, gtid, kmp_lf_critical_section );
1045 __kmp_init_user_lock_with_checks( lck );
1046 __kmp_set_user_lock_location( lck, loc );
1048 __kmp_itt_critical_creating( lck );
1061 int status = KMP_COMPARE_AND_STORE_PTR( lck_pp, 0, lck );
1063 if ( status == 0 ) {
1066 __kmp_itt_critical_destroyed( lck );
1070 __kmp_destroy_user_lock_with_checks( lck );
1071 __kmp_user_lock_free( &idx, gtid, lck );
1072 lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
1073 KMP_DEBUG_ASSERT( lck != NULL );
1092 kmp_user_lock_p lck;
1094 KC_TRACE( 10, (
"__kmpc_critical: called T#%d\n", global_tid ) );
1098 KMP_CHECK_USER_LOCK_INIT();
1100 if ( ( __kmp_user_lock_kind == lk_tas )
1101 && (
sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1102 lck = (kmp_user_lock_p)crit;
1104 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1105 else if ( ( __kmp_user_lock_kind == lk_futex )
1106 && (
sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1107 lck = (kmp_user_lock_p)crit;
1111 lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
1114 if ( __kmp_env_consistency_check )
1115 __kmp_push_sync( global_tid, ct_critical, loc, lck );
1124 __kmp_itt_critical_acquiring( lck );
1128 __kmp_acquire_user_lock_with_checks( lck, global_tid );
1131 __kmp_itt_critical_acquired( lck );
1134 KA_TRACE( 15, (
"__kmpc_critical: done T#%d\n", global_tid ));
1149 kmp_user_lock_p lck;
1151 KC_TRACE( 10, (
"__kmpc_end_critical: called T#%d\n", global_tid ));
1153 if ( ( __kmp_user_lock_kind == lk_tas )
1154 && (
sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1155 lck = (kmp_user_lock_p)crit;
1157 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1158 else if ( ( __kmp_user_lock_kind == lk_futex )
1159 && (
sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1160 lck = (kmp_user_lock_p)crit;
1164 lck = (kmp_user_lock_p) TCR_PTR(*((kmp_user_lock_p *)crit));
1167 KMP_ASSERT(lck != NULL);
1169 if ( __kmp_env_consistency_check )
1170 __kmp_pop_sync( global_tid, ct_critical, loc );
1173 __kmp_itt_critical_releasing( lck );
1177 __kmp_release_user_lock_with_checks( lck, global_tid );
1179 KA_TRACE( 15, (
"__kmpc_end_critical: done T#%d\n", global_tid ));
1195 KC_TRACE( 10, (
"__kmpc_barrier_master: called T#%d\n", global_tid ) );
1197 if (! TCR_4(__kmp_init_parallel))
1198 __kmp_parallel_initialize();
1200 if ( __kmp_env_consistency_check )
1201 __kmp_check_barrier( global_tid, ct_barrier, loc );
1203 status = __kmp_barrier( bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL );
1205 return (status != 0) ? 0 : 1;
1220 KC_TRACE( 10, (
"__kmpc_end_barrier_master: called T#%d\n", global_tid ));
1222 __kmp_end_split_barrier ( bs_plain_barrier, global_tid );
1240 KC_TRACE( 10, (
"__kmpc_barrier_master_nowait: called T#%d\n", global_tid ));
1242 if (! TCR_4(__kmp_init_parallel))
1243 __kmp_parallel_initialize();
1245 if ( __kmp_env_consistency_check ) {
1247 KMP_WARNING( ConstructIdentInvalid );
1249 __kmp_check_barrier( global_tid, ct_barrier, loc );
1252 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
1256 if ( __kmp_env_consistency_check ) {
1260 if ( global_tid < 0 ) {
1261 KMP_WARNING( ThreadIdentInvalid );
1267 __kmp_pop_sync( global_tid, ct_master, loc );
1289 kmp_int32 rc = __kmp_enter_single( global_tid, loc, TRUE );
1305 __kmp_exit_single( global_tid );
1318 KE_TRACE( 10, (
"__kmpc_for_static_fini called T#%d\n", global_tid));
1320 if ( __kmp_env_consistency_check )
1321 __kmp_pop_workshare( global_tid, ct_pdo, loc );
1330 ompc_set_num_threads(
int arg )
1333 __kmp_set_num_threads( arg, __kmp_entry_gtid() );
1337 ompc_set_dynamic(
int flag )
1342 thread = __kmp_entry_thread();
1344 __kmp_save_internal_controls( thread );
1346 set__dynamic( thread, flag ? TRUE : FALSE );
1350 ompc_set_nested(
int flag )
1355 thread = __kmp_entry_thread();
1357 __kmp_save_internal_controls( thread );
1359 set__nested( thread, flag ? TRUE : FALSE );
1365 ompc_set_max_active_levels(
int max_active_levels )
1371 __kmp_set_max_active_levels( __kmp_entry_gtid(), max_active_levels );
1375 ompc_set_schedule( omp_sched_t kind,
int modifier )
1378 __kmp_set_schedule( __kmp_entry_gtid(), ( kmp_sched_t ) kind, modifier );
1382 ompc_get_ancestor_thread_num(
int level )
1384 return __kmp_get_ancestor_thread_num( __kmp_entry_gtid(), level );
1388 ompc_get_team_size(
int level )
1390 return __kmp_get_team_size( __kmp_entry_gtid(), level );
1393 #endif // OMP_30_ENABLED
1396 kmpc_set_stacksize(
int arg )
1399 __kmp_aux_set_stacksize( arg );
1403 kmpc_set_stacksize_s(
size_t arg )
1406 __kmp_aux_set_stacksize( arg );
1410 kmpc_set_blocktime(
int arg )
1415 gtid = __kmp_entry_gtid();
1416 tid = __kmp_tid_from_gtid(gtid);
1417 thread = __kmp_thread_from_gtid(gtid);
1419 __kmp_aux_set_blocktime( arg, thread, tid );
1423 kmpc_set_library(
int arg )
1426 __kmp_user_set_library( (
enum library_type)arg );
1430 kmpc_set_defaults(
char const * str )
1433 __kmp_aux_set_defaults( str, strlen( str ) );
1436 #ifdef OMP_30_ENABLED
1439 kmpc_set_affinity_mask_proc(
int proc,
void **mask )
1441 #if defined(KMP_STUB) || !(KMP_OS_WINDOWS || KMP_OS_LINUX)
1444 if ( ! TCR_4(__kmp_init_middle) ) {
1445 __kmp_middle_initialize();
1447 return __kmp_aux_set_affinity_mask_proc( proc, mask );
1452 kmpc_unset_affinity_mask_proc(
int proc,
void **mask )
1454 #if defined(KMP_STUB) || !(KMP_OS_WINDOWS || KMP_OS_LINUX)
1457 if ( ! TCR_4(__kmp_init_middle) ) {
1458 __kmp_middle_initialize();
1460 return __kmp_aux_unset_affinity_mask_proc( proc, mask );
1465 kmpc_get_affinity_mask_proc(
int proc,
void **mask )
1467 #if defined(KMP_STUB) || !(KMP_OS_WINDOWS || KMP_OS_LINUX)
1470 if ( ! TCR_4(__kmp_init_middle) ) {
1471 __kmp_middle_initialize();
1473 return __kmp_aux_get_affinity_mask_proc( proc, mask );
1525 KC_TRACE( 10, (
"__kmpc_copyprivate: called T#%d\n", gtid ));
1529 data_ptr = & __kmp_team_from_gtid( gtid )->t.t_copypriv_data;
1531 if ( __kmp_env_consistency_check ) {
1533 KMP_WARNING( ConstructIdentInvalid );
1539 if (didit) *data_ptr = cpy_data;
1542 __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
1544 if (! didit) (*cpy_func)( cpy_data, *data_ptr );
1549 __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
1554 #define INIT_LOCK __kmp_init_user_lock_with_checks
1555 #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks
1556 #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks
1557 #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed
1558 #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks
1559 #define ACQUIRE_NESTED_LOCK_TIMED __kmp_acquire_nested_user_lock_with_checks_timed
1560 #define RELEASE_LOCK __kmp_release_user_lock_with_checks
1561 #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks
1562 #define TEST_LOCK __kmp_test_user_lock_with_checks
1563 #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks
1564 #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks
1565 #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks
1575 __kmpc_init_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1576 static char const *
const func =
"omp_init_lock";
1577 kmp_user_lock_p lck;
1578 KMP_DEBUG_ASSERT( __kmp_init_serial );
1580 if ( __kmp_env_consistency_check ) {
1581 if ( user_lock == NULL ) {
1582 KMP_FATAL( LockIsUninitialized, func );
1586 KMP_CHECK_USER_LOCK_INIT();
1588 if ( ( __kmp_user_lock_kind == lk_tas )
1589 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1590 lck = (kmp_user_lock_p)user_lock;
1592 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1593 else if ( ( __kmp_user_lock_kind == lk_futex )
1594 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1595 lck = (kmp_user_lock_p)user_lock;
1599 lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
1602 __kmp_set_user_lock_location( lck, loc );
1605 __kmp_itt_lock_creating( lck );
1611 __kmpc_init_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1612 static char const *
const func =
"omp_init_nest_lock";
1613 kmp_user_lock_p lck;
1614 KMP_DEBUG_ASSERT( __kmp_init_serial );
1616 if ( __kmp_env_consistency_check ) {
1617 if ( user_lock == NULL ) {
1618 KMP_FATAL( LockIsUninitialized, func );
1622 KMP_CHECK_USER_LOCK_INIT();
1624 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1625 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1626 lck = (kmp_user_lock_p)user_lock;
1628 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1629 else if ( ( __kmp_user_lock_kind == lk_futex )
1630 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1631 <= OMP_NEST_LOCK_T_SIZE ) ) {
1632 lck = (kmp_user_lock_p)user_lock;
1636 lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
1639 INIT_NESTED_LOCK( lck );
1640 __kmp_set_user_lock_location( lck, loc );
1643 __kmp_itt_lock_creating( lck );
1648 __kmpc_destroy_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1650 kmp_user_lock_p lck;
1652 if ( ( __kmp_user_lock_kind == lk_tas )
1653 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1654 lck = (kmp_user_lock_p)user_lock;
1656 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1657 else if ( ( __kmp_user_lock_kind == lk_futex )
1658 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1659 lck = (kmp_user_lock_p)user_lock;
1663 lck = __kmp_lookup_user_lock( user_lock,
"omp_destroy_lock" );
1667 __kmp_itt_lock_destroyed( lck );
1669 DESTROY_LOCK( lck );
1671 if ( ( __kmp_user_lock_kind == lk_tas )
1672 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1675 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1676 else if ( ( __kmp_user_lock_kind == lk_futex )
1677 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1682 __kmp_user_lock_free( user_lock, gtid, lck );
1688 __kmpc_destroy_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1690 kmp_user_lock_p lck;
1692 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1693 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1694 lck = (kmp_user_lock_p)user_lock;
1696 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1697 else if ( ( __kmp_user_lock_kind == lk_futex )
1698 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1699 <= OMP_NEST_LOCK_T_SIZE ) ) {
1700 lck = (kmp_user_lock_p)user_lock;
1704 lck = __kmp_lookup_user_lock( user_lock,
"omp_destroy_nest_lock" );
1708 __kmp_itt_lock_destroyed( lck );
1711 DESTROY_NESTED_LOCK( lck );
1713 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1714 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1717 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1718 else if ( ( __kmp_user_lock_kind == lk_futex )
1719 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1720 <= OMP_NEST_LOCK_T_SIZE ) ) {
1725 __kmp_user_lock_free( user_lock, gtid, lck );
1730 __kmpc_set_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1731 kmp_user_lock_p lck;
1733 if ( ( __kmp_user_lock_kind == lk_tas )
1734 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1735 lck = (kmp_user_lock_p)user_lock;
1737 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1738 else if ( ( __kmp_user_lock_kind == lk_futex )
1739 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1740 lck = (kmp_user_lock_p)user_lock;
1744 lck = __kmp_lookup_user_lock( user_lock,
"omp_set_lock" );
1748 __kmp_itt_lock_acquiring( lck );
1751 ACQUIRE_LOCK( lck, gtid );
1754 __kmp_itt_lock_acquired( lck );
1760 __kmpc_set_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1761 kmp_user_lock_p lck;
1763 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1764 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1765 lck = (kmp_user_lock_p)user_lock;
1767 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1768 else if ( ( __kmp_user_lock_kind == lk_futex )
1769 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1770 <= OMP_NEST_LOCK_T_SIZE ) ) {
1771 lck = (kmp_user_lock_p)user_lock;
1775 lck = __kmp_lookup_user_lock( user_lock,
"omp_set_nest_lock" );
1779 __kmp_itt_lock_acquiring( lck );
1782 ACQUIRE_NESTED_LOCK( lck, gtid );
1785 __kmp_itt_lock_acquired( lck );
1790 __kmpc_unset_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
1792 kmp_user_lock_p lck;
1797 if ( ( __kmp_user_lock_kind == lk_tas )
1798 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1799 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1802 __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
1804 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
1808 lck = (kmp_user_lock_p)user_lock;
1811 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1812 else if ( ( __kmp_user_lock_kind == lk_futex )
1813 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1814 lck = (kmp_user_lock_p)user_lock;
1818 lck = __kmp_lookup_user_lock( user_lock,
"omp_unset_lock" );
1822 __kmp_itt_lock_releasing( lck );
1825 RELEASE_LOCK( lck, gtid );
1830 __kmpc_unset_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
1832 kmp_user_lock_p lck;
1836 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1837 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1838 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1840 kmp_tas_lock_t *tl = (kmp_tas_lock_t*)user_lock;
1842 __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
1844 if ( --(tl->lk.depth_locked) == 0 ) {
1845 TCW_4(tl->lk.poll, 0);
1850 lck = (kmp_user_lock_p)user_lock;
1853 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1854 else if ( ( __kmp_user_lock_kind == lk_futex )
1855 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1856 <= OMP_NEST_LOCK_T_SIZE ) ) {
1857 lck = (kmp_user_lock_p)user_lock;
1861 lck = __kmp_lookup_user_lock( user_lock,
"omp_unset_nest_lock" );
1865 __kmp_itt_lock_releasing( lck );
1868 RELEASE_NESTED_LOCK( lck, gtid );
1873 __kmpc_test_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
1875 kmp_user_lock_p lck;
1878 if ( ( __kmp_user_lock_kind == lk_tas )
1879 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1880 lck = (kmp_user_lock_p)user_lock;
1882 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1883 else if ( ( __kmp_user_lock_kind == lk_futex )
1884 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1885 lck = (kmp_user_lock_p)user_lock;
1889 lck = __kmp_lookup_user_lock( user_lock,
"omp_test_lock" );
1893 __kmp_itt_lock_acquiring( lck );
1896 rc = TEST_LOCK( lck, gtid );
1899 __kmp_itt_lock_acquired( lck );
1901 __kmp_itt_lock_cancelled( lck );
1904 return ( rc ? FTN_TRUE : FTN_FALSE );
1911 __kmpc_test_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
1913 kmp_user_lock_p lck;
1916 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1917 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1918 lck = (kmp_user_lock_p)user_lock;
1920 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
1921 else if ( ( __kmp_user_lock_kind == lk_futex )
1922 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1923 <= OMP_NEST_LOCK_T_SIZE ) ) {
1924 lck = (kmp_user_lock_p)user_lock;
1928 lck = __kmp_lookup_user_lock( user_lock,
"omp_test_nest_lock" );
1932 __kmp_itt_lock_acquiring( lck );
1935 rc = TEST_NESTED_LOCK( lck, gtid );
1938 __kmp_itt_lock_acquired( lck );
1940 __kmp_itt_lock_cancelled( lck );
1958 #define __KMP_SET_REDUCTION_METHOD(gtid,rmethod) \
1959 ( ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method ) = ( rmethod ) )
1961 #define __KMP_GET_REDUCTION_METHOD(gtid) \
1962 ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method )
1968 static __forceinline
void
1969 __kmp_enter_critical_section_reduce_block(
ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
1975 kmp_user_lock_p lck;
1980 if ( __kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE ) {
1981 lck = (kmp_user_lock_p)crit;
1984 lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
1986 KMP_DEBUG_ASSERT( lck != NULL );
1988 if ( __kmp_env_consistency_check )
1989 __kmp_push_sync( global_tid, ct_critical, loc, lck );
1991 __kmp_acquire_user_lock_with_checks( lck, global_tid );
1995 static __forceinline
void
1996 __kmp_end_critical_section_reduce_block(
ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
1998 kmp_user_lock_p lck;
2002 if ( __kmp_base_user_lock_size > 32 ) {
2003 lck = *( (kmp_user_lock_p *) crit );
2004 KMP_ASSERT( lck != NULL );
2006 lck = (kmp_user_lock_p) crit;
2009 if ( __kmp_env_consistency_check )
2010 __kmp_pop_sync( global_tid, ct_critical, loc );
2012 __kmp_release_user_lock_with_checks( lck, global_tid );
2033 ident_t *loc, kmp_int32 global_tid,
2034 kmp_int32 num_vars,
size_t reduce_size,
void *reduce_data,
void (*reduce_func)(
void *lhs_data,
void *rhs_data),
2035 kmp_critical_name *lck ) {
2038 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2040 KA_TRACE( 10, (
"__kmpc_reduce_nowait() enter: called T#%d\n", global_tid ) );
2047 if( ! TCR_4( __kmp_init_parallel ) )
2048 __kmp_parallel_initialize();
2051 if ( __kmp_env_consistency_check )
2052 __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
2065 packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
2066 __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
2068 if( packed_reduction_method == critical_reduce_block ) {
2070 __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
2073 }
else if( packed_reduction_method == empty_reduce_block ) {
2078 }
else if( packed_reduction_method == atomic_reduce_block ) {
2085 if ( __kmp_env_consistency_check )
2086 __kmp_pop_sync( global_tid, ct_reduce, loc );
2088 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2100 retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, FALSE, reduce_size, reduce_data, reduce_func );
2101 retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
2105 if ( __kmp_env_consistency_check ) {
2107 __kmp_pop_sync( global_tid, ct_reduce, loc );
2118 KA_TRACE( 10, (
"__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
2134 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2136 KA_TRACE( 10, (
"__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid ) );
2138 packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
2140 if( packed_reduction_method == critical_reduce_block ) {
2142 __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
2144 }
else if( packed_reduction_method == empty_reduce_block ) {
2148 }
else if( packed_reduction_method == atomic_reduce_block ) {
2155 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2166 if ( __kmp_env_consistency_check )
2167 __kmp_pop_sync( global_tid, ct_reduce, loc );
2169 KA_TRACE( 10, (
"__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
2191 ident_t *loc, kmp_int32 global_tid,
2192 kmp_int32 num_vars,
size_t reduce_size,
void *reduce_data,
2193 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
2194 kmp_critical_name *lck )
2197 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2199 KA_TRACE( 10, (
"__kmpc_reduce() enter: called T#%d\n", global_tid ) );
2206 if( ! TCR_4( __kmp_init_parallel ) )
2207 __kmp_parallel_initialize();
2210 if ( __kmp_env_consistency_check )
2211 __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
2215 packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
2216 __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
2218 if( packed_reduction_method == critical_reduce_block ) {
2220 __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
2223 }
else if( packed_reduction_method == empty_reduce_block ) {
2228 }
else if( packed_reduction_method == atomic_reduce_block ) {
2232 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2237 retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, TRUE, reduce_size, reduce_data, reduce_func );
2238 retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
2242 if ( __kmp_env_consistency_check ) {
2244 __kmp_pop_sync( global_tid, ct_reduce, loc );
2255 KA_TRACE( 10, (
"__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
2272 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2274 KA_TRACE( 10, (
"__kmpc_end_reduce() enter: called T#%d\n", global_tid ) );
2276 packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
2281 if( packed_reduction_method == critical_reduce_block ) {
2283 __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
2286 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2288 }
else if( packed_reduction_method == empty_reduce_block ) {
2293 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2295 }
else if( packed_reduction_method == atomic_reduce_block ) {
2298 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2300 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2303 __kmp_end_split_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid );
2312 if ( __kmp_env_consistency_check )
2313 __kmp_pop_sync( global_tid, ct_reduce, loc );
2315 KA_TRACE( 10, (
"__kmpc_end_reduce() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
2320 #undef __KMP_GET_REDUCTION_METHOD
2321 #undef __KMP_SET_REDUCTION_METHOD
2326 __kmpc_get_taskid() {
2331 kmp_info_t * thread;
2333 gtid = __kmp_get_gtid();
2337 thread = __kmp_thread_from_gtid( gtid );
2338 return thread->th.th_current_task->td_task_id;
2350 __kmpc_get_parent_taskid() {
2355 kmp_info_t * thread;
2356 kmp_taskdata_t * parent_task;
2358 gtid = __kmp_get_gtid();
2362 thread = __kmp_thread_from_gtid( gtid );
2363 parent_task = thread->th.th_current_task->td_parent;
2364 return ( parent_task == NULL ? 0 : parent_task->td_task_id );
2374 void __kmpc_place_threads(
int nC,
int nT,
int nO)
2377 if ( ! __kmp_init_serial ) {
2378 __kmp_serial_initialize();
2380 __kmp_place_num_cores = nC;
2381 __kmp_place_num_threads_per_core = nT;
2382 __kmp_place_core_offset = nO;
kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end(ident_t *loc)
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
#define KMP_IDENT_AUTOPAR
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
kmp_int32 __kmpc_in_parallel(ident_t *loc)
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_flush(ident_t *loc,...)
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)