35 #ifndef KMP_WAIT_RELEASE_H
36 #define KMP_WAIT_RELEASE_H
76 volatile P *
get() {
return loc; }
99 static inline void __kmp_wait_template(kmp_info_t *this_thr, C *flag,
int final_spin
100 USE_ITT_BUILD_ARG(
void * itt_sync_obj) )
103 volatile typename C::flag_t *spin = flag->get();
105 kmp_uint32 hibernate;
107 int tasks_completed = FALSE;
109 KMP_FSYNC_SPIN_INIT(spin, NULL);
110 if (flag->done_check()) {
111 KMP_FSYNC_SPIN_ACQUIRED(spin);
114 th_gtid = this_thr->th.th_info.ds.ds_gtid;
115 KA_TRACE(20, (
"__kmp_wait_sleep: T#%d waiting for flag(%p)\n", th_gtid, flag));
118 KMP_INIT_YIELD(spins);
120 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
123 #ifdef KMP_ADJUST_BLOCKTIME
124 if (__kmp_zero_bt && !this_thr->th.th_team_bt_set)
128 hibernate = this_thr->th.th_team_bt_intervals;
130 hibernate = this_thr->th.th_team_bt_intervals;
140 hibernate += TCR_4(__kmp_global.g.g_time.dt.t_value);
141 KF_TRACE(20, (
"__kmp_wait_sleep: T#%d now=%d, hibernate=%d, intervals=%d\n",
142 th_gtid, __kmp_global.g.g_time.dt.t_value, hibernate,
143 hibernate - __kmp_global.g.g_time.dt.t_value));
148 while (flag->notdone_check()) {
158 kmp_task_team_t * task_team = NULL;
159 if (__kmp_tasking_mode != tskm_immediate_exec) {
160 task_team = this_thr->th.th_task_team;
161 if (task_team != NULL) {
162 if (!TCR_SYNC_4(task_team->tt.tt_active)) {
163 KMP_DEBUG_ASSERT(!KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid));
164 __kmp_unref_task_team(task_team, this_thr);
165 }
else if (KMP_TASKING_ENABLED(task_team)) {
166 flag->execute_tasks(this_thr, th_gtid, final_spin, &tasks_completed
167 USE_ITT_BUILD_ARG(itt_sync_obj), 0);
172 KMP_FSYNC_SPIN_PREPARE(spin);
173 if (TCR_4(__kmp_global.g.g_done)) {
174 if (__kmp_global.g.g_abort)
175 __kmp_abort_thread();
180 KMP_YIELD(TCR_4(__kmp_nth) > __kmp_avail_proc);
184 KMP_YIELD_SPIN(spins);
188 in_pool = !!TCR_4(this_thr->th.th_in_pool);
189 if (in_pool != !!this_thr->th.th_active_in_pool) {
191 KMP_TEST_THEN_INC32((kmp_int32 *)&__kmp_thread_pool_active_nth);
192 this_thr->th.th_active_in_pool = TRUE;
200 KMP_TEST_THEN_DEC32((kmp_int32 *) &__kmp_thread_pool_active_nth);
201 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
202 this_thr->th.th_active_in_pool = FALSE;
207 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME)
211 if ((task_team != NULL) && TCR_4(task_team->tt.tt_found_tasks))
215 if (TCR_4(__kmp_global.g.g_time.dt.t_value) < hibernate)
218 KF_TRACE(50, (
"__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid));
220 flag->suspend(th_gtid);
222 if (TCR_4(__kmp_global.g.g_done)) {
223 if (__kmp_global.g.g_abort)
224 __kmp_abort_thread();
229 KMP_FSYNC_SPIN_ACQUIRED(spin);
236 static inline void __kmp_release_template(C *flag)
240 kmp_info_t * wait_thr = flag->get_waiter(0);
241 int target_gtid = wait_thr->th.th_info.ds.ds_gtid;
242 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
244 KF_TRACE(20, (
"__kmp_release: T#%d releasing T#%d spin(%p)\n", gtid, target_gtid, flag->get()));
245 KMP_DEBUG_ASSERT(flag->get());
246 KMP_FSYNC_RELEASING(flag->get());
248 typename C::flag_t old_spin = flag->internal_release();
250 KF_TRACE(100, (
"__kmp_release: T#%d old spin(%p)=%d, set new spin=%d\n",
251 gtid, flag->get(), old_spin, *(flag->get())));
253 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
255 if (flag->is_sleeping_val(old_spin)) {
256 for (
unsigned int i=0; i<flag->get_num_waiters(); ++i) {
257 kmp_info_t * waiter = flag->get_waiter(i);
258 int wait_gtid = waiter->th.th_info.ds.ds_gtid;
260 KF_TRACE(50, (
"__kmp_release: T#%d waking up thread T#%d since sleep spin(%p) set\n",
261 gtid, wait_gtid, flag->get()));
262 flag->resume(wait_gtid);
265 KF_TRACE(50, (
"__kmp_release: T#%d don't wake up thread T#%d since sleep spin(%p) not set\n",
266 gtid, target_gtid, flag->get()));
271 template <
typename FlagType>
272 struct flag_traits {};
275 struct flag_traits<kmp_uint32> {
276 typedef kmp_uint32 flag_t;
278 static inline flag_t tcr(flag_t f) {
return TCR_4(f); }
279 static inline flag_t test_then_add4(
volatile flag_t *f) {
return KMP_TEST_THEN_ADD4_32((
volatile kmp_int32 *)f); }
280 static inline flag_t test_then_or(
volatile flag_t *f, flag_t v) {
return KMP_TEST_THEN_OR32((
volatile kmp_int32 *)f, v); }
281 static inline flag_t test_then_and(
volatile flag_t *f, flag_t v) {
return KMP_TEST_THEN_AND32((
volatile kmp_int32 *)f, v); }
285 struct flag_traits<kmp_uint64> {
286 typedef kmp_uint64 flag_t;
288 static inline flag_t tcr(flag_t f) {
return TCR_8(f); }
289 static inline flag_t test_then_add4(
volatile flag_t *f) {
return KMP_TEST_THEN_ADD4_64((
volatile kmp_int64 *)f); }
290 static inline flag_t test_then_or(
volatile flag_t *f, flag_t v) {
return KMP_TEST_THEN_OR64((
volatile kmp_int64 *)f, v); }
291 static inline flag_t test_then_and(
volatile flag_t *f, flag_t v) {
return KMP_TEST_THEN_AND64((
volatile kmp_int64 *)f, v); }
294 template <
typename FlagType>
295 class kmp_basic_flag :
public kmp_flag<FlagType> {
296 typedef flag_traits<FlagType> traits_type;
298 kmp_info_t * waiting_threads[1];
299 kmp_uint32 num_waiting_threads;
301 kmp_basic_flag(
volatile FlagType *p) :
kmp_flag<FlagType>(p, traits_type::t), num_waiting_threads(0) {}
302 kmp_basic_flag(
volatile FlagType *p, kmp_info_t *thr) :
kmp_flag<FlagType>(p, traits_type::t), num_waiting_threads(1) {
303 waiting_threads[0] = thr;
305 kmp_basic_flag(
volatile FlagType *p, FlagType c) :
kmp_flag<FlagType>(p, traits_type::t), checker(c), num_waiting_threads(0) {}
310 kmp_info_t * get_waiter(kmp_uint32 i) {
311 KMP_DEBUG_ASSERT(i<num_waiting_threads);
312 return waiting_threads[i];
317 kmp_uint32 get_num_waiters() {
return num_waiting_threads; }
323 void set_waiter(kmp_info_t *thr) {
324 waiting_threads[0] = thr;
325 num_waiting_threads = 1;
330 bool done_check() {
return traits_type::tcr(*(this->
get())) == checker; }
335 bool done_check_val(FlagType old_loc) {
return old_loc == checker; }
343 bool notdone_check() {
return traits_type::tcr(*(this->
get())) != checker; }
348 FlagType internal_release() {
349 return traits_type::test_then_add4((
volatile FlagType *)this->
get());
355 FlagType set_sleeping() {
356 return traits_type::test_then_or((
volatile FlagType *)this->
get(), KMP_BARRIER_SLEEP_STATE);
362 FlagType unset_sleeping() {
363 return traits_type::test_then_and((
volatile FlagType *)this->
get(), ~KMP_BARRIER_SLEEP_STATE);
369 bool is_sleeping_val(FlagType old_loc) {
return old_loc & KMP_BARRIER_SLEEP_STATE; }
373 bool is_sleeping() {
return is_sleeping_val(*(this->
get())); }
376 class kmp_flag_32 :
public kmp_basic_flag<kmp_uint32> {
378 kmp_flag_32(
volatile kmp_uint32 *p) : kmp_basic_flag<kmp_uint32>(p) {}
379 kmp_flag_32(
volatile kmp_uint32 *p, kmp_info_t *thr) : kmp_basic_flag<kmp_uint32>(p, thr) {}
380 kmp_flag_32(
volatile kmp_uint32 *p, kmp_uint32 c) : kmp_basic_flag<kmp_uint32>(p, c) {}
381 void suspend(
int th_gtid) { __kmp_suspend_32(th_gtid,
this); }
382 void resume(
int th_gtid) { __kmp_resume_32(th_gtid,
this); }
383 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid,
int final_spin,
int *thread_finished
384 USE_ITT_BUILD_ARG(
void * itt_sync_obj), kmp_int32 is_constrained) {
385 return __kmp_execute_tasks_32(this_thr, gtid,
this, final_spin, thread_finished
386 USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
388 void wait(kmp_info_t *this_thr,
int final_spin
389 USE_ITT_BUILD_ARG(
void * itt_sync_obj)) {
390 __kmp_wait_template(this_thr,
this, final_spin
391 USE_ITT_BUILD_ARG(itt_sync_obj));
393 void release() { __kmp_release_template(
this); }
396 class kmp_flag_64 :
public kmp_basic_flag<kmp_uint64> {
398 kmp_flag_64(
volatile kmp_uint64 *p) : kmp_basic_flag<kmp_uint64>(p) {}
399 kmp_flag_64(
volatile kmp_uint64 *p, kmp_info_t *thr) : kmp_basic_flag<kmp_uint64>(p, thr) {}
400 kmp_flag_64(
volatile kmp_uint64 *p, kmp_uint64 c) : kmp_basic_flag<kmp_uint64>(p, c) {}
401 void suspend(
int th_gtid) { __kmp_suspend_64(th_gtid,
this); }
402 void resume(
int th_gtid) { __kmp_resume_64(th_gtid,
this); }
403 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid,
int final_spin,
int *thread_finished
404 USE_ITT_BUILD_ARG(
void * itt_sync_obj), kmp_int32 is_constrained) {
405 return __kmp_execute_tasks_64(this_thr, gtid,
this, final_spin, thread_finished
406 USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
408 void wait(kmp_info_t *this_thr,
int final_spin
409 USE_ITT_BUILD_ARG(
void * itt_sync_obj)) {
410 __kmp_wait_template(this_thr,
this, final_spin
411 USE_ITT_BUILD_ARG(itt_sync_obj));
413 void release() { __kmp_release_template(
this); }
417 class kmp_flag_oncore :
public kmp_flag<kmp_uint64> {
419 kmp_info_t * waiting_threads[1];
420 kmp_uint32 num_waiting_threads;
423 enum barrier_type bt;
424 kmp_info_t * this_thr;
428 char& byteref(
volatile kmp_uint64* loc,
size_t offset) {
return ((
char *)loc)[offset]; }
430 kmp_flag_oncore(
volatile kmp_uint64 *p)
432 kmp_flag_oncore(
volatile kmp_uint64 *p, kmp_uint32 idx)
433 :
kmp_flag<kmp_uint64>(p,
flag_oncore), offset(idx), num_waiting_threads(0), flag_switch(false) {}
434 kmp_flag_oncore(
volatile kmp_uint64 *p, kmp_uint64 c, kmp_uint32 idx,
enum barrier_type bar_t,
444 , num_waiting_threads(0), flag_switch(false) {}
445 kmp_info_t * get_waiter(kmp_uint32 i) {
446 KMP_DEBUG_ASSERT(i<num_waiting_threads);
447 return waiting_threads[i];
449 kmp_uint32 get_num_waiters() {
return num_waiting_threads; }
450 void set_waiter(kmp_info_t *thr) {
451 waiting_threads[0] = thr;
452 num_waiting_threads = 1;
454 bool done_check_val(kmp_uint64 old_loc) {
return byteref(&old_loc,offset) == checker; }
455 bool done_check() {
return done_check_val(*
get()); }
456 bool notdone_check() {
458 if (this_thr->th.th_bar[bt].bb.wait_flag == KMP_BARRIER_SWITCH_TO_OWN_FLAG)
460 if (byteref(
get(),offset) != 1 && !flag_switch)
462 else if (flag_switch) {
463 this_thr->th.th_bar[bt].bb.wait_flag = KMP_BARRIER_SWITCHING;
464 kmp_flag_64 flag(&this_thr->th.th_bar[bt].bb.b_go, (kmp_uint64)KMP_BARRIER_STATE_BUMP);
465 __kmp_wait_64(this_thr, &flag, TRUE
473 kmp_uint64 internal_release() {
475 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
477 byteref(
get(),offset) = 1;
481 byteref(&mask,offset) = 1;
482 old_val = KMP_TEST_THEN_OR64((
volatile kmp_int64 *)
get(), mask);
486 kmp_uint64 set_sleeping() {
487 return KMP_TEST_THEN_OR64((kmp_int64
volatile *)
get(), KMP_BARRIER_SLEEP_STATE);
489 kmp_uint64 unset_sleeping() {
490 return KMP_TEST_THEN_AND64((kmp_int64
volatile *)
get(), ~KMP_BARRIER_SLEEP_STATE);
492 bool is_sleeping_val(kmp_uint64 old_loc) {
return old_loc & KMP_BARRIER_SLEEP_STATE; }
493 bool is_sleeping() {
return is_sleeping_val(*
get()); }
494 void wait(kmp_info_t *this_thr,
int final_spin
495 USE_ITT_BUILD_ARG(
void * itt_sync_obj)) {
496 __kmp_wait_template(this_thr,
this, final_spin
497 USE_ITT_BUILD_ARG(itt_sync_obj));
499 void release() { __kmp_release_template(
this); }
500 void suspend(
int th_gtid) { __kmp_suspend_oncore(th_gtid,
this); }
501 void resume(
int th_gtid) { __kmp_resume_oncore(th_gtid,
this); }
502 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid,
int final_spin,
int *thread_finished
503 USE_ITT_BUILD_ARG(
void * itt_sync_obj), kmp_int32 is_constrained) {
504 return __kmp_execute_tasks_oncore(this_thr, gtid,
this, final_spin, thread_finished
505 USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
513 #endif // KMP_WAIT_RELEASE_H