Kokkos Core Kernels Package  Version of the Day
Kokkos_ExecPolicy.hpp
1 /*
2 //@HEADER
3 // ************************************************************************
4 //
5 // Kokkos v. 2.0
6 // Copyright (2014) Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact H. Carter Edwards (hcedwar@sandia.gov)
39 //
40 // ************************************************************************
41 //@HEADER
42 */
43 
44 #ifndef KOKKOS_EXECPOLICY_HPP
45 #define KOKKOS_EXECPOLICY_HPP
46 
47 #include <Kokkos_Core_fwd.hpp>
48 #include <impl/Kokkos_Traits.hpp>
49 #include <impl/Kokkos_StaticAssert.hpp>
50 #include <impl/Kokkos_Tags.hpp>
51 
52 //----------------------------------------------------------------------------
53 
54 namespace Kokkos {
55 
77 template< class Arg0 = void , class Arg1 = void , class Arg2 = void
78  , class ExecSpace =
79  // The first argument is the execution space,
80  // otherwise use the default execution space.
81  typename std::conditional
82  < Impl::is_execution_space< Arg0 >::value , Arg0
83  , Kokkos::DefaultExecutionSpace >::type
84  >
85 class RangePolicy {
86 private:
87 
88  // Default integral type and blocking factor:
89  typedef int DefaultIntType ;
90  enum { DefaultIntValue = 8 };
91 
92  enum { Arg0_Void = Impl::is_same< Arg0 , void >::value };
93  enum { Arg1_Void = Impl::is_same< Arg1 , void >::value };
94  enum { Arg2_Void = Impl::is_same< Arg2 , void >::value };
95 
96  enum { Arg0_ExecSpace = Impl::is_execution_space< Arg0 >::value };
97 
98  enum { Arg0_IntConst = Impl::is_integral_constant< Arg0 >::value };
99  enum { Arg1_IntConst = Impl::is_integral_constant< Arg1 >::value };
100  enum { Arg2_IntConst = Impl::is_integral_constant< Arg2 >::value };
101 
102  enum { Arg0_IntType = Impl::is_integral< Arg0 >::value };
103  enum { Arg1_IntType = Impl::is_integral< Arg1 >::value };
104  enum { Arg2_IntType = Impl::is_integral< Arg2 >::value };
105 
106  enum { Arg0_WorkTag = ! Arg0_ExecSpace && ! Arg0_IntConst && ! Arg0_IntType && ! Arg0_Void };
107  enum { Arg1_WorkTag = Arg0_ExecSpace && ! Arg1_IntConst && ! Arg1_IntType && ! Arg1_Void };
108 
109  enum { ArgOption_OK = Impl::StaticAssert< (
110  ( Arg0_ExecSpace && Arg1_WorkTag && ( Arg2_IntConst || Arg2_IntType ) ) ||
111  ( Arg0_ExecSpace && Arg1_WorkTag && Arg2_Void ) ||
112  ( Arg0_ExecSpace && ( Arg1_IntConst || Arg1_IntType ) && Arg2_Void ) ||
113  ( Arg0_ExecSpace && Arg1_Void && Arg2_Void ) ||
114  ( Arg0_WorkTag && ( Arg1_IntConst || Arg1_IntType ) && Arg2_Void ) ||
115  ( Arg0_WorkTag && Arg1_Void && Arg2_Void ) ||
116  ( ( Arg0_IntConst || Arg0_IntType ) && Arg1_Void && Arg2_Void ) ||
117  ( Arg0_Void && Arg1_Void && Arg2_Void )
118  ) >::value };
119 
120  // The work argument tag is the first or second argument
121  typedef typename std::conditional< Arg0_WorkTag , Arg0 ,
122  typename std::conditional< Arg1_WorkTag , Arg1 , void
123  >::type >::type
124  WorkTag ;
125 
126  enum { Granularity = Arg0_IntConst ? unsigned(Impl::is_integral_constant<Arg0>::integral_value) : (
127  Arg1_IntConst ? unsigned(Impl::is_integral_constant<Arg1>::integral_value) : (
128  Arg2_IntConst ? unsigned(Impl::is_integral_constant<Arg2>::integral_value) : (
129  unsigned(DefaultIntValue) ))) };
130 
131  // Only accept the integral type if the blocking is a power of two
132  static_assert( Impl::is_integral_power_of_two( Granularity )
133  , "RangePolicy blocking granularity must be power of two" );
134 
135  typedef typename std::conditional< Arg0_IntType , Arg0 ,
136  typename std::conditional< Arg1_IntType , Arg1 ,
137  typename std::conditional< Arg2_IntType , Arg2 ,
138  typename std::conditional< Arg0_IntConst , typename Impl::is_integral_constant<Arg0>::integral_type ,
139  typename std::conditional< Arg1_IntConst , typename Impl::is_integral_constant<Arg1>::integral_type ,
140  typename std::conditional< Arg2_IntConst , typename Impl::is_integral_constant<Arg2>::integral_type ,
141  DefaultIntType
142  >::type >::type >::type
143  >::type >::type >::type
144  IntType ;
145 
146  enum { GranularityMask = IntType(Granularity) - 1 };
147 
148  ExecSpace m_space ;
149  IntType m_begin ;
150  IntType m_end ;
151 
152 public:
153 
155  typedef ExecSpace execution_space ;
156  typedef RangePolicy execution_policy ;
157  typedef WorkTag work_tag ;
158  typedef IntType member_type ;
159 
160  KOKKOS_INLINE_FUNCTION const execution_space & space() const { return m_space ; }
161  KOKKOS_INLINE_FUNCTION member_type begin() const { return m_begin ; }
162  KOKKOS_INLINE_FUNCTION member_type end() const { return m_end ; }
163 
164  inline RangePolicy() : m_space(), m_begin(0), m_end(0) {}
165 
167  inline
168  RangePolicy( const member_type work_begin
169  , const member_type work_end
170  )
171  : m_space()
172  , m_begin( work_begin < work_end ? work_begin : 0 )
173  , m_end( work_begin < work_end ? work_end : 0 )
174  {}
175 
177  inline
178  RangePolicy( const execution_space & work_space
179  , const member_type work_begin
180  , const member_type work_end
181  )
182  : m_space( work_space )
183  , m_begin( work_begin < work_end ? work_begin : 0 )
184  , m_end( work_begin < work_end ? work_end : 0 )
185  {}
186 
191  struct WorkRange {
192  typedef typename RangePolicy::work_tag work_tag ;
193  typedef typename RangePolicy::member_type member_type ;
194 
195  KOKKOS_INLINE_FUNCTION member_type begin() const { return m_begin ; }
196  KOKKOS_INLINE_FUNCTION member_type end() const { return m_end ; }
197 
202  KOKKOS_INLINE_FUNCTION
203  WorkRange( const RangePolicy & range
204  , const int part_rank
205  , const int part_size
206  )
207  : m_begin(0), m_end(0)
208  {
209  if ( part_size ) {
210 
211  // Split evenly among partitions, then round up to the granularity.
212  const member_type work_part =
213  ( ( ( ( range.end() - range.begin() ) + ( part_size - 1 ) ) / part_size )
214  + GranularityMask ) & ~member_type(GranularityMask);
215 
216  m_begin = range.begin() + work_part * part_rank ;
217  m_end = m_begin + work_part ;
218 
219  if ( range.end() < m_begin ) m_begin = range.end() ;
220  if ( range.end() < m_end ) m_end = range.end() ;
221  }
222  }
223  private:
224  member_type m_begin ;
225  member_type m_end ;
226  WorkRange();
227  WorkRange & operator = ( const WorkRange & );
228  };
229 };
230 
231 } // namespace Kokkos
232 
233 //----------------------------------------------------------------------------
234 //----------------------------------------------------------------------------
235 
236 namespace Kokkos {
237 
238 namespace Experimental {
239 
247 template< class MemorySpace >
249  size_t m_per_team;
250  size_t m_per_thread;
251 
252 public:
253  TeamScratchRequest(size_t per_team_, size_t per_thread_ = 0):
254  m_per_team(per_team_), m_per_thread(per_thread_) {
255  }
256 
257  size_t per_team() const {
258  return m_per_team;
259  }
260  size_t per_thread() const {
261  return m_per_thread;
262  }
263  size_t total(const size_t team_size) const {
264  return m_per_team + m_per_thread * team_size;
265  }
266 };
267 
268 }
269 
290 template< class Arg0 = void
291  , class Arg1 = void
292  , class ExecSpace =
293  // If the first argument is not an execution
294  // then use the default execution space.
295  typename std::conditional
296  < Impl::is_execution_space< Arg0 >::value , Arg0
297  , Kokkos::DefaultExecutionSpace >::type
298  >
299 class TeamPolicy {
300 private:
301 
302  enum { Arg0_ExecSpace = Impl::is_execution_space< Arg0 >::value };
303  enum { Arg1_Void = Impl::is_same< Arg1 , void >::value };
304  enum { ArgOption_OK = Impl::StaticAssert< ( Arg0_ExecSpace || Arg1_Void ) >::value };
305 
306  typedef typename std::conditional< Arg0_ExecSpace , Arg1 , Arg0 >::type WorkTag ;
307 
308 public:
309 
312  typedef ExecSpace execution_space ;
313  typedef WorkTag work_tag ;
314 
315  //----------------------------------------
322  template< class FunctorType >
323  static int team_size_max( const FunctorType & );
324 
331  template< class FunctorType >
332  static int team_size_recommended( const FunctorType & );
333 
334  template< class FunctorType >
335  static int team_size_recommended( const FunctorType & , const int&);
336  //----------------------------------------
338  TeamPolicy( const execution_space & , int league_size_request , int team_size_request , int vector_length_request = 1 );
339 
340  TeamPolicy( const execution_space & , int league_size_request , const Kokkos::AUTO_t & , int vector_length_request = 1 );
341 
343  TeamPolicy( int league_size_request , int team_size_request , int vector_length_request = 1 );
344 
345  TeamPolicy( int league_size_request , const Kokkos::AUTO_t & , int vector_length_request = 1 );
346 
347  template<class MemorySpace>
348  TeamPolicy( int league_size_request , int team_size_request , const Experimental::TeamScratchRequest<MemorySpace>& team_scratch_memory_request );
349 
350  template<class MemorySpace>
351  TeamPolicy( int league_size_request , const Kokkos::AUTO_t & , const Experimental::TeamScratchRequest<MemorySpace>& team_scratch_memory_request );
352 
358  KOKKOS_INLINE_FUNCTION int league_size() const ;
359 
365  KOKKOS_INLINE_FUNCTION int team_size() const ;
366 
370  struct member_type {
371 
373  KOKKOS_INLINE_FUNCTION
374  typename execution_space::scratch_memory_space team_shmem() const ;
375 
377  KOKKOS_INLINE_FUNCTION int league_rank() const ;
378 
380  KOKKOS_INLINE_FUNCTION int league_size() const ;
381 
383  KOKKOS_INLINE_FUNCTION int team_rank() const ;
384 
386  KOKKOS_INLINE_FUNCTION int team_size() const ;
387 
389  KOKKOS_INLINE_FUNCTION void team_barrier() const ;
390 
392  template< class JoinOp >
393  KOKKOS_INLINE_FUNCTION
394  typename JoinOp::value_type team_reduce( const typename JoinOp::value_type
395  , const JoinOp & ) const ;
396 
402  template< typename Type >
403  KOKKOS_INLINE_FUNCTION Type team_scan( const Type & value ) const ;
404 
414  template< typename Type >
415  KOKKOS_INLINE_FUNCTION Type team_scan( const Type & value , Type * const global_accum ) const ;
416  };
417 };
418 
419 } // namespace Kokkos
420 
421 namespace Kokkos {
422 
423 namespace Impl {
424 
425 template<typename iType, class TeamMemberType>
426 struct TeamThreadRangeBoundariesStruct {
427 private:
428 
429  KOKKOS_INLINE_FUNCTION static
430  iType ibegin( const iType & arg_begin
431  , const iType & arg_end
432  , const iType & arg_rank
433  , const iType & arg_size
434  )
435  {
436  return arg_begin + ( ( arg_end - arg_begin + arg_size - 1 ) / arg_size ) * arg_rank ;
437  }
438 
439  KOKKOS_INLINE_FUNCTION static
440  iType iend( const iType & arg_begin
441  , const iType & arg_end
442  , const iType & arg_rank
443  , const iType & arg_size
444  )
445  {
446  const iType end_ = arg_begin + ( ( arg_end - arg_begin + arg_size - 1 ) / arg_size ) * ( arg_rank + 1 );
447  return end_ < arg_end ? end_ : arg_end ;
448  }
449 
450 public:
451 
452  typedef iType index_type;
453  const iType start;
454  const iType end;
455  enum {increment = 1};
456  const TeamMemberType& thread;
457 
458  KOKKOS_INLINE_FUNCTION
459  TeamThreadRangeBoundariesStruct( const TeamMemberType& arg_thread
460  , const iType& arg_end
461  )
462  : start( ibegin( 0 , arg_end , arg_thread.team_rank() , arg_thread.team_size() ) )
463  , end( iend( 0 , arg_end , arg_thread.team_rank() , arg_thread.team_size() ) )
464  , thread( arg_thread )
465  {}
466 
467  KOKKOS_INLINE_FUNCTION
468  TeamThreadRangeBoundariesStruct( const TeamMemberType& arg_thread
469  , const iType& arg_begin
470  , const iType& arg_end
471  )
472  : start( ibegin( arg_begin , arg_end , arg_thread.team_rank() , arg_thread.team_size() ) )
473  , end( iend( arg_begin , arg_end , arg_thread.team_rank() , arg_thread.team_size() ) )
474  , thread( arg_thread )
475  {}
476 };
477 
478  template<typename iType, class TeamMemberType>
479  struct ThreadVectorRangeBoundariesStruct {
480  typedef iType index_type;
481  enum {start = 0};
482  const iType end;
483  enum {increment = 1};
484 
485  KOKKOS_INLINE_FUNCTION
486  ThreadVectorRangeBoundariesStruct (const TeamMemberType& thread, const iType& count):
487  end( count )
488  {}
489  };
490 
491  template<class TeamMemberType>
492  struct ThreadSingleStruct {
493  const TeamMemberType& team_member;
494  KOKKOS_INLINE_FUNCTION
495  ThreadSingleStruct(const TeamMemberType& team_member_):team_member(team_member_){}
496  };
497 
498  template<class TeamMemberType>
499  struct VectorSingleStruct {
500  const TeamMemberType& team_member;
501  KOKKOS_INLINE_FUNCTION
502  VectorSingleStruct(const TeamMemberType& team_member_):team_member(team_member_){}
503  };
504 } // namespace Impl
505 
512 template<typename iType, class TeamMemberType>
513 KOKKOS_INLINE_FUNCTION
514 Impl::TeamThreadRangeBoundariesStruct<iType,TeamMemberType> TeamThreadRange(const TeamMemberType&, const iType& count);
515 
522 template<typename iType, class TeamMemberType>
523 KOKKOS_INLINE_FUNCTION
524 Impl::TeamThreadRangeBoundariesStruct<iType,TeamMemberType> TeamThreadRange(const TeamMemberType&, const iType& begin, const iType& end);
525 
532 template<typename iType, class TeamMemberType>
533 KOKKOS_INLINE_FUNCTION
534 Impl::ThreadVectorRangeBoundariesStruct<iType,TeamMemberType> ThreadVectorRange(const TeamMemberType&, const iType& count);
535 
536 } // namespace Kokkos
537 
538 #endif /* #define KOKKOS_EXECPOLICY_HPP */
539 
540 //----------------------------------------------------------------------------
541 //----------------------------------------------------------------------------
542 
TeamPolicy execution_policy
Tag this class as an execution policy.
KOKKOS_INLINE_FUNCTION Impl::TeamThreadRangeBoundariesStruct< iType, TeamMemberType > TeamThreadRange(const TeamMemberType &, const iType &count)
Execution policy for parallel work over a threads within a team.
Scratch memory request accepting per team and per thread value.
RangePolicy(const execution_space &work_space, const member_type work_begin, const member_type work_end)
Total range.
Parallel execution of a functor calls the functor once with each member of the execution policy...
RangePolicy(const member_type work_begin, const member_type work_end)
Total range.
KOKKOS_INLINE_FUNCTION Impl::ThreadVectorRangeBoundariesStruct< iType, TeamMemberType > ThreadVectorRange(const TeamMemberType &, const iType &count)
Execution policy for a vector parallel loop.
Execution policy for work over a range of an integral type.
Subrange for a partition&#39;s rank and size.
Execution policy for parallel work over a league of teams of threads.
KOKKOS_INLINE_FUNCTION WorkRange(const RangePolicy &range, const int part_rank, const int part_size)
Subrange for a partition&#39;s rank and size.
ExecSpace execution_space
Tag this class as an execution policy.