11 #ifndef EIGEN_COEFFBASED_PRODUCT_H
12 #define EIGEN_COEFFBASED_PRODUCT_H
31 template<
int Traversal,
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename RetScalar>
32 struct product_coeff_impl;
34 template<
int StorageOrder,
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
35 struct product_packet_impl;
37 template<
typename LhsNested,
typename RhsNested,
int NestingFlags>
38 struct traits<CoeffBasedProduct<LhsNested,RhsNested,NestingFlags> >
40 typedef MatrixXpr XprKind;
41 typedef typename remove_all<LhsNested>::type _LhsNested;
42 typedef typename remove_all<RhsNested>::type _RhsNested;
43 typedef typename scalar_product_traits<typename _LhsNested::Scalar, typename _RhsNested::Scalar>::ReturnType Scalar;
44 typedef typename promote_storage_type<typename traits<_LhsNested>::StorageKind,
45 typename traits<_RhsNested>::StorageKind>::ret StorageKind;
46 typedef typename promote_index_type<typename traits<_LhsNested>::Index,
47 typename traits<_RhsNested>::Index>::type Index;
50 LhsCoeffReadCost = _LhsNested::CoeffReadCost,
51 RhsCoeffReadCost = _RhsNested::CoeffReadCost,
52 LhsFlags = _LhsNested::Flags,
53 RhsFlags = _RhsNested::Flags,
55 RowsAtCompileTime = _LhsNested::RowsAtCompileTime,
56 ColsAtCompileTime = _RhsNested::ColsAtCompileTime,
57 InnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(_LhsNested::ColsAtCompileTime, _RhsNested::RowsAtCompileTime),
59 MaxRowsAtCompileTime = _LhsNested::MaxRowsAtCompileTime,
60 MaxColsAtCompileTime = _RhsNested::MaxColsAtCompileTime,
65 SameType = is_same<typename _LhsNested::Scalar,typename _RhsNested::Scalar>::value,
68 && (ColsAtCompileTime ==
Dynamic
69 || ( (ColsAtCompileTime % packet_traits<Scalar>::size) == 0
75 && (RowsAtCompileTime ==
Dynamic
76 || ( (RowsAtCompileTime % packet_traits<Scalar>::size) == 0
81 EvalToRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1
82 : (MaxColsAtCompileTime==1&&MaxRowsAtCompileTime!=1) ? 0
83 : (RhsRowMajor && !CanVectorizeLhs),
85 Flags = ((
unsigned int)(LhsFlags | RhsFlags) & HereditaryBits & ~RowMajorBit)
86 | (EvalToRowMajor ? RowMajorBit : 0)
90 | (SameType && (CanVectorizeLhs || CanVectorizeRhs) ?
PacketAccessBit : 0),
93 : InnerSize * (NumTraits<Scalar>::MulCost + LhsCoeffReadCost + RhsCoeffReadCost)
94 + (InnerSize - 1) * NumTraits<Scalar>::AddCost,
101 CanVectorizeInner = SameType
106 && (InnerSize % packet_traits<Scalar>::size == 0)
112 template<
typename LhsNested,
typename RhsNested,
int NestingFlags>
113 class CoeffBasedProduct
114 : internal::no_assignment_operator,
115 public MatrixBase<CoeffBasedProduct<LhsNested, RhsNested, NestingFlags> >
119 typedef MatrixBase<CoeffBasedProduct> Base;
120 EIGEN_DENSE_PUBLIC_INTERFACE(CoeffBasedProduct)
121 typedef typename Base::PlainObject PlainObject;
125 typedef typename internal::traits<CoeffBasedProduct>::_LhsNested _LhsNested;
126 typedef typename internal::traits<CoeffBasedProduct>::_RhsNested _RhsNested;
129 PacketSize = internal::packet_traits<Scalar>::size,
130 InnerSize = internal::traits<CoeffBasedProduct>::InnerSize,
131 Unroll = CoeffReadCost !=
Dynamic && CoeffReadCost <= EIGEN_UNROLLING_LIMIT,
132 CanVectorizeInner = internal::traits<CoeffBasedProduct>::CanVectorizeInner
135 typedef internal::product_coeff_impl<CanVectorizeInner ? InnerVectorizedTraversal : DefaultTraversal,
136 Unroll ? InnerSize-1 :
Dynamic,
137 _LhsNested, _RhsNested, Scalar> ScalarCoeffImpl;
139 typedef CoeffBasedProduct<LhsNested,RhsNested,NestByRefBit> LazyCoeffBasedProductType;
143 inline CoeffBasedProduct(
const CoeffBasedProduct& other)
144 : Base(), m_lhs(other.m_lhs), m_rhs(other.m_rhs)
147 template<
typename Lhs,
typename Rhs>
148 inline CoeffBasedProduct(
const Lhs& lhs,
const Rhs& rhs)
149 : m_lhs(lhs), m_rhs(rhs)
153 EIGEN_STATIC_ASSERT((internal::scalar_product_traits<typename Lhs::RealScalar, typename Rhs::RealScalar>::Defined),
154 YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY)
155 eigen_assert(lhs.cols() == rhs.rows()
156 && "invalid matrix product"
157 && "if you wanted a coeff-wise or a dot product use the respective explicit functions");
160 EIGEN_STRONG_INLINE Index rows()
const {
return m_lhs.rows(); }
161 EIGEN_STRONG_INLINE Index cols()
const {
return m_rhs.cols(); }
163 EIGEN_STRONG_INLINE
const Scalar coeff(Index row, Index col)
const
166 ScalarCoeffImpl::run(row, col, m_lhs, m_rhs, res);
173 EIGEN_STRONG_INLINE
const Scalar coeff(Index index)
const
176 const Index row = RowsAtCompileTime == 1 ? 0 : index;
177 const Index col = RowsAtCompileTime == 1 ? index : 0;
178 ScalarCoeffImpl::run(row, col, m_lhs, m_rhs, res);
182 template<
int LoadMode>
183 EIGEN_STRONG_INLINE
const PacketScalar packet(Index row, Index col)
const
187 Unroll ? InnerSize-1 :
Dynamic,
188 _LhsNested, _RhsNested, PacketScalar, LoadMode>
189 ::run(row, col, m_lhs, m_rhs, res);
194 EIGEN_STRONG_INLINE
operator const PlainObject& ()
const
196 m_result.lazyAssign(*
this);
200 const _LhsNested& lhs()
const {
return m_lhs; }
201 const _RhsNested& rhs()
const {
return m_rhs; }
203 const Diagonal<const LazyCoeffBasedProductType,0> diagonal()
const
204 {
return reinterpret_cast<const LazyCoeffBasedProductType&
>(*this); }
206 template<
int DiagonalIndex>
207 const Diagonal<const LazyCoeffBasedProductType,DiagonalIndex> diagonal()
const
208 {
return reinterpret_cast<const LazyCoeffBasedProductType&
>(*this); }
210 const Diagonal<const LazyCoeffBasedProductType,Dynamic> diagonal(Index index)
const
211 {
return reinterpret_cast<const LazyCoeffBasedProductType&
>(*this).diagonal(index); }
214 typename internal::add_const_on_value_type<LhsNested>::type m_lhs;
215 typename internal::add_const_on_value_type<RhsNested>::type m_rhs;
217 mutable PlainObject m_result;
224 template<
typename Lhs,
typename Rhs,
int N,
typename PlainObject>
227 typedef PlainObject
const& type;
238 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename RetScalar>
239 struct product_coeff_impl<DefaultTraversal, UnrollingIndex, Lhs, Rhs, RetScalar>
241 typedef typename Lhs::Index Index;
242 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, RetScalar &res)
244 product_coeff_impl<DefaultTraversal, UnrollingIndex-1, Lhs, Rhs, RetScalar>::run(row, col, lhs, rhs, res);
245 res += lhs.coeff(row, UnrollingIndex) * rhs.coeff(UnrollingIndex, col);
249 template<
typename Lhs,
typename Rhs,
typename RetScalar>
250 struct product_coeff_impl<DefaultTraversal, 0, Lhs, Rhs, RetScalar>
252 typedef typename Lhs::Index Index;
253 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, RetScalar &res)
255 res = lhs.coeff(row, 0) * rhs.coeff(0, col);
259 template<
typename Lhs,
typename Rhs,
typename RetScalar>
260 struct product_coeff_impl<DefaultTraversal,
Dynamic, Lhs, Rhs, RetScalar>
262 typedef typename Lhs::Index Index;
263 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, RetScalar& res)
265 eigen_assert(lhs.cols()>0 &&
"you are using a non initialized matrix");
266 res = lhs.coeff(row, 0) * rhs.coeff(0, col);
267 for(Index i = 1; i < lhs.cols(); ++i)
268 res += lhs.coeff(row, i) * rhs.coeff(i, col);
276 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet>
277 struct product_coeff_vectorized_unroller
279 typedef typename Lhs::Index Index;
280 enum { PacketSize = packet_traits<typename Lhs::Scalar>::size };
281 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::PacketScalar &pres)
283 product_coeff_vectorized_unroller<UnrollingIndex-PacketSize, Lhs, Rhs, Packet>::run(row, col, lhs, rhs, pres);
284 pres = padd(pres, pmul( lhs.template packet<Aligned>(row, UnrollingIndex) , rhs.template packet<Aligned>(UnrollingIndex, col) ));
288 template<
typename Lhs,
typename Rhs,
typename Packet>
289 struct product_coeff_vectorized_unroller<0, Lhs, Rhs, Packet>
291 typedef typename Lhs::Index Index;
292 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::PacketScalar &pres)
294 pres = pmul(lhs.template packet<Aligned>(row, 0) , rhs.template packet<Aligned>(0, col));
298 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename RetScalar>
299 struct product_coeff_impl<InnerVectorizedTraversal, UnrollingIndex, Lhs, Rhs, RetScalar>
301 typedef typename Lhs::PacketScalar Packet;
302 typedef typename Lhs::Index Index;
303 enum { PacketSize = packet_traits<typename Lhs::Scalar>::size };
304 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, RetScalar &res)
307 product_coeff_vectorized_unroller<UnrollingIndex+1-PacketSize, Lhs, Rhs, Packet>::run(row, col, lhs, rhs, pres);
308 product_coeff_impl<DefaultTraversal,UnrollingIndex,Lhs,Rhs,RetScalar>::run(row, col, lhs, rhs, res);
313 template<
typename Lhs,
typename Rhs,
int LhsRows = Lhs::RowsAtCompileTime,
int RhsCols = Rhs::ColsAtCompileTime>
314 struct product_coeff_vectorized_dyn_selector
316 typedef typename Lhs::Index Index;
317 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
319 res = lhs.row(row).transpose().cwiseProduct(rhs.col(col)).sum();
325 template<
typename Lhs,
typename Rhs,
int RhsCols>
326 struct product_coeff_vectorized_dyn_selector<Lhs,Rhs,1,RhsCols>
328 typedef typename Lhs::Index Index;
329 static EIGEN_STRONG_INLINE
void run(Index , Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
331 res = lhs.transpose().cwiseProduct(rhs.col(col)).sum();
335 template<
typename Lhs,
typename Rhs,
int LhsRows>
336 struct product_coeff_vectorized_dyn_selector<Lhs,Rhs,LhsRows,1>
338 typedef typename Lhs::Index Index;
339 static EIGEN_STRONG_INLINE
void run(Index row, Index ,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
341 res = lhs.row(row).transpose().cwiseProduct(rhs).sum();
345 template<
typename Lhs,
typename Rhs>
346 struct product_coeff_vectorized_dyn_selector<Lhs,Rhs,1,1>
348 typedef typename Lhs::Index Index;
349 static EIGEN_STRONG_INLINE
void run(Index , Index ,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
351 res = lhs.transpose().cwiseProduct(rhs).sum();
355 template<
typename Lhs,
typename Rhs,
typename RetScalar>
356 struct product_coeff_impl<InnerVectorizedTraversal,
Dynamic, Lhs, Rhs, RetScalar>
358 typedef typename Lhs::Index Index;
359 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
361 product_coeff_vectorized_dyn_selector<Lhs,Rhs>::run(row, col, lhs, rhs, res);
369 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
370 struct product_packet_impl<
RowMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode>
372 typedef typename Lhs::Index Index;
373 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
375 product_packet_impl<RowMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, res);
376 res = pmadd(pset1<Packet>(lhs.coeff(row, UnrollingIndex)), rhs.template packet<LoadMode>(UnrollingIndex, col), res);
380 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
381 struct product_packet_impl<
ColMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode>
383 typedef typename Lhs::Index Index;
384 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
386 product_packet_impl<ColMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, res);
387 res = pmadd(lhs.template packet<LoadMode>(row, UnrollingIndex), pset1<Packet>(rhs.coeff(UnrollingIndex, col)), res);
391 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
392 struct product_packet_impl<
RowMajor, 0, Lhs, Rhs, Packet, LoadMode>
394 typedef typename Lhs::Index Index;
395 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
397 res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode>(0, col));
401 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
402 struct product_packet_impl<
ColMajor, 0, Lhs, Rhs, Packet, LoadMode>
404 typedef typename Lhs::Index Index;
405 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
407 res = pmul(lhs.template packet<LoadMode>(row, 0), pset1<Packet>(rhs.coeff(0, col)));
411 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
412 struct product_packet_impl<
RowMajor,
Dynamic, Lhs, Rhs, Packet, LoadMode>
414 typedef typename Lhs::Index Index;
415 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet& res)
417 eigen_assert(lhs.cols()>0 &&
"you are using a non initialized matrix");
418 res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode>(0, col));
419 for(Index i = 1; i < lhs.cols(); ++i)
420 res = pmadd(pset1<Packet>(lhs.coeff(row, i)), rhs.template packet<LoadMode>(i, col), res);
424 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
425 struct product_packet_impl<
ColMajor,
Dynamic, Lhs, Rhs, Packet, LoadMode>
427 typedef typename Lhs::Index Index;
428 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet& res)
430 eigen_assert(lhs.cols()>0 &&
"you are using a non initialized matrix");
431 res = pmul(lhs.template packet<LoadMode>(row, 0), pset1<Packet>(rhs.coeff(0, col)));
432 for(Index i = 1; i < lhs.cols(); ++i)
433 res = pmadd(lhs.template packet<LoadMode>(row, i), pset1<Packet>(rhs.coeff(i, col)), res);
441 #endif // EIGEN_COEFFBASED_PRODUCT_H