13 #ifndef EIGEN_PRODUCTEVALUATORS_H
14 #define EIGEN_PRODUCTEVALUATORS_H
28 template<
typename Lhs,
typename Rhs,
int Options>
29 struct evaluator<Product<Lhs, Rhs, Options> >
30 :
public product_evaluator<Product<Lhs, Rhs, Options> >
32 typedef Product<Lhs, Rhs, Options> XprType;
33 typedef product_evaluator<XprType> Base;
35 EIGEN_DEVICE_FUNC
explicit evaluator(
const XprType& xpr) : Base(xpr) {}
40 template<
typename Lhs,
typename Rhs,
typename Scalar>
41 struct evaluator_traits<CwiseUnaryOp<
internal::scalar_multiple_op<Scalar>, const Product<Lhs, Rhs, DefaultProduct> > >
42 : evaluator_traits_base<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const Product<Lhs, Rhs, DefaultProduct> > >
44 enum { AssumeAliasing = 1 };
46 template<
typename Lhs,
typename Rhs,
typename Scalar>
47 struct evaluator<CwiseUnaryOp<
internal::scalar_multiple_op<Scalar>, const Product<Lhs, Rhs, DefaultProduct> > >
48 :
public evaluator<Product<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>,const Lhs>, Rhs, DefaultProduct> >
50 typedef CwiseUnaryOp<internal::scalar_multiple_op<Scalar>,
const Product<Lhs, Rhs, DefaultProduct> > XprType;
51 typedef evaluator<Product<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>,
const Lhs>, Rhs, DefaultProduct> > Base;
53 EIGEN_DEVICE_FUNC
explicit evaluator(
const XprType& xpr)
54 : Base(xpr.functor().m_other * xpr.nestedExpression().lhs() * xpr.nestedExpression().rhs())
59 template<
typename Lhs,
typename Rhs,
int DiagIndex>
60 struct evaluator<Diagonal<const Product<Lhs, Rhs, DefaultProduct>, DiagIndex> >
61 :
public evaluator<Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex> >
63 typedef Diagonal<const Product<Lhs, Rhs, DefaultProduct>, DiagIndex> XprType;
64 typedef evaluator<Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex> > Base;
66 EIGEN_DEVICE_FUNC
explicit evaluator(
const XprType& xpr)
67 : Base(Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex>(
68 Product<Lhs, Rhs, LazyProduct>(xpr.nestedExpression().lhs(), xpr.nestedExpression().rhs()),
77 template<
typename Lhs,
typename Rhs,
78 typename LhsShape =
typename evaluator_traits<Lhs>::Shape,
79 typename RhsShape =
typename evaluator_traits<Rhs>::Shape,
80 int ProductType = internal::product_type<Lhs,Rhs>::value>
81 struct generic_product_impl;
83 template<
typename Lhs,
typename Rhs>
84 struct evaluator_traits<Product<Lhs, Rhs, DefaultProduct> >
85 : evaluator_traits_base<Product<Lhs, Rhs, DefaultProduct> >
87 enum { AssumeAliasing = 1 };
90 template<
typename Lhs,
typename Rhs>
91 struct evaluator_traits<Product<Lhs, Rhs, AliasFreeProduct> >
92 : evaluator_traits_base<Product<Lhs, Rhs, AliasFreeProduct> >
94 enum { AssumeAliasing = 0 };
99 template<
typename Lhs,
typename Rhs,
int Options,
int ProductTag,
typename LhsShape,
typename RhsShape>
100 struct product_evaluator<Product<Lhs, Rhs, Options>, ProductTag, LhsShape, RhsShape>
101 :
public evaluator<typename Product<Lhs, Rhs, Options>::PlainObject>
103 typedef Product<Lhs, Rhs, Options> XprType;
104 typedef typename XprType::PlainObject PlainObject;
105 typedef evaluator<PlainObject> Base;
110 EIGEN_DEVICE_FUNC
explicit product_evaluator(
const XprType& xpr)
111 : m_result(xpr.rows(), xpr.cols())
113 ::new (static_cast<Base*>(
this)) Base(m_result);
127 generic_product_impl<Lhs, Rhs, LhsShape, RhsShape, ProductTag>::evalTo(m_result, xpr.lhs(), xpr.rhs());
131 PlainObject m_result;
135 template< typename DstXprType, typename Lhs, typename Rhs,
int Options, typename Scalar>
136 struct Assignment<DstXprType, Product<Lhs,Rhs,Options>,
internal::assign_op<Scalar>, Dense2Dense,
137 typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct),Scalar>::type>
139 typedef Product<Lhs,Rhs,Options> SrcXprType;
140 static void run(DstXprType &dst,
const SrcXprType &src,
const internal::assign_op<Scalar> &)
143 generic_product_impl<Lhs, Rhs>::evalTo(dst, src.lhs(), src.rhs());
148 template<
typename DstXprType,
typename Lhs,
typename Rhs,
int Options,
typename Scalar>
149 struct Assignment<DstXprType, Product<Lhs,Rhs,Options>,
internal::add_assign_op<Scalar>, Dense2Dense,
150 typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct),Scalar>::type>
152 typedef Product<Lhs,Rhs,Options> SrcXprType;
153 static void run(DstXprType &dst,
const SrcXprType &src,
const internal::add_assign_op<Scalar> &)
156 generic_product_impl<Lhs, Rhs>::addTo(dst, src.lhs(), src.rhs());
161 template<
typename DstXprType,
typename Lhs,
typename Rhs,
int Options,
typename Scalar>
162 struct Assignment<DstXprType, Product<Lhs,Rhs,Options>,
internal::sub_assign_op<Scalar>, Dense2Dense,
163 typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct),Scalar>::type>
165 typedef Product<Lhs,Rhs,Options> SrcXprType;
166 static void run(DstXprType &dst,
const SrcXprType &src,
const internal::sub_assign_op<Scalar> &)
169 generic_product_impl<Lhs, Rhs>::subTo(dst, src.lhs(), src.rhs());
177 template<
typename DstXprType,
typename Lhs,
typename Rhs,
typename AssignFunc,
typename Scalar,
typename ScalarBis>
178 struct Assignment<DstXprType, CwiseUnaryOp<
internal::scalar_multiple_op<ScalarBis>,
179 const Product<Lhs,Rhs,DefaultProduct> >, AssignFunc, Dense2Dense, Scalar>
181 typedef CwiseUnaryOp<internal::scalar_multiple_op<ScalarBis>,
182 const Product<Lhs,Rhs,DefaultProduct> > SrcXprType;
183 static void run(DstXprType &dst,
const SrcXprType &src,
const AssignFunc& func)
185 call_assignment_no_alias(dst, (src.functor().m_other * src.nestedExpression().lhs())*src.nestedExpression().rhs(), func);
193 template<
typename DstXprType,
typename OtherXpr,
typename ProductType,
typename Scalar,
typename Func1,
typename Func2>
194 struct assignment_from_xpr_plus_product
196 typedef CwiseBinaryOp<internal::scalar_sum_op<Scalar>,
const OtherXpr,
const ProductType> SrcXprType;
197 static void run(DstXprType &dst,
const SrcXprType &src,
const Func1& func)
199 call_assignment_no_alias(dst, src.lhs(), func);
200 call_assignment_no_alias(dst, src.rhs(), Func2());
204 template<
typename DstXprType,
typename OtherXpr,
typename Lhs,
typename Rhs,
typename Scalar>
205 struct Assignment<DstXprType, CwiseBinaryOp<
internal::scalar_sum_op<Scalar>, const OtherXpr,
206 const Product<Lhs,Rhs,DefaultProduct> >, internal::assign_op<Scalar>, Dense2Dense>
207 : assignment_from_xpr_plus_product<DstXprType, OtherXpr, Product<Lhs,Rhs,DefaultProduct>, Scalar, internal::assign_op<Scalar>, internal::add_assign_op<Scalar> >
209 template<
typename DstXprType,
typename OtherXpr,
typename Lhs,
typename Rhs,
typename Scalar>
210 struct Assignment<DstXprType, CwiseBinaryOp<
internal::scalar_sum_op<Scalar>, const OtherXpr,
211 const Product<Lhs,Rhs,DefaultProduct> >, internal::add_assign_op<Scalar>, Dense2Dense>
212 : assignment_from_xpr_plus_product<DstXprType, OtherXpr, Product<Lhs,Rhs,DefaultProduct>, Scalar, internal::add_assign_op<Scalar>, internal::add_assign_op<Scalar> >
214 template<
typename DstXprType,
typename OtherXpr,
typename Lhs,
typename Rhs,
typename Scalar>
215 struct Assignment<DstXprType, CwiseBinaryOp<
internal::scalar_sum_op<Scalar>, const OtherXpr,
216 const Product<Lhs,Rhs,DefaultProduct> >, internal::sub_assign_op<Scalar>, Dense2Dense>
217 : assignment_from_xpr_plus_product<DstXprType, OtherXpr, Product<Lhs,Rhs,DefaultProduct>, Scalar, internal::sub_assign_op<Scalar>, internal::sub_assign_op<Scalar> >
221 template<
typename Lhs,
typename Rhs>
222 struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,InnerProduct>
224 template<
typename Dst>
225 static inline void evalTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
227 dst.coeffRef(0,0) = (lhs.transpose().cwiseProduct(rhs)).sum();
230 template<
typename Dst>
231 static inline void addTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
233 dst.coeffRef(0,0) += (lhs.transpose().cwiseProduct(rhs)).sum();
236 template<
typename Dst>
237 static void subTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
238 { dst.coeffRef(0,0) -= (lhs.transpose().cwiseProduct(rhs)).sum(); }
247 template<
typename Dst,
typename Lhs,
typename Rhs,
typename Func>
248 EIGEN_DONT_INLINE
void outer_product_selector_run(Dst& dst,
const Lhs &lhs,
const Rhs &rhs,
const Func& func,
const false_type&)
250 evaluator<Rhs> rhsEval(rhs);
251 typename nested_eval<Lhs,Rhs::SizeAtCompileTime>::type actual_lhs(lhs);
254 const Index cols = dst.cols();
255 for (Index j=0; j<cols; ++j)
256 func(dst.col(j), rhsEval.coeff(0,j) * actual_lhs);
260 template<
typename Dst,
typename Lhs,
typename Rhs,
typename Func>
261 EIGEN_DONT_INLINE
void outer_product_selector_run(Dst& dst,
const Lhs &lhs,
const Rhs &rhs,
const Func& func,
const true_type&)
263 evaluator<Lhs> lhsEval(lhs);
264 typename nested_eval<Rhs,Lhs::SizeAtCompileTime>::type actual_rhs(rhs);
267 const Index rows = dst.rows();
268 for (Index i=0; i<rows; ++i)
269 func(dst.row(i), lhsEval.coeff(i,0) * actual_rhs);
272 template<
typename Lhs,
typename Rhs>
273 struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,OuterProduct>
275 template<
typename T>
struct is_row_major : internal::conditional<(int(T::Flags)&RowMajorBit), internal::true_type, internal::false_type>::type {};
276 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
279 struct set {
template<
typename Dst,
typename Src>
void operator()(
const Dst& dst,
const Src& src)
const { dst.const_cast_derived() = src; } };
280 struct add {
template<
typename Dst,
typename Src>
void operator()(
const Dst& dst,
const Src& src)
const { dst.const_cast_derived() += src; } };
281 struct sub {
template<
typename Dst,
typename Src>
void operator()(
const Dst& dst,
const Src& src)
const { dst.const_cast_derived() -= src; } };
284 explicit adds(
const Scalar& s) : m_scale(s) {}
285 template<
typename Dst,
typename Src>
void operator()(
const Dst& dst,
const Src& src)
const {
286 dst.const_cast_derived() += m_scale * src;
290 template<
typename Dst>
291 static inline void evalTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
293 internal::outer_product_selector_run(dst, lhs, rhs, set(), is_row_major<Dst>());
296 template<
typename Dst>
297 static inline void addTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
299 internal::outer_product_selector_run(dst, lhs, rhs, add(), is_row_major<Dst>());
302 template<
typename Dst>
303 static inline void subTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
305 internal::outer_product_selector_run(dst, lhs, rhs, sub(), is_row_major<Dst>());
308 template<
typename Dst>
309 static inline void scaleAndAddTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
311 internal::outer_product_selector_run(dst, lhs, rhs, adds(alpha), is_row_major<Dst>());
318 template<
typename Lhs,
typename Rhs,
typename Derived>
319 struct generic_product_impl_base
321 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
323 template<
typename Dst>
324 static void evalTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
325 { dst.setZero(); scaleAndAddTo(dst, lhs, rhs, Scalar(1)); }
327 template<
typename Dst>
328 static void addTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
329 { scaleAndAddTo(dst,lhs, rhs, Scalar(1)); }
331 template<
typename Dst>
332 static void subTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
333 { scaleAndAddTo(dst, lhs, rhs, Scalar(-1)); }
335 template<
typename Dst>
336 static void scaleAndAddTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
337 { Derived::scaleAndAddTo(dst,lhs,rhs,alpha); }
341 template<
typename Lhs,
typename Rhs>
342 struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemvProduct>
343 : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemvProduct> >
345 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
347 typedef typename internal::conditional<int(Side)==OnTheRight,Lhs,Rhs>::type MatrixType;
349 template<
typename Dest>
350 static void scaleAndAddTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
352 internal::gemv_dense_selector<Side,
354 bool(internal::blas_traits<MatrixType>::HasUsableDirectAccess)
355 >::run(lhs, rhs, dst, alpha);
359 template<
typename Lhs,
typename Rhs>
360 struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,CoeffBasedProductMode>
362 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
364 template<
typename Dst>
365 static inline void evalTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
369 call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::assign_op<Scalar>());
372 template<
typename Dst>
373 static inline void addTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
376 call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::add_assign_op<Scalar>());
379 template<
typename Dst>
380 static inline void subTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
383 call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::sub_assign_op<Scalar>());
392 template<
typename Lhs,
typename Rhs>
393 struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,LazyCoeffBasedProductMode>
394 : generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,CoeffBasedProductMode> {};
402 template<
int Traversal,
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename RetScalar>
403 struct etor_product_coeff_impl;
405 template<
int StorageOrder,
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
406 struct etor_product_packet_impl;
408 template<
typename Lhs,
typename Rhs,
int ProductTag>
409 struct product_evaluator<Product<Lhs, Rhs, LazyProduct>, ProductTag, DenseShape, DenseShape>
410 : evaluator_base<Product<Lhs, Rhs, LazyProduct> >
412 typedef Product<Lhs, Rhs, LazyProduct> XprType;
413 typedef typename XprType::Scalar Scalar;
414 typedef typename XprType::CoeffReturnType CoeffReturnType;
415 typedef typename XprType::PacketScalar PacketScalar;
416 typedef typename XprType::PacketReturnType PacketReturnType;
418 EIGEN_DEVICE_FUNC
explicit product_evaluator(
const XprType& xpr)
424 m_innerDim(xpr.lhs().cols())
426 EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<Scalar>::MulCost);
427 EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<Scalar>::AddCost);
428 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
433 typedef typename internal::nested_eval<Lhs,Rhs::ColsAtCompileTime>::type LhsNested;
434 typedef typename internal::nested_eval<Rhs,Lhs::RowsAtCompileTime>::type RhsNested;
436 typedef typename internal::remove_all<LhsNested>::type LhsNestedCleaned;
437 typedef typename internal::remove_all<RhsNested>::type RhsNestedCleaned;
439 typedef evaluator<LhsNestedCleaned> LhsEtorType;
440 typedef evaluator<RhsNestedCleaned> RhsEtorType;
443 RowsAtCompileTime = LhsNestedCleaned::RowsAtCompileTime,
444 ColsAtCompileTime = RhsNestedCleaned::ColsAtCompileTime,
445 InnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(LhsNestedCleaned::ColsAtCompileTime, RhsNestedCleaned::RowsAtCompileTime),
446 MaxRowsAtCompileTime = LhsNestedCleaned::MaxRowsAtCompileTime,
447 MaxColsAtCompileTime = RhsNestedCleaned::MaxColsAtCompileTime,
449 PacketSize = packet_traits<Scalar>::size,
451 LhsCoeffReadCost = LhsEtorType::CoeffReadCost,
452 RhsCoeffReadCost = RhsEtorType::CoeffReadCost,
453 CoeffReadCost = InnerSize==0 ? NumTraits<Scalar>::ReadCost
454 : InnerSize == Dynamic ? HugeCost
455 : InnerSize * (NumTraits<Scalar>::MulCost + LhsCoeffReadCost + RhsCoeffReadCost)
456 + (InnerSize - 1) * NumTraits<Scalar>::AddCost,
458 Unroll = CoeffReadCost <= EIGEN_UNROLLING_LIMIT,
460 LhsFlags = LhsEtorType::Flags,
461 RhsFlags = RhsEtorType::Flags,
463 LhsAlignment = LhsEtorType::Alignment,
464 RhsAlignment = RhsEtorType::Alignment,
469 SameType = is_same<typename LhsNestedCleaned::Scalar,typename RhsNestedCleaned::Scalar>::value,
472 && (ColsAtCompileTime == Dynamic || ((ColsAtCompileTime % PacketSize) == 0) ),
475 && (RowsAtCompileTime == Dynamic || ((RowsAtCompileTime % PacketSize) == 0) ),
477 EvalToRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1
478 : (MaxColsAtCompileTime==1&&MaxRowsAtCompileTime!=1) ? 0
479 : (RhsRowMajor && !CanVectorizeLhs),
481 Flags = ((
unsigned int)(LhsFlags | RhsFlags) & HereditaryBits & ~RowMajorBit)
482 | (EvalToRowMajor ? RowMajorBit : 0)
484 | (SameType && (CanVectorizeLhs || CanVectorizeRhs) ? PacketAccessBit : 0)
487 LhsOuterStrideBytes =
int(LhsNestedCleaned::OuterStrideAtCompileTime) * int(
sizeof(
typename LhsNestedCleaned::Scalar)),
488 RhsOuterStrideBytes = int(RhsNestedCleaned::OuterStrideAtCompileTime) * int(
sizeof(
typename RhsNestedCleaned::Scalar)),
490 Alignment = CanVectorizeLhs ? (LhsOuterStrideBytes<0 || (int(LhsOuterStrideBytes) % EIGEN_PLAIN_ENUM_MAX(1,LhsAlignment))!=0 ? 0 : LhsAlignment)
491 : CanVectorizeRhs ? (RhsOuterStrideBytes<0 || (int(RhsOuterStrideBytes) % EIGEN_PLAIN_ENUM_MAX(1,RhsAlignment))!=0 ? 0 : RhsAlignment)
499 CanVectorizeInner = SameType
503 && (InnerSize % packet_traits<Scalar>::size == 0)
506 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const CoeffReturnType coeff(Index row, Index col)
const
508 return (m_lhs.row(row).transpose().cwiseProduct( m_rhs.col(col) )).sum();
515 EIGEN_DEVICE_FUNC
const CoeffReturnType coeff(Index index)
const
517 const Index row = RowsAtCompileTime == 1 ? 0 : index;
518 const Index col = RowsAtCompileTime == 1 ? index : 0;
519 return (m_lhs.row(row).transpose().cwiseProduct( m_rhs.col(col) )).sum();
522 template<
int LoadMode,
typename PacketType>
523 const PacketType packet(Index row, Index col)
const
526 typedef etor_product_packet_impl<bool(
int(Flags)&RowMajorBit) ?
RowMajor :
ColMajor,
527 Unroll ? int(InnerSize) : Dynamic,
528 LhsEtorType, RhsEtorType, PacketType, LoadMode> PacketImpl;
529 PacketImpl::run(row, col, m_lhsImpl, m_rhsImpl, m_innerDim, res);
533 template<
int LoadMode,
typename PacketType>
534 const PacketType packet(Index index)
const
536 const Index row = RowsAtCompileTime == 1 ? 0 : index;
537 const Index col = RowsAtCompileTime == 1 ? index : 0;
538 return packet<LoadMode,PacketType>(row,col);
542 const LhsNested m_lhs;
543 const RhsNested m_rhs;
545 LhsEtorType m_lhsImpl;
546 RhsEtorType m_rhsImpl;
552 template<
typename Lhs,
typename Rhs>
553 struct product_evaluator<Product<Lhs, Rhs, DefaultProduct>, LazyCoeffBasedProductMode, DenseShape, DenseShape>
554 : product_evaluator<Product<Lhs, Rhs, LazyProduct>, CoeffBasedProductMode, DenseShape, DenseShape>
556 typedef Product<Lhs, Rhs, DefaultProduct> XprType;
557 typedef Product<Lhs, Rhs, LazyProduct> BaseProduct;
558 typedef product_evaluator<BaseProduct, CoeffBasedProductMode, DenseShape, DenseShape> Base;
562 EIGEN_DEVICE_FUNC
explicit product_evaluator(
const XprType& xpr)
563 : Base(BaseProduct(xpr.lhs(),xpr.rhs()))
571 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
572 struct etor_product_packet_impl<
RowMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode>
574 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Index innerDim, Packet &res)
576 etor_product_packet_impl<RowMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res);
577 res = pmadd(pset1<Packet>(lhs.coeff(row, UnrollingIndex-1)), rhs.template packet<LoadMode,Packet>(UnrollingIndex-1, col), res);
581 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
582 struct etor_product_packet_impl<
ColMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode>
584 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Index innerDim, Packet &res)
586 etor_product_packet_impl<ColMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res);
587 res = pmadd(lhs.template packet<LoadMode,Packet>(row, UnrollingIndex-1), pset1<Packet>(rhs.coeff(UnrollingIndex-1, col)), res);
591 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
592 struct etor_product_packet_impl<
RowMajor, 1, Lhs, Rhs, Packet, LoadMode>
594 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Index , Packet &res)
596 res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode,Packet>(0, col));
600 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
601 struct etor_product_packet_impl<
ColMajor, 1, Lhs, Rhs, Packet, LoadMode>
603 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Index , Packet &res)
605 res = pmul(lhs.template packet<LoadMode,Packet>(row, 0), pset1<Packet>(rhs.coeff(0, col)));
609 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
610 struct etor_product_packet_impl<
RowMajor, 0, Lhs, Rhs, Packet, LoadMode>
612 static EIGEN_STRONG_INLINE
void run(Index , Index ,
const Lhs& ,
const Rhs& , Index , Packet &res)
614 res = pset1<Packet>(0);
618 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
619 struct etor_product_packet_impl<
ColMajor, 0, Lhs, Rhs, Packet, LoadMode>
621 static EIGEN_STRONG_INLINE
void run(Index , Index ,
const Lhs& ,
const Rhs& , Index , Packet &res)
623 res = pset1<Packet>(0);
627 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
628 struct etor_product_packet_impl<
RowMajor, Dynamic, Lhs, Rhs, Packet, LoadMode>
630 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Index innerDim, Packet& res)
632 res = pset1<Packet>(0);
633 for(Index i = 0; i < innerDim; ++i)
634 res = pmadd(pset1<Packet>(lhs.coeff(row, i)), rhs.template packet<LoadMode,Packet>(i, col), res);
638 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
639 struct etor_product_packet_impl<
ColMajor, Dynamic, Lhs, Rhs, Packet, LoadMode>
641 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Index innerDim, Packet& res)
643 res = pset1<Packet>(0);
644 for(Index i = 0; i < innerDim; ++i)
645 res = pmadd(lhs.template packet<LoadMode,Packet>(row, i), pset1<Packet>(rhs.coeff(i, col)), res);
653 template<
int Mode,
bool LhsIsTriangular,
654 typename Lhs,
bool LhsIsVector,
655 typename Rhs,
bool RhsIsVector>
656 struct triangular_product_impl;
658 template<
typename Lhs,
typename Rhs,
int ProductTag>
659 struct generic_product_impl<Lhs,Rhs,TriangularShape,DenseShape,ProductTag>
660 : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,TriangularShape,DenseShape,ProductTag> >
662 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
664 template<
typename Dest>
665 static void scaleAndAddTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
667 triangular_product_impl<Lhs::Mode,true,typename Lhs::MatrixType,false,Rhs, Rhs::ColsAtCompileTime==1>
668 ::run(dst, lhs.nestedExpression(), rhs, alpha);
672 template<
typename Lhs,
typename Rhs,
int ProductTag>
673 struct generic_product_impl<Lhs,Rhs,DenseShape,TriangularShape,ProductTag>
674 : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,TriangularShape,ProductTag> >
676 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
678 template<
typename Dest>
679 static void scaleAndAddTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
681 triangular_product_impl<Rhs::Mode,false,Lhs,Lhs::RowsAtCompileTime==1, typename Rhs::MatrixType, false>::run(dst, lhs, rhs.nestedExpression(), alpha);
689 template <
typename Lhs,
int LhsMode,
bool LhsIsVector,
690 typename Rhs,
int RhsMode,
bool RhsIsVector>
691 struct selfadjoint_product_impl;
693 template<
typename Lhs,
typename Rhs,
int ProductTag>
694 struct generic_product_impl<Lhs,Rhs,SelfAdjointShape,DenseShape,ProductTag>
695 : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,SelfAdjointShape,DenseShape,ProductTag> >
697 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
699 template<
typename Dest>
700 static void scaleAndAddTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
702 selfadjoint_product_impl<typename Lhs::MatrixType,Lhs::Mode,false,Rhs,0,Rhs::IsVectorAtCompileTime>::run(dst, lhs.nestedExpression(), rhs, alpha);
706 template<
typename Lhs,
typename Rhs,
int ProductTag>
707 struct generic_product_impl<Lhs,Rhs,DenseShape,SelfAdjointShape,ProductTag>
708 : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,SelfAdjointShape,ProductTag> >
710 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
712 template<
typename Dest>
713 static void scaleAndAddTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
715 selfadjoint_product_impl<Lhs,0,Lhs::IsVectorAtCompileTime,typename Rhs::MatrixType,Rhs::Mode,false>::run(dst, lhs, rhs.nestedExpression(), alpha);
724 template<
typename MatrixType,
typename DiagonalType,
typename Derived,
int ProductOrder>
725 struct diagonal_product_evaluator_base
726 : evaluator_base<Derived>
728 typedef typename scalar_product_traits<typename MatrixType::Scalar, typename DiagonalType::Scalar>::ReturnType Scalar;
731 CoeffReadCost = NumTraits<Scalar>::MulCost + evaluator<MatrixType>::CoeffReadCost + evaluator<DiagonalType>::CoeffReadCost,
733 MatrixFlags = evaluator<MatrixType>::Flags,
734 DiagFlags = evaluator<DiagonalType>::Flags,
736 _ScalarAccessOnDiag = !((int(_StorageOrder) ==
ColMajor && int(ProductOrder) ==
OnTheLeft)
738 _SameTypes = is_same<typename MatrixType::Scalar, typename DiagonalType::Scalar>::value,
741 _Vectorizable = bool(
int(MatrixFlags)&PacketAccessBit) && _SameTypes && (_ScalarAccessOnDiag || (bool(
int(DiagFlags)&PacketAccessBit))),
742 _LinearAccessMask = (MatrixType::RowsAtCompileTime==1 || MatrixType::ColsAtCompileTime==1) ?
LinearAccessBit : 0,
743 Flags = ((HereditaryBits|_LinearAccessMask) & (
unsigned int)(MatrixFlags)) | (_Vectorizable ? PacketAccessBit : 0),
744 Alignment = evaluator<MatrixType>::Alignment
747 diagonal_product_evaluator_base(
const MatrixType &mat,
const DiagonalType &diag)
748 : m_diagImpl(diag), m_matImpl(mat)
750 EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<Scalar>::MulCost);
751 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
754 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Scalar coeff(Index idx)
const
756 return m_diagImpl.coeff(idx) * m_matImpl.coeff(idx);
760 template<
int LoadMode,
typename PacketType>
761 EIGEN_STRONG_INLINE PacketType packet_impl(Index row, Index col, Index
id, internal::true_type)
const
763 return internal::pmul(m_matImpl.template packet<LoadMode,PacketType>(row, col),
764 internal::pset1<PacketType>(m_diagImpl.coeff(
id)));
767 template<
int LoadMode,
typename PacketType>
768 EIGEN_STRONG_INLINE PacketType packet_impl(Index row, Index col, Index
id, internal::false_type)
const
771 InnerSize = (MatrixType::Flags &
RowMajorBit) ? MatrixType::ColsAtCompileTime : MatrixType::RowsAtCompileTime,
772 DiagonalPacketLoadMode = EIGEN_PLAIN_ENUM_MIN(LoadMode,((InnerSize%16) == 0) ? int(
Aligned16) : int(evaluator<DiagonalType>::Alignment))
774 return internal::pmul(m_matImpl.template packet<LoadMode,PacketType>(row, col),
775 m_diagImpl.template packet<DiagonalPacketLoadMode,PacketType>(
id));
778 evaluator<DiagonalType> m_diagImpl;
779 evaluator<MatrixType> m_matImpl;
783 template<
typename Lhs,
typename Rhs,
int ProductKind,
int ProductTag>
784 struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DiagonalShape, DenseShape>
785 : diagonal_product_evaluator_base<Rhs, typename Lhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>, OnTheLeft>
787 typedef diagonal_product_evaluator_base<Rhs, typename Lhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>,
OnTheLeft> Base;
788 using Base::m_diagImpl;
789 using Base::m_matImpl;
791 typedef typename Base::Scalar Scalar;
793 typedef Product<Lhs, Rhs, ProductKind> XprType;
794 typedef typename XprType::PlainObject PlainObject;
800 EIGEN_DEVICE_FUNC
explicit product_evaluator(
const XprType& xpr)
801 : Base(xpr.rhs(), xpr.lhs().diagonal())
805 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Scalar coeff(Index row, Index col)
const
807 return m_diagImpl.coeff(row) * m_matImpl.coeff(row, col);
811 template<
int LoadMode,
typename PacketType>
812 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col)
const
816 return this->
template packet_impl<LoadMode,PacketType>(row,col, row,
817 typename internal::conditional<int(StorageOrder)==RowMajor, internal::true_type, internal::false_type>::type());
820 template<
int LoadMode,
typename PacketType>
821 EIGEN_STRONG_INLINE PacketType packet(Index idx)
const
823 return packet<LoadMode,PacketType>(int(StorageOrder)==
ColMajor?idx:0,int(StorageOrder)==
ColMajor?0:idx);
829 template<
typename Lhs,
typename Rhs,
int ProductKind,
int ProductTag>
830 struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DenseShape, DiagonalShape>
831 : diagonal_product_evaluator_base<Lhs, typename Rhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>,
OnTheRight>
833 typedef diagonal_product_evaluator_base<Lhs, typename Rhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>,
OnTheRight> Base;
834 using Base::m_diagImpl;
835 using Base::m_matImpl;
837 typedef typename Base::Scalar Scalar;
839 typedef Product<Lhs, Rhs, ProductKind> XprType;
840 typedef typename XprType::PlainObject PlainObject;
842 enum { StorageOrder = int(Lhs::Flags) & RowMajorBit ?
RowMajor :
ColMajor };
844 EIGEN_DEVICE_FUNC
explicit product_evaluator(
const XprType& xpr)
845 : Base(xpr.lhs(), xpr.rhs().diagonal())
849 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Scalar coeff(Index row, Index col)
const
851 return m_matImpl.coeff(row, col) * m_diagImpl.coeff(col);
855 template<
int LoadMode,
typename PacketType>
856 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col)
const
858 return this->
template packet_impl<LoadMode,PacketType>(row,col, col,
859 typename internal::conditional<int(StorageOrder)==ColMajor, internal::true_type, internal::false_type>::type());
862 template<
int LoadMode,
typename PacketType>
863 EIGEN_STRONG_INLINE PacketType packet(Index idx)
const
865 return packet<LoadMode,PacketType>(int(StorageOrder)==
ColMajor?idx:0,int(StorageOrder)==
ColMajor?0:idx);
879 template<
typename ExpressionType,
int S
ide,
bool Transposed,
typename ExpressionShape>
880 struct permutation_matrix_product;
882 template<
typename ExpressionType,
int S
ide,
bool Transposed>
883 struct permutation_matrix_product<ExpressionType, Side, Transposed, DenseShape>
885 typedef typename nested_eval<ExpressionType, 1>::type MatrixType;
886 typedef typename remove_all<MatrixType>::type MatrixTypeCleaned;
888 template<
typename Dest,
typename PermutationType>
889 static inline void run(Dest& dst,
const PermutationType& perm,
const ExpressionType& xpr)
892 const Index n = Side==
OnTheLeft ? mat.rows() : mat.cols();
896 if(is_same_dense(dst, mat))
899 Matrix<bool,PermutationType::RowsAtCompileTime,1,0,PermutationType::MaxRowsAtCompileTime> mask(perm.size());
902 while(r < perm.size())
905 while(r<perm.size() && mask[r]) r++;
911 mask.coeffRef(k0) =
true;
912 for(Index k=perm.indices().coeff(k0); k!=k0; k=perm.indices().coeff(k))
914 Block<Dest, Side==OnTheLeft ? 1 : Dest::RowsAtCompileTime, Side==OnTheRight ? 1 : Dest::ColsAtCompileTime>(dst, k)
915 .swap(Block<Dest, Side==OnTheLeft ? 1 : Dest::RowsAtCompileTime, Side==OnTheRight ? 1 : Dest::ColsAtCompileTime>
916 (dst,((Side==
OnTheLeft) ^ Transposed) ? k0 : kPrev));
918 mask.coeffRef(k) =
true;
925 for(Index i = 0; i < n; ++i)
927 Block<Dest, Side==OnTheLeft ? 1 : Dest::RowsAtCompileTime, Side==OnTheRight ? 1 : Dest::ColsAtCompileTime>
928 (dst, ((Side==
OnTheLeft) ^ Transposed) ? perm.indices().coeff(i) : i)
932 Block<const MatrixTypeCleaned,Side==OnTheLeft ? 1 : MatrixTypeCleaned::RowsAtCompileTime,Side==OnTheRight ? 1 : MatrixTypeCleaned::ColsAtCompileTime>
933 (mat, ((Side==
OnTheRight) ^ Transposed) ? perm.indices().coeff(i) : i);
939 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
940 struct generic_product_impl<Lhs, Rhs, PermutationShape, MatrixShape, ProductTag>
942 template<
typename Dest>
943 static void evalTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs)
945 permutation_matrix_product<Rhs, OnTheLeft, false, MatrixShape>::run(dst, lhs, rhs);
949 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
950 struct generic_product_impl<Lhs, Rhs, MatrixShape, PermutationShape, ProductTag>
952 template<
typename Dest>
953 static void evalTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs)
955 permutation_matrix_product<Lhs, OnTheRight, false, MatrixShape>::run(dst, rhs, lhs);
959 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
960 struct generic_product_impl<Inverse<Lhs>, Rhs, PermutationShape, MatrixShape, ProductTag>
962 template<
typename Dest>
963 static void evalTo(Dest& dst,
const Inverse<Lhs>& lhs,
const Rhs& rhs)
965 permutation_matrix_product<Rhs, OnTheLeft, true, MatrixShape>::run(dst, lhs.nestedExpression(), rhs);
969 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
970 struct generic_product_impl<Lhs, Inverse<Rhs>, MatrixShape, PermutationShape, ProductTag>
972 template<
typename Dest>
973 static void evalTo(Dest& dst,
const Lhs& lhs,
const Inverse<Rhs>& rhs)
975 permutation_matrix_product<Lhs, OnTheRight, true, MatrixShape>::run(dst, rhs.nestedExpression(), lhs);
990 template<
typename ExpressionType,
int S
ide,
bool Transposed,
typename ExpressionShape>
991 struct transposition_matrix_product
993 typedef typename nested_eval<ExpressionType, 1>::type MatrixType;
994 typedef typename remove_all<MatrixType>::type MatrixTypeCleaned;
996 template<
typename Dest,
typename TranspositionType>
997 static inline void run(Dest& dst,
const TranspositionType& tr,
const ExpressionType& xpr)
1000 typedef typename TranspositionType::StorageIndex StorageIndex;
1001 const Index size = tr.size();
1004 if(!(is_same<MatrixTypeCleaned,Dest>::value && extract_data(dst) == extract_data(mat)))
1007 for(Index k=(Transposed?size-1:0) ; Transposed?k>=0:k<size ; Transposed?--k:++k)
1008 if(Index(j=tr.coeff(k))!=k)
1010 if(Side==
OnTheLeft) dst.row(k).swap(dst.row(j));
1011 else if(Side==
OnTheRight) dst.col(k).swap(dst.col(j));
1016 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
1017 struct generic_product_impl<Lhs, Rhs, TranspositionsShape, MatrixShape, ProductTag>
1019 template<
typename Dest>
1020 static void evalTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs)
1022 transposition_matrix_product<Rhs, OnTheLeft, false, MatrixShape>::run(dst, lhs, rhs);
1026 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
1027 struct generic_product_impl<Lhs, Rhs, MatrixShape, TranspositionsShape, ProductTag>
1029 template<
typename Dest>
1030 static void evalTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs)
1032 transposition_matrix_product<Lhs, OnTheRight, false, MatrixShape>::run(dst, rhs, lhs);
1037 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
1038 struct generic_product_impl<Transpose<Lhs>, Rhs, TranspositionsShape, MatrixShape, ProductTag>
1040 template<
typename Dest>
1041 static void evalTo(Dest& dst,
const Transpose<Lhs>& lhs,
const Rhs& rhs)
1043 transposition_matrix_product<Rhs, OnTheLeft, true, MatrixShape>::run(dst, lhs.nestedExpression(), rhs);
1047 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
1048 struct generic_product_impl<Lhs, Transpose<Rhs>, MatrixShape, TranspositionsShape, ProductTag>
1050 template<
typename Dest>
1051 static void evalTo(Dest& dst,
const Lhs& lhs,
const Transpose<Rhs>& rhs)
1053 transposition_matrix_product<Lhs, OnTheRight, true, MatrixShape>::run(dst, rhs.nestedExpression(), lhs);
1061 #endif // EIGEN_PRODUCT_EVALUATORS_H
Definition: Constants.h:320
Definition: Constants.h:230
const unsigned int RowMajorBit
Definition: Constants.h:61
const unsigned int PacketAccessBit
Definition: Constants.h:88
Definition: Constants.h:335
Definition: Constants.h:322
Definition: Eigen_Colamd.h:54
const unsigned int EvalBeforeNestingBit
Definition: Constants.h:65
const unsigned int ActualPacketAccessBit
Definition: Constants.h:99
const unsigned int LinearAccessBit
Definition: Constants.h:124
Definition: Constants.h:333