429template <
typename Func,
typename Evaluator>
430struct redux_impl<Func, Evaluator, LinearTraversal, CompleteUnrolling>
431 : redux_novec_linear_unroller<Func, Evaluator, 0, Evaluator::SizeAtCompileTime> {
432 using Base = redux_novec_linear_unroller<Func, Evaluator, 0, Evaluator::SizeAtCompileTime>;
433 using Scalar =
typename Evaluator::Scalar;
434 template <
typename XprType>
435 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Scalar run(
const Evaluator& eval,
const Func& func,
437 return Base::run(eval, func);
441template <
typename Func,
typename Evaluator>
442struct redux_impl<Func, Evaluator, LinearVectorizedTraversal, NoUnrolling> {
443 using Scalar =
typename Evaluator::Scalar;
444 using PacketScalar =
typename redux_traits<Func, Evaluator>::PacketType;
446 template <
typename XprType>
447 static Scalar run(
const Evaluator& eval,
const Func& func,
const XprType& xpr) {
448 const Index size = xpr.size();
450 constexpr Index packetSize = redux_traits<Func, Evaluator>::PacketSize;
451 constexpr int packetAlignment = unpacket_traits<PacketScalar>::alignment;
452 constexpr int alignment0 =
453 (bool(Evaluator::Flags & DirectAccessBit) && bool(packet_traits<Scalar>::AlignedOnScalar))
454 ?
int(packetAlignment)
456 constexpr int alignment = plain_enum_max(alignment0, Evaluator::Alignment);
457 const Index alignedStart = internal::first_aligned<alignment>(xpr);
458 const Index alignedSize4 = numext::round_down(size - alignedStart, 4 * packetSize);
459 const Index alignedSize = numext::round_down(size - alignedStart, packetSize);
460 const Index alignedEnd4 = alignedStart + alignedSize4;
461 const Index alignedEnd = alignedStart + alignedSize;
463 constexpr Index maxSize = redux_max_size<XprType>::Size;
464 EIGEN_IF_CONSTEXPR (maxSize == Dynamic || maxSize >= packetSize) {
466 PacketScalar packet_res0 = eval.template packet<alignment, PacketScalar>(alignedStart);
467 EIGEN_IF_CONSTEXPR (maxSize == Dynamic || maxSize >= 4 * packetSize) {
470 PacketScalar packet_res1 = eval.template packet<alignment, PacketScalar>(alignedStart + packetSize);
471 PacketScalar packet_res2 = eval.template packet<alignment, PacketScalar>(alignedStart + 2 * packetSize);
472 PacketScalar packet_res3 = eval.template packet<alignment, PacketScalar>(alignedStart + 3 * packetSize);
473 for (Index index = alignedStart + 4 * packetSize; index < alignedEnd4; index += 4 * packetSize) {
474 packet_res0 = func.packetOp(packet_res0, eval.template packet<alignment, PacketScalar>(index));
476 func.packetOp(packet_res1, eval.template packet<alignment, PacketScalar>(index + packetSize));
478 func.packetOp(packet_res2, eval.template packet<alignment, PacketScalar>(index + 2 * packetSize));
480 func.packetOp(packet_res3, eval.template packet<alignment, PacketScalar>(index + 3 * packetSize));
485 const Index remSize = alignedSize - alignedSize4;
486 if (remSize >= packetSize) {
487 packet_res0 = func.packetOp(packet_res0, eval.template packet<alignment, PacketScalar>(alignedEnd4));
488 if (remSize >= 2 * packetSize) {
490 func.packetOp(packet_res1, eval.template packet<alignment, PacketScalar>(alignedEnd4 + packetSize));
491 if (remSize == 3 * packetSize)
492 packet_res2 = func.packetOp(
493 packet_res2, eval.template packet<alignment, PacketScalar>(alignedEnd4 + 2 * packetSize));
498 packet_res0 = func.packetOp(packet_res0, packet_res1);
499 packet_res2 = func.packetOp(packet_res2, packet_res3);
500 packet_res0 = func.packetOp(packet_res0, packet_res2);
503 EIGEN_IF_CONSTEXPR (maxSize == Dynamic || maxSize >= 2 * packetSize) {
504 if (!alignedSize4 && alignedSize > packetSize) {
507 func.packetOp(packet_res0, eval.template packet<alignment, PacketScalar>(alignedStart + packetSize));
508 EIGEN_IF_CONSTEXPR (maxSize == Dynamic || maxSize >= 3 * packetSize) {
509 if (alignedSize > 2 * packetSize)
510 packet_res0 = func.packetOp(
511 packet_res0, eval.template packet<alignment, PacketScalar>(alignedStart + 2 * packetSize));
515 res = func.predux(packet_res0);
517 for (Index index = 0; index < alignedStart; ++index) res = func(res, eval.coeff(index));
519 for (Index index = alignedEnd; index < size; ++index) res = func(res, eval.coeff(index));
525 for (Index index = 1; index < size; ++index) res = func(res, eval.coeff(index));
531template <
typename Func,
typename Evaluator,
int Unrolling>
532struct redux_impl<Func, Evaluator, SliceVectorizedTraversal, Unrolling> {
533 using Scalar =
typename Evaluator::Scalar;
534 using PacketType =
typename redux_traits<Func, Evaluator>::PacketType;
536 static constexpr Index PacketSize = redux_traits<Func, Evaluator>::PacketSize;
538 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE PacketType packetAt(
const Evaluator& eval, Index j, Index i) {
539 return eval.template packetByOuterInner<Unaligned, PacketType>(j, i);
542 template <
typename XprType>
543 EIGEN_DEVICE_FUNC
static Scalar run(
const Evaluator& eval,
const Func& func,
const XprType& xpr) {
544 EIGEN_INTERNAL_ASSERT_NONEMPTY(xpr);
545 const Index innerSize = xpr.innerSize();
546 const Index outerSize = xpr.outerSize();
547 const Index packetedInnerSize = numext::round_down(innerSize, PacketSize);
548 const Index quadInnerSize = numext::round_down(innerSize, 4 * PacketSize);
550 if (packetedInnerSize) {
553 PacketType packet_res0 = packetAt(eval, 0, 0);
556 PacketType packet_res1 = packetAt(eval, 0, PacketSize);
557 PacketType packet_res2 = packetAt(eval, 0, 2 * PacketSize);
558 PacketType packet_res3 = packetAt(eval, 0, 3 * PacketSize);
561 const Index remSize = packetedInnerSize - quadInnerSize;
562 for (Index j = 0; j < outerSize; ++j) {
563 for (Index i = (j == 0) ? 4 * PacketSize : 0; i < quadInnerSize; i += 4 * PacketSize) {
564 packet_res0 = func.packetOp(packet_res0, packetAt(eval, j, i));
565 packet_res1 = func.packetOp(packet_res1, packetAt(eval, j, i + PacketSize));
566 packet_res2 = func.packetOp(packet_res2, packetAt(eval, j, i + 2 * PacketSize));
567 packet_res3 = func.packetOp(packet_res3, packetAt(eval, j, i + 3 * PacketSize));
571 if (remSize >= PacketSize) {
572 packet_res0 = func.packetOp(packet_res0, packetAt(eval, j, quadInnerSize));
573 if (remSize >= 2 * PacketSize) {
574 packet_res1 = func.packetOp(packet_res1, packetAt(eval, j, quadInnerSize + PacketSize));
575 if (remSize == 3 * PacketSize)
576 packet_res2 = func.packetOp(packet_res2, packetAt(eval, j, quadInnerSize + 2 * PacketSize));
581 packet_res0 = func.packetOp(packet_res0, packet_res1);
582 packet_res2 = func.packetOp(packet_res2, packet_res3);
583 packet_res0 = func.packetOp(packet_res0, packet_res2);
584 }
else if (outerSize >= 4) {
587 PacketType packet_res1 = packetAt(eval, 1, 0);
588 PacketType packet_res2 = packetAt(eval, 2, 0);
589 PacketType packet_res3 = packetAt(eval, 3, 0);
590 for (Index i = PacketSize; i < packetedInnerSize; i += PacketSize) {
591 packet_res0 = func.packetOp(packet_res0, packetAt(eval, 0, i));
592 packet_res1 = func.packetOp(packet_res1, packetAt(eval, 1, i));
593 packet_res2 = func.packetOp(packet_res2, packetAt(eval, 2, i));
594 packet_res3 = func.packetOp(packet_res3, packetAt(eval, 3, i));
598 const Index quadOuterSize = numext::round_down(outerSize, 4);
599 for (Index j = 4; j < quadOuterSize; j += 4)
600 for (Index i = 0; i < packetedInnerSize; i += PacketSize) {
601 packet_res0 = func.packetOp(packet_res0, packetAt(eval, j, i));
602 packet_res1 = func.packetOp(packet_res1, packetAt(eval, j + 1, i));
603 packet_res2 = func.packetOp(packet_res2, packetAt(eval, j + 2, i));
604 packet_res3 = func.packetOp(packet_res3, packetAt(eval, j + 3, i));
606 for (Index j = quadOuterSize; j < outerSize; ++j)
607 for (Index i = 0; i < packetedInnerSize; i += PacketSize)
608 packet_res0 = func.packetOp(packet_res0, packetAt(eval, j, i));
610 packet_res0 = func.packetOp(packet_res0, packet_res1);
611 packet_res2 = func.packetOp(packet_res2, packet_res3);
612 packet_res0 = func.packetOp(packet_res0, packet_res2);
615 for (Index j = 0; j < outerSize; ++j)
616 for (Index i = (j == 0 ? PacketSize : 0); i < packetedInnerSize; i += PacketSize)
617 packet_res0 = func.packetOp(packet_res0, packetAt(eval, j, i));
620 res = func.predux(packet_res0);
621 for (Index j = 0; j < outerSize; ++j)
622 for (Index i = packetedInnerSize; i < innerSize; ++i) res = func(res, eval.coeffByOuterInner(j, i));
626 res = redux_impl<Func, Evaluator, DefaultTraversal, NoUnrolling>::run(eval, func, xpr);
633template <
typename Func,
typename Evaluator>
634struct redux_impl<Func, Evaluator, LinearVectorizedTraversal, CompleteUnrolling> {
635 using Scalar =
typename Evaluator::Scalar;
637 using PacketType =
typename redux_traits<Func, Evaluator>::PacketType;
638 static constexpr Index PacketSize = redux_traits<Func, Evaluator>::PacketSize;
639 static constexpr Index Size = Evaluator::SizeAtCompileTime;
640 static constexpr Index VectorizedSize = (int(Size) / int(PacketSize)) *
int(PacketSize);
642 template <
typename XprType>
643 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Scalar run(
const Evaluator& eval,
const Func& func,
const XprType& xpr) {
644 EIGEN_ONLY_USED_FOR_DEBUG(xpr);
645 EIGEN_INTERNAL_ASSERT_NONEMPTY(xpr);
646 if (VectorizedSize > 0) {
647 Scalar res = func.predux(
648 redux_vec_linear_unroller<Func, Evaluator, 0, Size / PacketSize>::template run<PacketType>(eval, func));
649 if (VectorizedSize != Size)
651 res, redux_novec_linear_unroller<Func, Evaluator, VectorizedSize, Size - VectorizedSize>::run(eval, func));
654 return redux_novec_linear_unroller<Func, Evaluator, 0, Size>::run(eval, func);
660template <
typename XprType_>
661class redux_evaluator :
public internal::evaluator<const XprType_> {
662 using Base = internal::evaluator<const XprType_>;
665 using XprType = XprType_;
666 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
explicit redux_evaluator(
const XprType& xpr) : Base(xpr) {}
668 using Scalar =
typename XprType::Scalar;
669 using CoeffReturnType =
typename XprType::CoeffReturnType;
670 using PacketScalar =
typename XprType::PacketScalar;
673 MaxRowsAtCompileTime = XprType::MaxRowsAtCompileTime,
674 MaxColsAtCompileTime = XprType::MaxColsAtCompileTime,
677 Flags = Base::Flags & ~DirectAccessBit,
678 IsRowMajor = XprType::IsRowMajor,
679 SizeAtCompileTime = XprType::SizeAtCompileTime,
680 InnerSizeAtCompileTime = XprType::InnerSizeAtCompileTime
683 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeffByOuterInner(Index outer, Index inner)
const {
684 return Base::coeff(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer);
687 template <
int LoadMode,
typename PacketType>
688 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetByOuterInner(Index outer, Index inner)
const {
689 return Base::template packet<LoadMode, PacketType>(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer);
692 template <
int LoadMode,
typename PacketType>
693 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegmentByOuterInner(Index outer, Index inner, Index begin,
695 return Base::template packetSegment<LoadMode, PacketType>(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer,
707template <
typename Func,
typename Evaluator>
708struct redux_has_runtime_unit_stride_path {
709 using XprType =
typename Evaluator::XprType;
710 using Scalar =
typename Evaluator::Scalar;
711 static constexpr bool value = bool(traits<XprType>::Flags & DirectAccessBit) &&
712 bool(functor_traits<Func>::PacketAccess) && bool(packet_traits<Scalar>::Vectorizable) &&
713 (int(inner_stride_at_compile_time<XprType>::value) != 1);
716template <
typename Func,
typename Evaluator,
typename XprType,
717 bool = redux_has_runtime_unit_stride_path<Func, Evaluator>::value>
718struct redux_dispatch {
719 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename Evaluator::Scalar run(
const Evaluator& thisEval,
720 const Func& func,
const XprType& xpr) {
721 return redux_impl<Func, Evaluator>::run(thisEval, func, xpr);
730template <
typename Func,
typename Evaluator,
typename XprType>
731struct redux_dispatch<Func, Evaluator, XprType, true> {
732 using Scalar =
typename Evaluator::Scalar;
733 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(
const Evaluator& thisEval,
const Func& func,
734 const XprType& xpr) {
735 if (xpr.innerStride() == 1 && (xpr.outerSize() == 1 || xpr.outerStride() == xpr.innerSize())) {
736 using PlainVector = Matrix<Scalar, Dynamic, 1>;
737 using MapType = Map<const PlainVector, Evaluator::Alignment>;
738 MapType contiguous(xpr.data(), xpr.size());
739 redux_evaluator<MapType> mapEval(contiguous);
740 return redux_impl<Func, redux_evaluator<MapType>>::run(mapEval, func, contiguous);
742 return redux_impl<Func, Evaluator>::run(thisEval, func, xpr);
749template <
typename Derived,
typename Func,
typename Identity>
750EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
typename Derived::Scalar redux_with_identity_impl(
const DenseBase<Derived>&,
754 return typename Derived::Scalar(std::forward<Identity>(identity));
759template <
typename Derived,
typename Func,
typename Identity,
760 std::enable_if_t<(Derived::SizeAtCompileTime < 0), int> = 0>
761EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
typename Derived::Scalar redux_with_identity_impl(
const DenseBase<Derived>& xpr,
765 if (xpr.size() == 0)
return typename Derived::Scalar(std::forward<Identity>(identity));
766 return xpr.derived().redux(func);
769template <
typename Derived,
typename Func,
typename Identity,
770 std::enable_if_t<(Derived::SizeAtCompileTime >= 0),
int> = 0>
771EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
typename Derived::Scalar redux_with_identity_impl(
const DenseBase<Derived>& xpr,
772 const Func& func, Identity&&,
774 return xpr.derived().redux(func);
777template <
typename Derived,
typename Func,
typename Identity>
778EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
typename Derived::Scalar redux_with_identity(
const DenseBase<Derived>& xpr,
780 Identity&& identity) {
781 return redux_with_identity_impl(xpr, func, std::forward<Identity>(identity),
782 bool_constant<Derived::MaxSizeAtCompileTime == 0>());
803template <
typename Derived>
804template <
typename Func>
805EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar DenseBase<Derived>::redux(
806 const Func& func)
const {
807 EIGEN_INTERNAL_ASSERT_NONEMPTY(*
this);
809 using ThisEvaluator =
typename internal::redux_evaluator<Derived>;
810 ThisEvaluator thisEval(derived());
816 return internal::redux_dispatch<Func, ThisEvaluator, Derived>::run(thisEval, func, derived());
826template <
typename Derived>
827template <
int NaNPropagation>
829 return derived().redux(Eigen::internal::scalar_min_op<Scalar, Scalar, NaNPropagation>());
839template <
typename Derived>
840template <
int NaNPropagation>
842 return derived().redux(Eigen::internal::scalar_max_op<Scalar, Scalar, NaNPropagation>());