13#ifndef EIGEN_ASSIGN_EVALUATOR_H
14#define EIGEN_ASSIGN_EVALUATOR_H
17#include "./InternalHeaderCheck.h"
31template <
typename DstEvaluator,
typename SrcEvaluator,
typename AssignFunc,
int MaxPacketSize = Dynamic>
32struct copy_using_evaluator_traits {
33 using Src =
typename SrcEvaluator::XprType;
34 using Dst =
typename DstEvaluator::XprType;
35 using DstScalar =
typename Dst::Scalar;
37 static constexpr int DstFlags = DstEvaluator::Flags;
38 static constexpr int SrcFlags = SrcEvaluator::Flags;
41 static constexpr int DstAlignment = DstEvaluator::Alignment;
42 static constexpr int SrcAlignment = SrcEvaluator::Alignment;
43 static constexpr int JointAlignment = plain_enum_min(DstAlignment, SrcAlignment);
44 static constexpr bool DstHasDirectAccess = bool(DstFlags &
DirectAccessBit);
45 static constexpr bool SrcIsRowMajor = bool(SrcFlags &
RowMajorBit);
46 static constexpr bool DstIsRowMajor = bool(DstFlags &
RowMajorBit);
47 static constexpr bool IsVectorAtCompileTime = Dst::IsVectorAtCompileTime;
48 static constexpr int RowsAtCompileTime = size_prefer_fixed(Src::RowsAtCompileTime, Dst::RowsAtCompileTime);
49 static constexpr int ColsAtCompileTime = size_prefer_fixed(Src::ColsAtCompileTime, Dst::ColsAtCompileTime);
50 static constexpr int SizeAtCompileTime = size_at_compile_time(RowsAtCompileTime, ColsAtCompileTime);
51 static constexpr int MaxRowsAtCompileTime =
52 min_size_prefer_fixed(Src::MaxRowsAtCompileTime, Dst::MaxRowsAtCompileTime);
53 static constexpr int MaxColsAtCompileTime =
54 min_size_prefer_fixed(Src::MaxColsAtCompileTime, Dst::MaxColsAtCompileTime);
55 static constexpr int MaxSizeAtCompileTime =
56 min_size_prefer_fixed(Src::MaxSizeAtCompileTime, Dst::MaxSizeAtCompileTime);
57 static constexpr int InnerSizeAtCompileTime = IsVectorAtCompileTime ? SizeAtCompileTime
58 : DstIsRowMajor ? ColsAtCompileTime
60 static constexpr int MaxInnerSizeAtCompileTime = IsVectorAtCompileTime ? MaxSizeAtCompileTime
61 : DstIsRowMajor ? MaxColsAtCompileTime
62 : MaxRowsAtCompileTime;
63 static constexpr int RestrictedInnerSize = min_size_prefer_fixed(MaxInnerSizeAtCompileTime, MaxPacketSize);
64 static constexpr int RestrictedLinearSize = min_size_prefer_fixed(MaxSizeAtCompileTime, MaxPacketSize);
65 static constexpr int OuterStride = outer_stride_at_compile_time<Dst>::value;
77 using LinearPacketType =
typename find_assign_linear_packet<DstScalar, RestrictedLinearSize>::type;
78 using InnerPacketType =
typename find_best_packet<DstScalar, RestrictedInnerSize>::type;
80 static constexpr int LinearPacketSize = unpacket_traits<LinearPacketType>::size;
81 static constexpr int InnerPacketSize = unpacket_traits<InnerPacketType>::size;
86 static constexpr int LinearTraversalPacketSize =
87 unpacket_traits<typename find_best_packet<DstScalar, RestrictedLinearSize>::type>::size;
90 static constexpr int LinearRequiredAlignment = unpacket_traits<LinearPacketType>::alignment;
91 static constexpr int InnerRequiredAlignment = unpacket_traits<InnerPacketType>::alignment;
94 static constexpr bool StorageOrdersAgree = DstIsRowMajor == SrcIsRowMajor;
95 static constexpr bool MightVectorize = StorageOrdersAgree && bool(DstFlags & SrcFlags &
ActualPacketAccessBit) &&
96 bool(functor_traits<AssignFunc>::PacketAccess);
98 static constexpr bool MayInnerVectorize =
99 MightVectorize && (DstHasDirectAccess || std::is_same<AssignFunc, swap_assign_op<DstScalar>>::value) &&
100 (InnerSizeAtCompileTime != Dynamic) && (InnerSizeAtCompileTime % InnerPacketSize == 0) &&
101 (OuterStride != Dynamic) && (OuterStride % InnerPacketSize == 0) &&
102 (EIGEN_UNALIGNED_VECTORIZE || JointAlignment >= InnerRequiredAlignment);
103 static constexpr bool MayLinearize = StorageOrdersAgree && (DstFlags & SrcFlags &
LinearAccessBit);
104 static constexpr bool MayLinearVectorize =
105 MightVectorize && MayLinearize && DstHasDirectAccess &&
106 (EIGEN_UNALIGNED_VECTORIZE || (DstAlignment >= LinearRequiredAlignment) || MaxSizeAtCompileTime == Dynamic) &&
107 (MaxSizeAtCompileTime == Dynamic || MaxSizeAtCompileTime >= LinearPacketSize);
110 static constexpr int InnerSizeThreshold = (EIGEN_UNALIGNED_VECTORIZE ? 1 : 3) * InnerPacketSize;
111 static constexpr bool MaySliceVectorize =
112 MightVectorize && DstHasDirectAccess &&
113 (MaxInnerSizeAtCompileTime == Dynamic || MaxInnerSizeAtCompileTime >= InnerSizeThreshold);
120 static constexpr int Traversal = SizeAtCompileTime == 0 ? AllAtOnceTraversal
121 : (MayLinearVectorize && (LinearTraversalPacketSize > InnerPacketSize))
122 ? LinearVectorizedTraversal
123 : MayInnerVectorize ? InnerVectorizedTraversal
124 : MayLinearVectorize ? LinearVectorizedTraversal
125 : MaySliceVectorize ? SliceVectorizedTraversal
126 : MayLinearize ? LinearTraversal
128 static constexpr bool Vectorized = Traversal == InnerVectorizedTraversal || Traversal == LinearVectorizedTraversal ||
129 Traversal == SliceVectorizedTraversal;
131 using PacketType = std::conditional_t<Traversal == LinearVectorizedTraversal, LinearPacketType, InnerPacketType>;
134 static constexpr int ActualPacketSize = Vectorized ? unpacket_traits<PacketType>::size : 1;
135 static constexpr int UnrollingLimit = EIGEN_UNROLLING_LIMIT * ActualPacketSize;
136 static constexpr int CoeffReadCost = int(DstEvaluator::CoeffReadCost) + int(SrcEvaluator::CoeffReadCost);
137 static constexpr bool MayUnrollInner =
138 (InnerSizeAtCompileTime != Dynamic) && (InnerSizeAtCompileTime * CoeffReadCost <= UnrollingLimit);
143 static constexpr bool MayUnrollCompletely =
144 (SizeAtCompileTime != Dynamic) && (SizeAtCompileTime * CoeffReadCost <= UnrollingLimit);
145 static constexpr int Unrolling =
146 (Traversal == InnerVectorizedTraversal || Traversal == DefaultTraversal)
147 ? (MayUnrollCompletely ? CompleteUnrolling
148 : MayUnrollInner ? InnerUnrolling
150 : Traversal == LinearVectorizedTraversal
151 ? (MayUnrollCompletely && (EIGEN_UNALIGNED_VECTORIZE || (DstAlignment >= LinearRequiredAlignment))
154 : Traversal == LinearTraversal ? (MayUnrollCompletely ? CompleteUnrolling : NoUnrolling)
155#if EIGEN_UNALIGNED_VECTORIZE
156 : Traversal == SliceVectorizedTraversal ? (MayUnrollInner ? InnerUnrolling : NoUnrolling)
163 static constexpr bool UsePacketSegment =
164 has_packet_segment<PacketType>::value &&
165 !(MaxSizeAtCompileTime != Dynamic && MaxSizeAtCompileTime <= 4 * int(unpacket_traits<PacketType>::size) &&
166 unpacket_traits<PacketType>::size <= 8 && int(SrcEvaluator::CoeffReadCost) < HugeCost);
168#ifdef EIGEN_DEBUG_ASSIGN
169 static void debug() {
170 std::cerr <<
"DstXpr: " <<
typeid(
typename DstEvaluator::XprType).name() << std::endl;
171 std::cerr <<
"SrcXpr: " <<
typeid(
typename SrcEvaluator::XprType).name() << std::endl;
172 std::cerr.setf(std::ios::hex, std::ios::basefield);
173 std::cerr <<
"DstFlags"
174 <<
" = " << DstFlags <<
" (" << demangle_flags(DstFlags) <<
" )" << std::endl;
175 std::cerr <<
"SrcFlags"
176 <<
" = " << SrcFlags <<
" (" << demangle_flags(SrcFlags) <<
" )" << std::endl;
177 std::cerr.unsetf(std::ios::hex);
178 EIGEN_DEBUG_VAR(DstAlignment)
179 EIGEN_DEBUG_VAR(SrcAlignment)
180 EIGEN_DEBUG_VAR(LinearRequiredAlignment)
181 EIGEN_DEBUG_VAR(InnerRequiredAlignment)
182 EIGEN_DEBUG_VAR(JointAlignment)
183 EIGEN_DEBUG_VAR(InnerSizeAtCompileTime)
184 EIGEN_DEBUG_VAR(MaxInnerSizeAtCompileTime)
185 EIGEN_DEBUG_VAR(LinearPacketSize)
186 EIGEN_DEBUG_VAR(InnerPacketSize)
187 EIGEN_DEBUG_VAR(ActualPacketSize)
188 EIGEN_DEBUG_VAR(StorageOrdersAgree)
189 EIGEN_DEBUG_VAR(MightVectorize)
190 EIGEN_DEBUG_VAR(MayLinearize)
191 EIGEN_DEBUG_VAR(MayInnerVectorize)
192 EIGEN_DEBUG_VAR(MayLinearVectorize)
193 EIGEN_DEBUG_VAR(MaySliceVectorize)
194 std::cerr <<
"Traversal"
195 <<
" = " << Traversal <<
" (" << demangle_traversal(Traversal) <<
")" << std::endl;
196 EIGEN_DEBUG_VAR(SrcEvaluator::CoeffReadCost)
197 EIGEN_DEBUG_VAR(DstEvaluator::CoeffReadCost)
198 EIGEN_DEBUG_VAR(Dst::SizeAtCompileTime)
199 EIGEN_DEBUG_VAR(UnrollingLimit)
200 EIGEN_DEBUG_VAR(MayUnrollCompletely)
201 EIGEN_DEBUG_VAR(MayUnrollInner)
202 std::cerr <<
"Unrolling"
203 <<
" = " << Unrolling <<
" (" << demangle_unrolling(Unrolling) <<
")" << std::endl;
204 std::cerr << std::endl;
217template <
typename Kernel,
int Index_,
int Stop>
218struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling {
219 template <
int... Offsets>
220 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run_impl(Kernel& kernel,
221 std::integer_sequence<int, Offsets...>) {
223 0, (kernel.assignCoeffByOuterInner((Index_ + Offsets) / Kernel::AssignmentTraits::InnerSizeAtCompileTime,
224 (Index_ + Offsets) % Kernel::AssignmentTraits::InnerSizeAtCompileTime),
226 EIGEN_UNUSED_VARIABLE(unused);
229 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel) {
230 run_impl(kernel, std::make_integer_sequence<int, Stop - Index_>{});
234template <
typename Kernel,
int Index_,
int Stop>
235struct copy_using_evaluator_DefaultTraversal_InnerUnrolling {
236 template <
int... Offsets>
237 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run_impl(Kernel& kernel, Index outer,
238 std::integer_sequence<int, Offsets...>) {
239 int unused[] = {0, (kernel.assignCoeffByOuterInner(outer, Index_ + Offsets), 0)...};
240 EIGEN_UNUSED_VARIABLE(unused);
243 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel, Index outer) {
244 run_impl(kernel, outer, std::make_integer_sequence<int, Stop - Index_>{});
252template <
typename Kernel,
int Index_,
int Stop>
253struct copy_using_evaluator_LinearTraversal_CompleteUnrolling {
254 template <
int... Offsets>
255 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run_impl(Kernel& kernel,
256 std::integer_sequence<int, Offsets...>) {
257 int unused[] = {0, (kernel.assignCoeff(Index_ + Offsets), 0)...};
258 EIGEN_UNUSED_VARIABLE(unused);
261 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel) {
262 run_impl(kernel, std::make_integer_sequence<int, Stop - Index_>{});
270template <
typename Kernel,
int Index_,
int Stop>
271struct copy_using_evaluator_innervec_CompleteUnrolling {
272 using PacketType =
typename Kernel::PacketType;
273 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
274 static constexpr int SrcAlignment = Kernel::AssignmentTraits::SrcAlignment;
275 static constexpr int DstAlignment = Kernel::AssignmentTraits::DstAlignment;
276 static_assert((Stop - Index_) % PacketSize == 0,
"Packet unrolling range must be divisible by packet size.");
278 template <
int... Offsets>
279 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run_impl(Kernel& kernel, std::integer_sequence<int, Offsets...>) {
280 int unused[] = {0, (kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(
281 (Index_ + Offsets * PacketSize) / Kernel::AssignmentTraits::InnerSizeAtCompileTime,
282 (Index_ + Offsets * PacketSize) % Kernel::AssignmentTraits::InnerSizeAtCompileTime),
284 EIGEN_UNUSED_VARIABLE(unused);
287 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel& kernel) {
288 run_impl(kernel, std::make_integer_sequence<
int, (Stop - Index_) / PacketSize>{});
292template <
typename Kernel,
int Index_,
int Stop,
int SrcAlignment,
int DstAlignment>
293struct copy_using_evaluator_innervec_InnerUnrolling {
294 using PacketType =
typename Kernel::PacketType;
295 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
296 static_assert((Stop - Index_) % PacketSize == 0,
"Packet unrolling range must be divisible by packet size.");
298 template <
int... Offsets>
299 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run_impl(Kernel& kernel, Index outer,
300 std::integer_sequence<int, Offsets...>) {
301 int unused[] = {0, (kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(
302 outer, Index_ + Offsets * PacketSize),
304 EIGEN_UNUSED_VARIABLE(unused);
305 EIGEN_UNUSED_VARIABLE(outer);
308 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel& kernel, Index outer) {
309 run_impl(kernel, outer, std::make_integer_sequence<
int, (Stop - Index_) / PacketSize>{});
313template <
typename Kernel,
int Start,
int Stop,
int SrcAlignment,
int DstAlignment,
bool UsePacketSegment>
314struct copy_using_evaluator_innervec_segment {
315 using PacketType =
typename Kernel::PacketType;
317 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel& kernel, Index outer) {
318 kernel.template assignPacketSegmentByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, Start, 0,
323template <
typename Kernel,
int Start,
int Stop,
int SrcAlignment,
int DstAlignment>
324struct copy_using_evaluator_innervec_segment<Kernel, Start, Stop, SrcAlignment, DstAlignment,
326 : copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Start, Stop> {};
328template <
typename Kernel,
int Stop,
int SrcAlignment,
int DstAlignment>
329struct copy_using_evaluator_innervec_segment<Kernel, Stop, Stop, SrcAlignment, DstAlignment,
331 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel&, Index) {}
334template <
typename Kernel,
int Stop,
int SrcAlignment,
int DstAlignment>
335struct copy_using_evaluator_innervec_segment<Kernel, Stop, Stop, SrcAlignment, DstAlignment,
337 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel&, Index) {}
346template <
typename Kernel,
int Traversal = Kernel::AssignmentTraits::Traversal,
347 int Unrolling = Kernel::AssignmentTraits::Unrolling>
348struct dense_assignment_loop_impl;
350template <
typename Kernel,
int Traversal = Kernel::AssignmentTraits::Traversal,
351 int Unrolling = Kernel::AssignmentTraits::Unrolling>
352struct dense_assignment_loop {
354 EIGEN_DEVICE_FUNC
static EIGEN_ALWAYS_INLINE
constexpr void run(Kernel& kernel) {
355#ifdef __cpp_lib_is_constant_evaluated
356 if (internal::is_constant_evaluated())
357 dense_assignment_loop_impl<Kernel, Traversal == AllAtOnceTraversal ? AllAtOnceTraversal : DefaultTraversal,
358 NoUnrolling>::run(kernel);
361 dense_assignment_loop_impl<Kernel, Traversal, Unrolling>::run(kernel);
370template <
typename Kernel,
int Unrolling>
371struct dense_assignment_loop_impl<Kernel, AllAtOnceTraversal, Unrolling> {
372 static constexpr int SizeAtCompileTime = Kernel::AssignmentTraits::SizeAtCompileTime;
374 EIGEN_DEVICE_FUNC
static void EIGEN_STRONG_INLINE
constexpr run(Kernel& ) {
375 EIGEN_STATIC_ASSERT(SizeAtCompileTime == 0, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT)
383template <
typename Kernel>
384struct dense_assignment_loop_impl<Kernel, DefaultTraversal, NoUnrolling> {
385 EIGEN_DEVICE_FUNC
static void EIGEN_STRONG_INLINE
constexpr run(Kernel& kernel) {
386 for (Index outer = 0; outer < kernel.outerSize(); ++outer) {
387 for (Index inner = 0; inner < kernel.innerSize(); ++inner) {
388 kernel.assignCoeffByOuterInner(outer, inner);
394template <
typename Kernel>
395struct dense_assignment_loop_impl<Kernel, DefaultTraversal, CompleteUnrolling> {
396 static constexpr int SizeAtCompileTime = Kernel::AssignmentTraits::SizeAtCompileTime;
398 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel) {
399 copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, 0, SizeAtCompileTime>::run(kernel);
403template <
typename Kernel>
404struct dense_assignment_loop_impl<Kernel, DefaultTraversal, InnerUnrolling> {
405 static constexpr int InnerSizeAtCompileTime = Kernel::AssignmentTraits::InnerSizeAtCompileTime;
407 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel) {
408 const Index outerSize = kernel.outerSize();
409 for (Index outer = 0; outer < outerSize; ++outer)
410 copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, 0, InnerSizeAtCompileTime>::run(kernel, outer);
421template <
typename PacketType,
int DstAlignment,
int SrcAlignment,
bool UsePacketSegment,
bool Skip>
422struct unaligned_dense_assignment_loop {
424 template <
typename Kernel>
425 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& , Index , Index ) {}
426 template <
typename Kernel>
427 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& , Index ,
431template <
typename PacketType,
int DstAlignment,
int SrcAlignment>
432struct unaligned_dense_assignment_loop<PacketType, DstAlignment, SrcAlignment, true,
434 template <
typename Kernel>
435 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel, Index start, Index end) {
436 Index count = end - start;
437 eigen_assert(count <= unpacket_traits<PacketType>::size);
438 if (count > 0) kernel.template assignPacketSegment<DstAlignment, SrcAlignment, PacketType>(start, 0, count);
440 template <
typename Kernel>
441 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel, Index outer, Index start, Index end) {
442 Index count = end - start;
443 eigen_assert(count <= unpacket_traits<PacketType>::size);
445 kernel.template assignPacketSegmentByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, start, 0, count);
449template <
typename PacketType,
int DstAlignment,
int SrcAlignment>
450struct unaligned_dense_assignment_loop<PacketType, DstAlignment, SrcAlignment, false,
452 template <
typename Kernel>
453 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel, Index start, Index end) {
454 const Index count = end - start;
455 for (Index offset = 0; offset < count; ++offset) kernel.assignCoeff(start + offset);
457 template <
typename Kernel>
458 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel, Index outer, Index innerStart,
460 for (Index inner = innerStart; inner < innerEnd; ++inner) kernel.assignCoeffByOuterInner(outer, inner);
464template <
typename Kernel,
int Index_,
int Stop>
465struct copy_using_evaluator_linearvec_CompleteUnrolling {
466 using PacketType =
typename Kernel::PacketType;
467 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
468 static constexpr int SrcAlignment = Kernel::AssignmentTraits::SrcAlignment;
469 static constexpr int DstAlignment = Kernel::AssignmentTraits::DstAlignment;
470 static_assert((Stop - Index_) % PacketSize == 0,
"Packet unrolling range must be divisible by packet size.");
472 template <
int... Offsets>
473 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run_impl(Kernel& kernel, std::integer_sequence<int, Offsets...>) {
475 0, (kernel.template assignPacket<DstAlignment, SrcAlignment, PacketType>(Index_ + Offsets * PacketSize), 0)...};
476 EIGEN_UNUSED_VARIABLE(unused);
479 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel& kernel) {
480 run_impl(kernel, std::make_integer_sequence<
int, (Stop - Index_) / PacketSize>{});
484template <
typename Kernel,
int Index_,
int Stop,
bool UsePacketSegment>
485struct copy_using_evaluator_linearvec_segment {
486 using PacketType =
typename Kernel::PacketType;
487 static constexpr int SrcAlignment = Kernel::AssignmentTraits::SrcAlignment;
488 static constexpr int DstAlignment = Kernel::AssignmentTraits::DstAlignment;
490 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel& kernel) {
491 kernel.template assignPacketSegment<DstAlignment, SrcAlignment, PacketType>(Index_, 0, Stop - Index_);
495template <
typename Kernel,
int Index_,
int Stop>
496struct copy_using_evaluator_linearvec_segment<Kernel, Index_, Stop, false>
497 : copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Index_, Stop> {};
499template <
typename Kernel,
int Stop>
500struct copy_using_evaluator_linearvec_segment<Kernel, Stop, Stop, true> {
501 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel&) {}
504template <
typename Kernel,
int Stop>
505struct copy_using_evaluator_linearvec_segment<Kernel, Stop, Stop, false> {
506 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel&) {}
509template <
typename Kernel>
510struct dense_assignment_loop_impl<Kernel, LinearVectorizedTraversal, NoUnrolling> {
511 using Scalar =
typename Kernel::Scalar;
512 using PacketType =
typename Kernel::PacketType;
513 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
514 static constexpr int SrcAlignment = Kernel::AssignmentTraits::JointAlignment;
515 static constexpr int DstAlignment = plain_enum_max(Kernel::AssignmentTraits::DstAlignment,
alignof(Scalar));
516 static constexpr int RequestedAlignment = unpacket_traits<PacketType>::alignment;
517 static constexpr bool Alignable = (DstAlignment >= RequestedAlignment) ||
518 (
static_cast<std::size_t
>(RequestedAlignment - DstAlignment) %
sizeof(Scalar) == 0);
519 static constexpr int Alignment = Alignable ? RequestedAlignment : DstAlignment;
520 static constexpr bool DstIsAligned = DstAlignment >= Alignment;
521 static constexpr bool UsePacketSegment = Kernel::AssignmentTraits::UsePacketSegment;
524 unaligned_dense_assignment_loop<PacketType, DstAlignment, SrcAlignment, UsePacketSegment, DstIsAligned>;
525 using tail_loop = unaligned_dense_assignment_loop<PacketType, Alignment, SrcAlignment, UsePacketSegment, false>;
527 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel) {
531 constexpr int MaxSize = Kernel::AssignmentTraits::MaxSizeAtCompileTime;
532 constexpr bool ClampToMaxSize = has_packet_segment<PacketType>::value && !UsePacketSegment && MaxSize != Dynamic;
533 eigen_assert(MaxSize == Dynamic || kernel.size() <= MaxSize);
534 const Index size = ClampToMaxSize ? numext::mini(kernel.size(), Index(MaxSize)) : kernel.size();
535 const Index alignedStart = DstIsAligned ? 0 : first_aligned<Alignment>(kernel.dstDataPtr(), size);
537 head_loop::run(kernel, 0, alignedStart);
539#if EIGEN_GNUC_STRICT_AT_LEAST(8, 0, 0) && EIGEN_ARCH_i386_OR_x86_64 && !defined(EIGEN_GPUCC)
542 using SrcEvaluator =
typename Kernel::SrcEvaluatorType;
543 EIGEN_IF_CONSTEXPR (
int(SrcEvaluator::CoeffReadCost) > NumTraits<Scalar>::ReadCost &&
544 int(SrcEvaluator::CoeffReadCost) <= 2 * NumTraits<Scalar>::ReadCost) {
546 for (Index index = alignedStart; index <= size - PacketSize; index += PacketSize)
547 kernel.template assignPacket<Alignment, SrcAlignment, PacketType>(index);
551 for (Index index = alignedStart; index <= size - PacketSize; index += PacketSize)
552 kernel.template assignPacket<Alignment, SrcAlignment, PacketType>(index);
556 const Index alignedEnd = size - (size - alignedStart) % PacketSize;
557 tail_loop::run(kernel, alignedEnd, size);
561template <
typename Kernel>
562struct dense_assignment_loop_impl<Kernel, LinearVectorizedTraversal, CompleteUnrolling> {
563 using PacketType =
typename Kernel::PacketType;
564 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
565 static constexpr int Size = Kernel::AssignmentTraits::SizeAtCompileTime;
566 static constexpr int AlignedSize = numext::round_down(Size, PacketSize);
568 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel) {
569 copy_using_evaluator_linearvec_CompleteUnrolling<Kernel, 0, AlignedSize>::run(kernel);
579 copy_using_evaluator_linearvec_segment<Kernel, AlignedSize, Size,
false>::run(kernel);
587template <
typename Kernel>
588struct dense_assignment_loop_impl<Kernel, InnerVectorizedTraversal, NoUnrolling> {
589 using PacketType =
typename Kernel::PacketType;
590 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
591 static constexpr int SrcAlignment = Kernel::AssignmentTraits::JointAlignment;
592 static constexpr int DstAlignment = Kernel::AssignmentTraits::DstAlignment;
594 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel) {
595 const Index innerSize = kernel.innerSize();
596 const Index outerSize = kernel.outerSize();
597 for (Index outer = 0; outer < outerSize; ++outer)
598 for (Index inner = 0; inner < innerSize; inner += PacketSize)
599 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
603template <
typename Kernel>
604struct dense_assignment_loop_impl<Kernel, InnerVectorizedTraversal, CompleteUnrolling> {
605 static constexpr int SizeAtCompileTime = Kernel::AssignmentTraits::SizeAtCompileTime;
607 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel& kernel) {
608 copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, SizeAtCompileTime>::run(kernel);
612template <
typename Kernel>
613struct dense_assignment_loop_impl<Kernel, InnerVectorizedTraversal, InnerUnrolling> {
614 static constexpr int InnerSize = Kernel::AssignmentTraits::InnerSizeAtCompileTime;
615 static constexpr int SrcAlignment = Kernel::AssignmentTraits::SrcAlignment;
616 static constexpr int DstAlignment = Kernel::AssignmentTraits::DstAlignment;
618 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel& kernel) {
619 const Index outerSize = kernel.outerSize();
620 for (Index outer = 0; outer < outerSize; ++outer)
621 copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, InnerSize, SrcAlignment, DstAlignment>::run(kernel,
630template <
typename Kernel>
631struct dense_assignment_loop_impl<Kernel, LinearTraversal, NoUnrolling> {
632 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel) {
633 const Index size = kernel.size();
634 for (Index i = 0; i < size; ++i) kernel.assignCoeff(i);
638template <
typename Kernel>
639struct dense_assignment_loop_impl<Kernel, LinearTraversal, CompleteUnrolling> {
640 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel) {
641 copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, 0, Kernel::AssignmentTraits::SizeAtCompileTime>::run(
650template <
typename Kernel>
651struct dense_assignment_loop_impl<Kernel, SliceVectorizedTraversal, NoUnrolling> {
652 using Scalar =
typename Kernel::Scalar;
653 using PacketType =
typename Kernel::PacketType;
654 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
655 static constexpr int SrcAlignment = Kernel::AssignmentTraits::JointAlignment;
656 static constexpr int DstAlignment = plain_enum_max(Kernel::AssignmentTraits::DstAlignment,
alignof(Scalar));
657 static constexpr int RequestedAlignment = unpacket_traits<PacketType>::alignment;
658 static constexpr bool Alignable = (DstAlignment >= RequestedAlignment) ||
659 (
static_cast<std::size_t
>(RequestedAlignment - DstAlignment) %
sizeof(Scalar) == 0);
660 static constexpr int Alignment = Alignable ? RequestedAlignment : DstAlignment;
661 static constexpr bool DstIsAligned = DstAlignment >= Alignment;
662 static constexpr bool UsePacketSegment = Kernel::AssignmentTraits::UsePacketSegment;
664 using head_loop = unaligned_dense_assignment_loop<PacketType, DstAlignment, Unaligned, UsePacketSegment, !Alignable>;
665 using tail_loop = unaligned_dense_assignment_loop<PacketType, Alignment, Unaligned, UsePacketSegment, false>;
670 EIGEN_DEVICE_FUNC
static EIGEN_ALWAYS_INLINE
constexpr void runInvariant(Kernel& kernel, Index alignedStart,
671 Index innerSize, Index outerSize) {
672 const Index alignedEnd = alignedStart + numext::round_down(innerSize - alignedStart, PacketSize);
673 for (Index outer = 0; outer < outerSize; ++outer) {
674 head_loop::run(kernel, outer, 0, alignedStart);
677 for (Index inner = alignedStart; inner < alignedEnd; inner += PacketSize)
678 kernel.template assignPacketByOuterInner<Alignment, Unaligned, PacketType>(outer, inner);
680 tail_loop::run(kernel, outer, alignedEnd, innerSize);
684#if EIGEN_UNALIGNED_VECTORIZE
687 using unaligned_tail_loop =
688 unaligned_dense_assignment_loop<PacketType, Unaligned, Unaligned, UsePacketSegment, false>;
690 EIGEN_DEVICE_FUNC
static EIGEN_ALWAYS_INLINE
constexpr void runUnaligned(Kernel& kernel, Index innerSize,
692 const Index packetEnd = numext::round_down(innerSize, PacketSize);
693 for (Index outer = 0; outer < outerSize; ++outer) {
694 for (Index inner = 0; inner < packetEnd; inner += PacketSize)
695 kernel.template assignPacketByOuterInner<Unaligned, Unaligned, PacketType>(outer, inner);
697 unaligned_tail_loop::run(kernel, outer, packetEnd, innerSize);
705 EIGEN_DEVICE_FUNC
static EIGEN_ALWAYS_INLINE
constexpr void run(Kernel& kernel) {
706 const Index innerSize = kernel.innerSize();
707 const Index outerSize = kernel.outerSize();
708#if EIGEN_UNALIGNED_VECTORIZE
709 EIGEN_IF_CONSTEXPR (DstIsAligned) {
710 runUnaligned(kernel, innerSize, outerSize);
714 const Scalar* dst_ptr = kernel.dstDataPtr();
715 const Index alignedStep = Alignable ? (PacketSize - kernel.outerStride() % PacketSize) % PacketSize : 0;
716 Index alignedStart = ((!Alignable) || DstIsAligned) ? 0 : internal::first_aligned<Alignment>(dst_ptr, innerSize);
718 if (alignedStep == 0) {
719 runInvariant(kernel, alignedStart, innerSize, outerSize);
722#if EIGEN_UNALIGNED_VECTORIZE
723 runUnaligned(kernel, innerSize, outerSize);
725 for (Index outer = 0; outer < outerSize; ++outer) {
726 const Index alignedEnd = alignedStart + numext::round_down(innerSize - alignedStart, PacketSize);
728 head_loop::run(kernel, outer, 0, alignedStart);
731 for (Index inner = alignedStart; inner < alignedEnd; inner += PacketSize)
732 kernel.template assignPacketByOuterInner<Alignment, Unaligned, PacketType>(outer, inner);
734 tail_loop::run(kernel, outer, alignedEnd, innerSize);
736 alignedStart = numext::mini((alignedStart + alignedStep) % PacketSize, innerSize);
743#if EIGEN_UNALIGNED_VECTORIZE
744template <
typename Kernel>
745struct dense_assignment_loop_impl<Kernel, SliceVectorizedTraversal, InnerUnrolling> {
746 using PacketType =
typename Kernel::PacketType;
747 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
748 static constexpr int InnerSize = Kernel::AssignmentTraits::InnerSizeAtCompileTime;
749 static constexpr int VectorizableSize = numext::round_down(InnerSize, PacketSize);
756 static constexpr bool UsePacketSegment =
757 Kernel::AssignmentTraits::UsePacketSegment && !Kernel::AssignmentTraits::MayUnrollCompletely;
759 using packet_loop = copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, VectorizableSize, Unaligned, Unaligned>;
760 using packet_segment_loop = copy_using_evaluator_innervec_segment<Kernel, VectorizableSize, InnerSize,
Unaligned,
763 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(Kernel& kernel) {
764 for (Index outer = 0; outer < kernel.outerSize(); ++outer) {
765 packet_loop::run(kernel, outer);
766 packet_segment_loop::run(kernel, outer);
782template <
typename DstEvaluatorTypeT,
typename SrcEvaluatorTypeT,
typename Functor,
int Version = Specialized>
783class generic_dense_assignment_kernel {
785 using DstXprType =
typename DstEvaluatorTypeT::XprType;
786 using SrcXprType =
typename SrcEvaluatorTypeT::XprType;
789 using DstEvaluatorType = DstEvaluatorTypeT;
790 using SrcEvaluatorType = SrcEvaluatorTypeT;
791 using Scalar =
typename DstEvaluatorType::Scalar;
792 using AssignmentTraits = copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor>;
793 using PacketType =
typename AssignmentTraits::PacketType;
795 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr generic_dense_assignment_kernel(DstEvaluatorType& dst,
796 const SrcEvaluatorType& src,
799 : m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr) {
800#ifdef EIGEN_DEBUG_ASSIGN
801 AssignmentTraits::debug();
805 EIGEN_DEVICE_FUNC
constexpr Index size() const noexcept {
return m_dstExpr.size(); }
806 EIGEN_DEVICE_FUNC
constexpr Index innerSize() const noexcept {
return m_dstExpr.innerSize(); }
807 EIGEN_DEVICE_FUNC
constexpr Index outerSize() const noexcept {
return m_dstExpr.outerSize(); }
808 EIGEN_DEVICE_FUNC
constexpr Index rows() const noexcept {
return m_dstExpr.rows(); }
809 EIGEN_DEVICE_FUNC
constexpr Index cols() const noexcept {
return m_dstExpr.cols(); }
810 EIGEN_DEVICE_FUNC
constexpr Index outerStride() const noexcept {
return m_dstExpr.outerStride(); }
812 EIGEN_DEVICE_FUNC
constexpr DstEvaluatorType& dstEvaluator() noexcept {
return m_dst; }
813 EIGEN_DEVICE_FUNC
constexpr const SrcEvaluatorType& srcEvaluator() const noexcept {
return m_src; }
816 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void assignCoeff(Index row, Index col) {
817 m_functor.assignCoeff(m_dst.coeffRef(row, col), m_src.coeff(row, col));
821 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void assignCoeff(Index index) {
822 m_functor.assignCoeff(m_dst.coeffRef(index), m_src.coeff(index));
826 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void assignCoeffByOuterInner(Index outer, Index inner) {
827 Index row = rowIndexByOuterInner(outer, inner);
828 Index col = colIndexByOuterInner(outer, inner);
829 assignCoeff(row, col);
832 template <
int StoreMode,
int LoadMode,
typename Packet>
833 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacket(Index row, Index col) {
834 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(row, col),
835 m_src.template packet<LoadMode, Packet>(row, col));
838 template <
int StoreMode,
int LoadMode,
typename Packet>
839 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacket(Index index) {
840 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(index), m_src.template packet<LoadMode, Packet>(index));
843 template <
int StoreMode,
int LoadMode,
typename Packet>
844 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacketByOuterInner(Index outer, Index inner) {
845 Index row = rowIndexByOuterInner(outer, inner);
846 Index col = colIndexByOuterInner(outer, inner);
847 assignPacket<StoreMode, LoadMode, Packet>(row, col);
850 template <
int StoreMode,
int LoadMode,
typename Packet>
851 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacketSegment(Index row, Index col, Index begin, Index count) {
852 m_functor.template assignPacketSegment<StoreMode>(
853 &m_dst.coeffRef(row, col), m_src.template packetSegment<LoadMode, Packet>(row, col, begin, count), begin,
857 template <
int StoreMode,
int LoadMode,
typename Packet>
858 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacketSegment(Index index, Index begin, Index count) {
859 m_functor.template assignPacketSegment<StoreMode>(
860 &m_dst.coeffRef(index), m_src.template packetSegment<LoadMode, Packet>(index, begin, count), begin, count);
863 template <
int StoreMode,
int LoadMode,
typename Packet>
864 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacketSegmentByOuterInner(Index outer, Index inner, Index begin,
866 Index row = rowIndexByOuterInner(outer, inner);
867 Index col = colIndexByOuterInner(outer, inner);
868 assignPacketSegment<StoreMode, LoadMode, Packet>(row, col, begin, count);
871 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr Index rowIndexByOuterInner(Index outer, Index inner) {
872 using Traits =
typename DstEvaluatorType::ExpressionTraits;
873 return int(Traits::RowsAtCompileTime) == 1 ? 0
874 : int(Traits::ColsAtCompileTime) == 1 ? inner
875 : int(DstEvaluatorType::Flags) &
RowMajorBit ? outer
879 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr Index colIndexByOuterInner(Index outer, Index inner) {
880 using Traits =
typename DstEvaluatorType::ExpressionTraits;
881 return int(Traits::ColsAtCompileTime) == 1 ? 0
882 : int(Traits::RowsAtCompileTime) == 1 ? inner
883 : int(DstEvaluatorType::Flags) &
RowMajorBit ? inner
887 EIGEN_DEVICE_FUNC
const Scalar* dstDataPtr()
const {
return m_dstExpr.data(); }
890 DstEvaluatorType& m_dst;
891 const SrcEvaluatorType& m_src;
892 const Functor& m_functor;
894 DstXprType& m_dstExpr;
901template <
typename DstEvaluatorTypeT,
typename SrcEvaluatorTypeT,
typename Functor>
902class restricted_packet_dense_assignment_kernel
903 :
public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn> {
905 using Base = generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn>;
908 using Scalar =
typename Base::Scalar;
909 using DstXprType =
typename Base::DstXprType;
910 using AssignmentTraits = copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, 4>;
911 using PacketType =
typename AssignmentTraits::PacketType;
913 EIGEN_DEVICE_FUNC restricted_packet_dense_assignment_kernel(DstEvaluatorTypeT& dst,
const SrcEvaluatorTypeT& src,
914 const Functor& func, DstXprType& dstExpr)
915 : Base(dst, src, func, dstExpr) {}
922template <
typename DstXprType,
typename SrcXprType,
typename Functor>
923EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void resize_if_allowed(DstXprType& dst,
const SrcXprType& src,
925 EIGEN_INTERNAL_ASSERT_SAME_SHAPE(dst, src);
928template <
typename DstXprType,
typename SrcXprType,
typename T1,
typename T2>
929EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void resize_if_allowed(DstXprType& dst,
const SrcXprType& src,
930 const internal::assign_op<T1, T2>& ) {
931 Index dstRows = src.rows();
932 Index dstCols = src.cols();
933 if (((dst.rows() != dstRows) || (dst.cols() != dstCols))) {
934#ifdef EIGEN_NO_AUTOMATIC_RESIZING
935 eigen_assert(dst.size() == 0 &&
936 "Size mismatch. Automatic resizing is disabled because EIGEN_NO_AUTOMATIC_RESIZING is defined");
937 if (dst.size() == 0) {
938 dst.resize(dstRows, dstCols);
941 dst.resize(dstRows, dstCols);
942 EIGEN_INTERNAL_CHECK_SHAPE(dst.rows() == dstRows && dst.cols() == dstCols, YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES,
943 internal::matching_shapes<DstXprType, SrcXprType>);
951template <
typename DstEvaluatorType,
typename SrcXprType,
typename Enable =
void>
952struct dense_assignment_source_evaluator {
953 using type = evaluator<SrcXprType>;
956template <
typename DstXprType,
typename SrcXprType,
typename Functor>
957EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
constexpr void call_dense_assignment_loop(DstXprType& dst,
const SrcXprType& src,
958 const Functor& func) {
959 using DstEvaluatorType = evaluator<DstXprType>;
960 using SrcEvaluatorType =
typename dense_assignment_source_evaluator<DstEvaluatorType, SrcXprType>::type;
962 SrcEvaluatorType srcEvaluator(src);
966 resize_if_allowed(dst, src, func);
968 DstEvaluatorType dstEvaluator(dst);
970 using Kernel = generic_dense_assignment_kernel<DstEvaluatorType, SrcEvaluatorType, Functor>;
971 Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
973 dense_assignment_loop<Kernel>::run(kernel);
976template <
typename DstXprType,
typename SrcXprType>
977EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void call_dense_assignment_loop(DstXprType& dst,
const SrcXprType& src) {
978 call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>());
988template <
typename DstShape,
typename SrcShape>
989struct AssignmentKind;
992struct Dense2Dense {};
993struct EigenBase2EigenBase {};
995template <
typename,
typename>
996struct AssignmentKind {
997 using Kind = EigenBase2EigenBase;
1000struct AssignmentKind<DenseShape, DenseShape> {
1001 using Kind = Dense2Dense;
1005template <
typename DstXprType,
typename SrcXprType,
typename Functor,
1006 typename Kind =
typename AssignmentKind<typename evaluator_traits<DstXprType>::Shape,
1007 typename evaluator_traits<SrcXprType>::Shape>::Kind,
1008 typename EnableIf =
void>
1011template <
typename Src,
typename Func>
1012struct assignment_assume_aliasing : evaluator_assume_aliasing<Src> {};
1019template <
typename Dst,
typename Src>
1020EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void call_assignment(Dst& dst,
const Src& src) {
1021 call_assignment(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
1023template <
typename Dst,
typename Src>
1024EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_assignment(
const Dst& dst,
const Src& src) {
1025 call_assignment(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
1029template <
typename Dst,
typename Src,
typename Func,
1030 std::enable_if_t<assignment_assume_aliasing<Src, Func>::value,
int> = 0>
1031EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void call_assignment(Dst& dst,
const Src& src,
const Func& func) {
1032 typename plain_matrix_type<Src>::type tmp(src);
1033 call_assignment_no_alias(dst, tmp, func);
1036template <
typename Dst,
typename Src,
typename Func,
1037 std::enable_if_t<!assignment_assume_aliasing<Src, Func>::value,
int> = 0>
1038EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void call_assignment(Dst& dst,
const Src& src,
const Func& func) {
1039 call_assignment_no_alias(dst, src, func);
1044template <
typename Dst,
template <
typename>
class StorageBase,
typename Src,
typename Func>
1045EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void call_assignment(NoAlias<Dst, StorageBase>& dst,
const Src& src,
1047 call_assignment_no_alias(dst.expression(), src, func);
1050template <
typename Dst,
typename Src,
typename Func>
1051EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void call_assignment_no_alias(Dst& dst,
const Src& src,
1054 NeedToTranspose = ((int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1) ||
1055 (int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)) &&
1056 int(Dst::SizeAtCompileTime) != 1
1059 using ActualDstTypeCleaned = std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst>;
1060 using ActualDstType = std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst&>;
1061 ActualDstType actualDst(dst);
1064 EIGEN_STATIC_ASSERT_LVALUE(Dst)
1065 EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned, Src)
1066 EIGEN_CHECK_BINARY_COMPATIBILITY(Func,
typename ActualDstTypeCleaned::Scalar,
typename Src::Scalar);
1068 Assignment<ActualDstTypeCleaned, Src, Func>::run(actualDst, src, func);
1071template <
typename Dst,
typename Src,
typename Func>
1072EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void call_restricted_packet_assignment_no_alias(Dst& dst,
const Src& src,
1074 using DstEvaluatorType = evaluator<Dst>;
1075 using SrcEvaluatorType =
typename dense_assignment_source_evaluator<DstEvaluatorType, Src>::type;
1076 using Kernel = restricted_packet_dense_assignment_kernel<DstEvaluatorType, SrcEvaluatorType, Func>;
1078 EIGEN_STATIC_ASSERT_LVALUE(Dst)
1079 EIGEN_CHECK_BINARY_COMPATIBILITY(Func,
typename Dst::Scalar,
typename Src::Scalar);
1081 SrcEvaluatorType srcEvaluator(src);
1082 resize_if_allowed(dst, src, func);
1084 DstEvaluatorType dstEvaluator(dst);
1085 Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
1087 dense_assignment_loop<Kernel>::run(kernel);
1090template <
typename Dst,
typename Src>
1091EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void call_assignment_no_alias(Dst& dst,
const Src& src) {
1092 call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
1095template <
typename Dst,
typename Src,
typename Func>
1096EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void call_assignment_no_alias_no_transpose(Dst& dst,
const Src& src,
1099 EIGEN_STATIC_ASSERT_LVALUE(Dst)
1100 EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst, Src)
1101 EIGEN_CHECK_BINARY_COMPATIBILITY(Func,
typename Dst::Scalar,
typename Src::Scalar);
1103 Assignment<Dst, Src, Func>::run(dst, src, func);
1105template <
typename Dst,
typename Src>
1106EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
constexpr void call_assignment_no_alias_no_transpose(Dst& dst,
const Src& src) {
1107 call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
1111template <
typename Dst,
typename Src>
1112EIGEN_DEVICE_FUNC
void check_for_aliasing(
const Dst& dst,
const Src& src);
1117template <
typename DstXprType,
typename SrcXprType,
typename Functor,
typename Weak>
1118struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Weak> {
1119 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
constexpr void run(DstXprType& dst,
const SrcXprType& src,
1120 const Functor& func) {
1121#ifndef EIGEN_NO_DEBUG
1122 if (!internal::is_constant_evaluated()) {
1123 internal::check_for_aliasing(dst, src);
1127 call_dense_assignment_loop(dst, src, func);
1131template <
typename DstXprType,
typename SrcPlainObject,
typename Weak>
1132struct Assignment<DstXprType, CwiseNullaryOp<scalar_constant_op<typename DstXprType::Scalar>, SrcPlainObject>,
1133 assign_op<typename DstXprType::Scalar, typename DstXprType::Scalar>, Dense2Dense, Weak> {
1134 using Scalar =
typename DstXprType::Scalar;
1135 using NullaryOp = scalar_constant_op<Scalar>;
1136 using SrcXprType = CwiseNullaryOp<NullaryOp, SrcPlainObject>;
1137 using Functor = assign_op<Scalar, Scalar>;
1138 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(DstXprType& dst,
const SrcXprType& src,
1140 eigen_fill_impl<DstXprType>::run(dst, src);
1144template <
typename DstXprType,
typename SrcPlainObject,
typename Weak>
1145struct Assignment<DstXprType, CwiseNullaryOp<scalar_zero_op<typename DstXprType::Scalar>, SrcPlainObject>,
1146 assign_op<typename DstXprType::Scalar, typename DstXprType::Scalar>, Dense2Dense, Weak> {
1147 using Scalar =
typename DstXprType::Scalar;
1148 using NullaryOp = scalar_zero_op<Scalar>;
1149 using SrcXprType = CwiseNullaryOp<NullaryOp, SrcPlainObject>;
1150 using Functor = assign_op<Scalar, Scalar>;
1151 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(DstXprType& dst,
const SrcXprType& src,
1153 eigen_zero_impl<DstXprType>::run(dst, src);
1161template <
typename DstXprType,
typename SrcXprType,
typename Functor,
typename Weak>
1162struct Assignment<DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Weak> {
1163 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(
1164 DstXprType& dst,
const SrcXprType& src,
1165 const internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>& ) {
1166 Index dstRows = src.rows();
1167 Index dstCols = src.cols();
1168 if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
1170 EIGEN_INTERNAL_ASSERT_SAME_SHAPE(dst, src);
1176 template <
typename SrcScalarType>
1177 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(
1178 DstXprType& dst,
const SrcXprType& src,
1179 const internal::add_assign_op<typename DstXprType::Scalar, SrcScalarType>& ) {
1180 Index dstRows = src.rows();
1181 Index dstCols = src.cols();
1182 if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
1184 EIGEN_INTERNAL_ASSERT_SAME_SHAPE(dst, src);
1188 template <
typename SrcScalarType>
1189 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(
1190 DstXprType& dst,
const SrcXprType& src,
1191 const internal::sub_assign_op<typename DstXprType::Scalar, SrcScalarType>& ) {
1192 Index dstRows = src.rows();
1193 Index dstCols = src.cols();
1194 if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
1196 EIGEN_INTERNAL_ASSERT_SAME_SHAPE(dst, src);
@ Unaligned
Definition Constants.h:236
constexpr unsigned int ActualPacketAccessBit
Definition Constants.h:109
constexpr unsigned int DirectAccessBit
Definition Constants.h:160
constexpr unsigned int LinearAccessBit
Definition Constants.h:134
constexpr unsigned int RowMajorBit
Definition Constants.h:71