Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
AssignEvaluator.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2011 Benoit Jacob <jacob.benoit.1@gmail.com>
5// Copyright (C) 2011-2014 Gael Guennebaud <gael.guennebaud@inria.fr>
6// Copyright (C) 2011-2012 Jitse Niesen <jitse@maths.leeds.ac.uk>
7//
8// This Source Code Form is subject to the terms of the Mozilla
9// Public License v. 2.0. If a copy of the MPL was not distributed
10// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
11// SPDX-License-Identifier: MPL-2.0
12
13#ifndef EIGEN_ASSIGN_EVALUATOR_H
14#define EIGEN_ASSIGN_EVALUATOR_H
15
16// IWYU pragma: private
17#include "./InternalHeaderCheck.h"
18
19namespace Eigen {
20
21// This implementation is based on Assign.h
22
23namespace internal {
24
25/***************************************************************************
26 * Part 1 : the logic deciding a strategy for traversal and unrolling *
27 ***************************************************************************/
28
29// copy_using_evaluator_traits is based on assign_traits
30
31template <typename DstEvaluator, typename SrcEvaluator, typename AssignFunc, int MaxPacketSize = Dynamic>
32struct copy_using_evaluator_traits {
33 using Src = typename SrcEvaluator::XprType;
34 using Dst = typename DstEvaluator::XprType;
35 using DstScalar = typename Dst::Scalar;
36
37 static constexpr int DstFlags = DstEvaluator::Flags;
38 static constexpr int SrcFlags = SrcEvaluator::Flags;
39
40 public:
41 static constexpr int DstAlignment = DstEvaluator::Alignment;
42 static constexpr int SrcAlignment = SrcEvaluator::Alignment;
43 static constexpr int JointAlignment = plain_enum_min(DstAlignment, SrcAlignment);
44 static constexpr bool DstHasDirectAccess = bool(DstFlags & DirectAccessBit);
45 static constexpr bool SrcIsRowMajor = bool(SrcFlags & RowMajorBit);
46 static constexpr bool DstIsRowMajor = bool(DstFlags & RowMajorBit);
47 static constexpr bool IsVectorAtCompileTime = Dst::IsVectorAtCompileTime;
48 static constexpr int RowsAtCompileTime = size_prefer_fixed(Src::RowsAtCompileTime, Dst::RowsAtCompileTime);
49 static constexpr int ColsAtCompileTime = size_prefer_fixed(Src::ColsAtCompileTime, Dst::ColsAtCompileTime);
50 static constexpr int SizeAtCompileTime = size_at_compile_time(RowsAtCompileTime, ColsAtCompileTime);
51 static constexpr int MaxRowsAtCompileTime =
52 min_size_prefer_fixed(Src::MaxRowsAtCompileTime, Dst::MaxRowsAtCompileTime);
53 static constexpr int MaxColsAtCompileTime =
54 min_size_prefer_fixed(Src::MaxColsAtCompileTime, Dst::MaxColsAtCompileTime);
55 static constexpr int MaxSizeAtCompileTime =
56 min_size_prefer_fixed(Src::MaxSizeAtCompileTime, Dst::MaxSizeAtCompileTime);
57 static constexpr int InnerSizeAtCompileTime = IsVectorAtCompileTime ? SizeAtCompileTime
58 : DstIsRowMajor ? ColsAtCompileTime
59 : RowsAtCompileTime;
60 static constexpr int MaxInnerSizeAtCompileTime = IsVectorAtCompileTime ? MaxSizeAtCompileTime
61 : DstIsRowMajor ? MaxColsAtCompileTime
62 : MaxRowsAtCompileTime;
63 static constexpr int RestrictedInnerSize = min_size_prefer_fixed(MaxInnerSizeAtCompileTime, MaxPacketSize);
64 static constexpr int RestrictedLinearSize = min_size_prefer_fixed(MaxSizeAtCompileTime, MaxPacketSize);
65 static constexpr int OuterStride = outer_stride_at_compile_time<Dst>::value;
66
67 // LinearVectorizedTraversal handles a partial-packet tail (scalar emits under
68 // complete unrolling, packet_segment under no-unrolling), so we can prefer a
69 // wider packet than find_best_packet's exact-divisor pick when that strictly
70 // reduces total op count -- e.g. N=9 float on AVX2 gets 1*Packet8f + 1 scalar
71 // instead of 2*Packet4f + 1 scalar. find_assign_linear_packet stays on the
72 // exact-divisor choice when the wider alternative is no better
73 // (e.g. N=6 double on AVX-512: 3*Packet2d ties with 1*Packet4d + 2 scalars,
74 // so Packet2d is kept and LLT/LDLT sub-vector kernels are not disturbed).
75 // InnerVectorizedTraversal still requires Size % PacketSize == 0, so its
76 // packet type continues to use find_best_packet.
77 using LinearPacketType = typename find_assign_linear_packet<DstScalar, RestrictedLinearSize>::type;
78 using InnerPacketType = typename find_best_packet<DstScalar, RestrictedInnerSize>::type;
79
80 static constexpr int LinearPacketSize = unpacket_traits<LinearPacketType>::size;
81 static constexpr int InnerPacketSize = unpacket_traits<InnerPacketType>::size;
82 // Use the exact-divisor packet size for the Inner-vs-Linear choice below, so
83 // find_assign_linear_packet's widening only changes the packet *within*
84 // LinearVectorizedTraversal and never flips an inner-vectorizable assignment
85 // onto the linear path (e.g. vectorization_logic Matrix57 on NEON int).
86 static constexpr int LinearTraversalPacketSize =
87 unpacket_traits<typename find_best_packet<DstScalar, RestrictedLinearSize>::type>::size;
88
89 public:
90 static constexpr int LinearRequiredAlignment = unpacket_traits<LinearPacketType>::alignment;
91 static constexpr int InnerRequiredAlignment = unpacket_traits<InnerPacketType>::alignment;
92
93 private:
94 static constexpr bool StorageOrdersAgree = DstIsRowMajor == SrcIsRowMajor;
95 static constexpr bool MightVectorize = StorageOrdersAgree && bool(DstFlags & SrcFlags & ActualPacketAccessBit) &&
96 bool(functor_traits<AssignFunc>::PacketAccess);
97 // Generic packet assignment stores forward from coeffRef(); the swap kernel uses writePacket().
98 static constexpr bool MayInnerVectorize =
99 MightVectorize && (DstHasDirectAccess || std::is_same<AssignFunc, swap_assign_op<DstScalar>>::value) &&
100 (InnerSizeAtCompileTime != Dynamic) && (InnerSizeAtCompileTime % InnerPacketSize == 0) &&
101 (OuterStride != Dynamic) && (OuterStride % InnerPacketSize == 0) &&
102 (EIGEN_UNALIGNED_VECTORIZE || JointAlignment >= InnerRequiredAlignment);
103 static constexpr bool MayLinearize = StorageOrdersAgree && (DstFlags & SrcFlags & LinearAccessBit);
104 static constexpr bool MayLinearVectorize =
105 MightVectorize && MayLinearize && DstHasDirectAccess &&
106 (EIGEN_UNALIGNED_VECTORIZE || (DstAlignment >= LinearRequiredAlignment) || MaxSizeAtCompileTime == Dynamic) &&
107 (MaxSizeAtCompileTime == Dynamic || MaxSizeAtCompileTime >= LinearPacketSize);
108 /* If the destination isn't aligned, we have to do runtime checks and we don't unroll,
109 so it's only good for large enough sizes. */
110 static constexpr int InnerSizeThreshold = (EIGEN_UNALIGNED_VECTORIZE ? 1 : 3) * InnerPacketSize;
111 static constexpr bool MaySliceVectorize =
112 MightVectorize && DstHasDirectAccess &&
113 (MaxInnerSizeAtCompileTime == Dynamic || MaxInnerSizeAtCompileTime >= InnerSizeThreshold);
114 /* slice vectorization can be slow, so we only want it if the slices are big, which is
115 indicated by InnerMaxSize rather than InnerSize, think of the case of a dynamic block
116 in a fixed-size matrix
117 However, with EIGEN_UNALIGNED_VECTORIZE and unrolling, slice vectorization is still worth it */
118
119 public:
120 static constexpr int Traversal = SizeAtCompileTime == 0 ? AllAtOnceTraversal
121 : (MayLinearVectorize && (LinearTraversalPacketSize > InnerPacketSize))
122 ? LinearVectorizedTraversal
123 : MayInnerVectorize ? InnerVectorizedTraversal
124 : MayLinearVectorize ? LinearVectorizedTraversal
125 : MaySliceVectorize ? SliceVectorizedTraversal
126 : MayLinearize ? LinearTraversal
127 : DefaultTraversal;
128 static constexpr bool Vectorized = Traversal == InnerVectorizedTraversal || Traversal == LinearVectorizedTraversal ||
129 Traversal == SliceVectorizedTraversal;
130
131 using PacketType = std::conditional_t<Traversal == LinearVectorizedTraversal, LinearPacketType, InnerPacketType>;
132
133 private:
134 static constexpr int ActualPacketSize = Vectorized ? unpacket_traits<PacketType>::size : 1;
135 static constexpr int UnrollingLimit = EIGEN_UNROLLING_LIMIT * ActualPacketSize;
136 static constexpr int CoeffReadCost = int(DstEvaluator::CoeffReadCost) + int(SrcEvaluator::CoeffReadCost);
137 static constexpr bool MayUnrollInner =
138 (InnerSizeAtCompileTime != Dynamic) && (InnerSizeAtCompileTime * CoeffReadCost <= UnrollingLimit);
139
140 public:
141 // True when the whole assignment is a fixed-size kernel cheap enough to emit as straight-line
142 // code. Selects CompleteUnrolling, and gates the scalar tail in the SliceVectorized loop below.
143 static constexpr bool MayUnrollCompletely =
144 (SizeAtCompileTime != Dynamic) && (SizeAtCompileTime * CoeffReadCost <= UnrollingLimit);
145 static constexpr int Unrolling =
146 (Traversal == InnerVectorizedTraversal || Traversal == DefaultTraversal)
147 ? (MayUnrollCompletely ? CompleteUnrolling
148 : MayUnrollInner ? InnerUnrolling
149 : NoUnrolling)
150 : Traversal == LinearVectorizedTraversal
151 ? (MayUnrollCompletely && (EIGEN_UNALIGNED_VECTORIZE || (DstAlignment >= LinearRequiredAlignment))
152 ? CompleteUnrolling
153 : NoUnrolling)
154 : Traversal == LinearTraversal ? (MayUnrollCompletely ? CompleteUnrolling : NoUnrolling)
155#if EIGEN_UNALIGNED_VECTORIZE
156 : Traversal == SliceVectorizedTraversal ? (MayUnrollInner ? InnerUnrolling : NoUnrolling)
157#endif
158 : NoUnrolling;
159 // Scalar tails for expressions bounded to <= 4 packets (runtime-sized blocks of small fixed matrices, as in the
160 // column steps of a 4x4 Cholesky): their results are reread almost at once, and a masked store does not forward to
161 // the following loads. That pays only while a tail is a few cheap coefficients: packets of <= 8 lanes, and a source
162 // cost below HugeCost (a lazy product with a runtime inner size computes a dot product per coefficient).
163 static constexpr bool UsePacketSegment =
164 has_packet_segment<PacketType>::value &&
165 !(MaxSizeAtCompileTime != Dynamic && MaxSizeAtCompileTime <= 4 * int(unpacket_traits<PacketType>::size) &&
166 unpacket_traits<PacketType>::size <= 8 && int(SrcEvaluator::CoeffReadCost) < HugeCost);
167
168#ifdef EIGEN_DEBUG_ASSIGN
169 static void debug() {
170 std::cerr << "DstXpr: " << typeid(typename DstEvaluator::XprType).name() << std::endl;
171 std::cerr << "SrcXpr: " << typeid(typename SrcEvaluator::XprType).name() << std::endl;
172 std::cerr.setf(std::ios::hex, std::ios::basefield);
173 std::cerr << "DstFlags"
174 << " = " << DstFlags << " (" << demangle_flags(DstFlags) << " )" << std::endl;
175 std::cerr << "SrcFlags"
176 << " = " << SrcFlags << " (" << demangle_flags(SrcFlags) << " )" << std::endl;
177 std::cerr.unsetf(std::ios::hex);
178 EIGEN_DEBUG_VAR(DstAlignment)
179 EIGEN_DEBUG_VAR(SrcAlignment)
180 EIGEN_DEBUG_VAR(LinearRequiredAlignment)
181 EIGEN_DEBUG_VAR(InnerRequiredAlignment)
182 EIGEN_DEBUG_VAR(JointAlignment)
183 EIGEN_DEBUG_VAR(InnerSizeAtCompileTime)
184 EIGEN_DEBUG_VAR(MaxInnerSizeAtCompileTime)
185 EIGEN_DEBUG_VAR(LinearPacketSize)
186 EIGEN_DEBUG_VAR(InnerPacketSize)
187 EIGEN_DEBUG_VAR(ActualPacketSize)
188 EIGEN_DEBUG_VAR(StorageOrdersAgree)
189 EIGEN_DEBUG_VAR(MightVectorize)
190 EIGEN_DEBUG_VAR(MayLinearize)
191 EIGEN_DEBUG_VAR(MayInnerVectorize)
192 EIGEN_DEBUG_VAR(MayLinearVectorize)
193 EIGEN_DEBUG_VAR(MaySliceVectorize)
194 std::cerr << "Traversal"
195 << " = " << Traversal << " (" << demangle_traversal(Traversal) << ")" << std::endl;
196 EIGEN_DEBUG_VAR(SrcEvaluator::CoeffReadCost)
197 EIGEN_DEBUG_VAR(DstEvaluator::CoeffReadCost)
198 EIGEN_DEBUG_VAR(Dst::SizeAtCompileTime)
199 EIGEN_DEBUG_VAR(UnrollingLimit)
200 EIGEN_DEBUG_VAR(MayUnrollCompletely)
201 EIGEN_DEBUG_VAR(MayUnrollInner)
202 std::cerr << "Unrolling"
203 << " = " << Unrolling << " (" << demangle_unrolling(Unrolling) << ")" << std::endl;
204 std::cerr << std::endl;
205 }
206#endif
207};
208
209/***************************************************************************
210 * Part 2 : meta-unrollers
211 ***************************************************************************/
212
213/************************
214*** Default traversal ***
215************************/
216
217template <typename Kernel, int Index_, int Stop>
218struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling {
219 template <int... Offsets>
220 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run_impl(Kernel& kernel,
221 std::integer_sequence<int, Offsets...>) {
222 int unused[] = {
223 0, (kernel.assignCoeffByOuterInner((Index_ + Offsets) / Kernel::AssignmentTraits::InnerSizeAtCompileTime,
224 (Index_ + Offsets) % Kernel::AssignmentTraits::InnerSizeAtCompileTime),
225 0)...};
226 EIGEN_UNUSED_VARIABLE(unused);
227 }
228
229 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) {
230 run_impl(kernel, std::make_integer_sequence<int, Stop - Index_>{});
231 }
232};
233
234template <typename Kernel, int Index_, int Stop>
235struct copy_using_evaluator_DefaultTraversal_InnerUnrolling {
236 template <int... Offsets>
237 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run_impl(Kernel& kernel, Index outer,
238 std::integer_sequence<int, Offsets...>) {
239 int unused[] = {0, (kernel.assignCoeffByOuterInner(outer, Index_ + Offsets), 0)...};
240 EIGEN_UNUSED_VARIABLE(unused);
241 }
242
243 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel, Index outer) {
244 run_impl(kernel, outer, std::make_integer_sequence<int, Stop - Index_>{});
245 }
246};
247
248/***********************
249*** Linear traversal ***
250***********************/
251
252template <typename Kernel, int Index_, int Stop>
253struct copy_using_evaluator_LinearTraversal_CompleteUnrolling {
254 template <int... Offsets>
255 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run_impl(Kernel& kernel,
256 std::integer_sequence<int, Offsets...>) {
257 int unused[] = {0, (kernel.assignCoeff(Index_ + Offsets), 0)...};
258 EIGEN_UNUSED_VARIABLE(unused);
259 }
260
261 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) {
262 run_impl(kernel, std::make_integer_sequence<int, Stop - Index_>{});
263 }
264};
265
266/**************************
267*** Inner vectorization ***
268**************************/
269
270template <typename Kernel, int Index_, int Stop>
271struct copy_using_evaluator_innervec_CompleteUnrolling {
272 using PacketType = typename Kernel::PacketType;
273 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
274 static constexpr int SrcAlignment = Kernel::AssignmentTraits::SrcAlignment;
275 static constexpr int DstAlignment = Kernel::AssignmentTraits::DstAlignment;
276 static_assert((Stop - Index_) % PacketSize == 0, "Packet unrolling range must be divisible by packet size.");
277
278 template <int... Offsets>
279 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run_impl(Kernel& kernel, std::integer_sequence<int, Offsets...>) {
280 int unused[] = {0, (kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(
281 (Index_ + Offsets * PacketSize) / Kernel::AssignmentTraits::InnerSizeAtCompileTime,
282 (Index_ + Offsets * PacketSize) % Kernel::AssignmentTraits::InnerSizeAtCompileTime),
283 0)...};
284 EIGEN_UNUSED_VARIABLE(unused);
285 }
286
287 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
288 run_impl(kernel, std::make_integer_sequence<int, (Stop - Index_) / PacketSize>{});
289 }
290};
291
292template <typename Kernel, int Index_, int Stop, int SrcAlignment, int DstAlignment>
293struct copy_using_evaluator_innervec_InnerUnrolling {
294 using PacketType = typename Kernel::PacketType;
295 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
296 static_assert((Stop - Index_) % PacketSize == 0, "Packet unrolling range must be divisible by packet size.");
297
298 template <int... Offsets>
299 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run_impl(Kernel& kernel, Index outer,
300 std::integer_sequence<int, Offsets...>) {
301 int unused[] = {0, (kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(
302 outer, Index_ + Offsets * PacketSize),
303 0)...};
304 EIGEN_UNUSED_VARIABLE(unused);
305 EIGEN_UNUSED_VARIABLE(outer);
306 }
307
308 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel, Index outer) {
309 run_impl(kernel, outer, std::make_integer_sequence<int, (Stop - Index_) / PacketSize>{});
310 }
311};
312
313template <typename Kernel, int Start, int Stop, int SrcAlignment, int DstAlignment, bool UsePacketSegment>
314struct copy_using_evaluator_innervec_segment {
315 using PacketType = typename Kernel::PacketType;
316
317 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel, Index outer) {
318 kernel.template assignPacketSegmentByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, Start, 0,
319 Stop - Start);
320 }
321};
322
323template <typename Kernel, int Start, int Stop, int SrcAlignment, int DstAlignment>
324struct copy_using_evaluator_innervec_segment<Kernel, Start, Stop, SrcAlignment, DstAlignment,
325 /*UsePacketSegment*/ false>
326 : copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Start, Stop> {};
327
328template <typename Kernel, int Stop, int SrcAlignment, int DstAlignment>
329struct copy_using_evaluator_innervec_segment<Kernel, Stop, Stop, SrcAlignment, DstAlignment,
330 /*UsePacketSegment*/ true> {
331 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel&, Index) {}
332};
333
334template <typename Kernel, int Stop, int SrcAlignment, int DstAlignment>
335struct copy_using_evaluator_innervec_segment<Kernel, Stop, Stop, SrcAlignment, DstAlignment,
336 /*UsePacketSegment*/ false> {
337 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel&, Index) {}
338};
339
340/***************************************************************************
341 * Part 3 : implementation of all cases
342 ***************************************************************************/
343
344// dense_assignment_loop is based on assign_impl
345
346template <typename Kernel, int Traversal = Kernel::AssignmentTraits::Traversal,
347 int Unrolling = Kernel::AssignmentTraits::Unrolling>
348struct dense_assignment_loop_impl;
349
350template <typename Kernel, int Traversal = Kernel::AssignmentTraits::Traversal,
351 int Unrolling = Kernel::AssignmentTraits::Unrolling>
352struct dense_assignment_loop {
353 // Always inlined so the loop lands in the caller that owns the evaluators (see the slice traversal).
354 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE constexpr void run(Kernel& kernel) {
355#ifdef __cpp_lib_is_constant_evaluated
356 if (internal::is_constant_evaluated())
357 dense_assignment_loop_impl<Kernel, Traversal == AllAtOnceTraversal ? AllAtOnceTraversal : DefaultTraversal,
358 NoUnrolling>::run(kernel);
359 else
360#endif
361 dense_assignment_loop_impl<Kernel, Traversal, Unrolling>::run(kernel);
362 }
363};
364
365/************************
366***** Special Cases *****
367************************/
368
369// Zero-sized assignment is a no-op.
370template <typename Kernel, int Unrolling>
371struct dense_assignment_loop_impl<Kernel, AllAtOnceTraversal, Unrolling> {
372 static constexpr int SizeAtCompileTime = Kernel::AssignmentTraits::SizeAtCompileTime;
373
374 EIGEN_DEVICE_FUNC static void EIGEN_STRONG_INLINE constexpr run(Kernel& /*kernel*/) {
375 EIGEN_STATIC_ASSERT(SizeAtCompileTime == 0, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT)
376 }
377};
378
379/************************
380*** Default traversal ***
381************************/
382
383template <typename Kernel>
384struct dense_assignment_loop_impl<Kernel, DefaultTraversal, NoUnrolling> {
385 EIGEN_DEVICE_FUNC static void EIGEN_STRONG_INLINE constexpr run(Kernel& kernel) {
386 for (Index outer = 0; outer < kernel.outerSize(); ++outer) {
387 for (Index inner = 0; inner < kernel.innerSize(); ++inner) {
388 kernel.assignCoeffByOuterInner(outer, inner);
389 }
390 }
391 }
392};
393
394template <typename Kernel>
395struct dense_assignment_loop_impl<Kernel, DefaultTraversal, CompleteUnrolling> {
396 static constexpr int SizeAtCompileTime = Kernel::AssignmentTraits::SizeAtCompileTime;
397
398 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) {
399 copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, 0, SizeAtCompileTime>::run(kernel);
400 }
401};
402
403template <typename Kernel>
404struct dense_assignment_loop_impl<Kernel, DefaultTraversal, InnerUnrolling> {
405 static constexpr int InnerSizeAtCompileTime = Kernel::AssignmentTraits::InnerSizeAtCompileTime;
406
407 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) {
408 const Index outerSize = kernel.outerSize();
409 for (Index outer = 0; outer < outerSize; ++outer)
410 copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, 0, InnerSizeAtCompileTime>::run(kernel, outer);
411 }
412};
413
414/***************************
415*** Linear vectorization ***
416***************************/
417
418// The goal of unaligned_dense_assignment_loop is simply to factorize the handling
419// of the non vectorizable beginning and ending parts
420
421template <typename PacketType, int DstAlignment, int SrcAlignment, bool UsePacketSegment, bool Skip>
422struct unaligned_dense_assignment_loop {
423 // if Skip == true, then do nothing
424 template <typename Kernel>
425 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& /*kernel*/, Index /*start*/, Index /*end*/) {}
426 template <typename Kernel>
427 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& /*kernel*/, Index /*outer*/,
428 Index /*innerStart*/, Index /*innerEnd*/) {}
429};
430
431template <typename PacketType, int DstAlignment, int SrcAlignment>
432struct unaligned_dense_assignment_loop<PacketType, DstAlignment, SrcAlignment, /*UsePacketSegment*/ true,
433 /*Skip*/ false> {
434 template <typename Kernel>
435 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel, Index start, Index end) {
436 Index count = end - start;
437 eigen_assert(count <= unpacket_traits<PacketType>::size);
438 if (count > 0) kernel.template assignPacketSegment<DstAlignment, SrcAlignment, PacketType>(start, 0, count);
439 }
440 template <typename Kernel>
441 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel, Index outer, Index start, Index end) {
442 Index count = end - start;
443 eigen_assert(count <= unpacket_traits<PacketType>::size);
444 if (count > 0)
445 kernel.template assignPacketSegmentByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, start, 0, count);
446 }
447};
448
449template <typename PacketType, int DstAlignment, int SrcAlignment>
450struct unaligned_dense_assignment_loop<PacketType, DstAlignment, SrcAlignment, /*UsePacketSegment*/ false,
451 /*Skip*/ false> {
452 template <typename Kernel>
453 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel, Index start, Index end) {
454 const Index count = end - start;
455 for (Index offset = 0; offset < count; ++offset) kernel.assignCoeff(start + offset);
456 }
457 template <typename Kernel>
458 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel, Index outer, Index innerStart,
459 Index innerEnd) {
460 for (Index inner = innerStart; inner < innerEnd; ++inner) kernel.assignCoeffByOuterInner(outer, inner);
461 }
462};
463
464template <typename Kernel, int Index_, int Stop>
465struct copy_using_evaluator_linearvec_CompleteUnrolling {
466 using PacketType = typename Kernel::PacketType;
467 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
468 static constexpr int SrcAlignment = Kernel::AssignmentTraits::SrcAlignment;
469 static constexpr int DstAlignment = Kernel::AssignmentTraits::DstAlignment;
470 static_assert((Stop - Index_) % PacketSize == 0, "Packet unrolling range must be divisible by packet size.");
471
472 template <int... Offsets>
473 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run_impl(Kernel& kernel, std::integer_sequence<int, Offsets...>) {
474 int unused[] = {
475 0, (kernel.template assignPacket<DstAlignment, SrcAlignment, PacketType>(Index_ + Offsets * PacketSize), 0)...};
476 EIGEN_UNUSED_VARIABLE(unused);
477 }
478
479 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
480 run_impl(kernel, std::make_integer_sequence<int, (Stop - Index_) / PacketSize>{});
481 }
482};
483
484template <typename Kernel, int Index_, int Stop, bool UsePacketSegment>
485struct copy_using_evaluator_linearvec_segment {
486 using PacketType = typename Kernel::PacketType;
487 static constexpr int SrcAlignment = Kernel::AssignmentTraits::SrcAlignment;
488 static constexpr int DstAlignment = Kernel::AssignmentTraits::DstAlignment;
489
490 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
491 kernel.template assignPacketSegment<DstAlignment, SrcAlignment, PacketType>(Index_, 0, Stop - Index_);
492 }
493};
494
495template <typename Kernel, int Index_, int Stop>
496struct copy_using_evaluator_linearvec_segment<Kernel, Index_, Stop, /*UsePacketSegment*/ false>
497 : copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Index_, Stop> {};
498
499template <typename Kernel, int Stop>
500struct copy_using_evaluator_linearvec_segment<Kernel, Stop, Stop, /*UsePacketSegment*/ true> {
501 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel&) {}
502};
503
504template <typename Kernel, int Stop>
505struct copy_using_evaluator_linearvec_segment<Kernel, Stop, Stop, /*UsePacketSegment*/ false> {
506 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel&) {}
507};
508
509template <typename Kernel>
510struct dense_assignment_loop_impl<Kernel, LinearVectorizedTraversal, NoUnrolling> {
511 using Scalar = typename Kernel::Scalar;
512 using PacketType = typename Kernel::PacketType;
513 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
514 static constexpr int SrcAlignment = Kernel::AssignmentTraits::JointAlignment;
515 static constexpr int DstAlignment = plain_enum_max(Kernel::AssignmentTraits::DstAlignment, alignof(Scalar));
516 static constexpr int RequestedAlignment = unpacket_traits<PacketType>::alignment;
517 static constexpr bool Alignable = (DstAlignment >= RequestedAlignment) ||
518 (static_cast<std::size_t>(RequestedAlignment - DstAlignment) % sizeof(Scalar) == 0);
519 static constexpr int Alignment = Alignable ? RequestedAlignment : DstAlignment;
520 static constexpr bool DstIsAligned = DstAlignment >= Alignment;
521 static constexpr bool UsePacketSegment = Kernel::AssignmentTraits::UsePacketSegment;
522
523 using head_loop =
524 unaligned_dense_assignment_loop<PacketType, DstAlignment, SrcAlignment, UsePacketSegment, DstIsAligned>;
525 using tail_loop = unaligned_dense_assignment_loop<PacketType, Alignment, SrcAlignment, UsePacketSegment, false>;
526
527 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) {
528 // When a small bound disables available packet segments, GCC's -Waggressive-loop-optimizations fires on the
529 // scalar head and tail loops unless they see that bound; the no-op clamp supplies it. Backends without segments
530 // keep the unclamped size and their existing codegen.
531 constexpr int MaxSize = Kernel::AssignmentTraits::MaxSizeAtCompileTime;
532 constexpr bool ClampToMaxSize = has_packet_segment<PacketType>::value && !UsePacketSegment && MaxSize != Dynamic;
533 eigen_assert(MaxSize == Dynamic || kernel.size() <= MaxSize);
534 const Index size = ClampToMaxSize ? numext::mini(kernel.size(), Index(MaxSize)) : kernel.size();
535 const Index alignedStart = DstIsAligned ? 0 : first_aligned<Alignment>(kernel.dstDataPtr(), size);
536
537 head_loop::run(kernel, 0, alignedStart);
538
539#if EIGEN_GNUC_STRICT_AT_LEAST(8, 0, 0) && EIGEN_ARCH_i386_OR_x86_64 && !defined(EIGEN_GPUCC)
540 // GCC leaves cheap unary packet operations rolled. Copies, multi-source expressions and expensive functors
541 // showed no consistent benefit, so use the coefficient cost as a conservative unrolling heuristic.
542 using SrcEvaluator = typename Kernel::SrcEvaluatorType;
543 EIGEN_IF_CONSTEXPR (int(SrcEvaluator::CoeffReadCost) > NumTraits<Scalar>::ReadCost &&
544 int(SrcEvaluator::CoeffReadCost) <= 2 * NumTraits<Scalar>::ReadCost) {
545#pragma GCC unroll 2
546 for (Index index = alignedStart; index <= size - PacketSize; index += PacketSize)
547 kernel.template assignPacket<Alignment, SrcAlignment, PacketType>(index);
548 } else
549#endif
550 {
551 for (Index index = alignedStart; index <= size - PacketSize; index += PacketSize)
552 kernel.template assignPacket<Alignment, SrcAlignment, PacketType>(index);
553 }
554
555 // Derive the tail bound directly; GCC can lose its range when it comes from the packet-loop induction variable.
556 const Index alignedEnd = size - (size - alignedStart) % PacketSize;
557 tail_loop::run(kernel, alignedEnd, size);
558 }
559};
560
561template <typename Kernel>
562struct dense_assignment_loop_impl<Kernel, LinearVectorizedTraversal, CompleteUnrolling> {
563 using PacketType = typename Kernel::PacketType;
564 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
565 static constexpr int Size = Kernel::AssignmentTraits::SizeAtCompileTime;
566 static constexpr int AlignedSize = numext::round_down(Size, PacketSize);
567
568 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) {
569 copy_using_evaluator_linearvec_CompleteUnrolling<Kernel, 0, AlignedSize>::run(kernel);
570 // Partial-packet tail. Unlike the NoUnrolling case above, here the size is a
571 // compile-time constant and the loop is fully unrolled, so the tail is a
572 // fixed handful (fewer than PacketSize) of scalar assignments. Emit those
573 // directly by forcing UsePacketSegment = false: a masked packet segment
574 // buys nothing when there is no runtime-variable trip count to collapse,
575 // and on the AVX backend it lowers to a masked store (vmaskmovps/vmaskmovpd)
576 // that is slow on current hardware and forwards poorly to later loads. The
577 // NoUnrolling loop keeps UsePacketSegment because there the tail length is a
578 // runtime value, where a single masked op does beat a scalar remainder loop.
579 copy_using_evaluator_linearvec_segment<Kernel, AlignedSize, Size, /*UsePacketSegment=*/false>::run(kernel);
580 }
581};
582
583/**************************
584*** Inner vectorization ***
585**************************/
586
587template <typename Kernel>
588struct dense_assignment_loop_impl<Kernel, InnerVectorizedTraversal, NoUnrolling> {
589 using PacketType = typename Kernel::PacketType;
590 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
591 static constexpr int SrcAlignment = Kernel::AssignmentTraits::JointAlignment;
592 static constexpr int DstAlignment = Kernel::AssignmentTraits::DstAlignment;
593
594 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) {
595 const Index innerSize = kernel.innerSize();
596 const Index outerSize = kernel.outerSize();
597 for (Index outer = 0; outer < outerSize; ++outer)
598 for (Index inner = 0; inner < innerSize; inner += PacketSize)
599 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
600 }
601};
602
603template <typename Kernel>
604struct dense_assignment_loop_impl<Kernel, InnerVectorizedTraversal, CompleteUnrolling> {
605 static constexpr int SizeAtCompileTime = Kernel::AssignmentTraits::SizeAtCompileTime;
606
607 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
608 copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, SizeAtCompileTime>::run(kernel);
609 }
610};
611
612template <typename Kernel>
613struct dense_assignment_loop_impl<Kernel, InnerVectorizedTraversal, InnerUnrolling> {
614 static constexpr int InnerSize = Kernel::AssignmentTraits::InnerSizeAtCompileTime;
615 static constexpr int SrcAlignment = Kernel::AssignmentTraits::SrcAlignment;
616 static constexpr int DstAlignment = Kernel::AssignmentTraits::DstAlignment;
617
618 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
619 const Index outerSize = kernel.outerSize();
620 for (Index outer = 0; outer < outerSize; ++outer)
621 copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, InnerSize, SrcAlignment, DstAlignment>::run(kernel,
622 outer);
623 }
624};
625
626/***********************
627*** Linear traversal ***
628***********************/
629
630template <typename Kernel>
631struct dense_assignment_loop_impl<Kernel, LinearTraversal, NoUnrolling> {
632 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) {
633 const Index size = kernel.size();
634 for (Index i = 0; i < size; ++i) kernel.assignCoeff(i);
635 }
636};
637
638template <typename Kernel>
639struct dense_assignment_loop_impl<Kernel, LinearTraversal, CompleteUnrolling> {
640 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) {
641 copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, 0, Kernel::AssignmentTraits::SizeAtCompileTime>::run(
642 kernel);
643 }
644};
645
646/**************************
647*** Slice vectorization ***
648***************************/
649
650template <typename Kernel>
651struct dense_assignment_loop_impl<Kernel, SliceVectorizedTraversal, NoUnrolling> {
652 using Scalar = typename Kernel::Scalar;
653 using PacketType = typename Kernel::PacketType;
654 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
655 static constexpr int SrcAlignment = Kernel::AssignmentTraits::JointAlignment;
656 static constexpr int DstAlignment = plain_enum_max(Kernel::AssignmentTraits::DstAlignment, alignof(Scalar));
657 static constexpr int RequestedAlignment = unpacket_traits<PacketType>::alignment;
658 static constexpr bool Alignable = (DstAlignment >= RequestedAlignment) ||
659 (static_cast<std::size_t>(RequestedAlignment - DstAlignment) % sizeof(Scalar) == 0);
660 static constexpr int Alignment = Alignable ? RequestedAlignment : DstAlignment;
661 static constexpr bool DstIsAligned = DstAlignment >= Alignment;
662 static constexpr bool UsePacketSegment = Kernel::AssignmentTraits::UsePacketSegment;
663
664 using head_loop = unaligned_dense_assignment_loop<PacketType, DstAlignment, Unaligned, UsePacketSegment, !Alignable>;
665 using tail_loop = unaligned_dense_assignment_loop<PacketType, Alignment, Unaligned, UsePacketSegment, false>;
666
667 // All inner slices share one alignment offset: the loop bounds are outer-invariant and stay
668 // hoisted out of the outer loop. Keeping that loop free of per-slice bookkeeping is what makes
669 // slice vectorization competitive with compiler-vectorized scalar code when slices are short.
670 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE constexpr void runInvariant(Kernel& kernel, Index alignedStart,
671 Index innerSize, Index outerSize) {
672 const Index alignedEnd = alignedStart + numext::round_down(innerSize - alignedStart, PacketSize);
673 for (Index outer = 0; outer < outerSize; ++outer) {
674 head_loop::run(kernel, outer, 0, alignedStart);
675
676 // do the vectorizable part of the assignment
677 for (Index inner = alignedStart; inner < alignedEnd; inner += PacketSize)
678 kernel.template assignPacketByOuterInner<Alignment, Unaligned, PacketType>(outer, inner);
679
680 tail_loop::run(kernel, outer, alignedEnd, innerSize);
681 }
682 }
683
684#if EIGEN_UNALIGNED_VECTORIZE
685 // Unaligned stores with outer-invariant bounds avoid chasing each slice's alignment offset.
686 // Also used for statically aligned destinations, so run() inlines one loop nest instead of two.
687 using unaligned_tail_loop =
688 unaligned_dense_assignment_loop<PacketType, Unaligned, Unaligned, UsePacketSegment, false>;
689
690 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE constexpr void runUnaligned(Kernel& kernel, Index innerSize,
691 Index outerSize) {
692 const Index packetEnd = numext::round_down(innerSize, PacketSize);
693 for (Index outer = 0; outer < outerSize; ++outer) {
694 for (Index inner = 0; inner < packetEnd; inner += PacketSize)
695 kernel.template assignPacketByOuterInner<Unaligned, Unaligned, PacketType>(outer, inner);
696
697 unaligned_tail_loop::run(kernel, outer, packetEnd, innerSize);
698 }
699 }
700#endif
701
702 // run(), runInvariant() and runUnaligned() must inline into the function that owns the evaluators.
703 // Out of line, `kernel` is a reference the packet stores may alias, so the evaluators' pointers
704 // and strides are reloaded after every store. Neither GCC nor Clang inlines them reliably unforced.
705 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE constexpr void run(Kernel& kernel) {
706 const Index innerSize = kernel.innerSize();
707 const Index outerSize = kernel.outerSize();
708#if EIGEN_UNALIGNED_VECTORIZE
709 EIGEN_IF_CONSTEXPR (DstIsAligned) {
710 runUnaligned(kernel, innerSize, outerSize);
711 } else
712#endif
713 {
714 const Scalar* dst_ptr = kernel.dstDataPtr();
715 const Index alignedStep = Alignable ? (PacketSize - kernel.outerStride() % PacketSize) % PacketSize : 0;
716 Index alignedStart = ((!Alignable) || DstIsAligned) ? 0 : internal::first_aligned<Alignment>(dst_ptr, innerSize);
717
718 if (alignedStep == 0) {
719 runInvariant(kernel, alignedStart, innerSize, outerSize);
720 return;
721 }
722#if EIGEN_UNALIGNED_VECTORIZE
723 runUnaligned(kernel, innerSize, outerSize);
724#else
725 for (Index outer = 0; outer < outerSize; ++outer) {
726 const Index alignedEnd = alignedStart + numext::round_down(innerSize - alignedStart, PacketSize);
727
728 head_loop::run(kernel, outer, 0, alignedStart);
729
730 // do the vectorizable part of the assignment
731 for (Index inner = alignedStart; inner < alignedEnd; inner += PacketSize)
732 kernel.template assignPacketByOuterInner<Alignment, Unaligned, PacketType>(outer, inner);
733
734 tail_loop::run(kernel, outer, alignedEnd, innerSize);
735
736 alignedStart = numext::mini((alignedStart + alignedStep) % PacketSize, innerSize);
737 }
738#endif
739 }
740 }
741};
742
743#if EIGEN_UNALIGNED_VECTORIZE
744template <typename Kernel>
745struct dense_assignment_loop_impl<Kernel, SliceVectorizedTraversal, InnerUnrolling> {
746 using PacketType = typename Kernel::PacketType;
747 static constexpr int PacketSize = unpacket_traits<PacketType>::size;
748 static constexpr int InnerSize = Kernel::AssignmentTraits::InnerSizeAtCompileTime;
749 static constexpr int VectorizableSize = numext::round_down(InnerSize, PacketSize);
750
751 // The tail length is a compile-time constant here, so it can be emitted as a masked packet
752 // segment or as scalars. Scalars win for the same fixed-size kernels LinearVectorizedTraversal
753 // already emits them for: their destination is a temporary the enclosing expression reloads by
754 // packet, and on AVX a masked store forwards poorly to those loads. Everything else keeps the
755 // segment, where one packet evaluation replaces up to PacketSize - 1 scalar ones.
756 static constexpr bool UsePacketSegment =
757 Kernel::AssignmentTraits::UsePacketSegment && !Kernel::AssignmentTraits::MayUnrollCompletely;
758
759 using packet_loop = copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, VectorizableSize, Unaligned, Unaligned>;
760 using packet_segment_loop = copy_using_evaluator_innervec_segment<Kernel, VectorizableSize, InnerSize, Unaligned,
761 Unaligned, UsePacketSegment>;
762
763 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) {
764 for (Index outer = 0; outer < kernel.outerSize(); ++outer) {
765 packet_loop::run(kernel, outer);
766 packet_segment_loop::run(kernel, outer);
767 }
768 }
769};
770#endif
771
772/***************************************************************************
773 * Part 4 : Generic dense assignment kernel
774 ***************************************************************************/
775
776// This class generalize the assignment of a coefficient (or packet) from one dense evaluator
777// to another dense writable evaluator.
778// It is parametrized by the two evaluators, and the actual assignment functor.
779// This abstraction level permits to keep the evaluation loops as simple and as generic as possible.
780// One can customize the assignment using this generic dense_assignment_kernel with different
781// functors, or by completely overloading it, by-passing a functor.
782template <typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor, int Version = Specialized>
783class generic_dense_assignment_kernel {
784 protected:
785 using DstXprType = typename DstEvaluatorTypeT::XprType;
786 using SrcXprType = typename SrcEvaluatorTypeT::XprType;
787
788 public:
789 using DstEvaluatorType = DstEvaluatorTypeT;
790 using SrcEvaluatorType = SrcEvaluatorTypeT;
791 using Scalar = typename DstEvaluatorType::Scalar;
792 using AssignmentTraits = copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor>;
793 using PacketType = typename AssignmentTraits::PacketType;
794
795 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr generic_dense_assignment_kernel(DstEvaluatorType& dst,
796 const SrcEvaluatorType& src,
797 const Functor& func,
798 DstXprType& dstExpr)
799 : m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr) {
800#ifdef EIGEN_DEBUG_ASSIGN
801 AssignmentTraits::debug();
802#endif
803 }
804
805 EIGEN_DEVICE_FUNC constexpr Index size() const noexcept { return m_dstExpr.size(); }
806 EIGEN_DEVICE_FUNC constexpr Index innerSize() const noexcept { return m_dstExpr.innerSize(); }
807 EIGEN_DEVICE_FUNC constexpr Index outerSize() const noexcept { return m_dstExpr.outerSize(); }
808 EIGEN_DEVICE_FUNC constexpr Index rows() const noexcept { return m_dstExpr.rows(); }
809 EIGEN_DEVICE_FUNC constexpr Index cols() const noexcept { return m_dstExpr.cols(); }
810 EIGEN_DEVICE_FUNC constexpr Index outerStride() const noexcept { return m_dstExpr.outerStride(); }
811
812 EIGEN_DEVICE_FUNC constexpr DstEvaluatorType& dstEvaluator() noexcept { return m_dst; }
813 EIGEN_DEVICE_FUNC constexpr const SrcEvaluatorType& srcEvaluator() const noexcept { return m_src; }
814
816 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void assignCoeff(Index row, Index col) {
817 m_functor.assignCoeff(m_dst.coeffRef(row, col), m_src.coeff(row, col));
818 }
819
821 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void assignCoeff(Index index) {
822 m_functor.assignCoeff(m_dst.coeffRef(index), m_src.coeff(index));
823 }
824
826 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void assignCoeffByOuterInner(Index outer, Index inner) {
827 Index row = rowIndexByOuterInner(outer, inner);
828 Index col = colIndexByOuterInner(outer, inner);
829 assignCoeff(row, col);
830 }
831
832 template <int StoreMode, int LoadMode, typename Packet>
833 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index row, Index col) {
834 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(row, col),
835 m_src.template packet<LoadMode, Packet>(row, col));
836 }
837
838 template <int StoreMode, int LoadMode, typename Packet>
839 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index index) {
840 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(index), m_src.template packet<LoadMode, Packet>(index));
841 }
842
843 template <int StoreMode, int LoadMode, typename Packet>
844 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner) {
845 Index row = rowIndexByOuterInner(outer, inner);
846 Index col = colIndexByOuterInner(outer, inner);
847 assignPacket<StoreMode, LoadMode, Packet>(row, col);
848 }
849
850 template <int StoreMode, int LoadMode, typename Packet>
851 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketSegment(Index row, Index col, Index begin, Index count) {
852 m_functor.template assignPacketSegment<StoreMode>(
853 &m_dst.coeffRef(row, col), m_src.template packetSegment<LoadMode, Packet>(row, col, begin, count), begin,
854 count);
855 }
856
857 template <int StoreMode, int LoadMode, typename Packet>
858 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketSegment(Index index, Index begin, Index count) {
859 m_functor.template assignPacketSegment<StoreMode>(
860 &m_dst.coeffRef(index), m_src.template packetSegment<LoadMode, Packet>(index, begin, count), begin, count);
861 }
862
863 template <int StoreMode, int LoadMode, typename Packet>
864 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketSegmentByOuterInner(Index outer, Index inner, Index begin,
865 Index count) {
866 Index row = rowIndexByOuterInner(outer, inner);
867 Index col = colIndexByOuterInner(outer, inner);
868 assignPacketSegment<StoreMode, LoadMode, Packet>(row, col, begin, count);
869 }
870
871 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr Index rowIndexByOuterInner(Index outer, Index inner) {
872 using Traits = typename DstEvaluatorType::ExpressionTraits;
873 return int(Traits::RowsAtCompileTime) == 1 ? 0
874 : int(Traits::ColsAtCompileTime) == 1 ? inner
875 : int(DstEvaluatorType::Flags) & RowMajorBit ? outer
876 : inner;
877 }
878
879 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr Index colIndexByOuterInner(Index outer, Index inner) {
880 using Traits = typename DstEvaluatorType::ExpressionTraits;
881 return int(Traits::ColsAtCompileTime) == 1 ? 0
882 : int(Traits::RowsAtCompileTime) == 1 ? inner
883 : int(DstEvaluatorType::Flags) & RowMajorBit ? inner
884 : outer;
885 }
886
887 EIGEN_DEVICE_FUNC const Scalar* dstDataPtr() const { return m_dstExpr.data(); }
888
889 protected:
890 DstEvaluatorType& m_dst;
891 const SrcEvaluatorType& m_src;
892 const Functor& m_functor;
893 // TODO: find a way to avoid the needs of the original expression
894 DstXprType& m_dstExpr;
895};
896
897// Special kernel used when computing small products whose operands have dynamic dimensions. It ensures that the
898// PacketSize used is no larger than 4, thereby increasing the chance that vectorized instructions will be used
899// when computing the product.
900
901template <typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor>
902class restricted_packet_dense_assignment_kernel
903 : public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn> {
904 protected:
905 using Base = generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn>;
906
907 public:
908 using Scalar = typename Base::Scalar;
909 using DstXprType = typename Base::DstXprType;
910 using AssignmentTraits = copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, 4>;
911 using PacketType = typename AssignmentTraits::PacketType;
912
913 EIGEN_DEVICE_FUNC restricted_packet_dense_assignment_kernel(DstEvaluatorTypeT& dst, const SrcEvaluatorTypeT& src,
914 const Functor& func, DstXprType& dstExpr)
915 : Base(dst, src, func, dstExpr) {}
916};
917
918/***************************************************************************
919 * Part 5 : Entry point for dense rectangular assignment
920 ***************************************************************************/
921
922template <typename DstXprType, typename SrcXprType, typename Functor>
923EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize_if_allowed(DstXprType& dst, const SrcXprType& src,
924 const Functor& /*func*/) {
925 EIGEN_INTERNAL_ASSERT_SAME_SHAPE(dst, src);
926}
927
928template <typename DstXprType, typename SrcXprType, typename T1, typename T2>
929EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize_if_allowed(DstXprType& dst, const SrcXprType& src,
930 const internal::assign_op<T1, T2>& /*func*/) {
931 Index dstRows = src.rows();
932 Index dstCols = src.cols();
933 if (((dst.rows() != dstRows) || (dst.cols() != dstCols))) {
934#ifdef EIGEN_NO_AUTOMATIC_RESIZING
935 eigen_assert(dst.size() == 0 &&
936 "Size mismatch. Automatic resizing is disabled because EIGEN_NO_AUTOMATIC_RESIZING is defined");
937 if (dst.size() == 0) {
938 dst.resize(dstRows, dstCols);
939 }
940#else
941 dst.resize(dstRows, dstCols);
942 EIGEN_INTERNAL_CHECK_SHAPE(dst.rows() == dstRows && dst.cols() == dstCols, YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES,
943 internal::matching_shapes<DstXprType, SrcXprType>);
944#endif
945 }
946}
947
948// The evaluator a dense assignment reads its source through. The assignment loops index the source by the
949// destination's storage order and use only the source evaluator's flags, so a source may be evaluated in another
950// storage order than its expression reports; see the coefficient-based product in ProductEvaluators.h.
951template <typename DstEvaluatorType, typename SrcXprType, typename Enable = void>
952struct dense_assignment_source_evaluator {
953 using type = evaluator<SrcXprType>;
954};
955
956template <typename DstXprType, typename SrcXprType, typename Functor>
957EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE constexpr void call_dense_assignment_loop(DstXprType& dst, const SrcXprType& src,
958 const Functor& func) {
959 using DstEvaluatorType = evaluator<DstXprType>;
960 using SrcEvaluatorType = typename dense_assignment_source_evaluator<DstEvaluatorType, SrcXprType>::type;
961
962 SrcEvaluatorType srcEvaluator(src);
963
964 // NOTE To properly handle A = (A*A.transpose())/s with A rectangular,
965 // we need to resize the destination after the source evaluator has been created.
966 resize_if_allowed(dst, src, func);
967
968 DstEvaluatorType dstEvaluator(dst);
969
970 using Kernel = generic_dense_assignment_kernel<DstEvaluatorType, SrcEvaluatorType, Functor>;
971 Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
972
973 dense_assignment_loop<Kernel>::run(kernel);
974}
975
976template <typename DstXprType, typename SrcXprType>
977EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void call_dense_assignment_loop(DstXprType& dst, const SrcXprType& src) {
978 call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>());
979}
980
981/***************************************************************************
982 * Part 6 : Generic assignment
983 ***************************************************************************/
984
985// Based on the respective shapes of the destination and source,
986// the class AssignmentKind determine the kind of assignment mechanism.
987// AssignmentKind must define a Kind typedef.
988template <typename DstShape, typename SrcShape>
989struct AssignmentKind;
990
991// Assignment kind defined in this file:
992struct Dense2Dense {};
993struct EigenBase2EigenBase {};
994
995template <typename, typename>
996struct AssignmentKind {
997 using Kind = EigenBase2EigenBase;
998};
999template <>
1000struct AssignmentKind<DenseShape, DenseShape> {
1001 using Kind = Dense2Dense;
1002};
1003
1004// This is the main assignment class
1005template <typename DstXprType, typename SrcXprType, typename Functor,
1006 typename Kind = typename AssignmentKind<typename evaluator_traits<DstXprType>::Shape,
1007 typename evaluator_traits<SrcXprType>::Shape>::Kind,
1008 typename EnableIf = void>
1009struct Assignment;
1010
1011template <typename Src, typename Func>
1012struct assignment_assume_aliasing : evaluator_assume_aliasing<Src> {};
1013
1014// The only purpose of this call_assignment() function is to deal with noalias() / "assume-aliasing" and automatic
1015// transposition. Indeed, I (Gael) think that this concept of "assume-aliasing" was a mistake, and it makes thing quite
1016// complicated. So this intermediate function removes everything related to "assume-aliasing" such that Assignment does
1017// not has to bother about these annoying details.
1018
1019template <typename Dst, typename Src>
1020EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_assignment(Dst& dst, const Src& src) {
1021 call_assignment(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
1022}
1023template <typename Dst, typename Src>
1024EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment(const Dst& dst, const Src& src) {
1025 call_assignment(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
1026}
1027
1028// Deal with "assume-aliasing"
1029template <typename Dst, typename Src, typename Func,
1030 std::enable_if_t<assignment_assume_aliasing<Src, Func>::value, int> = 0>
1031EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_assignment(Dst& dst, const Src& src, const Func& func) {
1032 typename plain_matrix_type<Src>::type tmp(src);
1033 call_assignment_no_alias(dst, tmp, func);
1034}
1035
1036template <typename Dst, typename Src, typename Func,
1037 std::enable_if_t<!assignment_assume_aliasing<Src, Func>::value, int> = 0>
1038EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_assignment(Dst& dst, const Src& src, const Func& func) {
1039 call_assignment_no_alias(dst, src, func);
1040}
1041
1042// by-pass "assume-aliasing"
1043// When there is no aliasing, we require that 'dst' has been properly resized
1044template <typename Dst, template <typename> class StorageBase, typename Src, typename Func>
1045EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_assignment(NoAlias<Dst, StorageBase>& dst, const Src& src,
1046 const Func& func) {
1047 call_assignment_no_alias(dst.expression(), src, func);
1048}
1049
1050template <typename Dst, typename Src, typename Func>
1051EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_assignment_no_alias(Dst& dst, const Src& src,
1052 const Func& func) {
1053 enum {
1054 NeedToTranspose = ((int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1) ||
1055 (int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)) &&
1056 int(Dst::SizeAtCompileTime) != 1
1057 };
1058
1059 using ActualDstTypeCleaned = std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst>;
1060 using ActualDstType = std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst&>;
1061 ActualDstType actualDst(dst);
1062
1063 // TODO: check whether this is the right place to perform these checks:
1064 EIGEN_STATIC_ASSERT_LVALUE(Dst)
1065 EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned, Src)
1066 EIGEN_CHECK_BINARY_COMPATIBILITY(Func, typename ActualDstTypeCleaned::Scalar, typename Src::Scalar);
1067
1068 Assignment<ActualDstTypeCleaned, Src, Func>::run(actualDst, src, func);
1069}
1070
1071template <typename Dst, typename Src, typename Func>
1072EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void call_restricted_packet_assignment_no_alias(Dst& dst, const Src& src,
1073 const Func& func) {
1074 using DstEvaluatorType = evaluator<Dst>;
1075 using SrcEvaluatorType = typename dense_assignment_source_evaluator<DstEvaluatorType, Src>::type;
1076 using Kernel = restricted_packet_dense_assignment_kernel<DstEvaluatorType, SrcEvaluatorType, Func>;
1077
1078 EIGEN_STATIC_ASSERT_LVALUE(Dst)
1079 EIGEN_CHECK_BINARY_COMPATIBILITY(Func, typename Dst::Scalar, typename Src::Scalar);
1080
1081 SrcEvaluatorType srcEvaluator(src);
1082 resize_if_allowed(dst, src, func);
1083
1084 DstEvaluatorType dstEvaluator(dst);
1085 Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
1086
1087 dense_assignment_loop<Kernel>::run(kernel);
1088}
1089
1090template <typename Dst, typename Src>
1091EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_assignment_no_alias(Dst& dst, const Src& src) {
1092 call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
1093}
1094
1095template <typename Dst, typename Src, typename Func>
1096EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src,
1097 const Func& func) {
1098 // TODO: check whether this is the right place to perform these checks:
1099 EIGEN_STATIC_ASSERT_LVALUE(Dst)
1100 EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst, Src)
1101 EIGEN_CHECK_BINARY_COMPATIBILITY(Func, typename Dst::Scalar, typename Src::Scalar);
1102
1103 Assignment<Dst, Src, Func>::run(dst, src, func);
1104}
1105template <typename Dst, typename Src>
1106EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src) {
1107 call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
1108}
1109
1110// forward declaration
1111template <typename Dst, typename Src>
1112EIGEN_DEVICE_FUNC void check_for_aliasing(const Dst& dst, const Src& src);
1113
1114// Generic Dense to Dense assignment
1115// Note that the last template argument "Weak" is needed to make it possible to perform
1116// both partial specialization+SFINAE without ambiguous specialization
1117template <typename DstXprType, typename SrcXprType, typename Functor, typename Weak>
1118struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Weak> {
1119 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(DstXprType& dst, const SrcXprType& src,
1120 const Functor& func) {
1121#ifndef EIGEN_NO_DEBUG
1122 if (!internal::is_constant_evaluated()) {
1123 internal::check_for_aliasing(dst, src);
1124 }
1125#endif
1126
1127 call_dense_assignment_loop(dst, src, func);
1128 }
1129};
1130
1131template <typename DstXprType, typename SrcPlainObject, typename Weak>
1132struct Assignment<DstXprType, CwiseNullaryOp<scalar_constant_op<typename DstXprType::Scalar>, SrcPlainObject>,
1133 assign_op<typename DstXprType::Scalar, typename DstXprType::Scalar>, Dense2Dense, Weak> {
1134 using Scalar = typename DstXprType::Scalar;
1135 using NullaryOp = scalar_constant_op<Scalar>;
1136 using SrcXprType = CwiseNullaryOp<NullaryOp, SrcPlainObject>;
1137 using Functor = assign_op<Scalar, Scalar>;
1138 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src,
1139 const Functor& /*func*/) {
1140 eigen_fill_impl<DstXprType>::run(dst, src);
1141 }
1142};
1143
1144template <typename DstXprType, typename SrcPlainObject, typename Weak>
1145struct Assignment<DstXprType, CwiseNullaryOp<scalar_zero_op<typename DstXprType::Scalar>, SrcPlainObject>,
1146 assign_op<typename DstXprType::Scalar, typename DstXprType::Scalar>, Dense2Dense, Weak> {
1147 using Scalar = typename DstXprType::Scalar;
1148 using NullaryOp = scalar_zero_op<Scalar>;
1149 using SrcXprType = CwiseNullaryOp<NullaryOp, SrcPlainObject>;
1150 using Functor = assign_op<Scalar, Scalar>;
1151 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src,
1152 const Functor& /*func*/) {
1153 eigen_zero_impl<DstXprType>::run(dst, src);
1154 }
1155};
1156
1157// Generic assignment through evalTo.
1158// TODO: evaluate whether this generic evalTo-based assignment path is still needed.
1159// Note that the last template argument "Weak" is needed to make it possible to perform
1160// both partial specialization+SFINAE without ambiguous specialization
1161template <typename DstXprType, typename SrcXprType, typename Functor, typename Weak>
1162struct Assignment<DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Weak> {
1163 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(
1164 DstXprType& dst, const SrcXprType& src,
1165 const internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>& /*func*/) {
1166 Index dstRows = src.rows();
1167 Index dstCols = src.cols();
1168 if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
1169
1170 EIGEN_INTERNAL_ASSERT_SAME_SHAPE(dst, src);
1171 src.evalTo(dst);
1172 }
1173
1174 // NOTE The following two functions are templated to avoid their instantiation if not needed
1175 // This is needed because some expressions supports evalTo only and/or have 'void' as scalar type.
1176 template <typename SrcScalarType>
1177 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(
1178 DstXprType& dst, const SrcXprType& src,
1179 const internal::add_assign_op<typename DstXprType::Scalar, SrcScalarType>& /*func*/) {
1180 Index dstRows = src.rows();
1181 Index dstCols = src.cols();
1182 if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
1183
1184 EIGEN_INTERNAL_ASSERT_SAME_SHAPE(dst, src);
1185 src.addTo(dst);
1186 }
1187
1188 template <typename SrcScalarType>
1189 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(
1190 DstXprType& dst, const SrcXprType& src,
1191 const internal::sub_assign_op<typename DstXprType::Scalar, SrcScalarType>& /*func*/) {
1192 Index dstRows = src.rows();
1193 Index dstCols = src.cols();
1194 if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
1195
1196 EIGEN_INTERNAL_ASSERT_SAME_SHAPE(dst, src);
1197 src.subTo(dst);
1198 }
1199};
1200
1201} // namespace internal
1202
1203} // end namespace Eigen
1204
1205#endif // EIGEN_ASSIGN_EVALUATOR_H
@ Unaligned
Definition Constants.h:236
constexpr unsigned int ActualPacketAccessBit
Definition Constants.h:109
constexpr unsigned int DirectAccessBit
Definition Constants.h:160
constexpr unsigned int LinearAccessBit
Definition Constants.h:134
constexpr unsigned int RowMajorBit
Definition Constants.h:71