Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
BlasUtil.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2009-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_BLASUTIL_H
12#define EIGEN_BLASUTIL_H
13
14// This file contains many lightweight helper classes used to
15// implement and control fast level 2 and level 3 BLAS-like routines.
16
17// IWYU pragma: private
18#include "../InternalHeaderCheck.h"
19
20namespace Eigen {
21
22namespace internal {
23
24// A singleton stored column never uses its outer stride, but BLAS still checks ld >= max(1, inner).
25// Normalize before narrowing: an unused stride need not fit BlasIndex.
26template <int StorageOrder, typename Index>
27EIGEN_STRONG_INLINE BlasIndex blas_leading_dim(Index rows, Index cols, Index stride) {
28 const Index inner = StorageOrder == ColMajor ? rows : cols;
29 const Index outer = StorageOrder == ColMajor ? cols : rows;
30 return convert_index<BlasIndex>(outer <= 1 ? numext::maxi(inner, Index(1)) : stride);
31}
32
33template <int StorageOrder, typename Index>
34EIGEN_STRONG_INLINE bool blas_compatible_stride(Index rows, Index cols, Index stride) {
35 const Index inner = StorageOrder == ColMajor ? rows : cols;
36 const Index outer = StorageOrder == ColMajor ? cols : rows;
37 return outer <= 1 || stride >= numext::maxi(inner, Index(1));
38}
39
40// BLAS rejects zero increments even when the vector has only one coefficient.
41template <typename Index>
42EIGEN_STRONG_INLINE BlasIndex blas_vector_increment(Index size, Index stride) {
43 return convert_index<BlasIndex>(size <= 1 ? Index(1) : stride);
44}
45
46// forward declarations
47template <typename LhsScalar, typename RhsScalar, typename Index, typename DataMapper, int mr, int nr,
48 bool ConjugateLhs = false, bool ConjugateRhs = false>
49struct gebp_kernel;
50
51template <typename Scalar, typename Index, typename DataMapper, int nr, int StorageOrder, bool Conjugate = false,
52 bool PanelMode = false>
53struct gemm_pack_rhs;
54
55template <typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, int StorageOrder,
56 bool Conjugate = false, bool PanelMode = false>
57struct gemm_pack_lhs;
58
59// LhsStorage and RhsStorage are the scalar types the operands are stored in. The packers convert each coefficient to
60// LhsScalar or RhsScalar as they read it, so that A.cast<float>() * B.cast<float>() runs the float kernel on half
61// operands without materializing the casts.
62template <typename Index, typename LhsScalar, int LhsStorageOrder, bool ConjugateLhs, typename RhsScalar,
63 int RhsStorageOrder, bool ConjugateRhs, int ResStorageOrder, int ResInnerStride,
64 typename LhsStorage = LhsScalar, typename RhsStorage = RhsScalar>
65struct general_matrix_matrix_product;
66
67template <typename Index, typename LhsScalar, typename LhsMapper, int LhsStorageOrder, bool ConjugateLhs,
68 typename RhsScalar, typename RhsMapper, bool ConjugateRhs, int Version = Specialized>
69struct general_matrix_vector_product;
70
71template <typename From, typename To>
72struct get_factor {
73 EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE To run(const From& x) { return To(x); }
74};
75
76template <typename Scalar>
77struct get_factor<Scalar, typename NumTraits<Scalar>::Real> {
78 EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE typename NumTraits<Scalar>::Real run(const Scalar& x) {
79 return numext::real(x);
80 }
81};
82
83template <typename Scalar, typename Index>
84class BlasVectorMapper {
85 public:
86 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE BlasVectorMapper(Scalar* data) : m_data(data) {}
87
88 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE Scalar operator()(Index i) const { return m_data[i]; }
89 template <typename Packet, int AlignmentType>
90 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet load(Index i) const {
91 return ploadt<Packet, AlignmentType>(m_data + i);
92 }
93
94 template <typename Packet>
95 EIGEN_DEVICE_FUNC bool aligned(Index i) const {
96 return (std::uintptr_t(m_data + i) % sizeof(Packet)) == 0;
97 }
98
99 protected:
100 Scalar* m_data;
101};
102
103template <typename Scalar, typename Index, int AlignmentType, int Incr = 1>
104class BlasLinearMapper;
105
106template <typename Scalar, typename Index, int AlignmentType>
107class BlasLinearMapper<Scalar, Index, AlignmentType> {
108 public:
109 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE BlasLinearMapper(Scalar* data, Index incr = 1) : m_data(data) {
110 EIGEN_ONLY_USED_FOR_DEBUG(incr);
111 eigen_assert(incr == 1);
112 }
113
114 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void prefetch(Index i) const { internal::prefetch(&operator()(i)); }
115
116 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE Scalar& operator()(Index i) const { return m_data[i]; }
117
118 template <typename PacketType>
119 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i) const {
120 return ploadt<PacketType, AlignmentType>(m_data + i);
121 }
122
123 template <typename PacketType>
124 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index n, Index offset = 0) const {
125 return ploadt_partial<PacketType, AlignmentType>(m_data + i, n, offset);
126 }
127
128 template <typename PacketType, int AlignmentT>
129 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType load(Index i) const {
130 return ploadt<PacketType, AlignmentT>(m_data + i);
131 }
132
133 template <typename PacketType>
134 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, const PacketType& p) const {
135 pstoret<Scalar, PacketType, AlignmentType>(m_data + i, p);
136 }
137
138 template <typename PacketType>
139 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketPartial(Index i, const PacketType& p, Index n,
140 Index offset = 0) const {
141 pstoret_partial<Scalar, PacketType, AlignmentType>(m_data + i, p, n, offset);
142 }
143
144 protected:
145 Scalar* m_data;
146};
147
148// Lightweight helper class to access matrix coefficients.
149template <typename Scalar, typename Index, int StorageOrder, int AlignmentType = Unaligned, int Incr = 1>
150class blas_data_mapper;
151
152// TMP to help PacketBlock store implementation.
153// There's currently no known use case for PacketBlock load.
154// The default implementation assumes ColMajor order.
155// It always store each packet sequentially one `stride` apart.
156template <typename Index, typename Scalar, typename Packet, int n, int idx, int StorageOrder>
157struct PacketBlockManagement {
158 PacketBlockManagement<Index, Scalar, Packet, n, idx - 1, StorageOrder> pbm;
159 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(Scalar* to, const Index stride, Index i, Index j,
160 const PacketBlock<Packet, n>& block) const {
161 pbm.store(to, stride, i, j, block);
162 pstoreu<Scalar>(to + i + (j + idx) * stride, block.packet[idx]);
163 }
164};
165
166// PacketBlockManagement specialization to take care of RowMajor order without ifs.
167template <typename Index, typename Scalar, typename Packet, int n, int idx>
168struct PacketBlockManagement<Index, Scalar, Packet, n, idx, RowMajor> {
169 PacketBlockManagement<Index, Scalar, Packet, n, idx - 1, RowMajor> pbm;
170 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(Scalar* to, const Index stride, Index i, Index j,
171 const PacketBlock<Packet, n>& block) const {
172 pbm.store(to, stride, i, j, block);
173 pstoreu<Scalar>(to + j + (i + idx) * stride, block.packet[idx]);
174 }
175};
176
177template <typename Index, typename Scalar, typename Packet, int n, int StorageOrder>
178struct PacketBlockManagement<Index, Scalar, Packet, n, -1, StorageOrder> {
179 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(Scalar* to, const Index stride, Index i, Index j,
180 const PacketBlock<Packet, n>& block) const {
181 EIGEN_UNUSED_VARIABLE(to);
182 EIGEN_UNUSED_VARIABLE(stride);
183 EIGEN_UNUSED_VARIABLE(i);
184 EIGEN_UNUSED_VARIABLE(j);
185 EIGEN_UNUSED_VARIABLE(block);
186 }
187};
188
189template <typename Index, typename Scalar, typename Packet, int n>
190struct PacketBlockManagement<Index, Scalar, Packet, n, -1, RowMajor> {
191 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(Scalar* to, const Index stride, Index i, Index j,
192 const PacketBlock<Packet, n>& block) const {
193 EIGEN_UNUSED_VARIABLE(to);
194 EIGEN_UNUSED_VARIABLE(stride);
195 EIGEN_UNUSED_VARIABLE(i);
196 EIGEN_UNUSED_VARIABLE(j);
197 EIGEN_UNUSED_VARIABLE(block);
198 }
199};
200
201template <typename Scalar, typename Index, int StorageOrder, int AlignmentType>
202class blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, 1> {
203 public:
204 using LinearMapper = BlasLinearMapper<Scalar, Index, AlignmentType>;
205 using SubMapper = blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType>;
206 using VectorMapper = BlasVectorMapper<Scalar, Index>;
207
208 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE blas_data_mapper(Scalar* data, Index stride, Index incr = 1)
209 : m_data(data), m_stride(stride) {
210 EIGEN_ONLY_USED_FOR_DEBUG(incr);
211 eigen_assert(incr == 1);
212 }
213
214 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE SubMapper getSubMapper(Index i, Index j) const {
215 return SubMapper(&operator()(i, j), m_stride);
216 }
217
218 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j) const {
219 return LinearMapper(&operator()(i, j));
220 }
221
222 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE VectorMapper getVectorMapper(Index i, Index j) const {
223 return VectorMapper(&operator()(i, j));
224 }
225
226 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void prefetch(Index i, Index j) const { internal::prefetch(&operator()(i, j)); }
227
228 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE Scalar& operator()(Index i, Index j) const {
229 return m_data[StorageOrder == RowMajor ? j + i * m_stride : i + j * m_stride];
230 }
231
232 template <typename PacketType>
233 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i, Index j) const {
234 return ploadt<PacketType, AlignmentType>(&operator()(i, j));
235 }
236
237 template <typename PacketType>
238 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index j, Index n,
239 Index offset = 0) const {
240 return ploadt_partial<PacketType, AlignmentType>(&operator()(i, j), n, offset);
241 }
242
243 template <typename PacketT, int AlignmentT>
244 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i, Index j) const {
245 return ploadt<PacketT, AlignmentT>(&operator()(i, j));
246 }
247
248 template <typename PacketType>
249 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, Index j, const PacketType& p) const {
250 pstoret<Scalar, PacketType, AlignmentType>(&operator()(i, j), p);
251 }
252
253 template <typename PacketType>
254 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketPartial(Index i, Index j, const PacketType& p, Index n,
255 Index offset = 0) const {
256 pstoret_partial<Scalar, PacketType, AlignmentType>(&operator()(i, j), p, n, offset);
257 }
258
259 template <typename SubPacket>
260 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void scatterPacket(Index i, Index j, const SubPacket& p) const {
261 pscatter<Scalar, SubPacket>(&operator()(i, j), p, m_stride);
262 }
263
264 template <typename SubPacket>
265 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SubPacket gatherPacket(Index i, Index j) const {
266 return pgather<Scalar, SubPacket>(&operator()(i, j), m_stride);
267 }
268
269 EIGEN_DEVICE_FUNC constexpr const Index stride() const { return m_stride; }
270 EIGEN_DEVICE_FUNC constexpr const Index incr() const { return 1; }
271 EIGEN_DEVICE_FUNC constexpr const Scalar* data() const { return m_data; }
272
273 EIGEN_DEVICE_FUNC Index firstAligned(Index size) const {
274 if (std::uintptr_t(m_data) % sizeof(Scalar)) {
275 return -1;
276 }
277 return internal::first_default_aligned(m_data, size);
278 }
279
280 template <typename SubPacket, int n>
281 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketBlock(Index i, Index j,
282 const PacketBlock<SubPacket, n>& block) const {
283 PacketBlockManagement<Index, Scalar, SubPacket, n, n - 1, StorageOrder> pbm;
284 pbm.store(m_data, m_stride, i, j, block);
285 }
286
287 protected:
288 Scalar* EIGEN_RESTRICT m_data;
289 const Index m_stride;
290};
291
292// Implementation of non-natural increment (i.e. inner-stride != 1)
293// The exposed API is not complete yet compared to the Incr==1 case
294// because some features makes less sense in this case.
295template <typename Scalar, typename Index, int AlignmentType, int Incr>
296class BlasLinearMapper {
297 public:
298 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE BlasLinearMapper(Scalar* data, Index incr)
299 : m_data(data), m_incr(incr) {}
300
301 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void prefetch(int i) const { internal::prefetch(&operator()(i)); }
302
303 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE Scalar& operator()(Index i) const {
304 return m_data[i * m_incr.value()];
305 }
306
307 template <typename PacketType>
308 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i) const {
309 return pgather<Scalar, PacketType>(m_data + i * m_incr.value(), m_incr.value());
310 }
311
312 template <typename PacketType>
313 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index n, Index /*offset*/ = 0) const {
314 return pgather_partial<Scalar, PacketType>(m_data + i * m_incr.value(), m_incr.value(), n);
315 }
316
317 template <typename PacketType>
318 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, const PacketType& p) const {
319 pscatter<Scalar, PacketType>(m_data + i * m_incr.value(), p, m_incr.value());
320 }
321
322 template <typename PacketType>
323 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketPartial(Index i, const PacketType& p, Index n,
324 Index /*offset*/ = 0) const {
325 pscatter_partial<Scalar, PacketType>(m_data + i * m_incr.value(), p, m_incr.value(), n);
326 }
327
328 protected:
329 Scalar* m_data;
330 const internal::variable_if_dynamic<Index, Incr> m_incr;
331};
332
333template <typename Scalar, typename Index, int StorageOrder, int AlignmentType, int Incr>
334class blas_data_mapper {
335 public:
336 using LinearMapper = BlasLinearMapper<Scalar, Index, AlignmentType, Incr>;
337 using SubMapper = blas_data_mapper;
338
339 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE blas_data_mapper(Scalar* data, Index stride, Index incr)
340 : m_data(data), m_stride(stride), m_incr(incr) {}
341
342 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE SubMapper getSubMapper(Index i, Index j) const {
343 return SubMapper(&operator()(i, j), m_stride, m_incr.value());
344 }
345
346 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j) const {
347 return LinearMapper(&operator()(i, j), m_incr.value());
348 }
349
350 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void prefetch(Index i, Index j) const { internal::prefetch(&operator()(i, j)); }
351
352 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE Scalar& operator()(Index i, Index j) const {
353 return m_data[StorageOrder == RowMajor ? j * m_incr.value() + i * m_stride : i * m_incr.value() + j * m_stride];
354 }
355
356 template <typename PacketType>
357 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i, Index j) const {
358 return pgather<Scalar, PacketType>(&operator()(i, j), m_incr.value());
359 }
360
361 template <typename PacketType>
362 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index j, Index n,
363 Index /*offset*/ = 0) const {
364 return pgather_partial<Scalar, PacketType>(&operator()(i, j), m_incr.value(), n);
365 }
366
367 template <typename PacketT, int AlignmentT>
368 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i, Index j) const {
369 return pgather<Scalar, PacketT>(&operator()(i, j), m_incr.value());
370 }
371
372 template <typename PacketType>
373 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, Index j, const PacketType& p) const {
374 pscatter<Scalar, PacketType>(&operator()(i, j), p, m_incr.value());
375 }
376
377 template <typename PacketType>
378 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketPartial(Index i, Index j, const PacketType& p, Index n,
379 Index /*offset*/ = 0) const {
380 pscatter_partial<Scalar, PacketType>(&operator()(i, j), p, m_incr.value(), n);
381 }
382
383 template <typename SubPacket>
384 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void scatterPacket(Index i, Index j, const SubPacket& p) const {
385 pscatter<Scalar, SubPacket>(&operator()(i, j), p, m_stride);
386 }
387
388 template <typename SubPacket>
389 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SubPacket gatherPacket(Index i, Index j) const {
390 return pgather<Scalar, SubPacket>(&operator()(i, j), m_stride);
391 }
392
393 // storePacketBlock_helper defines a way to access values inside the PacketBlock, this is essentially required by the
394 // Complex types.
395 template <typename SubPacket, typename Scalar_, int n, int idx>
396 struct storePacketBlock_helper {
397 storePacketBlock_helper<SubPacket, Scalar_, n, idx - 1> spbh;
398 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(
399 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>* sup, Index i, Index j,
400 const PacketBlock<SubPacket, n>& block) const {
401 spbh.store(sup, i, j, block);
402 sup->template storePacket<SubPacket>(i, j + idx, block.packet[idx]);
403 }
404 };
405
406 template <typename SubPacket, int n, int idx>
407 struct storePacketBlock_helper<SubPacket, std::complex<float>, n, idx> {
408 storePacketBlock_helper<SubPacket, std::complex<float>, n, idx - 1> spbh;
409 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(
410 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>* sup, Index i, Index j,
411 const PacketBlock<SubPacket, n>& block) const {
412 spbh.store(sup, i, j, block);
413 sup->template storePacket<SubPacket>(i, j + idx, block.packet[idx]);
414 }
415 };
416
417 template <typename SubPacket, int n, int idx>
418 struct storePacketBlock_helper<SubPacket, std::complex<double>, n, idx> {
419 storePacketBlock_helper<SubPacket, std::complex<double>, n, idx - 1> spbh;
420 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(
421 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>* sup, Index i, Index j,
422 const PacketBlock<SubPacket, n>& block) const {
423 spbh.store(sup, i, j, block);
424 for (int l = 0; l < unpacket_traits<SubPacket>::size; l++) {
425 std::complex<double>* v = &sup->operator()(i + l, j + idx);
426 v->real(block.packet[idx].v[2 * l + 0]);
427 v->imag(block.packet[idx].v[2 * l + 1]);
428 }
429 }
430 };
431
432 template <typename SubPacket, typename Scalar_, int n>
433 struct storePacketBlock_helper<SubPacket, Scalar_, n, -1> {
434 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(
435 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>*, Index, Index,
436 const PacketBlock<SubPacket, n>&) const {}
437 };
438
439 template <typename SubPacket, int n>
440 struct storePacketBlock_helper<SubPacket, std::complex<float>, n, -1> {
441 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(
442 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>*, Index, Index,
443 const PacketBlock<SubPacket, n>&) const {}
444 };
445
446 template <typename SubPacket, int n>
447 struct storePacketBlock_helper<SubPacket, std::complex<double>, n, -1> {
448 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(
449 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>*, Index, Index,
450 const PacketBlock<SubPacket, n>&) const {}
451 };
452 // This function stores a PacketBlock on m_data, this approach is really quite slow compare to Incr=1 and should be
453 // avoided when possible.
454 template <typename SubPacket, int n>
455 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketBlock(Index i, Index j,
456 const PacketBlock<SubPacket, n>& block) const {
457 storePacketBlock_helper<SubPacket, Scalar, n, n - 1> spb;
458 spb.store(this, i, j, block);
459 }
460
461 EIGEN_DEVICE_FUNC constexpr const Index stride() const { return m_stride; }
462 EIGEN_DEVICE_FUNC constexpr const Index incr() const { return m_incr.value(); }
463 EIGEN_DEVICE_FUNC constexpr Scalar* data() const { return m_data; }
464
465 protected:
466 Scalar* EIGEN_RESTRICT m_data;
467 const Index m_stride;
468 const internal::variable_if_dynamic<Index, Incr> m_incr;
469};
470
471// lightweight helper class to access matrix coefficients (const version)
472template <typename Scalar, typename Index, int StorageOrder>
473class const_blas_data_mapper : public blas_data_mapper<const Scalar, Index, StorageOrder> {
474 public:
475 using SubMapper = const_blas_data_mapper<Scalar, Index, StorageOrder>;
476
477 EIGEN_ALWAYS_INLINE const_blas_data_mapper(const Scalar* data, Index stride)
478 : blas_data_mapper<const Scalar, Index, StorageOrder>(data, stride) {}
479
480 EIGEN_ALWAYS_INLINE SubMapper getSubMapper(Index i, Index j) const {
481 return SubMapper(&(this->operator()(i, j)), this->m_stride);
482 }
483};
484
485// Loads a Packet from the SrcScalar coefficients at `from`, each converted as cast<SrcScalar, Scalar> converts it.
486// Vectorized when a source packet of as many coefficients casts to Packet, as in the core_cast_op evaluator.
487template <typename SrcScalar, typename Packet, typename = void>
488struct cast_ploadu_impl {
489 using Scalar = typename unpacket_traits<Packet>::type;
490 static constexpr int Size = unpacket_traits<Packet>::size;
491 static EIGEN_ALWAYS_INLINE Packet run(const SrcScalar* from) {
492 Scalar to[Size];
493 for (int i = 0; i < Size; ++i) to[i] = cast<SrcScalar, Scalar>(from[i]);
494 return ploadu<Packet>(to);
495 }
496};
497template <typename SrcScalar, typename Packet>
498struct cast_ploadu_impl<
499 SrcScalar, Packet,
500 std::enable_if_t<type_casting_traits<SrcScalar, typename unpacket_traits<Packet>::type>::VectorizedCast &&
501 find_packet_by_size<SrcScalar, unpacket_traits<Packet>::size>::value>> {
502 using SrcPacket = typename find_packet_by_size<SrcScalar, unpacket_traits<Packet>::size>::type;
503 static EIGEN_ALWAYS_INLINE Packet run(const SrcScalar* from) {
504 return pcast<SrcPacket, Packet>(ploadu<SrcPacket>(from));
505 }
506};
507#ifdef EIGEN_VECTORIZE_VSX
508// Packet8bf is its own half, so no bfloat16 packet holds four coefficients. pcast<Packet8bf, Packet4f> widens the
509// first four, so splatting them as one 64-bit lane suffices.
510template <>
511struct cast_ploadu_impl<bfloat16, Packet4f> {
512 static EIGEN_ALWAYS_INLINE Packet4f run(const bfloat16* from) {
513 unsigned long long bits;
514 std::memcpy(&bits, from, sizeof(bits));
515 return pcast<Packet8bf, Packet4f>(Packet8bf(reinterpret_cast<Packet8us>(vec_splats(bits))));
516 }
517};
518#endif
519
520template <typename Scalar, typename SrcScalar, typename Index>
521class BlasCastLinearMapper {
522 using SrcMapper = BlasLinearMapper<const SrcScalar, Index, Unaligned>;
523
524 public:
525 EIGEN_ALWAYS_INLINE explicit BlasCastLinearMapper(const SrcMapper& src) : m_src(src) {}
526
527 EIGEN_ALWAYS_INLINE void prefetch(Index i) const { m_src.prefetch(i); }
528
529 EIGEN_ALWAYS_INLINE Scalar operator()(Index i) const { return cast<SrcScalar, Scalar>(m_src(i)); }
530
531 template <typename PacketType>
532 EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i) const {
533 return cast_ploadu_impl<SrcScalar, PacketType>::run(&m_src(i));
534 }
535
536 private:
537 SrcMapper m_src;
538};
539
540// Read-only counterpart of const_blas_data_mapper over SrcScalar storage that reads each coefficient as Scalar. It
541// provides what the GEMM packers read through, so that they pack A.cast<Scalar>() straight from A. It holds the
542// SrcScalar mapper rather than deriving from it, so that a packer cannot reach the unconverted loads of the base.
543template <typename Scalar, typename SrcScalar, typename Index, int StorageOrder>
544class blas_cast_data_mapper {
545 using SrcMapper = const_blas_data_mapper<SrcScalar, Index, StorageOrder>;
546
547 public:
548 using LinearMapper = BlasCastLinearMapper<Scalar, SrcScalar, Index>;
549 using SubMapper = blas_cast_data_mapper;
550
551 EIGEN_ALWAYS_INLINE blas_cast_data_mapper(const SrcScalar* data, Index stride) : m_src(data, stride) {}
552 EIGEN_ALWAYS_INLINE explicit blas_cast_data_mapper(const SrcMapper& src) : m_src(src) {}
553
554 EIGEN_ALWAYS_INLINE SubMapper getSubMapper(Index i, Index j) const { return SubMapper(m_src.getSubMapper(i, j)); }
555
556 EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j) const {
557 return LinearMapper(m_src.getLinearMapper(i, j));
558 }
559
560 EIGEN_ALWAYS_INLINE void prefetch(Index i, Index j) const { m_src.prefetch(i, j); }
561
562 EIGEN_ALWAYS_INLINE Scalar operator()(Index i, Index j) const { return cast<SrcScalar, Scalar>(m_src(i, j)); }
563
564 template <typename PacketType>
565 EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i, Index j) const {
566 return cast_ploadu_impl<SrcScalar, PacketType>::run(&m_src(i, j));
567 }
568
569 EIGEN_ALWAYS_INLINE Index stride() const { return m_src.stride(); }
570
571 private:
572 SrcMapper m_src;
573};
574
575// The mapper the GEMM packers read an operand stored as StorageScalar through, as Scalar.
576template <typename Scalar, typename StorageScalar, typename Index, int StorageOrder>
577using gemm_operand_mapper =
578 std::conditional_t<std::is_same<Scalar, StorageScalar>::value, const_blas_data_mapper<Scalar, Index, StorageOrder>,
579 blas_cast_data_mapper<Scalar, StorageScalar, Index, StorageOrder>>;
580
581/* Helper class to analyze the factors of a Product expression.
582 * In particular it allows to pop out operator-, scalar multiples,
583 * and conjugate */
584template <typename XprType>
585struct blas_traits {
586 using Scalar = typename traits<XprType>::Scalar;
587 using ExtractType = const XprType&;
588 using ExtractType_ = XprType;
589 enum {
590 IsComplex = NumTraits<Scalar>::IsComplex,
591 IsTransposed = false,
592 NeedToConjugate = false,
593 HasUsableDirectAccess =
594 ((int(XprType::Flags) & DirectAccessBit) &&
595 (bool(XprType::IsVectorAtCompileTime) || int(inner_stride_at_compile_time<XprType>::value) == 1))
596 ? 1
597 : 0,
598 HasScalarFactor = false
599 };
600 using DirectLinearAccessType =
601 std::conditional_t<bool(HasUsableDirectAccess), ExtractType, typename ExtractType_::PlainObject>;
602 EIGEN_DEVICE_FUNC static inline EIGEN_DEVICE_FUNC ExtractType extract(const XprType& x) { return x; }
603 EIGEN_DEVICE_FUNC static inline EIGEN_DEVICE_FUNC const Scalar extractScalarFactor(const XprType&) {
604 return Scalar(1);
605 }
606 // Inspect explicit factors without combining them across noncommuting coefficients.
607 EIGEN_DEVICE_FUNC static bool hasZeroScalarFactor(const XprType&) { return false; }
608};
609
610// pop conjugate
611template <typename Scalar, typename NestedXpr>
612struct blas_traits<CwiseUnaryOp<scalar_conjugate_op<Scalar>, NestedXpr> > : blas_traits<NestedXpr> {
613 using Base = blas_traits<NestedXpr>;
614 using XprType = CwiseUnaryOp<scalar_conjugate_op<Scalar>, NestedXpr>;
615 using ExtractType = typename Base::ExtractType;
616
617 enum { IsComplex = NumTraits<Scalar>::IsComplex, NeedToConjugate = Base::NeedToConjugate ? 0 : IsComplex };
618 EIGEN_DEVICE_FUNC static inline ExtractType extract(const XprType& x) { return Base::extract(x.nestedExpression()); }
619 EIGEN_DEVICE_FUNC static inline Scalar extractScalarFactor(const XprType& x) {
620 return numext::conj(Base::extractScalarFactor(x.nestedExpression()));
621 }
622 EIGEN_DEVICE_FUNC static bool hasZeroScalarFactor(const XprType& x) {
623 return Base::hasZeroScalarFactor(x.nestedExpression());
624 }
625};
626
627// pop scalar multiple
628template <typename Scalar, typename NestedXpr, typename Plain>
629struct blas_traits<
630 CwiseBinaryOp<scalar_product_op<Scalar>, const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain>, NestedXpr> >
631 : blas_traits<NestedXpr> {
632 enum { HasScalarFactor = true };
633 using Base = blas_traits<NestedXpr>;
634 using XprType =
635 CwiseBinaryOp<scalar_product_op<Scalar>, const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain>, NestedXpr>;
636 using ExtractType = typename Base::ExtractType;
637 EIGEN_DEVICE_FUNC static inline EIGEN_DEVICE_FUNC ExtractType extract(const XprType& x) {
638 return Base::extract(x.rhs());
639 }
640 EIGEN_DEVICE_FUNC static inline EIGEN_DEVICE_FUNC Scalar extractScalarFactor(const XprType& x) {
641 return x.lhs().functor().m_other * Base::extractScalarFactor(x.rhs());
642 }
643 EIGEN_DEVICE_FUNC static bool hasZeroScalarFactor(const XprType& x) {
644 return numext::is_exactly_zero(x.lhs().functor().m_other) || Base::hasZeroScalarFactor(x.rhs());
645 }
646};
647template <typename Scalar, typename NestedXpr, typename Plain>
648struct blas_traits<
649 CwiseBinaryOp<scalar_product_op<Scalar>, NestedXpr, const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain> > >
650 : blas_traits<NestedXpr> {
651 enum { HasScalarFactor = true };
652 using Base = blas_traits<NestedXpr>;
653 using XprType =
654 CwiseBinaryOp<scalar_product_op<Scalar>, NestedXpr, const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain>>;
655 using ExtractType = typename Base::ExtractType;
656 EIGEN_DEVICE_FUNC static inline ExtractType extract(const XprType& x) { return Base::extract(x.lhs()); }
657 EIGEN_DEVICE_FUNC static inline Scalar extractScalarFactor(const XprType& x) {
658 return Base::extractScalarFactor(x.lhs()) * x.rhs().functor().m_other;
659 }
660 EIGEN_DEVICE_FUNC static bool hasZeroScalarFactor(const XprType& x) {
661 return numext::is_exactly_zero(x.rhs().functor().m_other) || Base::hasZeroScalarFactor(x.lhs());
662 }
663};
664// Constant * Constant matches both specializations above; pop the left factor, as the first one does.
665template <typename Scalar, typename Plain1, typename Plain2>
666struct blas_traits<CwiseBinaryOp<scalar_product_op<Scalar>, const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain1>,
667 const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain2>>>
668 : blas_traits<const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain2>> {
669 static constexpr bool HasScalarFactor = true;
670 using Base = blas_traits<const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain2>>;
671 using XprType = CwiseBinaryOp<scalar_product_op<Scalar>, const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain1>,
672 const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain2>>;
673 using ExtractType = typename Base::ExtractType;
674 EIGEN_DEVICE_FUNC static inline ExtractType extract(const XprType& x) { return Base::extract(x.rhs()); }
675 EIGEN_DEVICE_FUNC static inline Scalar extractScalarFactor(const XprType& x) {
676 return x.lhs().functor().m_other * Base::extractScalarFactor(x.rhs());
677 }
678 // Either constant can represent coefficients; preserve zero/NaN multiplication.
679 EIGEN_DEVICE_FUNC static bool hasZeroScalarFactor(const XprType&) { return false; }
680};
681
682// pop opposite
683template <typename Scalar, typename NestedXpr>
684struct blas_traits<CwiseUnaryOp<scalar_opposite_op<Scalar>, NestedXpr> > : blas_traits<NestedXpr> {
685 enum { HasScalarFactor = true };
686 using Base = blas_traits<NestedXpr>;
687 using XprType = CwiseUnaryOp<scalar_opposite_op<Scalar>, NestedXpr>;
688 using ExtractType = typename Base::ExtractType;
689 EIGEN_DEVICE_FUNC static inline ExtractType extract(const XprType& x) { return Base::extract(x.nestedExpression()); }
690 EIGEN_DEVICE_FUNC static inline Scalar extractScalarFactor(const XprType& x) {
691 return -Base::extractScalarFactor(x.nestedExpression());
692 }
693 EIGEN_DEVICE_FUNC static bool hasZeroScalarFactor(const XprType& x) {
694 return Base::hasZeroScalarFactor(x.nestedExpression());
695 }
696};
697
698// pop/push transpose
699template <typename NestedXpr>
700struct blas_traits<Transpose<NestedXpr> > : blas_traits<NestedXpr> {
701 using Scalar = typename NestedXpr::Scalar;
702 using Base = blas_traits<NestedXpr>;
703 using XprType = Transpose<NestedXpr>;
704 using ExtractType = Transpose<const typename Base::ExtractType_>; // const to get rid of a compile error; anyway blas
705 // traits are only used on the RHS
706 using ExtractType_ = Transpose<const typename Base::ExtractType_>;
707 using DirectLinearAccessType =
708 std::conditional_t<bool(Base::HasUsableDirectAccess), ExtractType, typename ExtractType::PlainObject>;
709 enum { IsTransposed = Base::IsTransposed ? 0 : 1 };
710 EIGEN_DEVICE_FUNC static inline ExtractType extract(const XprType& x) {
711 return ExtractType(Base::extract(x.nestedExpression()));
712 }
713 EIGEN_DEVICE_FUNC static inline Scalar extractScalarFactor(const XprType& x) {
714 return Base::extractScalarFactor(x.nestedExpression());
715 }
716 EIGEN_DEVICE_FUNC static bool hasZeroScalarFactor(const XprType& x) {
717 return Base::hasZeroScalarFactor(x.nestedExpression());
718 }
719};
720
721template <typename T>
722struct blas_traits<const T> : blas_traits<T> {};
723
724template <typename T, bool HasUsableDirectAccess = blas_traits<T>::HasUsableDirectAccess>
725struct extract_data_selector {
726 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE static const typename T::Scalar* run(const T& m) {
727 return blas_traits<T>::extract(m).data();
728 }
729};
730
731template <typename T>
732struct extract_data_selector<T, false> {
733 EIGEN_DEVICE_FUNC constexpr static typename T::Scalar* run(const T&) { return 0; }
734};
735
736template <typename T>
737EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE const typename T::Scalar* extract_data(const T& m) {
738 return extract_data_selector<T>::run(m);
739}
740
741// Factor extraction can move scales through either input and the result.
742template <typename LhsScalar, typename RhsScalar>
743struct blas_product_is_commutative
744 : bool_constant<functor_is_commutative<scalar_product_op<LhsScalar, RhsScalar>>::value &&
745 functor_is_commutative<scalar_product_op<LhsScalar>>::value &&
746 functor_is_commutative<scalar_product_op<RhsScalar>>::value &&
747 functor_is_commutative<
748 scalar_product_op<typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType>>::value> {};
749
754template <typename ResScalar, typename Lhs, typename Rhs>
756 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE static ResScalar run(const Lhs& lhs, const Rhs& rhs) {
757 return blas_traits<Lhs>::extractScalarFactor(lhs) * blas_traits<Rhs>::extractScalarFactor(rhs);
758 }
759 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE static ResScalar run(const ResScalar& alpha, const Lhs& lhs,
760 const Rhs& rhs) {
761 return alpha * blas_traits<Lhs>::extractScalarFactor(lhs) * blas_traits<Rhs>::extractScalarFactor(rhs);
762 }
763};
764template <typename Lhs, typename Rhs>
765struct combine_scalar_factors_impl<bool, Lhs, Rhs> {
766 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE static bool run(const Lhs& lhs, const Rhs& rhs) {
767 return blas_traits<Lhs>::extractScalarFactor(lhs) && blas_traits<Rhs>::extractScalarFactor(rhs);
768 }
769 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE static bool run(const bool& alpha, const Lhs& lhs, const Rhs& rhs) {
770 return alpha && blas_traits<Lhs>::extractScalarFactor(lhs) && blas_traits<Rhs>::extractScalarFactor(rhs);
771 }
772};
773
774template <typename ResScalar, typename Lhs, typename Rhs>
775EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE ResScalar combine_scalar_factors(const ResScalar& alpha, const Lhs& lhs,
776 const Rhs& rhs) {
777 return combine_scalar_factors_impl<ResScalar, Lhs, Rhs>::run(alpha, lhs, rhs);
778}
779template <typename ResScalar, typename Lhs, typename Rhs>
780EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE ResScalar combine_scalar_factors(const Lhs& lhs, const Rhs& rhs) {
781 return combine_scalar_factors_impl<ResScalar, Lhs, Rhs>::run(lhs, rhs);
782}
783
784} // end namespace internal
785
786} // end namespace Eigen
787
788#endif // EIGEN_BLASUTIL_H
AlignmentType
Definition Constants.h:235
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321
constexpr unsigned int DirectAccessBit
Definition Constants.h:160