11#ifndef EIGEN_BLASUTIL_H
12#define EIGEN_BLASUTIL_H
18#include "../InternalHeaderCheck.h"
26template <
int StorageOrder,
typename Index>
27EIGEN_STRONG_INLINE BlasIndex blas_leading_dim(Index rows, Index cols, Index stride) {
28 const Index inner = StorageOrder ==
ColMajor ? rows : cols;
29 const Index outer = StorageOrder ==
ColMajor ? cols : rows;
30 return convert_index<BlasIndex>(outer <= 1 ? numext::maxi(inner, Index(1)) : stride);
33template <
int StorageOrder,
typename Index>
34EIGEN_STRONG_INLINE
bool blas_compatible_stride(Index rows, Index cols, Index stride) {
35 const Index inner = StorageOrder ==
ColMajor ? rows : cols;
36 const Index outer = StorageOrder ==
ColMajor ? cols : rows;
37 return outer <= 1 || stride >= numext::maxi(inner, Index(1));
41template <
typename Index>
42EIGEN_STRONG_INLINE BlasIndex blas_vector_increment(Index size, Index stride) {
43 return convert_index<BlasIndex>(size <= 1 ? Index(1) : stride);
47template <
typename LhsScalar,
typename RhsScalar,
typename Index,
typename DataMapper,
int mr,
int nr,
48 bool ConjugateLhs =
false,
bool ConjugateRhs =
false>
51template <
typename Scalar,
typename Index,
typename DataMapper,
int nr,
int StorageOrder,
bool Conjugate =
false,
52 bool PanelMode =
false>
55template <
typename Scalar,
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
int StorageOrder,
56 bool Conjugate =
false,
bool PanelMode =
false>
62template <
typename Index,
typename LhsScalar,
int LhsStorageOrder,
bool ConjugateLhs,
typename RhsScalar,
63 int RhsStorageOrder,
bool ConjugateRhs,
int ResStorageOrder,
int ResInnerStride,
64 typename LhsStorage = LhsScalar,
typename RhsStorage = RhsScalar>
65struct general_matrix_matrix_product;
67template <
typename Index,
typename LhsScalar,
typename LhsMapper,
int LhsStorageOrder,
bool ConjugateLhs,
68 typename RhsScalar,
typename RhsMapper,
bool ConjugateRhs,
int Version = Specialized>
69struct general_matrix_vector_product;
71template <
typename From,
typename To>
73 EIGEN_DEVICE_FUNC
constexpr static EIGEN_STRONG_INLINE To run(
const From& x) {
return To(x); }
76template <
typename Scalar>
77struct get_factor<Scalar, typename NumTraits<Scalar>::Real> {
78 EIGEN_DEVICE_FUNC
constexpr static EIGEN_STRONG_INLINE
typename NumTraits<Scalar>::Real run(
const Scalar& x) {
79 return numext::real(x);
83template <
typename Scalar,
typename Index>
84class BlasVectorMapper {
86 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE BlasVectorMapper(Scalar* data) : m_data(data) {}
88 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE Scalar operator()(Index i)
const {
return m_data[i]; }
89 template <
typename Packet,
int AlignmentType>
90 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet load(Index i)
const {
91 return ploadt<Packet, AlignmentType>(m_data + i);
94 template <
typename Packet>
95 EIGEN_DEVICE_FUNC
bool aligned(Index i)
const {
96 return (std::uintptr_t(m_data + i) %
sizeof(Packet)) == 0;
103template <
typename Scalar,
typename Index,
int AlignmentType,
int Incr = 1>
104class BlasLinearMapper;
106template <
typename Scalar,
typename Index,
int AlignmentType>
109 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE BlasLinearMapper(Scalar* data, Index incr = 1) : m_data(data) {
110 EIGEN_ONLY_USED_FOR_DEBUG(incr);
111 eigen_assert(incr == 1);
114 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void prefetch(Index i)
const { internal::prefetch(&
operator()(i)); }
116 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE Scalar& operator()(Index i)
const {
return m_data[i]; }
118 template <
typename PacketType>
119 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i)
const {
120 return ploadt<PacketType, AlignmentType>(m_data + i);
123 template <
typename PacketType>
124 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index n, Index offset = 0)
const {
125 return ploadt_partial<PacketType, AlignmentType>(m_data + i, n, offset);
128 template <
typename PacketType,
int AlignmentT>
129 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType load(Index i)
const {
130 return ploadt<PacketType, AlignmentT>(m_data + i);
133 template <
typename PacketType>
134 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void storePacket(Index i,
const PacketType& p)
const {
135 pstoret<Scalar, PacketType, AlignmentType>(m_data + i, p);
138 template <
typename PacketType>
139 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void storePacketPartial(Index i,
const PacketType& p, Index n,
140 Index offset = 0)
const {
141 pstoret_partial<Scalar, PacketType, AlignmentType>(m_data + i, p, n, offset);
149template <
typename Scalar,
typename Index,
int StorageOrder,
int AlignmentType = Unaligned,
int Incr = 1>
150class blas_data_mapper;
156template <
typename Index,
typename Scalar,
typename Packet,
int n,
int idx,
int StorageOrder>
157struct PacketBlockManagement {
158 PacketBlockManagement<Index, Scalar, Packet, n, idx - 1, StorageOrder> pbm;
159 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void store(Scalar* to,
const Index stride, Index i, Index j,
160 const PacketBlock<Packet, n>& block)
const {
161 pbm.store(to, stride, i, j, block);
162 pstoreu<Scalar>(to + i + (j + idx) * stride, block.packet[idx]);
167template <
typename Index,
typename Scalar,
typename Packet,
int n,
int idx>
168struct PacketBlockManagement<Index, Scalar, Packet, n, idx,
RowMajor> {
169 PacketBlockManagement<Index, Scalar, Packet, n, idx - 1,
RowMajor> pbm;
170 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void store(Scalar* to,
const Index stride, Index i, Index j,
171 const PacketBlock<Packet, n>& block)
const {
172 pbm.store(to, stride, i, j, block);
173 pstoreu<Scalar>(to + j + (i + idx) * stride, block.packet[idx]);
177template <
typename Index,
typename Scalar,
typename Packet,
int n,
int StorageOrder>
178struct PacketBlockManagement<Index, Scalar, Packet, n, -1, StorageOrder> {
179 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void store(Scalar* to,
const Index stride, Index i, Index j,
180 const PacketBlock<Packet, n>& block)
const {
181 EIGEN_UNUSED_VARIABLE(to);
182 EIGEN_UNUSED_VARIABLE(stride);
183 EIGEN_UNUSED_VARIABLE(i);
184 EIGEN_UNUSED_VARIABLE(j);
185 EIGEN_UNUSED_VARIABLE(block);
189template <
typename Index,
typename Scalar,
typename Packet,
int n>
190struct PacketBlockManagement<Index, Scalar, Packet, n, -1,
RowMajor> {
191 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void store(Scalar* to,
const Index stride, Index i, Index j,
192 const PacketBlock<Packet, n>& block)
const {
193 EIGEN_UNUSED_VARIABLE(to);
194 EIGEN_UNUSED_VARIABLE(stride);
195 EIGEN_UNUSED_VARIABLE(i);
196 EIGEN_UNUSED_VARIABLE(j);
197 EIGEN_UNUSED_VARIABLE(block);
201template <
typename Scalar,
typename Index,
int StorageOrder,
int AlignmentType>
202class blas_data_mapper<Scalar, Index, StorageOrder,
AlignmentType, 1> {
204 using LinearMapper = BlasLinearMapper<Scalar, Index, AlignmentType>;
205 using SubMapper = blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType>;
206 using VectorMapper = BlasVectorMapper<Scalar, Index>;
208 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE blas_data_mapper(Scalar* data, Index stride, Index incr = 1)
209 : m_data(data), m_stride(stride) {
210 EIGEN_ONLY_USED_FOR_DEBUG(incr);
211 eigen_assert(incr == 1);
214 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE SubMapper getSubMapper(Index i, Index j)
const {
215 return SubMapper(&
operator()(i, j), m_stride);
218 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j)
const {
219 return LinearMapper(&
operator()(i, j));
222 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE VectorMapper getVectorMapper(Index i, Index j)
const {
223 return VectorMapper(&
operator()(i, j));
226 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void prefetch(Index i, Index j)
const { internal::prefetch(&
operator()(i, j)); }
228 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE Scalar& operator()(Index i, Index j)
const {
229 return m_data[StorageOrder ==
RowMajor ? j + i * m_stride : i + j * m_stride];
232 template <
typename PacketType>
233 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i, Index j)
const {
234 return ploadt<PacketType, AlignmentType>(&
operator()(i, j));
237 template <
typename PacketType>
238 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index j, Index n,
239 Index offset = 0)
const {
240 return ploadt_partial<PacketType, AlignmentType>(&
operator()(i, j), n, offset);
243 template <
typename PacketT,
int AlignmentT>
244 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i, Index j)
const {
245 return ploadt<PacketT, AlignmentT>(&
operator()(i, j));
248 template <
typename PacketType>
249 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void storePacket(Index i, Index j,
const PacketType& p)
const {
250 pstoret<Scalar, PacketType, AlignmentType>(&
operator()(i, j), p);
253 template <
typename PacketType>
254 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void storePacketPartial(Index i, Index j,
const PacketType& p, Index n,
255 Index offset = 0)
const {
256 pstoret_partial<Scalar, PacketType, AlignmentType>(&
operator()(i, j), p, n, offset);
259 template <
typename SubPacket>
260 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void scatterPacket(Index i, Index j,
const SubPacket& p)
const {
261 pscatter<Scalar, SubPacket>(&
operator()(i, j), p, m_stride);
264 template <
typename SubPacket>
265 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SubPacket gatherPacket(Index i, Index j)
const {
266 return pgather<Scalar, SubPacket>(&
operator()(i, j), m_stride);
269 EIGEN_DEVICE_FUNC
constexpr const Index stride()
const {
return m_stride; }
270 EIGEN_DEVICE_FUNC
constexpr const Index incr()
const {
return 1; }
271 EIGEN_DEVICE_FUNC
constexpr const Scalar* data()
const {
return m_data; }
273 EIGEN_DEVICE_FUNC Index firstAligned(Index size)
const {
274 if (std::uintptr_t(m_data) %
sizeof(Scalar)) {
277 return internal::first_default_aligned(m_data, size);
280 template <
typename SubPacket,
int n>
281 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void storePacketBlock(Index i, Index j,
282 const PacketBlock<SubPacket, n>& block)
const {
283 PacketBlockManagement<Index, Scalar, SubPacket, n, n - 1, StorageOrder> pbm;
284 pbm.store(m_data, m_stride, i, j, block);
288 Scalar* EIGEN_RESTRICT m_data;
289 const Index m_stride;
295template <
typename Scalar,
typename Index,
int AlignmentType,
int Incr>
296class BlasLinearMapper {
298 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE BlasLinearMapper(Scalar* data, Index incr)
299 : m_data(data), m_incr(incr) {}
301 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void prefetch(
int i)
const { internal::prefetch(&
operator()(i)); }
303 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE Scalar& operator()(Index i)
const {
304 return m_data[i * m_incr.value()];
307 template <
typename PacketType>
308 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i)
const {
309 return pgather<Scalar, PacketType>(m_data + i * m_incr.value(), m_incr.value());
312 template <
typename PacketType>
313 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index n, Index = 0)
const {
314 return pgather_partial<Scalar, PacketType>(m_data + i * m_incr.value(), m_incr.value(), n);
317 template <
typename PacketType>
318 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void storePacket(Index i,
const PacketType& p)
const {
319 pscatter<Scalar, PacketType>(m_data + i * m_incr.value(), p, m_incr.value());
322 template <
typename PacketType>
323 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void storePacketPartial(Index i,
const PacketType& p, Index n,
325 pscatter_partial<Scalar, PacketType>(m_data + i * m_incr.value(), p, m_incr.value(), n);
330 const internal::variable_if_dynamic<Index, Incr> m_incr;
333template <
typename Scalar,
typename Index,
int StorageOrder,
int AlignmentType,
int Incr>
334class blas_data_mapper {
336 using LinearMapper = BlasLinearMapper<Scalar, Index, AlignmentType, Incr>;
337 using SubMapper = blas_data_mapper;
339 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE blas_data_mapper(Scalar* data, Index stride, Index incr)
340 : m_data(data), m_stride(stride), m_incr(incr) {}
342 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE SubMapper getSubMapper(Index i, Index j)
const {
343 return SubMapper(&
operator()(i, j), m_stride, m_incr.value());
346 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j)
const {
347 return LinearMapper(&
operator()(i, j), m_incr.value());
350 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void prefetch(Index i, Index j)
const { internal::prefetch(&
operator()(i, j)); }
352 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE Scalar& operator()(Index i, Index j)
const {
353 return m_data[StorageOrder ==
RowMajor ? j * m_incr.value() + i * m_stride : i * m_incr.value() + j * m_stride];
356 template <
typename PacketType>
357 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i, Index j)
const {
358 return pgather<Scalar, PacketType>(&
operator()(i, j), m_incr.value());
361 template <
typename PacketType>
362 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index j, Index n,
364 return pgather_partial<Scalar, PacketType>(&
operator()(i, j), m_incr.value(), n);
367 template <
typename PacketT,
int AlignmentT>
368 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i, Index j)
const {
369 return pgather<Scalar, PacketT>(&
operator()(i, j), m_incr.value());
372 template <
typename PacketType>
373 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void storePacket(Index i, Index j,
const PacketType& p)
const {
374 pscatter<Scalar, PacketType>(&
operator()(i, j), p, m_incr.value());
377 template <
typename PacketType>
378 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void storePacketPartial(Index i, Index j,
const PacketType& p, Index n,
380 pscatter_partial<Scalar, PacketType>(&
operator()(i, j), p, m_incr.value(), n);
383 template <
typename SubPacket>
384 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void scatterPacket(Index i, Index j,
const SubPacket& p)
const {
385 pscatter<Scalar, SubPacket>(&
operator()(i, j), p, m_stride);
388 template <
typename SubPacket>
389 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SubPacket gatherPacket(Index i, Index j)
const {
390 return pgather<Scalar, SubPacket>(&
operator()(i, j), m_stride);
395 template <
typename SubPacket,
typename Scalar_,
int n,
int idx>
396 struct storePacketBlock_helper {
397 storePacketBlock_helper<SubPacket, Scalar_, n, idx - 1> spbh;
398 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void store(
399 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>* sup, Index i, Index j,
400 const PacketBlock<SubPacket, n>& block)
const {
401 spbh.store(sup, i, j, block);
402 sup->template storePacket<SubPacket>(i, j + idx, block.packet[idx]);
406 template <
typename SubPacket,
int n,
int idx>
407 struct storePacketBlock_helper<SubPacket, std::complex<float>, n, idx> {
408 storePacketBlock_helper<SubPacket, std::complex<float>, n, idx - 1> spbh;
409 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void store(
410 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>* sup, Index i, Index j,
411 const PacketBlock<SubPacket, n>& block)
const {
412 spbh.store(sup, i, j, block);
413 sup->template storePacket<SubPacket>(i, j + idx, block.packet[idx]);
417 template <
typename SubPacket,
int n,
int idx>
418 struct storePacketBlock_helper<SubPacket, std::complex<double>, n, idx> {
419 storePacketBlock_helper<SubPacket, std::complex<double>, n, idx - 1> spbh;
420 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void store(
421 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>* sup, Index i, Index j,
422 const PacketBlock<SubPacket, n>& block)
const {
423 spbh.store(sup, i, j, block);
424 for (
int l = 0; l < unpacket_traits<SubPacket>::size; l++) {
425 std::complex<double>* v = &sup->operator()(i + l, j + idx);
426 v->real(block.packet[idx].v[2 * l + 0]);
427 v->imag(block.packet[idx].v[2 * l + 1]);
432 template <
typename SubPacket,
typename Scalar_,
int n>
433 struct storePacketBlock_helper<SubPacket, Scalar_, n, -1> {
434 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void store(
435 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>*, Index, Index,
436 const PacketBlock<SubPacket, n>&)
const {}
439 template <
typename SubPacket,
int n>
440 struct storePacketBlock_helper<SubPacket, std::complex<float>, n, -1> {
441 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void store(
442 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>*, Index, Index,
443 const PacketBlock<SubPacket, n>&)
const {}
446 template <
typename SubPacket,
int n>
447 struct storePacketBlock_helper<SubPacket, std::complex<double>, n, -1> {
448 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void store(
449 const blas_data_mapper<Scalar, Index, StorageOrder, AlignmentType, Incr>*, Index, Index,
450 const PacketBlock<SubPacket, n>&)
const {}
454 template <
typename SubPacket,
int n>
455 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void storePacketBlock(Index i, Index j,
456 const PacketBlock<SubPacket, n>& block)
const {
457 storePacketBlock_helper<SubPacket, Scalar, n, n - 1> spb;
458 spb.store(
this, i, j, block);
461 EIGEN_DEVICE_FUNC
constexpr const Index stride()
const {
return m_stride; }
462 EIGEN_DEVICE_FUNC
constexpr const Index incr()
const {
return m_incr.value(); }
463 EIGEN_DEVICE_FUNC
constexpr Scalar* data()
const {
return m_data; }
466 Scalar* EIGEN_RESTRICT m_data;
467 const Index m_stride;
468 const internal::variable_if_dynamic<Index, Incr> m_incr;
472template <
typename Scalar,
typename Index,
int StorageOrder>
473class const_blas_data_mapper :
public blas_data_mapper<const Scalar, Index, StorageOrder> {
475 using SubMapper = const_blas_data_mapper<Scalar, Index, StorageOrder>;
477 EIGEN_ALWAYS_INLINE const_blas_data_mapper(
const Scalar* data, Index stride)
478 : blas_data_mapper<const Scalar, Index, StorageOrder>(data, stride) {}
480 EIGEN_ALWAYS_INLINE SubMapper getSubMapper(Index i, Index j)
const {
481 return SubMapper(&(this->
operator()(i, j)), this->m_stride);
487template <
typename SrcScalar,
typename Packet,
typename =
void>
488struct cast_ploadu_impl {
489 using Scalar =
typename unpacket_traits<Packet>::type;
490 static constexpr int Size = unpacket_traits<Packet>::size;
491 static EIGEN_ALWAYS_INLINE Packet run(
const SrcScalar* from) {
493 for (
int i = 0; i < Size; ++i) to[i] = cast<SrcScalar, Scalar>(from[i]);
494 return ploadu<Packet>(to);
497template <
typename SrcScalar,
typename Packet>
498struct cast_ploadu_impl<
500 std::enable_if_t<type_casting_traits<SrcScalar, typename unpacket_traits<Packet>::type>::VectorizedCast &&
501 find_packet_by_size<SrcScalar, unpacket_traits<Packet>::size>::value>> {
502 using SrcPacket =
typename find_packet_by_size<SrcScalar, unpacket_traits<Packet>::size>::type;
503 static EIGEN_ALWAYS_INLINE Packet run(
const SrcScalar* from) {
504 return pcast<SrcPacket, Packet>(ploadu<SrcPacket>(from));
507#ifdef EIGEN_VECTORIZE_VSX
511struct cast_ploadu_impl<bfloat16, Packet4f> {
512 static EIGEN_ALWAYS_INLINE Packet4f run(
const bfloat16* from) {
513 unsigned long long bits;
514 std::memcpy(&bits, from,
sizeof(bits));
515 return pcast<Packet8bf, Packet4f>(Packet8bf(
reinterpret_cast<Packet8us
>(vec_splats(bits))));
520template <
typename Scalar,
typename SrcScalar,
typename Index>
521class BlasCastLinearMapper {
522 using SrcMapper = BlasLinearMapper<const SrcScalar, Index, Unaligned>;
525 EIGEN_ALWAYS_INLINE
explicit BlasCastLinearMapper(
const SrcMapper& src) : m_src(src) {}
527 EIGEN_ALWAYS_INLINE
void prefetch(Index i)
const { m_src.prefetch(i); }
529 EIGEN_ALWAYS_INLINE Scalar operator()(Index i)
const {
return cast<SrcScalar, Scalar>(m_src(i)); }
531 template <
typename PacketType>
532 EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i)
const {
533 return cast_ploadu_impl<SrcScalar, PacketType>::run(&m_src(i));
543template <
typename Scalar,
typename SrcScalar,
typename Index,
int StorageOrder>
544class blas_cast_data_mapper {
545 using SrcMapper = const_blas_data_mapper<SrcScalar, Index, StorageOrder>;
548 using LinearMapper = BlasCastLinearMapper<Scalar, SrcScalar, Index>;
549 using SubMapper = blas_cast_data_mapper;
551 EIGEN_ALWAYS_INLINE blas_cast_data_mapper(
const SrcScalar* data, Index stride) : m_src(data, stride) {}
552 EIGEN_ALWAYS_INLINE
explicit blas_cast_data_mapper(
const SrcMapper& src) : m_src(src) {}
554 EIGEN_ALWAYS_INLINE SubMapper getSubMapper(Index i, Index j)
const {
return SubMapper(m_src.getSubMapper(i, j)); }
556 EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j)
const {
557 return LinearMapper(m_src.getLinearMapper(i, j));
560 EIGEN_ALWAYS_INLINE
void prefetch(Index i, Index j)
const { m_src.prefetch(i, j); }
562 EIGEN_ALWAYS_INLINE Scalar operator()(Index i, Index j)
const {
return cast<SrcScalar, Scalar>(m_src(i, j)); }
564 template <
typename PacketType>
565 EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i, Index j)
const {
566 return cast_ploadu_impl<SrcScalar, PacketType>::run(&m_src(i, j));
569 EIGEN_ALWAYS_INLINE Index stride()
const {
return m_src.stride(); }
576template <
typename Scalar,
typename StorageScalar,
typename Index,
int StorageOrder>
577using gemm_operand_mapper =
578 std::conditional_t<std::is_same<Scalar, StorageScalar>::value, const_blas_data_mapper<Scalar, Index, StorageOrder>,
579 blas_cast_data_mapper<Scalar, StorageScalar, Index, StorageOrder>>;
584template <
typename XprType>
586 using Scalar =
typename traits<XprType>::Scalar;
587 using ExtractType =
const XprType&;
588 using ExtractType_ = XprType;
590 IsComplex = NumTraits<Scalar>::IsComplex,
591 IsTransposed =
false,
592 NeedToConjugate =
false,
593 HasUsableDirectAccess =
595 (bool(XprType::IsVectorAtCompileTime) || int(inner_stride_at_compile_time<XprType>::value) == 1))
598 HasScalarFactor = false
600 using DirectLinearAccessType =
601 std::conditional_t<bool(HasUsableDirectAccess), ExtractType,
typename ExtractType_::PlainObject>;
602 EIGEN_DEVICE_FUNC
static inline EIGEN_DEVICE_FUNC ExtractType extract(
const XprType& x) {
return x; }
603 EIGEN_DEVICE_FUNC
static inline EIGEN_DEVICE_FUNC
const Scalar extractScalarFactor(
const XprType&) {
607 EIGEN_DEVICE_FUNC
static bool hasZeroScalarFactor(
const XprType&) {
return false; }
611template <
typename Scalar,
typename NestedXpr>
612struct blas_traits<CwiseUnaryOp<scalar_conjugate_op<Scalar>, NestedXpr> > : blas_traits<NestedXpr> {
613 using Base = blas_traits<NestedXpr>;
614 using XprType = CwiseUnaryOp<scalar_conjugate_op<Scalar>, NestedXpr>;
615 using ExtractType =
typename Base::ExtractType;
617 enum { IsComplex = NumTraits<Scalar>::IsComplex, NeedToConjugate = Base::NeedToConjugate ? 0 : IsComplex };
618 EIGEN_DEVICE_FUNC
static inline ExtractType extract(
const XprType& x) {
return Base::extract(x.nestedExpression()); }
619 EIGEN_DEVICE_FUNC
static inline Scalar extractScalarFactor(
const XprType& x) {
620 return numext::conj(Base::extractScalarFactor(x.nestedExpression()));
622 EIGEN_DEVICE_FUNC
static bool hasZeroScalarFactor(
const XprType& x) {
623 return Base::hasZeroScalarFactor(x.nestedExpression());
628template <
typename Scalar,
typename NestedXpr,
typename Plain>
630 CwiseBinaryOp<scalar_product_op<Scalar>, const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain>, NestedXpr> >
631 : blas_traits<NestedXpr> {
632 enum { HasScalarFactor =
true };
633 using Base = blas_traits<NestedXpr>;
635 CwiseBinaryOp<scalar_product_op<Scalar>,
const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain>, NestedXpr>;
636 using ExtractType =
typename Base::ExtractType;
637 EIGEN_DEVICE_FUNC
static inline EIGEN_DEVICE_FUNC ExtractType extract(
const XprType& x) {
638 return Base::extract(x.rhs());
640 EIGEN_DEVICE_FUNC
static inline EIGEN_DEVICE_FUNC Scalar extractScalarFactor(
const XprType& x) {
641 return x.lhs().functor().m_other * Base::extractScalarFactor(x.rhs());
643 EIGEN_DEVICE_FUNC
static bool hasZeroScalarFactor(
const XprType& x) {
644 return numext::is_exactly_zero(x.lhs().functor().m_other) || Base::hasZeroScalarFactor(x.rhs());
647template <
typename Scalar,
typename NestedXpr,
typename Plain>
649 CwiseBinaryOp<scalar_product_op<Scalar>, NestedXpr, const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain> > >
650 : blas_traits<NestedXpr> {
651 enum { HasScalarFactor =
true };
652 using Base = blas_traits<NestedXpr>;
654 CwiseBinaryOp<scalar_product_op<Scalar>, NestedXpr,
const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain>>;
655 using ExtractType =
typename Base::ExtractType;
656 EIGEN_DEVICE_FUNC
static inline ExtractType extract(
const XprType& x) {
return Base::extract(x.lhs()); }
657 EIGEN_DEVICE_FUNC
static inline Scalar extractScalarFactor(
const XprType& x) {
658 return Base::extractScalarFactor(x.lhs()) * x.rhs().functor().m_other;
660 EIGEN_DEVICE_FUNC
static bool hasZeroScalarFactor(
const XprType& x) {
661 return numext::is_exactly_zero(x.rhs().functor().m_other) || Base::hasZeroScalarFactor(x.lhs());
665template <
typename Scalar,
typename Plain1,
typename Plain2>
666struct blas_traits<CwiseBinaryOp<scalar_product_op<Scalar>, const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain1>,
667 const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain2>>>
668 : blas_traits<const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain2>> {
669 static constexpr bool HasScalarFactor =
true;
670 using Base = blas_traits<const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain2>>;
671 using XprType = CwiseBinaryOp<scalar_product_op<Scalar>,
const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain1>,
672 const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain2>>;
673 using ExtractType =
typename Base::ExtractType;
674 EIGEN_DEVICE_FUNC
static inline ExtractType extract(
const XprType& x) {
return Base::extract(x.rhs()); }
675 EIGEN_DEVICE_FUNC
static inline Scalar extractScalarFactor(
const XprType& x) {
676 return x.lhs().functor().m_other * Base::extractScalarFactor(x.rhs());
679 EIGEN_DEVICE_FUNC
static bool hasZeroScalarFactor(
const XprType&) {
return false; }
683template <
typename Scalar,
typename NestedXpr>
684struct blas_traits<CwiseUnaryOp<scalar_opposite_op<Scalar>, NestedXpr> > : blas_traits<NestedXpr> {
685 enum { HasScalarFactor =
true };
686 using Base = blas_traits<NestedXpr>;
687 using XprType = CwiseUnaryOp<scalar_opposite_op<Scalar>, NestedXpr>;
688 using ExtractType =
typename Base::ExtractType;
689 EIGEN_DEVICE_FUNC
static inline ExtractType extract(
const XprType& x) {
return Base::extract(x.nestedExpression()); }
690 EIGEN_DEVICE_FUNC
static inline Scalar extractScalarFactor(
const XprType& x) {
691 return -Base::extractScalarFactor(x.nestedExpression());
693 EIGEN_DEVICE_FUNC
static bool hasZeroScalarFactor(
const XprType& x) {
694 return Base::hasZeroScalarFactor(x.nestedExpression());
699template <
typename NestedXpr>
700struct blas_traits<Transpose<NestedXpr> > : blas_traits<NestedXpr> {
701 using Scalar =
typename NestedXpr::Scalar;
702 using Base = blas_traits<NestedXpr>;
703 using XprType = Transpose<NestedXpr>;
704 using ExtractType = Transpose<const typename Base::ExtractType_>;
706 using ExtractType_ = Transpose<const typename Base::ExtractType_>;
707 using DirectLinearAccessType =
708 std::conditional_t<bool(Base::HasUsableDirectAccess), ExtractType,
typename ExtractType::PlainObject>;
709 enum { IsTransposed = Base::IsTransposed ? 0 : 1 };
710 EIGEN_DEVICE_FUNC
static inline ExtractType extract(
const XprType& x) {
711 return ExtractType(Base::extract(x.nestedExpression()));
713 EIGEN_DEVICE_FUNC
static inline Scalar extractScalarFactor(
const XprType& x) {
714 return Base::extractScalarFactor(x.nestedExpression());
716 EIGEN_DEVICE_FUNC
static bool hasZeroScalarFactor(
const XprType& x) {
717 return Base::hasZeroScalarFactor(x.nestedExpression());
722struct blas_traits<const T> : blas_traits<T> {};
724template <typename T, bool HasUsableDirectAccess = blas_traits<T>::HasUsableDirectAccess>
725struct extract_data_selector {
726 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE
static const typename T::Scalar* run(
const T& m) {
727 return blas_traits<T>::extract(m).data();
732struct extract_data_selector<T, false> {
733 EIGEN_DEVICE_FUNC
constexpr static typename T::Scalar* run(
const T&) {
return 0; }
737EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE
const typename T::Scalar* extract_data(
const T& m) {
738 return extract_data_selector<T>::run(m);
742template <
typename LhsScalar,
typename RhsScalar>
743struct blas_product_is_commutative
744 : bool_constant<functor_is_commutative<scalar_product_op<LhsScalar, RhsScalar>>::value &&
745 functor_is_commutative<scalar_product_op<LhsScalar>>::value &&
746 functor_is_commutative<scalar_product_op<RhsScalar>>::value &&
747 functor_is_commutative<
748 scalar_product_op<typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType>>::value> {};
754template <
typename ResScalar,
typename Lhs,
typename Rhs>
756 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE
static ResScalar run(
const Lhs& lhs,
const Rhs& rhs) {
757 return blas_traits<Lhs>::extractScalarFactor(lhs) * blas_traits<Rhs>::extractScalarFactor(rhs);
759 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE
static ResScalar run(
const ResScalar& alpha,
const Lhs& lhs,
761 return alpha * blas_traits<Lhs>::extractScalarFactor(lhs) * blas_traits<Rhs>::extractScalarFactor(rhs);
764template <
typename Lhs,
typename Rhs>
766 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE
static bool run(
const Lhs& lhs,
const Rhs& rhs) {
767 return blas_traits<Lhs>::extractScalarFactor(lhs) && blas_traits<Rhs>::extractScalarFactor(rhs);
769 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE
static bool run(
const bool& alpha,
const Lhs& lhs,
const Rhs& rhs) {
770 return alpha && blas_traits<Lhs>::extractScalarFactor(lhs) && blas_traits<Rhs>::extractScalarFactor(rhs);
774template <
typename ResScalar,
typename Lhs,
typename Rhs>
775EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE ResScalar combine_scalar_factors(
const ResScalar& alpha,
const Lhs& lhs,
777 return combine_scalar_factors_impl<ResScalar, Lhs, Rhs>::run(alpha, lhs, rhs);
779template <
typename ResScalar,
typename Lhs,
typename Rhs>
780EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE ResScalar combine_scalar_factors(
const Lhs& lhs,
const Rhs& rhs) {
781 return combine_scalar_factors_impl<ResScalar, Lhs, Rhs>::run(lhs, rhs);
AlignmentType
Definition Constants.h:235
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321
constexpr unsigned int DirectAccessBit
Definition Constants.h:160
Definition BlasUtil.h:755