Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TensorContractionSycl.h
1// This file is part of Eigen, a lightweight C++ template library for linear algebra.
2//
3// Mehdi Goli Codeplay Software Ltd.
4// Ralph Potter Codeplay Software Ltd.
5// Luke Iwanski Codeplay Software Ltd.
6// Contact: <eigen@codeplay.com>
7//
8// This Source Code Form is subject to the terms of the Mozilla Public License v. 2.0. If a copy of the MPL was not
9// distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-FileCopyrightText: The Eigen Authors
11// SPDX-License-Identifier: MPL-2.0
12
13/*****************************************************************
14 * TensorContractionSycl.h
15 *
16 * \brief:
17 * TensorContractionSycl.h, provides various tensor contraction kernel for SYCL backend
18 *
19 *****************************************************************/
20
21#ifndef EIGEN_TENSOR_TENSOR_CONTRACTION_SYCL_H
22#define EIGEN_TENSOR_TENSOR_CONTRACTION_SYCL_H
23
24// IWYU pragma: private
25#include "./InternalHeaderCheck.h"
26
27namespace Eigen {
28
29namespace TensorSycl {
30namespace internal {
31
32#ifndef EIGEN_SYCL_DISABLE_GEMV
47template <typename Scalar, typename StorageIndex, StorageIndex NCWindow, StorageIndex CFactor, StorageIndex NCFactor>
49 // LocalThreadSizeC: determines total number of thread per workgroup for the contracting dimension
50 static constexpr StorageIndex LocalThreadSizeC = EIGEN_SYCL_LOCAL_THREAD_DIM0;
51 // LocalThreadSizeNC: determines total number of thread per workgroup for the non-contracting dimension
52 static constxpr StorageIndex LocalThreadSizeNC = EIGEN_SYCL_LOCAL_THREAD_DIM1;
53 // TileSizeDimNC: determines the tile size for the non-contracting dimension
54 static constexpr StorageIndex TileSizeDimNC = NCWindow / NCFactor;
55 // TileSizeDimC: determines the tile size for the contracting dimension
56 static constexpr StorageIndex TileSizeDimC = CFactor * LocalThreadSizeNC * LocalThreadSizeC;
57 // WorkLoadPerThreadNC : determines workload per thread for loading the non-contracting dimension
58 static constexpr StorageIndex WorkLoadPerThreadNC = TileSizeDimNC / LocalThreadSizeNC;
59 // WorkLoadPerThreadC: determines workload per thread for loading the non-contracting dimension
60 static constexpr StorageIndex WorkLoadPerThreadC = TileSizeDimC / LocalThreadSizeC;
61 // BC : determines if supporting bank conflict is required
62 static constexpr bool BC = false;
63};
64#endif
65
82
83template <typename Scalar, typename StorageIndex, StorageIndex REG_SIZE_M, StorageIndex REG_SIZE_N, StorageIndex TSDK>
85 // TileSizeDimK: determines Tile size for dimension K. The packet size is assumed to be considered
86 static constexpr StorageIndex TileSizeDimK = TSDK;
87 // WorkLoadPerThreadM : determines workload per thread for loading the M dimension This can be varied based on the
88 // available register on a chosen device(can be controlled by EIGEN_SYCL_REG_M macro
89#ifndef EIGEN_SYCL_REG_M
90 static constexpr StorageIndex WorkLoadPerThreadM = REG_SIZE_M;
91#else
92 static constexpr StorageIndex WorkLoadPerThreadM = EIGEN_SYCL_REG_M;
93#endif
94// WorkLoadPerThreadN : determines workload per thread for loading the N dimension This can be varied based on the
95// available register on a chosen device(can be controlled by EIGEN_SYCL_REG_N macro
96#ifndef EIGEN_SYCL_REG_N
97 static constexpr StorageIndex WorkLoadPerThreadN = REG_SIZE_N;
98#else
99 static constexpr StorageIndex WorkLoadPerThreadN = EIGEN_SYCL_REG_N;
100#endif
101 // LocalThreadSizeM: determines total number of thread per workgroup for the m dimension
102 static constexpr StorageIndex LocalThreadSizeM = EIGEN_SYCL_LOCAL_THREAD_DIM0;
103 // LocalThreadSizeN: determines total number of thread per workgroup for the n dimension
104 static constexpr StorageIndex LocalThreadSizeN = EIGEN_SYCL_LOCAL_THREAD_DIM1;
105 // TileSizeDimM: determines the tile size for the m dimension
106 static constexpr StorageIndex TileSizeDimM = LocalThreadSizeM * WorkLoadPerThreadM;
107 // TileSizeDimN: determines the tile size for the n dimension
108 static constexpr StorageIndex TileSizeDimN = LocalThreadSizeN * WorkLoadPerThreadN;
109 // LoadPerThreadLhs: determines workload per thread for loading Lhs Tensor. This must be divisible by packetsize
110 static constexpr StorageIndex LoadPerThreadLhs =
111 ((TileSizeDimK * WorkLoadPerThreadM * WorkLoadPerThreadN) / (TileSizeDimN));
112 // LoadPerThreadRhs: determines workload per thread for loading Rhs Tensor. This must be divisible by packetsize
113 static constexpr StorageIndex LoadPerThreadRhs =
114 ((TileSizeDimK * WorkLoadPerThreadM * WorkLoadPerThreadN) / (TileSizeDimM));
115 // BC : determines if supporting bank conflict is required
116 static constexpr bool BC = true;
117 // DoubleBuffer: determines if double buffering technique should be used (This can be disabled by
118 // EIGEN_SYCL_DISABLE_DOUBLE_BUFFER macro when the device does not have sufficient local memory)
119 static constexpr bool DoubleBuffer =
120#ifdef EIGEN_SYCL_DISABLE_DOUBLE_BUFFER
121 false;
122#else
123 true;
124#endif
125};
126
127/* !
128 * \brief contraction_type: an enum class representing the Tensor Contraction implementation algorithm. This is used to
129 * specialize the contraction algorithm based on device support for dedicated local memory.
130 */
131enum class contraction_type { local, no_local };
132/* !
133 * \brief data_source an enum class determining the location of the data in a memory hierarchy (global, local, private).
134 */
135enum class data_source { global_mem, local_mem, private_mem };
136
162template <bool PacketLoad, bool is_coalesced_layout, bool, typename PacketType, typename TensorMapper,
163 typename StorageIndex>
164static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<PacketLoad, PacketType> read(
165 const TensorMapper &tensorMapper, const StorageIndex &NCIndex, const StorageIndex &CIndex, const StorageIndex &ld) {
166 const StorageIndex row = (is_coalesced_layout) ? NCIndex : CIndex;
167 const StorageIndex col = (is_coalesced_layout) ? CIndex : NCIndex;
168 return tensorMapper.get_tensor().template packet<Unaligned>(row + (col * ld));
169}
170
193template <bool PacketLoad, bool, bool IsRhs, typename PacketType, typename TensorMapper, typename StorageIndex>
194static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<!PacketLoad, PacketType> read(
195 const TensorMapper &tensorMapper, const StorageIndex &NCIndex, const StorageIndex &CIndex, const StorageIndex &) {
196 const StorageIndex row = (IsRhs) ? CIndex : NCIndex;
197 const StorageIndex col = (IsRhs) ? NCIndex : CIndex;
198 return tensorMapper(row, col);
199}
200
221
222template <typename StorageIndex, StorageIndex ld, data_source dt, typename PacketType, typename DataScalar>
223static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<dt != data_source::global_mem, void> write(
224 PacketType &packet_data, DataScalar ptr) {
225 constexpr int PacketSize = Eigen::internal::unpacket_traits<PacketType>::size;
226 EIGEN_UNROLL_LOOP
227 for (int i = 0; i < PacketSize; i++) {
228 *ptr = PacketWrapper<PacketType, PacketSize>::scalarize(i, packet_data);
229 ptr += ld;
230 }
231}
232
247
248template <data_source dt, typename PacketType, typename DataScalar>
249static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
250 std::enable_if_t<Eigen::internal::unpacket_traits<PacketType>::size != 1 && dt == data_source::global_mem, void>
251 write(PacketType &packet_data, DataScalar *ptr) {
252 ::Eigen::internal::pstoreu<DataScalar, PacketType>(ptr, packet_data);
253}
254
268template <data_source dt, typename PacketType, typename DataScalar>
269static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
270 std::enable_if_t<Eigen::internal::unpacket_traits<PacketType>::size == 1 && dt == data_source::global_mem, void>
271 write(PacketType &packet_data, DataScalar *ptr) {
272 *ptr = packet_data;
273}
274
280template <bool is_internal>
281EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool check_boundary(bool) {
282 return true;
283}
284
290template <>
291EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool check_boundary<false>(bool cond) {
292 return cond;
293}
294
321template <bool is_transposed, bool is_rhs_, bool packet_load_, typename PacketType>
323 static constexpr bool packet_load = packet_load_;
324 using OutScalar = typename Eigen::internal::unpacket_traits<PacketType>::type;
325 static constexpr bool is_rhs = is_rhs_;
326 using OutType = std::conditional_t<packet_load, PacketType, OutScalar>;
327 static constexpr int elements_per_access = Eigen::internal::unpacket_traits<OutType>::size;
328 static constexpr bool is_coalesced_layout = !(is_transposed ^ is_rhs);
329 static constexpr int nc_stride = (is_coalesced_layout ? elements_per_access : 1);
330 static constexpr int c_stride = (is_coalesced_layout ? 1 : elements_per_access);
331};
332
372template <typename StorageIndex>
373struct ThreadProperties {
374 const StorageIndex linearLocalThreadId;
375 const StorageIndex kGroupId;
376 const StorageIndex mGroupOffset;
377 const StorageIndex nGroupOffset;
378 const StorageIndex kGroupOffset;
379 const StorageIndex mLocalOffset;
380 const StorageIndex nLocalOffset;
381 const StorageIndex mGlobalOffset;
382 const StorageIndex nGlobalOffset;
383 StorageIndex kSize;
384 const bool is_internal;
385 // this is used to adjust the last block
386 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ThreadProperties(
387 const StorageIndex linearLocalThreadId_, const StorageIndex kGroupId_, const StorageIndex mGroupOffset_,
388 const StorageIndex nGroupOffset_, const StorageIndex kGroupOffset_, const StorageIndex mLocalOffset_,
389 const StorageIndex nLocalOffset_, const StorageIndex mGlobalOffset_, const StorageIndex nGlobalOffset_,
390 StorageIndex kSize_, const bool is_internal_)
391 : linearLocalThreadId(linearLocalThreadId_),
392 kGroupId(kGroupId_),
393 mGroupOffset(mGroupOffset_),
394 nGroupOffset(nGroupOffset_),
395 kGroupOffset(kGroupOffset_),
396 mLocalOffset(mLocalOffset_),
397 nLocalOffset(nLocalOffset_),
398 mGlobalOffset(mGlobalOffset_),
399 nGlobalOffset(nGlobalOffset_),
400 kSize(kSize_),
401 is_internal(is_internal_) {}
402};
403
454template <typename OutScalar, typename LhsScalar, typename RhsScalar, typename OutAccessor, typename LhsMapper,
455 typename RhsMapper, typename StorageIndex, typename Properties, typename TripleDim, bool Vectorizable,
456 typename input_mapper_properties, bool IsFinal, contraction_type contraction_tp>
457class TensorContractionKernel {
458 public:
459 using PacketReturnType =
460 typename Eigen::TensorSycl::internal::Vectorise<OutScalar, Eigen::SyclDevice, Vectorizable>::PacketReturnType;
461 static constexpr int PacketSize =
462 Eigen::TensorSycl::internal::Vectorise<OutScalar, Eigen::SyclDevice, Vectorizable>::PacketSize;
463 static constexpr bool is_lhs_transposed =
464 !::Eigen::internal::TensorContractionInputMapperTrait<LhsMapper>::inner_dim_contiguous;
465 static constexpr bool is_rhs_transposed =
466 !::Eigen::internal::TensorContractionInputMapperTrait<RhsMapper>::inner_dim_contiguous;
467
468 using LHSBlockProperties = BlockProperties<is_lhs_transposed, false,
469 input_mapper_properties::is_lhs_matrix && Vectorizable, PacketReturnType>;
470
471 using RHSBlockProperties = BlockProperties<is_rhs_transposed, true,
472 input_mapper_properties::is_rhs_matrix && Vectorizable, PacketReturnType>;
473
474 static constexpr StorageIndex NStride =
475 contraction_tp == contraction_type::local ? Properties::WorkLoadPerThreadN : RHSBlockProperties::nc_stride;
476
477 using Scratch = cl::sycl::accessor<OutScalar, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::local>;
478 using local_ptr = cl::sycl::multi_ptr<OutScalar, cl::sycl::access::address_space::local_space>;
479 using private_ptr = OutScalar * /*cl::sycl::multi_ptr<OutScalar, cl::sycl::access::address_space::private_space>*/;
480 using tile_ptr = std::conditional_t<contraction_tp == contraction_type::local, local_ptr, private_ptr>;
481 static constexpr StorageIndex LSDL = contraction_tp == contraction_type::local
482 ? Properties::TileSizeDimM + Properties::BC
483 : Properties::WorkLoadPerThreadM;
484 static constexpr StorageIndex LSDR = contraction_tp == contraction_type::local
485 ? Properties::TileSizeDimN + Properties::BC
486 : Properties::WorkLoadPerThreadN;
487 static constexpr StorageIndex LocalOffset = Properties::LocalThreadSizeM * Properties::LocalThreadSizeN;
488
501 template <contraction_type, StorageIndex>
502 struct MemHolder {
503 tile_ptr ptr;
504 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE MemHolder(local_ptr block_start_ptr) : ptr(block_start_ptr) {}
505 };
506
509 template <StorageIndex MemSize>
510 struct MemHolder<contraction_type::no_local, MemSize> {
511 OutScalar ptr[MemSize] = {OutScalar{0}};
512 };
513
535 struct TiledMemory {
538 tile_ptr lhs_scratch_ptr_compute;
539 tile_ptr rhs_scratch_ptr_compute;
540 const std::pair<StorageIndex, StorageIndex> lhs_extract_index;
541 const std::pair<StorageIndex, StorageIndex> rhs_extract_index;
542 template <contraction_type tp = contraction_tp, std::enable_if_t<tp == contraction_type::no_local, int> = 0>
543 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TiledMemory(const ThreadProperties<StorageIndex> &, local_ptr)
544 : lhs_scratch_extract{},
545 rhs_scratch_extract{},
546 lhs_scratch_ptr_compute(lhs_scratch_extract.ptr),
547 rhs_scratch_ptr_compute(rhs_scratch_extract.ptr),
548 lhs_extract_index(std::pair<StorageIndex, StorageIndex>(StorageIndex{0}, StorageIndex{0})),
549 rhs_extract_index(std::pair<StorageIndex, StorageIndex>(StorageIndex{0}, StorageIndex{0})) {}
550
551 template <contraction_type tp = contraction_tp, std::enable_if_t<tp == contraction_type::local, int> = 0>
552 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TiledMemory(const ThreadProperties<StorageIndex> &thread_properties,
553 local_ptr block_start_ptr)
554 : lhs_scratch_extract{block_start_ptr},
555 rhs_scratch_extract{lhs_scratch_extract.ptr +
556 ((Properties::DoubleBuffer + 1) * LSDL * Properties::TileSizeDimK)},
557 lhs_scratch_ptr_compute(lhs_scratch_extract.ptr + thread_properties.mLocalOffset),
558 rhs_scratch_ptr_compute(rhs_scratch_extract.ptr + thread_properties.nLocalOffset),
559 lhs_extract_index(
560 local_id_extract<LHSBlockProperties, Properties::TileSizeDimM>(thread_properties.linearLocalThreadId)),
561 rhs_extract_index(
562 local_id_extract<RHSBlockProperties, Properties::TileSizeDimN>(thread_properties.linearLocalThreadId)) {}
563 };
564
565 Scratch scratch;
566 const LhsMapper lhs;
567 const RhsMapper rhs;
568 OutAccessor out_res;
569 const StorageIndex groupSizeM;
570 const StorageIndex groupSizeN;
571 const StorageIndex numTiles;
572 const TripleDim triple_dim;
573
574 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorContractionKernel(Scratch scratch_, const LhsMapper lhs_,
575 const RhsMapper rhs_, OutAccessor out_res_,
576 const StorageIndex groupSizeM_,
577 const StorageIndex groupSizeN_,
578 const StorageIndex numTiles_,
579 const TripleDim triple_dim_)
580 : scratch(scratch_),
581 lhs(lhs_),
582 rhs(rhs_),
583 out_res(out_res_),
584 groupSizeM(groupSizeM_),
585 groupSizeN(groupSizeN_),
586 numTiles(numTiles_),
587 triple_dim(triple_dim_) {}
588
589 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorContractionKernel(Scratch scratch_, const LhsMapper lhs_,
590 const RhsMapper rhs_, OutAccessor out_res_,
591 const StorageIndex groupSizeM_,
592 const StorageIndex numTiles_,
593 const TripleDim triple_dim_)
594 : TensorContractionKernel(scratch_, lhs_, rhs_, out_res_, groupSizeM_, 1, numTiles_, triple_dim_) {}
595
596 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(cl::sycl::nd_item<1> itemID) const {
597 const StorageIndex linearLocalThreadId = itemID.get_local_id(0);
598 const StorageIndex nLocalThreadId = linearLocalThreadId / Properties::LocalThreadSizeM;
599 const StorageIndex mLocalThreadId = linearLocalThreadId % Properties::LocalThreadSizeM;
600 const StorageIndex mGroupId = itemID.get_group(0) % groupSizeM;
601 const StorageIndex tmp = itemID.get_group(0) / groupSizeM;
602 const StorageIndex nGroupId = IsFinal ? tmp : tmp % groupSizeN;
603 const StorageIndex kGroupId = IsFinal ? 0 : tmp / groupSizeN;
604 const StorageIndex mGroupOffset = mGroupId * Properties::TileSizeDimM;
605 const StorageIndex nGroupOffset = nGroupId * Properties::TileSizeDimN;
606 const StorageIndex mLocalOffset = PacketSize * mLocalThreadId;
607 const StorageIndex nLocalOffset = NStride * nLocalThreadId;
608 const StorageIndex mGlobalOffset = mGroupOffset + mLocalOffset;
609 const StorageIndex nGlobalOffset = nGroupOffset + nLocalOffset;
610
611 const StorageIndex kSizePerWG = IsFinal ? triple_dim.K : numTiles * Properties::TileSizeDimK;
612 StorageIndex kGroupOffset = kGroupId * kSizePerWG;
613 const bool is_internal = triple_dim.M - mGroupOffset >= Properties::TileSizeDimM &&
614 triple_dim.N - nGroupOffset >= Properties::TileSizeDimN &&
615 triple_dim.K - kGroupOffset >= kSizePerWG;
616 // this is used to adjust the last block
617 StorageIndex kSize = IsFinal ? triple_dim.K : std::min(kSizePerWG, triple_dim.K - kGroupOffset);
618 // This is used to find out the last K offset so that kGroupOffset -kSize can compute the coffset for loading to
619 // tile
620 kGroupOffset += kSize;
621
622 auto thread_properties =
623 ThreadProperties<StorageIndex>(linearLocalThreadId, kGroupId, mGroupOffset, nGroupOffset, kGroupOffset,
624 mLocalOffset, nLocalOffset, mGlobalOffset, nGlobalOffset, kSize, is_internal);
625
626 auto out_ptr = out_res + (IsFinal ? 0 : thread_properties.kGroupId * triple_dim.M * triple_dim.N);
627
628 (thread_properties.is_internal) ? compute_panel<true>(itemID, thread_properties, out_ptr)
629 : compute_panel<false>(itemID, thread_properties, out_ptr);
630 }
631 // The compute block computes the contraction operation private block for each thread and store the result in the
632 // privateRes memory of Each computation the compute block function is independent of local and no local concepts as
633 // it only compute the block on each thread's private memory space
634 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void compute_block_per_tile(OutScalar *lhs_block_ptr, OutScalar *rhs_block_ptr,
635 PacketReturnType *privateRes) const {
636 StorageIndex idx = 0;
637 constexpr StorageIndex lhs_stride =
638 contraction_tp == contraction_type::local ? (PacketSize * Properties::LocalThreadSizeM) : 1;
639 EIGEN_UNROLL_LOOP
640 for (StorageIndex wLPTN = 0; wLPTN < Properties::WorkLoadPerThreadN; wLPTN++) {
641 auto rhsPacket = PacketReturnType{*(rhs_block_ptr + wLPTN)};
642 StorageIndex lhs_index = 0;
643 EIGEN_UNROLL_LOOP
644 for (StorageIndex wLPTM = 0; wLPTM < Properties::WorkLoadPerThreadM / PacketSize; wLPTM++) {
645 PacketReturnType lhsPack{};
646 Eigen::TensorSycl::internal::PacketWrapper<PacketReturnType, PacketSize>::set_packet(lhsPack,
647 lhs_block_ptr + lhs_index);
648 privateRes[idx] = ::Eigen::internal::pmadd(lhsPack, rhsPacket, privateRes[idx]);
649
650 lhs_index += lhs_stride;
651 idx++;
652 }
653 }
654 }
655 // The store function write the computed contraction operation in the private memory of each thread to the global
656 // memory. The store function is independent of local and no local concepts so that it can be abstracted out in the
657 // base class.
658 template <bool is_internal_block, StorageIndex PrivateNStride, typename OutPtr>
659 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void store(OutPtr *out_ptr, PacketReturnType *privateRes,
660 StorageIndex mGlobalOffset, StorageIndex nGlobalOffset) const {
661 auto chk_bound = [&](const StorageIndex &mIndex, const StorageIndex &nIndex) EIGEN_DEVICE_FUNC {
662 return (mIndex + PacketSize - 1 < triple_dim.M && nGlobalOffset + nIndex < triple_dim.N);
663 };
664 // when local memory is not used M and N are both accessed in a coalesced way. However, when local memory is
665 // available the k*N is transposed in the local to N*K therefore, each blocks operates on blockId*
666 // WorkLoadPerThreadN slice of N
667 constexpr StorageIndex GlobalNStride = contraction_tp == contraction_type::local ? 1 : Properties::LocalThreadSizeN;
668 EIGEN_UNROLL_LOOP
669 for (StorageIndex wLPTN = 0; wLPTN < Properties::WorkLoadPerThreadN / PrivateNStride; wLPTN++) {
670 // output leading dimension
671 StorageIndex outputLD = 0;
672 // When local memory is used the PrivateNstride is always 1 because the coalesced access on N is loaded into Local
673 // memory and extracting from local to global is the same as no transposed version. However, when local memory is
674 // not used and RHS is transposed we packetize the load for RHS.
675 EIGEN_UNROLL_LOOP
676 for (StorageIndex nId = 0; nId < PrivateNStride; nId++) {
677 StorageIndex globalRow = mGlobalOffset;
678 EIGEN_UNROLL_LOOP
679 for (StorageIndex wLPTM = 0; wLPTM < Properties::WorkLoadPerThreadM / PacketSize; wLPTM++) {
680 PacketReturnType privetOut = privateRes[wLPTM];
681 if (check_boundary<is_internal_block>(chk_bound(globalRow, nId))) {
682 // Store the final results in C. The C matrix has always M as a first StorageIndex and N as a second
683 // StorageIndex Therefore it is always coalesced layout
684 write<data_source::global_mem>(privetOut, out_ptr + outputLD + globalRow);
685 } else {
686 EIGEN_UNROLL_LOOP
687 for (StorageIndex mId = 0; mId < PacketSize; mId++) {
688 StorageIndex mOffset = globalRow + mId;
689 if (mOffset < triple_dim.M && (nGlobalOffset + nId < triple_dim.N)) {
690 out_ptr[mOffset + outputLD] =
691 Eigen::TensorSycl::internal::PacketWrapper<PacketReturnType, PacketSize>::scalarize(mId, privetOut);
692 }
693 }
694 }
695 globalRow += (PacketSize * Properties::LocalThreadSizeM);
696 }
697 outputLD += triple_dim.M;
698 privateRes += Properties::WorkLoadPerThreadM / PacketSize;
699 }
700 out_ptr += (GlobalNStride * outputLD);
701
702 nGlobalOffset += (PrivateNStride * GlobalNStride);
703 }
704 }
705 // when no local memory is used the following extract_block will be enabled
706 template <typename InputBlockProperties, bool is_internal_block, typename Input, typename PrivateReg,
707 contraction_type contract_tp = contraction_tp>
708 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<contract_tp == contraction_type::no_local> extract_block(
709 const Input &inpt, PrivateReg private_ptr, const std::pair<StorageIndex, StorageIndex> &,
710 const StorageIndex &ncOffset, const StorageIndex cOffset) const {
711 constexpr StorageIndex LocalThreadSizeNC =
712 InputBlockProperties::is_rhs ? Properties::LocalThreadSizeN : Properties::LocalThreadSizeM;
713 constexpr StorageIndex WorkLoadPerThreadNC =
714 InputBlockProperties::is_rhs ? Properties::WorkLoadPerThreadN : Properties::WorkLoadPerThreadM;
715 const StorageIndex &NC = InputBlockProperties::is_rhs ? triple_dim.N : triple_dim.M;
716
717 auto chk_bound = [&](const StorageIndex &CIndex, const StorageIndex &NCIndex) EIGEN_DEVICE_FUNC {
718 return ((CIndex + InputBlockProperties::c_stride - 1 < triple_dim.K) &&
719 (NCIndex + InputBlockProperties::nc_stride - 1 < NC));
720 };
721 const StorageIndex ld = InputBlockProperties::is_coalesced_layout ? NC : triple_dim.K;
722 StorageIndex cIndex = cOffset;
723
724 EIGEN_UNROLL_LOOP
725 for (StorageIndex cId = 0; cId < Properties::TileSizeDimK / InputBlockProperties::c_stride; cId++) {
726 StorageIndex ncIndex = ncOffset;
727 EIGEN_UNROLL_LOOP
728 for (StorageIndex ncId = 0; ncId < WorkLoadPerThreadNC / InputBlockProperties::nc_stride; ncId++) {
729 if (check_boundary<is_internal_block>(chk_bound(cIndex, ncIndex))) {
730 auto val =
731 read<InputBlockProperties::packet_load, InputBlockProperties::is_coalesced_layout,
732 InputBlockProperties::is_rhs, typename InputBlockProperties::OutType>(inpt, ncIndex, cIndex, ld);
733
734 write<StorageIndex, (InputBlockProperties::is_coalesced_layout ? 1 : WorkLoadPerThreadNC),
735 data_source::private_mem>(val, private_ptr);
736 } else {
737 EIGEN_UNROLL_LOOP
738 for (StorageIndex i = 0; i < InputBlockProperties::elements_per_access; i++) {
739 const StorageIndex ncInd = ncIndex + (InputBlockProperties::is_coalesced_layout ? i : 0);
740 const StorageIndex cInd = cIndex + (InputBlockProperties::is_coalesced_layout ? 0 : i);
741 OutScalar val =
742 (ncInd < NC && cInd < triple_dim.K)
743 ? read<false, InputBlockProperties::is_coalesced_layout, InputBlockProperties::is_rhs, OutScalar>(
744 inpt, ncInd, cInd, ld)
745 : OutScalar(0);
746 write<StorageIndex, (InputBlockProperties::is_coalesced_layout ? 1 : WorkLoadPerThreadNC),
747 data_source::private_mem>(
748 val, private_ptr + (InputBlockProperties::is_coalesced_layout ? i : 0) +
749 ((InputBlockProperties::is_coalesced_layout ? 0 : i) * WorkLoadPerThreadNC));
750 }
751 }
752
753 // if it is lhs we have to load it packetised when the packet size is > 1, because the output is coalesced. So
754 // even if M is not accessed in a coalesced mode, we have to load packet_size number of m per thread.
755 ncIndex = (!InputBlockProperties::is_rhs && InputBlockProperties::nc_stride == 1 && PacketSize != 1)
756 ? ncOffset + (ncId + 1) % PacketSize + ((ncId + 1) / PacketSize) * LocalThreadSizeNC
757 : (ncIndex + InputBlockProperties::nc_stride * LocalThreadSizeNC);
758 private_ptr += InputBlockProperties::nc_stride;
759 }
760 // the previous for loop ( private_ptr += (ncId * nc_stride)) has already moved ptr with one WorkLoadPerThreadNC
761 private_ptr += (InputBlockProperties::c_stride - 1) * WorkLoadPerThreadNC;
762 cIndex += InputBlockProperties::c_stride;
763 }
764 }
765 template <typename InputBlockProperties, StorageIndex TileSizeDimNC>
766 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::pair<StorageIndex, StorageIndex> local_id_extract(
767 const StorageIndex &linearLocalThreadId) {
768 const StorageIndex localThreadNC =
769 (InputBlockProperties::is_coalesced_layout)
770 ? linearLocalThreadId % (TileSizeDimNC / InputBlockProperties::nc_stride)
771 : linearLocalThreadId / (Properties::TileSizeDimK / InputBlockProperties::c_stride);
772 const StorageIndex localThreadC =
773 (InputBlockProperties::is_coalesced_layout)
774 ? linearLocalThreadId / (TileSizeDimNC / InputBlockProperties::nc_stride)
775 : linearLocalThreadId % (Properties::TileSizeDimK / InputBlockProperties::c_stride);
776 return std::pair<StorageIndex, StorageIndex>(localThreadNC, localThreadC);
777 }
778
779 template <bool db = Properties::DoubleBuffer, contraction_type ctp = contraction_tp>
780 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<db && ctp == contraction_type::local> sync_mem(
781 const cl::sycl::nd_item<1> &, bool &db_offset) noexcept {
782 db_offset = !db_offset;
783 }
784
785 template <bool db = Properties::DoubleBuffer, contraction_type ctp = contraction_tp>
786 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<!db && ctp == contraction_type::local> sync_mem(
787 const cl::sycl::nd_item<1> &itemID, bool &) noexcept {
788 itemID.barrier(cl::sycl::access::fence_space::local_space);
789 }
790
791 template <contraction_type ctp = contraction_tp>
792 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<ctp == contraction_type::no_local> sync_mem(
793 const cl::sycl::nd_item<1> &, bool &) noexcept {
794 return;
795 }
796
797 template <bool need_sync, contraction_type ctp = contraction_tp>
798 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<need_sync && ctp == contraction_type::no_local>
799 sync_thread(const cl::sycl::nd_item<1> &
800#ifdef EIGEN_SYCL_ARM_GPU_CACHE_OPTIMISATION
801 itemID
802#endif
803 ) noexcept {
804#ifdef EIGEN_SYCL_ARM_GPU_CACHE_OPTIMISATION
805 itemID.barrier(cl::sycl::access::fence_spacce::local_space);
806#else
807 return;
808#endif
809 }
810 template <bool need_sync, contraction_type ctp = contraction_tp>
811 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<need_sync && ctp == contraction_type::local>
812 sync_thread(const cl::sycl::nd_item<1> &itemID) {
813 itemID.barrier(cl::sycl::access::fence_space::local_space);
814 }
815 template <bool need_sync>
816 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<!need_sync> sync_thread(const cl::sycl::nd_item<1> &) {
817 return;
818 }
819
820 template <bool is_internal_block>
821 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void compute_tile_per_panel(const cl::sycl::nd_item<1> &itemID,
822 ThreadProperties<StorageIndex> &thread_properties,
823 TiledMemory &tiled_input_block,
824 PacketReturnType *privateRes,
825 bool &db_offset) const {
826 // Tiling the Rhs block from global to local memory
827 extract_block<RHSBlockProperties, is_internal_block>(
828 rhs, tiled_input_block.rhs_scratch_extract.ptr + (db_offset * Properties::TileSizeDimK * LSDR),
829 tiled_input_block.rhs_extract_index,
830 contraction_tp == contraction_type::local ? thread_properties.nGroupOffset : thread_properties.nGlobalOffset,
831 thread_properties.kGroupOffset - thread_properties.kSize);
832
833 sync_thread<contraction_tp == contraction_type::no_local>(itemID);
834
835 // Tiling the Lhs block from global to local memory
836 extract_block<LHSBlockProperties, is_internal_block>(
837 lhs, tiled_input_block.lhs_scratch_extract.ptr + (db_offset * LSDL * Properties::TileSizeDimK),
838 tiled_input_block.lhs_extract_index,
839 contraction_tp == contraction_type::local ? thread_properties.mGroupOffset : thread_properties.mGlobalOffset,
840 thread_properties.kGroupOffset - thread_properties.kSize);
841
842 sync_thread<contraction_tp == contraction_type::local>(itemID);
843 // switch to compute mode
844 StorageIndex lhs_offset = (db_offset * LSDL * Properties::TileSizeDimK);
845 StorageIndex rhs_offset = (db_offset * Properties::TileSizeDimK * LSDR);
846 // Loop over the values of a single tile
847 for (StorageIndex k = 0; k < Properties::TileSizeDimK; k++) {
848 compute_block_per_tile(tiled_input_block.lhs_scratch_ptr_compute + lhs_offset,
849 tiled_input_block.rhs_scratch_ptr_compute + rhs_offset, privateRes);
850 lhs_offset += LSDL;
851 rhs_offset += LSDR;
852 }
853 // computing the K index for the next tile
854 thread_properties.kSize -= Properties::TileSizeDimK;
855 sync_mem(itemID, db_offset);
856 }
857
858 // when local memory is available the following compute_panel will be enabled
859 template <bool is_internal_block, typename OutPtr>
860 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void compute_panel(const cl::sycl::nd_item<1> &itemID,
861 ThreadProperties<StorageIndex> &thread_properties,
862 OutPtr out_ptr) const {
863 auto tiled_input_block = TiledMemory{thread_properties, scratch.get_pointer()};
864 // Allocate register space
865 PacketReturnType privateRes[Properties::WorkLoadPerThreadM * Properties::WorkLoadPerThreadN / PacketSize] = {
866 PacketReturnType{0}};
867 bool db_offset = 0;
868
869 while (thread_properties.kSize >= Properties::TileSizeDimK) {
870 compute_tile_per_panel<is_internal_block>(itemID, thread_properties, tiled_input_block, privateRes, db_offset);
871 }
872 if (thread_properties.kSize > 0) {
873 compute_tile_per_panel<false>(itemID, thread_properties, tiled_input_block, privateRes, db_offset);
874 }
875
876 // Storing the final results in the output
877 store<is_internal_block,
878 contraction_tp == contraction_type::local ? static_cast<StorageIndex>(1) : RHSBlockProperties::nc_stride>(
879 out_ptr + thread_properties.nGlobalOffset * triple_dim.M, privateRes, thread_properties.mGlobalOffset,
880 thread_properties.nGlobalOffset);
881 }
882 // When local memory is available the following extract_block will be enabled
883 template <typename InputBlockProperties, bool is_internal_block, typename Input, typename Local,
884 contraction_type contract_tp = contraction_tp>
885 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<contract_tp == contraction_type::local> extract_block(
886 const Input &inpt, Local local_ptr, const std::pair<StorageIndex, StorageIndex> &local_index,
887 const StorageIndex &ncOffset, const StorageIndex cOffset) const {
888 constexpr StorageIndex TileSizeDimNC =
889 InputBlockProperties::is_rhs ? Properties::TileSizeDimN : Properties::TileSizeDimM;
890 constexpr StorageIndex LoadPerThread =
891 InputBlockProperties::is_rhs ? Properties::LoadPerThreadRhs : Properties::LoadPerThreadLhs;
892 constexpr StorageIndex LSD = InputBlockProperties::is_rhs ? LSDR : LSDL;
893 static_assert(((LocalOffset % (TileSizeDimNC / InputBlockProperties::nc_stride) == 0) &&
894 (LocalOffset % (Properties::TileSizeDimK / InputBlockProperties::c_stride) == 0)),
895 " LocalOffset must be divisible by stride");
896 const StorageIndex &NC = InputBlockProperties::is_rhs ? triple_dim.N : triple_dim.M;
897 StorageIndex localThreadNC = local_index.first;
898 StorageIndex localThreadC = local_index.second;
899 auto chk_bound = [&](const StorageIndex &CIndex, const StorageIndex &NCIndex) EIGEN_DEVICE_FUNC {
900 return ((CIndex + InputBlockProperties::c_stride - 1 < triple_dim.K) &&
901 (NCIndex + InputBlockProperties::nc_stride - 1 < NC));
902 };
903 EIGEN_UNROLL_LOOP
904 for (StorageIndex lPT = 0; lPT < LoadPerThread / InputBlockProperties::elements_per_access; lPT++) {
905 const StorageIndex CIndex = cOffset + (InputBlockProperties::c_stride * localThreadC);
906 const StorageIndex NCIndex = ncOffset + (InputBlockProperties::nc_stride * localThreadNC);
907 const StorageIndex ld = InputBlockProperties::is_coalesced_layout ? NC : triple_dim.K;
908 if (check_boundary<is_internal_block>(chk_bound(CIndex, NCIndex))) {
909 auto val =
910 read<InputBlockProperties::packet_load, InputBlockProperties::is_coalesced_layout,
911 InputBlockProperties::is_rhs, typename InputBlockProperties::OutType>(inpt, NCIndex, CIndex, ld);
912 write<StorageIndex, (InputBlockProperties::is_coalesced_layout ? 1 : LSD), data_source::local_mem>(
913 val, local_ptr + (InputBlockProperties::nc_stride * localThreadNC) +
914 (InputBlockProperties::c_stride * localThreadC * LSD));
915 } else {
916 EIGEN_UNROLL_LOOP
917 for (StorageIndex i = 0; i < InputBlockProperties::elements_per_access; i++) {
918 const StorageIndex nCInd = NCIndex + (InputBlockProperties::is_coalesced_layout ? i : 0);
919 const StorageIndex cInd = CIndex + (InputBlockProperties::is_coalesced_layout ? 0 : i);
920 OutScalar val =
921 (nCInd < NC && cInd < triple_dim.K)
922 ? read<false, InputBlockProperties::is_coalesced_layout, InputBlockProperties::is_rhs, OutScalar>(
923 inpt, nCInd, cInd, ld)
924 : OutScalar(0);
925
926 write<StorageIndex, (InputBlockProperties::is_coalesced_layout ? 1 : LSD), data_source::local_mem>(
927 val, local_ptr + (InputBlockProperties::nc_stride * localThreadNC) +
928 (InputBlockProperties::is_coalesced_layout ? i : 0) +
929 ((InputBlockProperties::c_stride * localThreadC +
930 (InputBlockProperties::is_coalesced_layout ? 0 : i)) *
931 LSD));
932 }
933 }
934 localThreadNC += (InputBlockProperties::is_coalesced_layout)
935 ? LocalOffset % (TileSizeDimNC / InputBlockProperties::nc_stride)
936 : LocalOffset / (Properties::TileSizeDimK / InputBlockProperties::c_stride);
937 localThreadC += (InputBlockProperties::is_coalesced_layout)
938 ? LocalOffset / (TileSizeDimNC / InputBlockProperties::nc_stride)
939 : LocalOffset % (Properties::TileSizeDimK / InputBlockProperties::c_stride);
940 }
941 }
942};
943
944#ifndef EIGEN_SYCL_DISABLE_GEMV
945
987template <typename OutScalar, typename OutAccessor, typename VectorMapper, typename TensorMapper, typename StorageIndex,
988 typename Properties, StorageIndex KFactor, bool Vectorizable, bool is_lhs_vec, bool IsFinal>
989struct GeneralVectorTensor {
990 using PacketReturnType =
991 typename Eigen::TensorSycl::internal::Vectorise<OutScalar, Eigen::SyclDevice, Vectorizable>::PacketReturnType;
992 static constexpr int PacketSize =
993 Eigen::TensorSycl::internal::Vectorise<OutScalar, Eigen::SyclDevice, Vectorizable>::PacketSize;
994 using Scratch = cl::sycl::accessor<OutScalar, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::local>;
995
996 static constexpr StorageIndex OutScratchOffset =
997 KFactor * Properties::LocalThreadSizeC * Properties::LocalThreadSizeNC;
998
999 // Since the access layout for a vector can always be coalesced, when LHS is a vector, we pass false and false to make
1000 // sure that the !^ is true When RHS is a vector, we pass true and true to make sure that the !^ is true.
1002
1003 Scratch scratch;
1004 const VectorMapper vec;
1005 const TensorMapper mat;
1006 OutAccessor out_res;
1007 const StorageIndex nonContractGroupSize;
1008 const StorageIndex nonContractDim;
1009 const StorageIndex contractDim;
1010
1011 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE GeneralVectorTensor(Scratch scratch_, const VectorMapper vec_,
1012 const TensorMapper mat_, OutAccessor out_res_,
1013 const StorageIndex nonContractGroupSize_,
1014 const StorageIndex nonContractDim_,
1015 const StorageIndex contractDim_)
1016 : scratch(scratch_),
1017 vec(vec_),
1018 mat(mat_),
1019 out_res(out_res_),
1020 nonContractGroupSize(nonContractGroupSize_),
1021 nonContractDim(nonContractDim_),
1022 contractDim(contractDim_) {}
1023
1024 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(cl::sycl::nd_item<1> itemID) const {
1025 auto scratch_ptr = scratch.get_pointer();
1026 const StorageIndex linearLocalThreadId = itemID.get_local_id(0);
1027 StorageIndex nonContractId = is_lhs_vec ? linearLocalThreadId / Properties::LocalThreadSizeC
1028 : linearLocalThreadId % Properties::LocalThreadSizeNC;
1029 StorageIndex contractId = is_lhs_vec ? linearLocalThreadId % Properties::LocalThreadSizeC
1030 : linearLocalThreadId / Properties::LocalThreadSizeNC;
1031 const StorageIndex cGroupSize = itemID.get_group_range(0) / nonContractGroupSize;
1032 const StorageIndex nonContractGroupId =
1033 is_lhs_vec ? itemID.get_group(0) / cGroupSize : itemID.get_group(0) % nonContractGroupSize;
1034 const StorageIndex contractGroupId =
1035 is_lhs_vec ? itemID.get_group(0) % cGroupSize : itemID.get_group(0) / nonContractGroupSize;
1036 auto out_ptr = out_res + (IsFinal ? 0 : contractGroupId * nonContractDim);
1037
1038 const StorageIndex nonContractGroupOffset = nonContractGroupId * Properties::TileSizeDimNC;
1039 const StorageIndex contractGroupOffset = contractGroupId * Properties::TileSizeDimC;
1040 auto outScratchIndex = nonContractId + contractId * Properties::LocalThreadSizeNC;
1041 const StorageIndex globalNonContractDimOffset = nonContractGroupOffset + nonContractId;
1042 const StorageIndex globalContractDimOffset = contractGroupOffset + contractId;
1043 auto local_output = scratch_ptr + OutScratchOffset;
1044 const bool is_internal = nonContractDim - nonContractGroupOffset >= Properties::TileSizeDimNC &&
1045 contractDim - contractGroupOffset >= Properties::TileSizeDimC;
1046 is_internal
1047 ? compute_panel<true>(itemID, vec, mat, local_output, out_ptr,
1048#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1049 scratch_ptr, contractGroupOffset,
1050#endif
1051 nonContractGroupOffset, linearLocalThreadId, contractDim, nonContractDim, contractId,
1052 nonContractId, globalContractDimOffset, globalNonContractDimOffset, outScratchIndex)
1053 : compute_panel<false>(itemID, vec, mat, local_output, out_ptr,
1054#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1055 scratch_ptr, contractGroupOffset,
1056#endif
1057 nonContractGroupOffset, linearLocalThreadId, contractDim, nonContractDim, contractId,
1058 nonContractId, globalContractDimOffset, globalNonContractDimOffset, outScratchIndex);
1059 }
1060 template <bool is_internal_block, typename OutPtr>
1061 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void compute_panel(
1062 const cl::sycl::nd_item<1> &itemID, const VectorMapper &vec, const TensorMapper &mat, OutScalar *local_output,
1063 OutPtr out_ptr,
1064#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1065 OutScalar *scratch_ptr, const StorageIndex contractGroupOffset,
1066#endif
1067 const StorageIndex nonContractGroupOffset, const StorageIndex linearLocalThreadId, StorageIndex contractDim,
1068 StorageIndex nonContractDim, StorageIndex contractId, StorageIndex nonContractId,
1069 StorageIndex globalContractDimOffset, StorageIndex globalNonContractDimOffset, StorageIndex outScratchIndex) {
1070 OutScalar outScalar[Properties::WorkLoadPerThreadNC] = {OutScalar(0)};
1071 // Reading the vector
1072#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1073 const StorageIndex vectorOffset = contractGroupOffset + linearLocalThreadId;
1074 extract_block<VecBlockProperties, is_internal_block, KFactor,
1075 Properties::LocalThreadSizeNC * Properties::LocalThreadSizeC>(vec, scratch_ptr, linearLocalThreadId,
1076 vectorOffset, contractDim);
1077
1078 itemID.barrier(cl::sycl::access::fence_space::local_space);
1079 auto in_scratch_ptr = scratch_ptr + contractId;
1080#endif
1081
1082 StorageIndex privateOffsetC = 0;
1083 EIGEN_UNROLL_LOOP
1084 for (StorageIndex i = 0; i < Properties::WorkLoadPerThreadC; i++) {
1085 StorageIndex privateOffsetNC = 0;
1086 bool contract_conds = ((globalContractDimOffset + privateOffsetC) < contractDim);
1087#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1088 auto vecScalar = *in_scratch_ptr;
1089#else
1090 auto vecScalar = (check_boundary<is_internal_block>(contract_conds))
1091 ? vec(is_lhs_vec ? StorageIndex(0) : globalContractDimOffset + privateOffsetC,
1092 is_lhs_vec ? globalContractDimOffset + privateOffsetC : StorageIndex(0))
1093 : OutScalar(0);
1094#endif
1095 EIGEN_UNROLL_LOOP
1096 for (StorageIndex j = 0; j < Properties::WorkLoadPerThreadNC; j++) {
1097 auto matScalar = (check_boundary<is_internal_block>(
1098 contract_conds && ((globalNonContractDimOffset + privateOffsetNC) < nonContractDim)))
1099 ? mat(is_lhs_vec ? globalContractDimOffset + privateOffsetC
1100 : globalNonContractDimOffset + privateOffsetNC,
1101 is_lhs_vec ? globalNonContractDimOffset + privateOffsetNC
1102 : globalContractDimOffset + privateOffsetC)
1103 : OutScalar(0);
1104
1105 outScalar[j] = ::Eigen::internal::pmadd(matScalar, vecScalar, outScalar[j]);
1106 privateOffsetNC += Properties::LocalThreadSizeNC;
1107 }
1108 privateOffsetC += Properties::LocalThreadSizeC;
1109#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1110 in_scratch_ptr += Properties::LocalThreadSizeC;
1111#endif
1112 }
1113
1114 auto out_scratch_ptr = local_output + outScratchIndex;
1115 // Each block of 16*16 element in shared memory should reduce to 16*1
1116 EIGEN_UNROLL_LOOP
1117 for (StorageIndex j = 0; j < Properties::WorkLoadPerThreadNC; j++) {
1118 *out_scratch_ptr = outScalar[j];
1119
1120 out_scratch_ptr += (Properties::LocalThreadSizeNC * Properties::LocalThreadSizeC);
1121 }
1122 EIGEN_IF_CONSTEXPR (is_lhs_vec) {
1123 nonContractId = linearLocalThreadId % Properties::LocalThreadSizeNC;
1124 contractId = linearLocalThreadId / Properties::LocalThreadSizeNC;
1125 outScratchIndex = nonContractId + contractId * Properties::LocalThreadSizeNC;
1126 }
1127
1128 out_scratch_ptr = local_output + outScratchIndex;
1129 EIGEN_UNROLL_LOOP
1130 for (StorageIndex j = 0; j < Properties::WorkLoadPerThreadNC; j++) {
1131 EIGEN_UNROLL_LOOP
1132 for (StorageIndex offset = Properties::LocalThreadSizeC >> 1; offset > 0; offset >>= 1) {
1133 itemID.barrier(cl::sycl::access::fence_space::local_space);
1134 if (contractId < offset) {
1135 StorageIndex myNeigbourId = (Properties::LocalThreadSizeNC * offset);
1136 *out_scratch_ptr += out_scratch_ptr[myNeigbourId];
1137 }
1138 }
1139 // moving to next 16 by 16 block
1140 out_scratch_ptr += (Properties::LocalThreadSizeNC * Properties::LocalThreadSizeC);
1141 }
1142
1143 if (contractId == 0) {
1144 out_scratch_ptr = local_output + nonContractId;
1145 StorageIndex global_final_offset = nonContractGroupOffset + nonContractId;
1146 out_ptr += global_final_offset;
1147 EIGEN_UNROLL_LOOP
1148 for (StorageIndex j = 0; j < Properties::WorkLoadPerThreadNC; j++) {
1149 if (check_boundary<is_internal_block>(global_final_offset < nonContractDim)) {
1150 auto res = *out_scratch_ptr;
1151
1152 *out_ptr = res;
1153 out_ptr += Properties::LocalThreadSizeNC;
1154 }
1155 // moving to next 16 by 16 block to get the next 16 reduced elements
1156 out_scratch_ptr += (Properties::LocalThreadSizeNC * Properties::LocalThreadSizeC);
1157 if (!(is_internal_block)) global_final_offset += Properties::LocalThreadSizeNC;
1158 }
1159 }
1160 }
1161
1162 template <typename InputBlockProperties, bool is_internal_block, int CFactor, int GroupSize, typename Input,
1163 typename Local>
1164 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void extract_block(const Input &inpt, Local *local_ptr,
1165 const StorageIndex &linearLocalThreadId,
1166 const StorageIndex &cOffset, const StorageIndex &C) {
1167 local_ptr += InputBlockProperties::c_stride * linearLocalThreadId;
1168 StorageIndex cIndex = cOffset;
1169 for (StorageIndex cId = 0; cId < CFactor / InputBlockProperties::c_stride; cId++) {
1170 if (check_boundary<is_internal_block>(cIndex + InputBlockProperties::c_stride - 1 < C)) {
1171 auto val = read<InputBlockProperties::packet_load, InputBlockProperties::is_coalesced_layout,
1172 InputBlockProperties::is_rhs, typename InputBlockProperties::OutType>(inpt, StorageIndex(0),
1173 cIndex, StorageIndex(1));
1174 write<StorageIndex, 1, data_source::local_mem>(val, local_ptr);
1175 } else {
1176 EIGEN_UNROLL_LOOP
1177 for (StorageIndex i = 0; i < InputBlockProperties::elements_per_access; i++) {
1178 OutScalar val =
1179 (cIndex + i < C)
1180 ? read<false, InputBlockProperties::is_coalesced_layout, InputBlockProperties::is_rhs, OutScalar>(
1181 inpt, StorageIndex(0), cIndex + i, StorageIndex(1))
1182 : OutScalar(0);
1183 write<StorageIndex, 1, data_source::local_mem>(val, local_ptr + i);
1184 }
1185 }
1186 local_ptr += InputBlockProperties::c_stride * GroupSize;
1187 cIndex += InputBlockProperties::c_stride * GroupSize;
1188 }
1189 }
1190};
1191#endif
1192
1193#ifndef EIGEN_SYCL_DISABLE_SCALAR
1194
1226template <typename OutScalar, typename LhsScalar, typename RhsScalar, typename OutAccessor, typename LhsMapper,
1227 typename RhsMapper, typename StorageIndex, bool Vectorizable>
1228struct GeneralScalarContraction {
1229 using Scratch = cl::sycl::accessor<OutScalar, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::local>;
1230 Scratch scratch;
1231 const LhsMapper lhs;
1232 const RhsMapper rhs;
1233 OutAccessor out_res;
1234 const StorageIndex rng;
1235
1236 EIGEN_DEVICE_FUNC GeneralScalarContraction(Scratch scratch_, const LhsMapper lhs_, const RhsMapper rhs_,
1237 OutAccessor out_res_, const StorageIndex rng_)
1238 : scratch(scratch_), lhs(lhs_), rhs(rhs_), out_res(out_res_), rng(rng_) {}
1239
1240 EIGEN_DEVICE_FUNC void operator()(cl::sycl::nd_item<1> itemID) const {
1241 auto out_ptr = out_res;
1242 OutScalar *scratch_ptr = scratch.get_pointer();
1243
1244 StorageIndex globalid = itemID.get_global_id(0);
1245 StorageIndex localid = itemID.get_local_id(0);
1246 OutScalar accumulator = OutScalar(0);
1247 for (StorageIndex i = globalid; i < rng; i += itemID.get_global_range(0)) {
1248 accumulator = Eigen::internal::pmadd(lhs(0, i), rhs(i, 0), accumulator);
1249 }
1250 auto out_scratch_ptr = scratch_ptr + localid;
1251 *out_scratch_ptr = accumulator;
1252 for (StorageIndex offset = itemID.get_local_range(0) >> 1; offset > 0; offset >>= 1) {
1253 itemID.barrier(cl::sycl::access::fence_space::local_space);
1254 if (localid < offset) {
1255 *out_scratch_ptr = (accumulator += out_scratch_ptr[offset]);
1256 }
1257 }
1258 if (localid == 0) {
1259 out_ptr[itemID.get_group(0)] = accumulator;
1260 }
1261 }
1262};
1263#endif
1264
1265} // namespace internal
1266} // namespace TensorSycl
1267
1268template <typename Indices, typename LeftArgType, typename RightArgType, typename OutputKernelType>
1269struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>,
1270 Eigen::SyclDevice>
1271 : public TensorContractionEvaluatorBase<TensorEvaluator<
1272 const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Eigen::SyclDevice>> {
1273 static_assert(std::is_same<OutputKernelType, const NoOpOutputKernel>::value,
1274 "SYCL tensor contraction does not support output kernels.");
1275
1276 using Device = Eigen::SyclDevice;
1277
1279 using Base = TensorContractionEvaluatorBase<Self>;
1281 using Scalar = std::remove_const_t<typename XprType::Scalar>;
1282 using StorageIndex = typename XprType::Index;
1283 using CoeffReturnType = typename XprType::CoeffReturnType;
1284 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
1285 using Storage = typename Base::Storage;
1286 using EvaluatorPointerType = typename Base::EvaluatorPointerType;
1287 struct TripleDim {
1288 const StorageIndex M;
1289 const StorageIndex N;
1290 const StorageIndex K;
1291 TripleDim(const StorageIndex M_, const StorageIndex N_, const StorageIndex K_) : M(M_), N(N_), K(K_) {}
1292 };
1293 enum {
1294 PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1),
1295 BlockAccess = false,
1296 };
1297
1298 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
1299 static constexpr int LDims = Base::LDims;
1300 static constexpr int RDims = Base::RDims;
1301 static constexpr int ContractDims = Base::ContractDims;
1302
1303 using left_dim_mapper_t = array<StorageIndex, LDims>;
1304 using right_dim_mapper_t = array<StorageIndex, RDims>;
1305
1306 using contract_t = array<StorageIndex, ContractDims>;
1307 using left_nocontract_t = array<StorageIndex, LDims - ContractDims>;
1308 using right_nocontract_t = array<StorageIndex, RDims - ContractDims>;
1309
1310 static constexpr int NumDims = LDims + RDims - 2 * ContractDims;
1311
1312 using Dimensions = DSizes<StorageIndex, NumDims>;
1313
1314 using LeftEvaluator = TensorEvaluator<typename Base::EvalLeftArgType, Device>;
1315 using RightEvaluator = TensorEvaluator<typename Base::EvalRightArgType, Device>;
1316 using LhsScalar = std::remove_const_t<typename LeftEvaluator::CoeffReturnType>;
1317 using RhsScalar = std::remove_const_t<typename RightEvaluator::CoeffReturnType>;
1318
1319 using LeftDimensions = typename LeftEvaluator::Dimensions;
1320 using RightDimensions = typename RightEvaluator::Dimensions;
1321
1322 template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered>
1323 struct input_mapper_propertis {
1324 static constexpr bool is_lhs_matrix = (LDims == 2 && ContractDims == 1) || lhs_inner_dim_contiguous;
1325 static constexpr bool is_rhs_matrix =
1326 (RDims == 2 && ContractDims == 1) || (rhs_inner_dim_contiguous && !rhs_inner_dim_reordered);
1327 };
1328
1329 TensorEvaluator(const XprType &op, const Device &device) : Base(op, device) {}
1330
1331 // We need to redefine this method to make nvcc happy
1332 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(typename Base::EvaluatorPointerType data) {
1333 this->m_leftImpl.evalSubExprsIfNeeded(nullptr);
1334 this->m_rightImpl.evalSubExprsIfNeeded(nullptr);
1335 if (!data) {
1336 this->m_result = this->m_device.get(
1337 static_cast<Scalar *>(this->m_device.allocate_temp(this->dimensions().TotalSize() * sizeof(Scalar))));
1338 data = this->m_result;
1339 }
1340 evalToSycl(data);
1341 return (this->m_result != nullptr);
1342 }
1343 const Eigen::SyclDevice &device() const { return this->m_device; }
1344 void evalToSycl(typename Base::EvaluatorPointerType buffer) const {
1345 if (this->m_lhs_inner_dim_contiguous) {
1346 if (this->m_rhs_inner_dim_contiguous) {
1347 if (this->m_rhs_inner_dim_reordered) {
1348 evalTyped<true, true, true, Unaligned>(buffer);
1349 } else {
1350 evalTyped<true, true, false, Unaligned>(buffer);
1351 }
1352 } else {
1353 if (this->m_rhs_inner_dim_reordered) {
1354 evalTyped<true, false, true, Unaligned>(buffer);
1355 } else {
1356 evalTyped<true, false, false, Unaligned>(buffer);
1357 }
1358 }
1359 } else {
1360 if (this->m_rhs_inner_dim_contiguous) {
1361 if (this->m_rhs_inner_dim_reordered) {
1362 evalTyped<false, true, true, Unaligned>(buffer);
1363 } else {
1364 evalTyped<false, true, false, Unaligned>(buffer);
1365 }
1366 } else {
1367 if (this->m_rhs_inner_dim_reordered) {
1368 evalTyped<false, false, true, Unaligned>(buffer);
1369 } else {
1370 evalTyped<false, false, false, Unaligned>(buffer);
1371 }
1372 }
1373 }
1374 }
1375
1376 template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment>
1377 void evalTyped(typename Base::EvaluatorPointerType buffer) const {
1378 const auto triple_dim = TripleDim{this->m_i_size, this->m_j_size, this->m_k_size};
1379 using LhsMapper =
1380 internal::TensorContractionInputMapper<LhsScalar, StorageIndex, internal::Lhs, LeftEvaluator, left_nocontract_t,
1381 contract_t, PacketType<CoeffReturnType, Device>::size,
1382 lhs_inner_dim_contiguous, false, Unaligned, MakePointer>;
1383
1384 using RhsMapper =
1385 internal::TensorContractionInputMapper<RhsScalar, StorageIndex, internal::Rhs, RightEvaluator,
1386 right_nocontract_t, contract_t,
1387 PacketType<CoeffReturnType, Device>::size, rhs_inner_dim_contiguous,
1388 rhs_inner_dim_reordered, Unaligned, MakePointer>;
1389
1390 // initialize data mappers
1391 LhsMapper lhs(this->m_leftImpl, this->m_left_nocontract_strides, this->m_i_strides,
1392 this->m_left_contracting_strides, this->m_k_strides);
1393
1394 RhsMapper rhs(this->m_rightImpl, this->m_right_nocontract_strides, this->m_j_strides,
1395 this->m_right_contracting_strides, this->m_k_strides);
1396
1397#ifndef EIGEN_SYCL_DISABLE_SCALAR
1398 if (triple_dim.M == 1 && triple_dim.N == 1) {
1399 launchSC(buffer, lhs, rhs, triple_dim.K);
1400 } else
1401#endif
1402#ifndef EIGEN_SYCL_DISABLE_GEMV
1403 if (triple_dim.M != 1 && triple_dim.N == 1) {
1404 LaunchVT<false>(buffer, rhs, lhs, triple_dim.M, triple_dim.K);
1405 } else if (triple_dim.M == 1 && triple_dim.N != 1) {
1406 LaunchVT<true>(buffer, lhs, rhs, triple_dim.N, triple_dim.K);
1407 } else // This is equivalent of if (m!=1 && n!=1)
1408#endif
1409 {
1410 using inpt_mapper_properties =
1411 input_mapper_propertis<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered>;
1412#ifndef EIGEN_SYCL_DISABLE_SKINNY
1413 bool skinny = false;
1414 auto platform_name = this->device().getPlatformName();
1415 // This is based on empirical calculation for AMD r9-nano and Fiji
1416 if (platform_name.find("AMD") == 0) {
1417 skinny = (triple_dim.M < triple_dim.K || triple_dim.N < triple_dim.K) &&
1418 ((triple_dim.M < 1024 && triple_dim.N < 1024) ||
1419 (uint64_t(triple_dim.M * triple_dim.N) < uint64_t(triple_dim.K)));
1420 } else {
1421 skinny = (((std::max(triple_dim.K, triple_dim.N) / std::min(triple_dim.K, triple_dim.N)) > 100) ||
1422 ((std::max(triple_dim.K, triple_dim.M) / std::min(triple_dim.K, triple_dim.M)) > 100) ||
1423 ((std::max(triple_dim.N, triple_dim.M) / std::min(triple_dim.N, triple_dim.M)) > 100));
1424 }
1425 if (skinny)
1426 adjustTT<true, inpt_mapper_properties>(buffer, lhs, rhs, triple_dim);
1427 else
1428#endif // EIGEN_SYCL_DISABLE_SKINNY
1429 adjustTT<false, inpt_mapper_properties>(buffer, lhs, rhs, triple_dim);
1430 }
1431 }
1432
1433 template <bool skinny, typename input_mapper_properties, typename LhsMapper, typename RhsMapper>
1434 void EIGEN_ALWAYS_INLINE adjustTT(EvaluatorPointerType buffer, const LhsMapper &lhs, const RhsMapper &rhs,
1435 const TripleDim &triple_dim) const {
1436#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1437 if (device().has_local_memory()) {
1438 using PanelParameters = TensorSycl::internal::TTPanelSize<CoeffReturnType, StorageIndex, 4, 4, 16>;
1439 launchTT<TensorSycl::internal::contraction_type::local, skinny, input_mapper_properties, PanelParameters>(
1440 buffer, lhs, rhs, triple_dim);
1441 }
1442#endif
1443#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_OFF
1444 if (!(device().has_local_memory())) {
1445 using PanelParameters = TensorSycl::internal::TTPanelSize<CoeffReturnType, StorageIndex, 4, 4, 4>;
1446 launchTT<TensorSycl::internal::contraction_type::no_local, skinny, input_mapper_properties, PanelParameters>(
1447 buffer, lhs, rhs, triple_dim);
1448 }
1449#endif
1450 }
1451
1452 template <TensorSycl::internal::contraction_type ct, bool skinny, typename input_mapper_properties,
1453 typename Properties, typename LhsMapper, typename RhsMapper>
1454 void launchTT(EvaluatorPointerType buffer, const LhsMapper &lhs, const RhsMapper &rhs,
1455 const TripleDim &triple_dim) const {
1456 const StorageIndex roundUpM = Eigen::TensorSycl::internal::roundUp(triple_dim.M, Properties::TileSizeDimM);
1457 const StorageIndex roundUpN = Eigen::TensorSycl::internal::roundUp(triple_dim.N, Properties::TileSizeDimN);
1458 const StorageIndex groupSizeM = roundUpM / Properties::TileSizeDimM;
1459 const StorageIndex groupSizeN = roundUpN / Properties::TileSizeDimN;
1460
1461 const StorageIndex roundUpK = Eigen::TensorSycl::internal::roundUp(triple_dim.K, Properties::TileSizeDimK);
1462 StorageIndex totalTilesK = roundUpK / Properties::TileSizeDimK;
1463 StorageIndex groupSizeK =
1464 skinny
1465 ? std::max(std::min(totalTilesK,
1466 (StorageIndex)(device().getPowerOfTwo(device().getNumSyclMultiProcessors(), true) * 4) /
1467 (groupSizeM * groupSizeN)),
1468 StorageIndex(1))
1469 : StorageIndex(1);
1470
1471 const StorageIndex numTilesPerGroup = Eigen::TensorSycl::internal::roundUp(totalTilesK, groupSizeK) / groupSizeK;
1472
1473 const StorageIndex totalGroupSize = groupSizeM * groupSizeN * groupSizeK;
1474
1475 const StorageIndex localRange = Properties::LocalThreadSizeM * Properties::LocalThreadSizeN;
1476 const StorageIndex globalRange = totalGroupSize * localRange;
1477
1478 const StorageIndex scratchSize = (ct == TensorSycl::internal::contraction_type::local)
1479 ? ((Properties::DoubleBuffer + 1) *
1480 (Properties::TileSizeDimM + Properties::BC) * (Properties::TileSizeDimK)) +
1481 ((Properties::DoubleBuffer + 1) * (Properties::TileSizeDimK) *
1482 (Properties::TileSizeDimN + Properties::BC))
1483 : StorageIndex(1);
1484
1485 auto thread_range = cl::sycl::nd_range<1>(cl::sycl::range<1>(globalRange), cl::sycl::range<1>(localRange));
1486 if (groupSizeK == 1) {
1487 using ContractKernelName =
1488 TensorSycl::internal::TensorContractionKernel<CoeffReturnType, LhsScalar, RhsScalar, EvaluatorPointerType,
1489 LhsMapper, RhsMapper, StorageIndex, Properties, TripleDim,
1490 PacketAccess, input_mapper_properties, true, ct>;
1491 device()
1492 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(
1493 lhs, rhs, buffer, thread_range, scratchSize, groupSizeM, groupSizeN, numTilesPerGroup, triple_dim)
1494 .wait();
1495 } else {
1496 using ContractKernelName =
1497 TensorSycl::internal::TensorContractionKernel<CoeffReturnType, LhsScalar, RhsScalar, EvaluatorPointerType,
1498 LhsMapper, RhsMapper, StorageIndex, Properties, TripleDim,
1499 PacketAccess, input_mapper_properties, false, ct>;
1500 CoeffReturnType *temp_pointer = static_cast<CoeffReturnType *>(
1501 device().allocate_temp(triple_dim.M * triple_dim.N * groupSizeK * sizeof(CoeffReturnType)));
1502 EvaluatorPointerType tmp_global_accessor = device().get(temp_pointer);
1503
1504 device()
1505 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(
1506 lhs, rhs, tmp_global_accessor, thread_range, scratchSize, groupSizeM, groupSizeN, numTilesPerGroup,
1507 triple_dim)
1508 .wait();
1509
1510 using Op = Eigen::internal::SumReducer<CoeffReturnType>;
1511 auto op = Op();
1512 using ReductionKernel =
1513 TensorSycl::internal::SecondStepPartialReduction<CoeffReturnType, StorageIndex, EvaluatorPointerType,
1514 EvaluatorPointerType, Op>;
1515
1516 device()
1517 .template unary_kernel_launcher<CoeffReturnType, ReductionKernel>(
1518 tmp_global_accessor, buffer,
1519 cl::sycl::nd_range<1>(cl::sycl::range<1>(StorageIndex(
1520 Eigen::TensorSycl::internal::roundUp(triple_dim.M * triple_dim.N, localRange))),
1521 cl::sycl::range<1>(localRange)),
1522 StorageIndex(1), op, StorageIndex(triple_dim.M * triple_dim.N), groupSizeK)
1523 .wait();
1524 device().deallocate_temp(temp_pointer);
1525 }
1526 }
1527
1528#ifndef EIGEN_SYCL_DISABLE_GEMV
1529 template <bool is_lhs_vec, typename VectorMapper, typename TensorMapper, typename StorageIndex>
1530 void EIGEN_ALWAYS_INLINE LaunchVT(EvaluatorPointerType buffer, const VectorMapper &vec, const TensorMapper &mat,
1531 StorageIndex NC, StorageIndex C) const {
1532 const StorageIndex nonContractDim = NC;
1533 constexpr StorageIndex NCFactor = 1;
1534 constexpr StorageIndex CFactor = 1;
1535 constexpr StorageIndex NCWindow = 16;
1536 using Properties =
1537 Eigen::TensorSycl::internal::TVPanelSize<CoeffReturnType, StorageIndex, NCWindow, CFactor, NCFactor>;
1538 const StorageIndex roundUpC = Eigen::TensorSycl::internal::roundUp(C, Properties::TileSizeDimC);
1539 const StorageIndex cNumGroups = roundUpC / (Properties::LocalThreadSizeC * Properties::WorkLoadPerThreadC);
1540 const StorageIndex roundUpNC = Eigen::TensorSycl::internal::roundUp(nonContractDim, Properties::TileSizeDimNC);
1541 const StorageIndex nCNumGroups = roundUpNC / (Properties::LocalThreadSizeNC * Properties::WorkLoadPerThreadNC);
1542 const StorageIndex globalRange =
1543 (roundUpNC / (Properties::WorkLoadPerThreadNC)) * (roundUpC / (Properties::WorkLoadPerThreadC));
1544 const StorageIndex localRange = Properties::LocalThreadSizeNC * Properties::LocalThreadSizeC;
1545 const StorageIndex scratchSize =
1546 (Properties::WorkLoadPerThreadNC + CFactor) * Properties::LocalThreadSizeC * Properties::LocalThreadSizeNC;
1547 auto thread_range = cl::sycl::nd_range<1>(cl::sycl::range<1>(globalRange), cl::sycl::range<1>(localRange));
1548 if (cNumGroups > 1) {
1549 using ContractKernelName =
1550 Eigen::TensorSycl::internal::GeneralVectorTensor<CoeffReturnType, EvaluatorPointerType, VectorMapper,
1551 TensorMapper, StorageIndex, Properties, CFactor, false,
1552 is_lhs_vec, false>;
1553 CoeffReturnType *temp_pointer =
1554 static_cast<CoeffReturnType *>(device().allocate_temp(nonContractDim * cNumGroups * sizeof(CoeffReturnType)));
1555 EvaluatorPointerType tmp_global_accessor = device().get(temp_pointer);
1556
1557 device()
1558 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(
1559 vec, mat, tmp_global_accessor, thread_range, scratchSize, nCNumGroups, nonContractDim, C)
1560 .wait();
1561
1562 using Op = Eigen::internal::SumReducer<CoeffReturnType>;
1563 using ReductionKernel =
1564 TensorSycl::internal::SecondStepPartialReduction<CoeffReturnType, StorageIndex, EvaluatorPointerType,
1565 EvaluatorPointerType, Op>;
1566
1567 device()
1568 .template unary_kernel_launcher<CoeffReturnType, ReductionKernel>(
1569 tmp_global_accessor, buffer,
1570 cl::sycl::nd_range<1>(
1571 cl::sycl::range<1>(Eigen::TensorSycl::internal::roundUp(nonContractDim, localRange)),
1572 cl::sycl::range<1>(localRange)),
1573 StorageIndex(1), Op(), nonContractDim, cNumGroups)
1574 .wait();
1575 device().deallocate_temp(temp_pointer);
1576 } else {
1577 using ContractKernelName =
1578 Eigen::TensorSycl::internal::GeneralVectorTensor<CoeffReturnType, EvaluatorPointerType, VectorMapper,
1579 TensorMapper, StorageIndex, Properties, CFactor, false,
1580 is_lhs_vec, true>;
1581 device()
1582 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(
1583 vec, mat, buffer, thread_range, scratchSize, nCNumGroups, nonContractDim, C)
1584 .wait();
1585 }
1586 }
1587#endif
1588
1589#ifndef EIGEN_SYCL_DISABLE_SCALAR
1590 template <typename LhsMapper, typename RhsMapper>
1591 EIGEN_ALWAYS_INLINE void launchSC(EvaluatorPointerType buffer, const LhsMapper &lhs, const RhsMapper &rhs,
1592 StorageIndex K) const {
1593 EIGEN_STATIC_ASSERT(!((EIGEN_SYCL_LOCAL_THREAD_DIM0 * EIGEN_SYCL_LOCAL_THREAD_DIM1) &
1594 (EIGEN_SYCL_LOCAL_THREAD_DIM0 * EIGEN_SYCL_LOCAL_THREAD_DIM1 - 1)),
1595 "The Local thread size must be a power of 2 for the reduction "
1596 "operation");
1597 constexpr StorageIndex local_range = EIGEN_SYCL_LOCAL_THREAD_DIM0 * EIGEN_SYCL_LOCAL_THREAD_DIM1;
1598
1599 // Here we force the code not to be more than 2-step reduction: Our empirical research shows that if each thread
1600 // reduces at least 512 elements individually, we get better performance.
1601 const StorageIndex num_work_group = ((K + (512 * local_range - 1)) / (512 * local_range) > 1 ? local_range : 1);
1602 const StorageIndex global_range = num_work_group * local_range;
1603
1604 using ContractKernelName = Eigen::TensorSycl::internal::GeneralScalarContraction<
1605 CoeffReturnType, LhsScalar, RhsScalar, EvaluatorPointerType, LhsMapper, RhsMapper, StorageIndex, false>;
1606 auto thread_range = cl::sycl::nd_range<1>(cl::sycl::range<1>(global_range), cl::sycl::range<1>(local_range));
1607 if (num_work_group > 1) {
1608 CoeffReturnType *temp_pointer =
1609 static_cast<CoeffReturnType *>(device().allocate_temp(num_work_group * sizeof(CoeffReturnType)));
1610 EvaluatorPointerType tmp_global_accessor = device().get(temp_pointer);
1611 device()
1612 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(lhs, rhs, tmp_global_accessor,
1613 thread_range, local_range, K)
1614 .wait();
1615 using Op = Eigen::internal::SumReducer<CoeffReturnType>;
1616 using GenericRKernel =
1617 TensorSycl::internal::SecondStepFullReducer<CoeffReturnType, Op, EvaluatorPointerType, EvaluatorPointerType,
1618 StorageIndex, local_range>;
1619 device()
1620 .template unary_kernel_launcher<CoeffReturnType, GenericRKernel>(
1621 tmp_global_accessor, buffer,
1622 cl::sycl::nd_range<1>(cl::sycl::range<1>(local_range), cl::sycl::range<1>(local_range)), local_range,
1623 Op())
1624 .wait();
1625 device().deallocate_temp(temp_pointer);
1626 } else {
1627 device()
1628 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(lhs, rhs, buffer, thread_range,
1629 local_range, K)
1630 .wait();
1631 }
1632 }
1633#endif
1634
1635 EIGEN_STRONG_INLINE void cleanup() {
1636 this->m_leftImpl.cleanup();
1637 this->m_rightImpl.cleanup();
1638
1639 if (this->m_result) {
1640 this->m_device.deallocate_temp(this->m_result);
1641 this->m_result = nullptr;
1642 }
1643 }
1644};
1645} // namespace Eigen
1646#endif // EIGEN_TENSOR_TENSOR_CONTRACTION_SYCL_H
Definition TensorContraction.h:418
TensorContractionKernel is a template class that provides Tensor -Tensor contraction operation.
Definition TensorContractionSycl.h:457
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47
BlockProperties is a template class that provides different characteristic of a block of each Tensor ...
Definition TensorContractionSycl.h:322
TTPanelSize, a template class used for setting the panel size required for launching General Tensor T...
Definition TensorContractionSycl.h:84
TVPanelSize, a template class used for setting the panel size required for launching General TensorVe...
Definition TensorContractionSycl.h:48
MemHolder this is a place holder struct for creating memory hierarchy in SYCL. Inside SYCL kernel it ...
Definition TensorContractionSycl.h:502
TiledMemory: contains required memory pointer for loading each tile of the TensorContraction panel fr...
Definition TensorContractionSycl.h:535
ThreadProperties is a template class that provides each thread's properties within a workgroup....
Definition TensorContractionSycl.h:373