21#ifndef EIGEN_TENSOR_TENSOR_CONTRACTION_SYCL_H
22#define EIGEN_TENSOR_TENSOR_CONTRACTION_SYCL_H
25#include "./InternalHeaderCheck.h"
32#ifndef EIGEN_SYCL_DISABLE_GEMV
47template <
typename Scalar,
typename StorageIndex, StorageIndex NCWindow, StorageIndex CFactor, StorageIndex NCFactor>
50 static constexpr StorageIndex LocalThreadSizeC = EIGEN_SYCL_LOCAL_THREAD_DIM0;
52 static constxpr StorageIndex LocalThreadSizeNC = EIGEN_SYCL_LOCAL_THREAD_DIM1;
54 static constexpr StorageIndex TileSizeDimNC = NCWindow / NCFactor;
56 static constexpr StorageIndex TileSizeDimC = CFactor * LocalThreadSizeNC * LocalThreadSizeC;
58 static constexpr StorageIndex WorkLoadPerThreadNC = TileSizeDimNC / LocalThreadSizeNC;
60 static constexpr StorageIndex WorkLoadPerThreadC = TileSizeDimC / LocalThreadSizeC;
62 static constexpr bool BC =
false;
83template <
typename Scalar,
typename StorageIndex, StorageIndex REG_SIZE_M, StorageIndex REG_SIZE_N, StorageIndex TSDK>
86 static constexpr StorageIndex TileSizeDimK = TSDK;
89#ifndef EIGEN_SYCL_REG_M
90 static constexpr StorageIndex WorkLoadPerThreadM = REG_SIZE_M;
92 static constexpr StorageIndex WorkLoadPerThreadM = EIGEN_SYCL_REG_M;
96#ifndef EIGEN_SYCL_REG_N
97 static constexpr StorageIndex WorkLoadPerThreadN = REG_SIZE_N;
99 static constexpr StorageIndex WorkLoadPerThreadN = EIGEN_SYCL_REG_N;
102 static constexpr StorageIndex LocalThreadSizeM = EIGEN_SYCL_LOCAL_THREAD_DIM0;
104 static constexpr StorageIndex LocalThreadSizeN = EIGEN_SYCL_LOCAL_THREAD_DIM1;
106 static constexpr StorageIndex TileSizeDimM = LocalThreadSizeM * WorkLoadPerThreadM;
108 static constexpr StorageIndex TileSizeDimN = LocalThreadSizeN * WorkLoadPerThreadN;
110 static constexpr StorageIndex LoadPerThreadLhs =
111 ((TileSizeDimK * WorkLoadPerThreadM * WorkLoadPerThreadN) / (TileSizeDimN));
113 static constexpr StorageIndex LoadPerThreadRhs =
114 ((TileSizeDimK * WorkLoadPerThreadM * WorkLoadPerThreadN) / (TileSizeDimM));
116 static constexpr bool BC =
true;
119 static constexpr bool DoubleBuffer =
120#ifdef EIGEN_SYCL_DISABLE_DOUBLE_BUFFER
131enum class contraction_type { local, no_local };
135enum class data_source { global_mem, local_mem, private_mem };
162template <
bool PacketLoad,
bool is_coalesced_layout, bool,
typename PacketType,
typename TensorMapper,
163 typename StorageIndex>
164static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<PacketLoad, PacketType> read(
165 const TensorMapper &tensorMapper,
const StorageIndex &NCIndex,
const StorageIndex &CIndex,
const StorageIndex &ld) {
166 const StorageIndex row = (is_coalesced_layout) ? NCIndex : CIndex;
167 const StorageIndex col = (is_coalesced_layout) ? CIndex : NCIndex;
168 return tensorMapper.get_tensor().template packet<Unaligned>(row + (col * ld));
193template <
bool PacketLoad,
bool,
bool IsRhs,
typename PacketType,
typename TensorMapper,
typename StorageIndex>
194static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<!PacketLoad, PacketType> read(
195 const TensorMapper &tensorMapper,
const StorageIndex &NCIndex,
const StorageIndex &CIndex,
const StorageIndex &) {
196 const StorageIndex row = (IsRhs) ? CIndex : NCIndex;
197 const StorageIndex col = (IsRhs) ? NCIndex : CIndex;
198 return tensorMapper(row, col);
222template <
typename StorageIndex, StorageIndex ld, data_source dt,
typename PacketType,
typename DataScalar>
223static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<dt != data_source::global_mem, void> write(
224 PacketType &packet_data, DataScalar ptr) {
225 constexpr int PacketSize = Eigen::internal::unpacket_traits<PacketType>::size;
227 for (
int i = 0; i < PacketSize; i++) {
228 *ptr = PacketWrapper<PacketType, PacketSize>::scalarize(i, packet_data);
248template <data_source dt,
typename PacketType,
typename DataScalar>
249static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
250 std::enable_if_t<Eigen::internal::unpacket_traits<PacketType>::size != 1 && dt == data_source::global_mem,
void>
251 write(PacketType &packet_data, DataScalar *ptr) {
252 ::Eigen::internal::pstoreu<DataScalar, PacketType>(ptr, packet_data);
268template <data_source dt,
typename PacketType,
typename DataScalar>
269static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
270 std::enable_if_t<Eigen::internal::unpacket_traits<PacketType>::size == 1 && dt == data_source::global_mem,
void>
271 write(PacketType &packet_data, DataScalar *ptr) {
280template <
bool is_
internal>
281EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool check_boundary(
bool) {
291EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool check_boundary<false>(
bool cond) {
321template <
bool is_transposed,
bool is_rhs_,
bool packet_load_,
typename PacketType>
323 static constexpr bool packet_load = packet_load_;
324 using OutScalar =
typename Eigen::internal::unpacket_traits<PacketType>::type;
325 static constexpr bool is_rhs = is_rhs_;
326 using OutType = std::conditional_t<packet_load, PacketType, OutScalar>;
327 static constexpr int elements_per_access = Eigen::internal::unpacket_traits<OutType>::size;
328 static constexpr bool is_coalesced_layout = !(is_transposed ^ is_rhs);
329 static constexpr int nc_stride = (is_coalesced_layout ? elements_per_access : 1);
330 static constexpr int c_stride = (is_coalesced_layout ? 1 : elements_per_access);
372template <
typename StorageIndex>
373struct ThreadProperties {
374 const StorageIndex linearLocalThreadId;
375 const StorageIndex kGroupId;
376 const StorageIndex mGroupOffset;
377 const StorageIndex nGroupOffset;
378 const StorageIndex kGroupOffset;
379 const StorageIndex mLocalOffset;
380 const StorageIndex nLocalOffset;
381 const StorageIndex mGlobalOffset;
382 const StorageIndex nGlobalOffset;
384 const bool is_internal;
386 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ThreadProperties(
387 const StorageIndex linearLocalThreadId_,
const StorageIndex kGroupId_,
const StorageIndex mGroupOffset_,
388 const StorageIndex nGroupOffset_,
const StorageIndex kGroupOffset_,
const StorageIndex mLocalOffset_,
389 const StorageIndex nLocalOffset_,
const StorageIndex mGlobalOffset_,
const StorageIndex nGlobalOffset_,
390 StorageIndex kSize_,
const bool is_internal_)
391 : linearLocalThreadId(linearLocalThreadId_),
393 mGroupOffset(mGroupOffset_),
394 nGroupOffset(nGroupOffset_),
395 kGroupOffset(kGroupOffset_),
396 mLocalOffset(mLocalOffset_),
397 nLocalOffset(nLocalOffset_),
398 mGlobalOffset(mGlobalOffset_),
399 nGlobalOffset(nGlobalOffset_),
401 is_internal(is_internal_) {}
454template <
typename OutScalar,
typename LhsScalar,
typename RhsScalar,
typename OutAccessor,
typename LhsMapper,
455 typename RhsMapper,
typename StorageIndex,
typename Properties,
typename TripleDim,
bool Vectorizable,
456 typename input_mapper_properties,
bool IsFinal, contraction_type contraction_tp>
457class TensorContractionKernel {
459 using PacketReturnType =
460 typename Eigen::TensorSycl::internal::Vectorise<OutScalar, Eigen::SyclDevice, Vectorizable>::PacketReturnType;
461 static constexpr int PacketSize =
462 Eigen::TensorSycl::internal::Vectorise<OutScalar, Eigen::SyclDevice, Vectorizable>::PacketSize;
463 static constexpr bool is_lhs_transposed =
464 !::Eigen::internal::TensorContractionInputMapperTrait<LhsMapper>::inner_dim_contiguous;
465 static constexpr bool is_rhs_transposed =
466 !::Eigen::internal::TensorContractionInputMapperTrait<RhsMapper>::inner_dim_contiguous;
469 input_mapper_properties::is_lhs_matrix && Vectorizable, PacketReturnType>;
472 input_mapper_properties::is_rhs_matrix && Vectorizable, PacketReturnType>;
474 static constexpr StorageIndex NStride =
475 contraction_tp == contraction_type::local ? Properties::WorkLoadPerThreadN : RHSBlockProperties::nc_stride;
477 using Scratch = cl::sycl::accessor<OutScalar, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::local>;
478 using local_ptr = cl::sycl::multi_ptr<OutScalar, cl::sycl::access::address_space::local_space>;
479 using private_ptr = OutScalar * ;
480 using tile_ptr = std::conditional_t<contraction_tp == contraction_type::local, local_ptr, private_ptr>;
481 static constexpr StorageIndex LSDL = contraction_tp == contraction_type::local
482 ? Properties::TileSizeDimM + Properties::BC
483 : Properties::WorkLoadPerThreadM;
484 static constexpr StorageIndex LSDR = contraction_tp == contraction_type::local
485 ? Properties::TileSizeDimN + Properties::BC
486 : Properties::WorkLoadPerThreadN;
487 static constexpr StorageIndex LocalOffset = Properties::LocalThreadSizeM * Properties::LocalThreadSizeN;
501 template <contraction_type, StorageIndex>
504 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE MemHolder(local_ptr block_start_ptr) : ptr(block_start_ptr) {}
509 template <StorageIndex MemSize>
510 struct MemHolder<contraction_type::no_local, MemSize> {
511 OutScalar ptr[MemSize] = {OutScalar{0}};
538 tile_ptr lhs_scratch_ptr_compute;
539 tile_ptr rhs_scratch_ptr_compute;
540 const std::pair<StorageIndex, StorageIndex> lhs_extract_index;
541 const std::pair<StorageIndex, StorageIndex> rhs_extract_index;
542 template <contraction_type tp = contraction_tp, std::enable_if_t<tp == contraction_type::no_local,
int> = 0>
544 : lhs_scratch_extract{},
545 rhs_scratch_extract{},
546 lhs_scratch_ptr_compute(lhs_scratch_extract.ptr),
547 rhs_scratch_ptr_compute(rhs_scratch_extract.ptr),
548 lhs_extract_index(std::pair<StorageIndex, StorageIndex>(StorageIndex{0}, StorageIndex{0})),
549 rhs_extract_index(std::pair<StorageIndex, StorageIndex>(StorageIndex{0}, StorageIndex{0})) {}
551 template <contraction_type tp = contraction_tp, std::enable_if_t<tp == contraction_type::local,
int> = 0>
553 local_ptr block_start_ptr)
554 : lhs_scratch_extract{block_start_ptr},
555 rhs_scratch_extract{lhs_scratch_extract.ptr +
556 ((Properties::DoubleBuffer + 1) * LSDL * Properties::TileSizeDimK)},
557 lhs_scratch_ptr_compute(lhs_scratch_extract.ptr + thread_properties.mLocalOffset),
558 rhs_scratch_ptr_compute(rhs_scratch_extract.ptr + thread_properties.nLocalOffset),
560 local_id_extract<LHSBlockProperties, Properties::TileSizeDimM>(thread_properties.linearLocalThreadId)),
562 local_id_extract<RHSBlockProperties, Properties::TileSizeDimN>(thread_properties.linearLocalThreadId)) {}
569 const StorageIndex groupSizeM;
570 const StorageIndex groupSizeN;
571 const StorageIndex numTiles;
572 const TripleDim triple_dim;
574 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorContractionKernel(Scratch scratch_,
const LhsMapper lhs_,
575 const RhsMapper rhs_, OutAccessor out_res_,
576 const StorageIndex groupSizeM_,
577 const StorageIndex groupSizeN_,
578 const StorageIndex numTiles_,
579 const TripleDim triple_dim_)
584 groupSizeM(groupSizeM_),
585 groupSizeN(groupSizeN_),
587 triple_dim(triple_dim_) {}
590 const RhsMapper rhs_, OutAccessor out_res_,
591 const StorageIndex groupSizeM_,
592 const StorageIndex numTiles_,
593 const TripleDim triple_dim_)
596 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void operator()(cl::sycl::nd_item<1> itemID)
const {
597 const StorageIndex linearLocalThreadId = itemID.get_local_id(0);
598 const StorageIndex nLocalThreadId = linearLocalThreadId / Properties::LocalThreadSizeM;
599 const StorageIndex mLocalThreadId = linearLocalThreadId % Properties::LocalThreadSizeM;
600 const StorageIndex mGroupId = itemID.get_group(0) % groupSizeM;
601 const StorageIndex tmp = itemID.get_group(0) / groupSizeM;
602 const StorageIndex nGroupId = IsFinal ? tmp : tmp % groupSizeN;
603 const StorageIndex kGroupId = IsFinal ? 0 : tmp / groupSizeN;
604 const StorageIndex mGroupOffset = mGroupId * Properties::TileSizeDimM;
605 const StorageIndex nGroupOffset = nGroupId * Properties::TileSizeDimN;
606 const StorageIndex mLocalOffset = PacketSize * mLocalThreadId;
607 const StorageIndex nLocalOffset = NStride * nLocalThreadId;
608 const StorageIndex mGlobalOffset = mGroupOffset + mLocalOffset;
609 const StorageIndex nGlobalOffset = nGroupOffset + nLocalOffset;
611 const StorageIndex kSizePerWG = IsFinal ? triple_dim.K : numTiles * Properties::TileSizeDimK;
612 StorageIndex kGroupOffset = kGroupId * kSizePerWG;
613 const bool is_internal = triple_dim.M - mGroupOffset >= Properties::TileSizeDimM &&
614 triple_dim.N - nGroupOffset >= Properties::TileSizeDimN &&
615 triple_dim.K - kGroupOffset >= kSizePerWG;
617 StorageIndex kSize = IsFinal ? triple_dim.K : std::min(kSizePerWG, triple_dim.K - kGroupOffset);
620 kGroupOffset += kSize;
622 auto thread_properties =
623 ThreadProperties<StorageIndex>(linearLocalThreadId, kGroupId, mGroupOffset, nGroupOffset, kGroupOffset,
624 mLocalOffset, nLocalOffset, mGlobalOffset, nGlobalOffset, kSize, is_internal);
626 auto out_ptr = out_res + (IsFinal ? 0 : thread_properties.kGroupId * triple_dim.M * triple_dim.N);
628 (thread_properties.is_internal) ? compute_panel<true>(itemID, thread_properties, out_ptr)
629 : compute_panel<false>(itemID, thread_properties, out_ptr);
634 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void compute_block_per_tile(OutScalar *lhs_block_ptr, OutScalar *rhs_block_ptr,
635 PacketReturnType *privateRes)
const {
636 StorageIndex idx = 0;
637 constexpr StorageIndex lhs_stride =
638 contraction_tp == contraction_type::local ? (PacketSize * Properties::LocalThreadSizeM) : 1;
640 for (StorageIndex wLPTN = 0; wLPTN < Properties::WorkLoadPerThreadN; wLPTN++) {
641 auto rhsPacket = PacketReturnType{*(rhs_block_ptr + wLPTN)};
642 StorageIndex lhs_index = 0;
644 for (StorageIndex wLPTM = 0; wLPTM < Properties::WorkLoadPerThreadM / PacketSize; wLPTM++) {
645 PacketReturnType lhsPack{};
646 Eigen::TensorSycl::internal::PacketWrapper<PacketReturnType, PacketSize>::set_packet(lhsPack,
647 lhs_block_ptr + lhs_index);
648 privateRes[idx] = ::Eigen::internal::pmadd(lhsPack, rhsPacket, privateRes[idx]);
650 lhs_index += lhs_stride;
658 template <
bool is_
internal_block, StorageIndex PrivateNStr
ide,
typename OutPtr>
659 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void store(OutPtr *out_ptr, PacketReturnType *privateRes,
660 StorageIndex mGlobalOffset, StorageIndex nGlobalOffset)
const {
661 auto chk_bound = [&](
const StorageIndex &mIndex,
const StorageIndex &nIndex) EIGEN_DEVICE_FUNC {
662 return (mIndex + PacketSize - 1 < triple_dim.M && nGlobalOffset + nIndex < triple_dim.N);
667 constexpr StorageIndex GlobalNStride = contraction_tp == contraction_type::local ? 1 : Properties::LocalThreadSizeN;
669 for (StorageIndex wLPTN = 0; wLPTN < Properties::WorkLoadPerThreadN / PrivateNStride; wLPTN++) {
671 StorageIndex outputLD = 0;
676 for (StorageIndex nId = 0; nId < PrivateNStride; nId++) {
677 StorageIndex globalRow = mGlobalOffset;
679 for (StorageIndex wLPTM = 0; wLPTM < Properties::WorkLoadPerThreadM / PacketSize; wLPTM++) {
680 PacketReturnType privetOut = privateRes[wLPTM];
681 if (check_boundary<is_internal_block>(chk_bound(globalRow, nId))) {
684 write<data_source::global_mem>(privetOut, out_ptr + outputLD + globalRow);
687 for (StorageIndex mId = 0; mId < PacketSize; mId++) {
688 StorageIndex mOffset = globalRow + mId;
689 if (mOffset < triple_dim.M && (nGlobalOffset + nId < triple_dim.N)) {
690 out_ptr[mOffset + outputLD] =
691 Eigen::TensorSycl::internal::PacketWrapper<PacketReturnType, PacketSize>::scalarize(mId, privetOut);
695 globalRow += (PacketSize * Properties::LocalThreadSizeM);
697 outputLD += triple_dim.M;
698 privateRes += Properties::WorkLoadPerThreadM / PacketSize;
700 out_ptr += (GlobalNStride * outputLD);
702 nGlobalOffset += (PrivateNStride * GlobalNStride);
706 template <
typename InputBlockProperties,
bool is_internal_block,
typename Input,
typename PrivateReg,
707 contraction_type contract_tp = contraction_tp>
708 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<contract_tp == contraction_type::no_local> extract_block(
709 const Input &inpt, PrivateReg private_ptr,
const std::pair<StorageIndex, StorageIndex> &,
710 const StorageIndex &ncOffset,
const StorageIndex cOffset)
const {
711 constexpr StorageIndex LocalThreadSizeNC =
712 InputBlockProperties::is_rhs ? Properties::LocalThreadSizeN : Properties::LocalThreadSizeM;
713 constexpr StorageIndex WorkLoadPerThreadNC =
714 InputBlockProperties::is_rhs ? Properties::WorkLoadPerThreadN : Properties::WorkLoadPerThreadM;
715 const StorageIndex &NC = InputBlockProperties::is_rhs ? triple_dim.N : triple_dim.M;
717 auto chk_bound = [&](
const StorageIndex &CIndex,
const StorageIndex &NCIndex) EIGEN_DEVICE_FUNC {
718 return ((CIndex + InputBlockProperties::c_stride - 1 < triple_dim.K) &&
719 (NCIndex + InputBlockProperties::nc_stride - 1 < NC));
721 const StorageIndex ld = InputBlockProperties::is_coalesced_layout ? NC : triple_dim.K;
722 StorageIndex cIndex = cOffset;
725 for (StorageIndex cId = 0; cId < Properties::TileSizeDimK / InputBlockProperties::c_stride; cId++) {
726 StorageIndex ncIndex = ncOffset;
728 for (StorageIndex ncId = 0; ncId < WorkLoadPerThreadNC / InputBlockProperties::nc_stride; ncId++) {
729 if (check_boundary<is_internal_block>(chk_bound(cIndex, ncIndex))) {
731 read<InputBlockProperties::packet_load, InputBlockProperties::is_coalesced_layout,
732 InputBlockProperties::is_rhs,
typename InputBlockProperties::OutType>(inpt, ncIndex, cIndex, ld);
734 write<StorageIndex, (InputBlockProperties::is_coalesced_layout ? 1 : WorkLoadPerThreadNC),
735 data_source::private_mem>(val, private_ptr);
738 for (StorageIndex i = 0; i < InputBlockProperties::elements_per_access; i++) {
739 const StorageIndex ncInd = ncIndex + (InputBlockProperties::is_coalesced_layout ? i : 0);
740 const StorageIndex cInd = cIndex + (InputBlockProperties::is_coalesced_layout ? 0 : i);
742 (ncInd < NC && cInd < triple_dim.K)
743 ? read<false, InputBlockProperties::is_coalesced_layout, InputBlockProperties::is_rhs, OutScalar>(
744 inpt, ncInd, cInd, ld)
746 write<StorageIndex, (InputBlockProperties::is_coalesced_layout ? 1 : WorkLoadPerThreadNC),
747 data_source::private_mem>(
748 val, private_ptr + (InputBlockProperties::is_coalesced_layout ? i : 0) +
749 ((InputBlockProperties::is_coalesced_layout ? 0 : i) * WorkLoadPerThreadNC));
755 ncIndex = (!InputBlockProperties::is_rhs && InputBlockProperties::nc_stride == 1 && PacketSize != 1)
756 ? ncOffset + (ncId + 1) % PacketSize + ((ncId + 1) / PacketSize) * LocalThreadSizeNC
757 : (ncIndex + InputBlockProperties::nc_stride * LocalThreadSizeNC);
758 private_ptr += InputBlockProperties::nc_stride;
761 private_ptr += (InputBlockProperties::c_stride - 1) * WorkLoadPerThreadNC;
762 cIndex += InputBlockProperties::c_stride;
765 template <
typename InputBlockProperties, StorageIndex TileSizeDimNC>
766 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::pair<StorageIndex, StorageIndex> local_id_extract(
767 const StorageIndex &linearLocalThreadId) {
768 const StorageIndex localThreadNC =
769 (InputBlockProperties::is_coalesced_layout)
770 ? linearLocalThreadId % (TileSizeDimNC / InputBlockProperties::nc_stride)
771 : linearLocalThreadId / (Properties::TileSizeDimK / InputBlockProperties::c_stride);
772 const StorageIndex localThreadC =
773 (InputBlockProperties::is_coalesced_layout)
774 ? linearLocalThreadId / (TileSizeDimNC / InputBlockProperties::nc_stride)
775 : linearLocalThreadId % (Properties::TileSizeDimK / InputBlockProperties::c_stride);
776 return std::pair<StorageIndex, StorageIndex>(localThreadNC, localThreadC);
779 template <
bool db = Properties::DoubleBuffer, contraction_type ctp = contraction_tp>
780 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<db && ctp == contraction_type::local> sync_mem(
781 const cl::sycl::nd_item<1> &,
bool &db_offset)
noexcept {
782 db_offset = !db_offset;
785 template <
bool db = Properties::DoubleBuffer, contraction_type ctp = contraction_tp>
786 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<!db && ctp == contraction_type::local> sync_mem(
787 const cl::sycl::nd_item<1> &itemID,
bool &)
noexcept {
788 itemID.barrier(cl::sycl::access::fence_space::local_space);
791 template <contraction_type ctp = contraction_tp>
792 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<ctp == contraction_type::no_local> sync_mem(
793 const cl::sycl::nd_item<1> &,
bool &)
noexcept {
797 template <
bool need_sync, contraction_type ctp = contraction_tp>
798 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<need_sync && ctp == contraction_type::no_local>
799 sync_thread(
const cl::sycl::nd_item<1> &
800#ifdef EIGEN_SYCL_ARM_GPU_CACHE_OPTIMISATION
804#ifdef EIGEN_SYCL_ARM_GPU_CACHE_OPTIMISATION
805 itemID.barrier(cl::sycl::access::fence_spacce::local_space);
810 template <
bool need_sync, contraction_type ctp = contraction_tp>
811 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<need_sync && ctp == contraction_type::local>
812 sync_thread(
const cl::sycl::nd_item<1> &itemID) {
813 itemID.barrier(cl::sycl::access::fence_space::local_space);
815 template <
bool need_sync>
816 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<!need_sync> sync_thread(
const cl::sycl::nd_item<1> &) {
820 template <
bool is_
internal_block>
821 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void compute_tile_per_panel(
const cl::sycl::nd_item<1> &itemID,
822 ThreadProperties<StorageIndex> &thread_properties,
824 PacketReturnType *privateRes,
825 bool &db_offset)
const {
827 extract_block<RHSBlockProperties, is_internal_block>(
828 rhs, tiled_input_block.rhs_scratch_extract.ptr + (db_offset * Properties::TileSizeDimK * LSDR),
829 tiled_input_block.rhs_extract_index,
830 contraction_tp == contraction_type::local ? thread_properties.nGroupOffset : thread_properties.nGlobalOffset,
831 thread_properties.kGroupOffset - thread_properties.kSize);
833 sync_thread<contraction_tp == contraction_type::no_local>(itemID);
836 extract_block<LHSBlockProperties, is_internal_block>(
837 lhs, tiled_input_block.lhs_scratch_extract.ptr + (db_offset * LSDL * Properties::TileSizeDimK),
838 tiled_input_block.lhs_extract_index,
839 contraction_tp == contraction_type::local ? thread_properties.mGroupOffset : thread_properties.mGlobalOffset,
840 thread_properties.kGroupOffset - thread_properties.kSize);
842 sync_thread<contraction_tp == contraction_type::local>(itemID);
844 StorageIndex lhs_offset = (db_offset * LSDL * Properties::TileSizeDimK);
845 StorageIndex rhs_offset = (db_offset * Properties::TileSizeDimK * LSDR);
847 for (StorageIndex k = 0; k < Properties::TileSizeDimK; k++) {
848 compute_block_per_tile(tiled_input_block.lhs_scratch_ptr_compute + lhs_offset,
849 tiled_input_block.rhs_scratch_ptr_compute + rhs_offset, privateRes);
854 thread_properties.kSize -= Properties::TileSizeDimK;
855 sync_mem(itemID, db_offset);
859 template <
bool is_
internal_block,
typename OutPtr>
860 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void compute_panel(
const cl::sycl::nd_item<1> &itemID,
861 ThreadProperties<StorageIndex> &thread_properties,
862 OutPtr out_ptr)
const {
863 auto tiled_input_block =
TiledMemory{thread_properties, scratch.get_pointer()};
865 PacketReturnType privateRes[Properties::WorkLoadPerThreadM * Properties::WorkLoadPerThreadN / PacketSize] = {
866 PacketReturnType{0}};
869 while (thread_properties.kSize >= Properties::TileSizeDimK) {
870 compute_tile_per_panel<is_internal_block>(itemID, thread_properties, tiled_input_block, privateRes, db_offset);
872 if (thread_properties.kSize > 0) {
873 compute_tile_per_panel<false>(itemID, thread_properties, tiled_input_block, privateRes, db_offset);
877 store<is_internal_block,
878 contraction_tp == contraction_type::local ?
static_cast<StorageIndex
>(1) : RHSBlockProperties::nc_stride>(
879 out_ptr + thread_properties.nGlobalOffset * triple_dim.M, privateRes, thread_properties.mGlobalOffset,
880 thread_properties.nGlobalOffset);
883 template <
typename InputBlockProperties,
bool is_internal_block,
typename Input,
typename Local,
884 contraction_type contract_tp = contraction_tp>
885 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<contract_tp == contraction_type::local> extract_block(
886 const Input &inpt, Local local_ptr,
const std::pair<StorageIndex, StorageIndex> &local_index,
887 const StorageIndex &ncOffset,
const StorageIndex cOffset)
const {
888 constexpr StorageIndex TileSizeDimNC =
889 InputBlockProperties::is_rhs ? Properties::TileSizeDimN : Properties::TileSizeDimM;
890 constexpr StorageIndex LoadPerThread =
891 InputBlockProperties::is_rhs ? Properties::LoadPerThreadRhs : Properties::LoadPerThreadLhs;
892 constexpr StorageIndex LSD = InputBlockProperties::is_rhs ? LSDR : LSDL;
893 static_assert(((LocalOffset % (TileSizeDimNC / InputBlockProperties::nc_stride) == 0) &&
894 (LocalOffset % (Properties::TileSizeDimK / InputBlockProperties::c_stride) == 0)),
895 " LocalOffset must be divisible by stride");
896 const StorageIndex &NC = InputBlockProperties::is_rhs ? triple_dim.N : triple_dim.M;
897 StorageIndex localThreadNC = local_index.first;
898 StorageIndex localThreadC = local_index.second;
899 auto chk_bound = [&](
const StorageIndex &CIndex,
const StorageIndex &NCIndex) EIGEN_DEVICE_FUNC {
900 return ((CIndex + InputBlockProperties::c_stride - 1 < triple_dim.K) &&
901 (NCIndex + InputBlockProperties::nc_stride - 1 < NC));
904 for (StorageIndex lPT = 0; lPT < LoadPerThread / InputBlockProperties::elements_per_access; lPT++) {
905 const StorageIndex CIndex = cOffset + (InputBlockProperties::c_stride * localThreadC);
906 const StorageIndex NCIndex = ncOffset + (InputBlockProperties::nc_stride * localThreadNC);
907 const StorageIndex ld = InputBlockProperties::is_coalesced_layout ? NC : triple_dim.K;
908 if (check_boundary<is_internal_block>(chk_bound(CIndex, NCIndex))) {
910 read<InputBlockProperties::packet_load, InputBlockProperties::is_coalesced_layout,
911 InputBlockProperties::is_rhs,
typename InputBlockProperties::OutType>(inpt, NCIndex, CIndex, ld);
912 write<StorageIndex, (InputBlockProperties::is_coalesced_layout ? 1 : LSD), data_source::local_mem>(
913 val, local_ptr + (InputBlockProperties::nc_stride * localThreadNC) +
914 (InputBlockProperties::c_stride * localThreadC * LSD));
917 for (StorageIndex i = 0; i < InputBlockProperties::elements_per_access; i++) {
918 const StorageIndex nCInd = NCIndex + (InputBlockProperties::is_coalesced_layout ? i : 0);
919 const StorageIndex cInd = CIndex + (InputBlockProperties::is_coalesced_layout ? 0 : i);
921 (nCInd < NC && cInd < triple_dim.K)
922 ? read<false, InputBlockProperties::is_coalesced_layout, InputBlockProperties::is_rhs, OutScalar>(
923 inpt, nCInd, cInd, ld)
926 write<StorageIndex, (InputBlockProperties::is_coalesced_layout ? 1 : LSD), data_source::local_mem>(
927 val, local_ptr + (InputBlockProperties::nc_stride * localThreadNC) +
928 (InputBlockProperties::is_coalesced_layout ? i : 0) +
929 ((InputBlockProperties::c_stride * localThreadC +
930 (InputBlockProperties::is_coalesced_layout ? 0 : i)) *
934 localThreadNC += (InputBlockProperties::is_coalesced_layout)
935 ? LocalOffset % (TileSizeDimNC / InputBlockProperties::nc_stride)
936 : LocalOffset / (Properties::TileSizeDimK / InputBlockProperties::c_stride);
937 localThreadC += (InputBlockProperties::is_coalesced_layout)
938 ? LocalOffset / (TileSizeDimNC / InputBlockProperties::nc_stride)
939 : LocalOffset % (Properties::TileSizeDimK / InputBlockProperties::c_stride);
944#ifndef EIGEN_SYCL_DISABLE_GEMV
987template <
typename OutScalar,
typename OutAccessor,
typename VectorMapper,
typename TensorMapper,
typename StorageIndex,
988 typename Properties, StorageIndex KFactor,
bool Vectorizable,
bool is_lhs_vec,
bool IsFinal>
989struct GeneralVectorTensor {
990 using PacketReturnType =
991 typename Eigen::TensorSycl::internal::Vectorise<OutScalar, Eigen::SyclDevice, Vectorizable>::PacketReturnType;
992 static constexpr int PacketSize =
993 Eigen::TensorSycl::internal::Vectorise<OutScalar, Eigen::SyclDevice, Vectorizable>::PacketSize;
994 using Scratch = cl::sycl::accessor<OutScalar, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::local>;
996 static constexpr StorageIndex OutScratchOffset =
997 KFactor * Properties::LocalThreadSizeC * Properties::LocalThreadSizeNC;
1004 const VectorMapper vec;
1005 const TensorMapper mat;
1006 OutAccessor out_res;
1007 const StorageIndex nonContractGroupSize;
1008 const StorageIndex nonContractDim;
1009 const StorageIndex contractDim;
1011 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE GeneralVectorTensor(Scratch scratch_,
const VectorMapper vec_,
1012 const TensorMapper mat_, OutAccessor out_res_,
1013 const StorageIndex nonContractGroupSize_,
1014 const StorageIndex nonContractDim_,
1015 const StorageIndex contractDim_)
1016 : scratch(scratch_),
1020 nonContractGroupSize(nonContractGroupSize_),
1021 nonContractDim(nonContractDim_),
1022 contractDim(contractDim_) {}
1024 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void operator()(cl::sycl::nd_item<1> itemID)
const {
1025 auto scratch_ptr = scratch.get_pointer();
1026 const StorageIndex linearLocalThreadId = itemID.get_local_id(0);
1027 StorageIndex nonContractId = is_lhs_vec ? linearLocalThreadId / Properties::LocalThreadSizeC
1028 : linearLocalThreadId % Properties::LocalThreadSizeNC;
1029 StorageIndex contractId = is_lhs_vec ? linearLocalThreadId % Properties::LocalThreadSizeC
1030 : linearLocalThreadId / Properties::LocalThreadSizeNC;
1031 const StorageIndex cGroupSize = itemID.get_group_range(0) / nonContractGroupSize;
1032 const StorageIndex nonContractGroupId =
1033 is_lhs_vec ? itemID.get_group(0) / cGroupSize : itemID.get_group(0) % nonContractGroupSize;
1034 const StorageIndex contractGroupId =
1035 is_lhs_vec ? itemID.get_group(0) % cGroupSize : itemID.get_group(0) / nonContractGroupSize;
1036 auto out_ptr = out_res + (IsFinal ? 0 : contractGroupId * nonContractDim);
1038 const StorageIndex nonContractGroupOffset = nonContractGroupId * Properties::TileSizeDimNC;
1039 const StorageIndex contractGroupOffset = contractGroupId * Properties::TileSizeDimC;
1040 auto outScratchIndex = nonContractId + contractId * Properties::LocalThreadSizeNC;
1041 const StorageIndex globalNonContractDimOffset = nonContractGroupOffset + nonContractId;
1042 const StorageIndex globalContractDimOffset = contractGroupOffset + contractId;
1043 auto local_output = scratch_ptr + OutScratchOffset;
1044 const bool is_internal = nonContractDim - nonContractGroupOffset >= Properties::TileSizeDimNC &&
1045 contractDim - contractGroupOffset >= Properties::TileSizeDimC;
1047 ? compute_panel<true>(itemID, vec, mat, local_output, out_ptr,
1048#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1049 scratch_ptr, contractGroupOffset,
1051 nonContractGroupOffset, linearLocalThreadId, contractDim, nonContractDim, contractId,
1052 nonContractId, globalContractDimOffset, globalNonContractDimOffset, outScratchIndex)
1053 : compute_panel<false>(itemID, vec, mat, local_output, out_ptr,
1054#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1055 scratch_ptr, contractGroupOffset,
1057 nonContractGroupOffset, linearLocalThreadId, contractDim, nonContractDim, contractId,
1058 nonContractId, globalContractDimOffset, globalNonContractDimOffset, outScratchIndex);
1060 template <
bool is_
internal_block,
typename OutPtr>
1061 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void compute_panel(
1062 const cl::sycl::nd_item<1> &itemID,
const VectorMapper &vec,
const TensorMapper &mat, OutScalar *local_output,
1064#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1065 OutScalar *scratch_ptr,
const StorageIndex contractGroupOffset,
1067 const StorageIndex nonContractGroupOffset,
const StorageIndex linearLocalThreadId, StorageIndex contractDim,
1068 StorageIndex nonContractDim, StorageIndex contractId, StorageIndex nonContractId,
1069 StorageIndex globalContractDimOffset, StorageIndex globalNonContractDimOffset, StorageIndex outScratchIndex) {
1070 OutScalar outScalar[Properties::WorkLoadPerThreadNC] = {OutScalar(0)};
1072#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1073 const StorageIndex vectorOffset = contractGroupOffset + linearLocalThreadId;
1074 extract_block<VecBlockProperties, is_internal_block, KFactor,
1075 Properties::LocalThreadSizeNC * Properties::LocalThreadSizeC>(vec, scratch_ptr, linearLocalThreadId,
1076 vectorOffset, contractDim);
1078 itemID.barrier(cl::sycl::access::fence_space::local_space);
1079 auto in_scratch_ptr = scratch_ptr + contractId;
1082 StorageIndex privateOffsetC = 0;
1084 for (StorageIndex i = 0; i < Properties::WorkLoadPerThreadC; i++) {
1085 StorageIndex privateOffsetNC = 0;
1086 bool contract_conds = ((globalContractDimOffset + privateOffsetC) < contractDim);
1087#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1088 auto vecScalar = *in_scratch_ptr;
1090 auto vecScalar = (check_boundary<is_internal_block>(contract_conds))
1091 ? vec(is_lhs_vec ? StorageIndex(0) : globalContractDimOffset + privateOffsetC,
1092 is_lhs_vec ? globalContractDimOffset + privateOffsetC : StorageIndex(0))
1096 for (StorageIndex j = 0; j < Properties::WorkLoadPerThreadNC; j++) {
1097 auto matScalar = (check_boundary<is_internal_block>(
1098 contract_conds && ((globalNonContractDimOffset + privateOffsetNC) < nonContractDim)))
1099 ? mat(is_lhs_vec ? globalContractDimOffset + privateOffsetC
1100 : globalNonContractDimOffset + privateOffsetNC,
1101 is_lhs_vec ? globalNonContractDimOffset + privateOffsetNC
1102 : globalContractDimOffset + privateOffsetC)
1105 outScalar[j] = ::Eigen::internal::pmadd(matScalar, vecScalar, outScalar[j]);
1106 privateOffsetNC += Properties::LocalThreadSizeNC;
1108 privateOffsetC += Properties::LocalThreadSizeC;
1109#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1110 in_scratch_ptr += Properties::LocalThreadSizeC;
1114 auto out_scratch_ptr = local_output + outScratchIndex;
1117 for (StorageIndex j = 0; j < Properties::WorkLoadPerThreadNC; j++) {
1118 *out_scratch_ptr = outScalar[j];
1120 out_scratch_ptr += (Properties::LocalThreadSizeNC * Properties::LocalThreadSizeC);
1122 EIGEN_IF_CONSTEXPR (is_lhs_vec) {
1123 nonContractId = linearLocalThreadId % Properties::LocalThreadSizeNC;
1124 contractId = linearLocalThreadId / Properties::LocalThreadSizeNC;
1125 outScratchIndex = nonContractId + contractId * Properties::LocalThreadSizeNC;
1128 out_scratch_ptr = local_output + outScratchIndex;
1130 for (StorageIndex j = 0; j < Properties::WorkLoadPerThreadNC; j++) {
1132 for (StorageIndex offset = Properties::LocalThreadSizeC >> 1; offset > 0; offset >>= 1) {
1133 itemID.barrier(cl::sycl::access::fence_space::local_space);
1134 if (contractId < offset) {
1135 StorageIndex myNeigbourId = (Properties::LocalThreadSizeNC * offset);
1136 *out_scratch_ptr += out_scratch_ptr[myNeigbourId];
1140 out_scratch_ptr += (Properties::LocalThreadSizeNC * Properties::LocalThreadSizeC);
1143 if (contractId == 0) {
1144 out_scratch_ptr = local_output + nonContractId;
1145 StorageIndex global_final_offset = nonContractGroupOffset + nonContractId;
1146 out_ptr += global_final_offset;
1148 for (StorageIndex j = 0; j < Properties::WorkLoadPerThreadNC; j++) {
1149 if (check_boundary<is_internal_block>(global_final_offset < nonContractDim)) {
1150 auto res = *out_scratch_ptr;
1153 out_ptr += Properties::LocalThreadSizeNC;
1156 out_scratch_ptr += (Properties::LocalThreadSizeNC * Properties::LocalThreadSizeC);
1157 if (!(is_internal_block)) global_final_offset += Properties::LocalThreadSizeNC;
1162 template <
typename InputBlockProperties,
bool is_internal_block,
int CFactor,
int GroupSize,
typename Input,
1164 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void extract_block(
const Input &inpt, Local *local_ptr,
1165 const StorageIndex &linearLocalThreadId,
1166 const StorageIndex &cOffset,
const StorageIndex &C) {
1167 local_ptr += InputBlockProperties::c_stride * linearLocalThreadId;
1168 StorageIndex cIndex = cOffset;
1169 for (StorageIndex cId = 0; cId < CFactor / InputBlockProperties::c_stride; cId++) {
1170 if (check_boundary<is_internal_block>(cIndex + InputBlockProperties::c_stride - 1 < C)) {
1171 auto val = read<InputBlockProperties::packet_load, InputBlockProperties::is_coalesced_layout,
1172 InputBlockProperties::is_rhs,
typename InputBlockProperties::OutType>(inpt, StorageIndex(0),
1173 cIndex, StorageIndex(1));
1174 write<StorageIndex, 1, data_source::local_mem>(val, local_ptr);
1177 for (StorageIndex i = 0; i < InputBlockProperties::elements_per_access; i++) {
1180 ? read<false, InputBlockProperties::is_coalesced_layout, InputBlockProperties::is_rhs, OutScalar>(
1181 inpt, StorageIndex(0), cIndex + i, StorageIndex(1))
1183 write<StorageIndex, 1, data_source::local_mem>(val, local_ptr + i);
1186 local_ptr += InputBlockProperties::c_stride * GroupSize;
1187 cIndex += InputBlockProperties::c_stride * GroupSize;
1193#ifndef EIGEN_SYCL_DISABLE_SCALAR
1226template <
typename OutScalar,
typename LhsScalar,
typename RhsScalar,
typename OutAccessor,
typename LhsMapper,
1227 typename RhsMapper,
typename StorageIndex,
bool Vectorizable>
1228struct GeneralScalarContraction {
1229 using Scratch = cl::sycl::accessor<OutScalar, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::local>;
1231 const LhsMapper lhs;
1232 const RhsMapper rhs;
1233 OutAccessor out_res;
1234 const StorageIndex rng;
1236 EIGEN_DEVICE_FUNC GeneralScalarContraction(Scratch scratch_,
const LhsMapper lhs_,
const RhsMapper rhs_,
1237 OutAccessor out_res_,
const StorageIndex rng_)
1238 : scratch(scratch_), lhs(lhs_), rhs(rhs_), out_res(out_res_), rng(rng_) {}
1240 EIGEN_DEVICE_FUNC
void operator()(cl::sycl::nd_item<1> itemID)
const {
1241 auto out_ptr = out_res;
1242 OutScalar *scratch_ptr = scratch.get_pointer();
1244 StorageIndex globalid = itemID.get_global_id(0);
1245 StorageIndex localid = itemID.get_local_id(0);
1246 OutScalar accumulator = OutScalar(0);
1247 for (StorageIndex i = globalid; i < rng; i += itemID.get_global_range(0)) {
1248 accumulator = Eigen::internal::pmadd(lhs(0, i), rhs(i, 0), accumulator);
1250 auto out_scratch_ptr = scratch_ptr + localid;
1251 *out_scratch_ptr = accumulator;
1252 for (StorageIndex offset = itemID.get_local_range(0) >> 1; offset > 0; offset >>= 1) {
1253 itemID.barrier(cl::sycl::access::fence_space::local_space);
1254 if (localid < offset) {
1255 *out_scratch_ptr = (accumulator += out_scratch_ptr[offset]);
1259 out_ptr[itemID.get_group(0)] = accumulator;
1268template <
typename Indices,
typename LeftArgType,
typename RightArgType,
typename OutputKernelType>
1271 :
public TensorContractionEvaluatorBase<TensorEvaluator<
1272 const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Eigen::SyclDevice>> {
1273 static_assert(std::is_same<OutputKernelType, const NoOpOutputKernel>::value,
1274 "SYCL tensor contraction does not support output kernels.");
1276 using Device = Eigen::SyclDevice;
1279 using Base = TensorContractionEvaluatorBase<Self>;
1281 using Scalar = std::remove_const_t<typename XprType::Scalar>;
1282 using StorageIndex =
typename XprType::Index;
1283 using CoeffReturnType =
typename XprType::CoeffReturnType;
1284 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
1285 using Storage =
typename Base::Storage;
1286 using EvaluatorPointerType =
typename Base::EvaluatorPointerType;
1288 const StorageIndex M;
1289 const StorageIndex N;
1290 const StorageIndex K;
1291 TripleDim(
const StorageIndex M_,
const StorageIndex N_,
const StorageIndex K_) : M(M_), N(N_), K(K_) {}
1294 PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1),
1295 BlockAccess =
false,
1298 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
1299 static constexpr int LDims = Base::LDims;
1300 static constexpr int RDims = Base::RDims;
1301 static constexpr int ContractDims = Base::ContractDims;
1303 using left_dim_mapper_t = array<StorageIndex, LDims>;
1304 using right_dim_mapper_t = array<StorageIndex, RDims>;
1306 using contract_t = array<StorageIndex, ContractDims>;
1307 using left_nocontract_t = array<StorageIndex, LDims - ContractDims>;
1308 using right_nocontract_t = array<StorageIndex, RDims - ContractDims>;
1310 static constexpr int NumDims = LDims + RDims - 2 * ContractDims;
1312 using Dimensions = DSizes<StorageIndex, NumDims>;
1314 using LeftEvaluator = TensorEvaluator<typename Base::EvalLeftArgType, Device>;
1315 using RightEvaluator = TensorEvaluator<typename Base::EvalRightArgType, Device>;
1316 using LhsScalar = std::remove_const_t<typename LeftEvaluator::CoeffReturnType>;
1317 using RhsScalar = std::remove_const_t<typename RightEvaluator::CoeffReturnType>;
1319 using LeftDimensions =
typename LeftEvaluator::Dimensions;
1320 using RightDimensions =
typename RightEvaluator::Dimensions;
1322 template <
bool lhs_inner_dim_contiguous,
bool rhs_inner_dim_contiguous,
bool rhs_inner_dim_reordered>
1323 struct input_mapper_propertis {
1324 static constexpr bool is_lhs_matrix = (LDims == 2 && ContractDims == 1) || lhs_inner_dim_contiguous;
1325 static constexpr bool is_rhs_matrix =
1326 (RDims == 2 && ContractDims == 1) || (rhs_inner_dim_contiguous && !rhs_inner_dim_reordered);
1329 TensorEvaluator(
const XprType &op,
const Device &device) : Base(op, device) {}
1332 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(
typename Base::EvaluatorPointerType data) {
1333 this->m_leftImpl.evalSubExprsIfNeeded(
nullptr);
1334 this->m_rightImpl.evalSubExprsIfNeeded(
nullptr);
1336 this->m_result = this->m_device.get(
1337 static_cast<Scalar *
>(this->m_device.allocate_temp(this->dimensions().TotalSize() *
sizeof(Scalar))));
1338 data = this->m_result;
1341 return (this->m_result !=
nullptr);
1343 const Eigen::SyclDevice &device()
const {
return this->m_device; }
1344 void evalToSycl(
typename Base::EvaluatorPointerType buffer)
const {
1345 if (this->m_lhs_inner_dim_contiguous) {
1346 if (this->m_rhs_inner_dim_contiguous) {
1347 if (this->m_rhs_inner_dim_reordered) {
1348 evalTyped<true, true, true, Unaligned>(buffer);
1350 evalTyped<true, true, false, Unaligned>(buffer);
1353 if (this->m_rhs_inner_dim_reordered) {
1354 evalTyped<true, false, true, Unaligned>(buffer);
1356 evalTyped<true, false, false, Unaligned>(buffer);
1360 if (this->m_rhs_inner_dim_contiguous) {
1361 if (this->m_rhs_inner_dim_reordered) {
1362 evalTyped<false, true, true, Unaligned>(buffer);
1364 evalTyped<false, true, false, Unaligned>(buffer);
1367 if (this->m_rhs_inner_dim_reordered) {
1368 evalTyped<false, false, true, Unaligned>(buffer);
1370 evalTyped<false, false, false, Unaligned>(buffer);
1376 template <
bool lhs_inner_dim_contiguous,
bool rhs_inner_dim_contiguous,
bool rhs_inner_dim_reordered,
int Alignment>
1377 void evalTyped(
typename Base::EvaluatorPointerType buffer)
const {
1378 const auto triple_dim = TripleDim{this->m_i_size, this->m_j_size, this->m_k_size};
1380 internal::TensorContractionInputMapper<LhsScalar, StorageIndex, internal::Lhs, LeftEvaluator, left_nocontract_t,
1381 contract_t, PacketType<CoeffReturnType, Device>::size,
1382 lhs_inner_dim_contiguous,
false,
Unaligned, MakePointer>;
1385 internal::TensorContractionInputMapper<RhsScalar, StorageIndex, internal::Rhs, RightEvaluator,
1386 right_nocontract_t, contract_t,
1387 PacketType<CoeffReturnType, Device>::size, rhs_inner_dim_contiguous,
1388 rhs_inner_dim_reordered,
Unaligned, MakePointer>;
1391 LhsMapper lhs(this->m_leftImpl, this->m_left_nocontract_strides, this->m_i_strides,
1392 this->m_left_contracting_strides, this->m_k_strides);
1394 RhsMapper rhs(this->m_rightImpl, this->m_right_nocontract_strides, this->m_j_strides,
1395 this->m_right_contracting_strides, this->m_k_strides);
1397#ifndef EIGEN_SYCL_DISABLE_SCALAR
1398 if (triple_dim.M == 1 && triple_dim.N == 1) {
1399 launchSC(buffer, lhs, rhs, triple_dim.K);
1402#ifndef EIGEN_SYCL_DISABLE_GEMV
1403 if (triple_dim.M != 1 && triple_dim.N == 1) {
1404 LaunchVT<false>(buffer, rhs, lhs, triple_dim.M, triple_dim.K);
1405 }
else if (triple_dim.M == 1 && triple_dim.N != 1) {
1406 LaunchVT<true>(buffer, lhs, rhs, triple_dim.N, triple_dim.K);
1410 using inpt_mapper_properties =
1411 input_mapper_propertis<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered>;
1412#ifndef EIGEN_SYCL_DISABLE_SKINNY
1413 bool skinny =
false;
1414 auto platform_name = this->device().getPlatformName();
1416 if (platform_name.find(
"AMD") == 0) {
1417 skinny = (triple_dim.M < triple_dim.K || triple_dim.N < triple_dim.K) &&
1418 ((triple_dim.M < 1024 && triple_dim.N < 1024) ||
1419 (uint64_t(triple_dim.M * triple_dim.N) < uint64_t(triple_dim.K)));
1421 skinny = (((std::max(triple_dim.K, triple_dim.N) / std::min(triple_dim.K, triple_dim.N)) > 100) ||
1422 ((std::max(triple_dim.K, triple_dim.M) / std::min(triple_dim.K, triple_dim.M)) > 100) ||
1423 ((std::max(triple_dim.N, triple_dim.M) / std::min(triple_dim.N, triple_dim.M)) > 100));
1426 adjustTT<true, inpt_mapper_properties>(buffer, lhs, rhs, triple_dim);
1429 adjustTT<false, inpt_mapper_properties>(buffer, lhs, rhs, triple_dim);
1433 template <
bool skinny,
typename input_mapper_properties,
typename LhsMapper,
typename RhsMapper>
1434 void EIGEN_ALWAYS_INLINE adjustTT(EvaluatorPointerType buffer,
const LhsMapper &lhs,
const RhsMapper &rhs,
1435 const TripleDim &triple_dim)
const {
1436#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_ON
1437 if (device().has_local_memory()) {
1438 using PanelParameters = TensorSycl::internal::TTPanelSize<CoeffReturnType, StorageIndex, 4, 4, 16>;
1439 launchTT<TensorSycl::internal::contraction_type::local, skinny, input_mapper_properties, PanelParameters>(
1440 buffer, lhs, rhs, triple_dim);
1443#ifdef EIGEN_SYCL_LOCAL_MEM_UNSET_OR_OFF
1444 if (!(device().has_local_memory())) {
1445 using PanelParameters = TensorSycl::internal::TTPanelSize<CoeffReturnType, StorageIndex, 4, 4, 4>;
1446 launchTT<TensorSycl::internal::contraction_type::no_local, skinny, input_mapper_properties, PanelParameters>(
1447 buffer, lhs, rhs, triple_dim);
1452 template <TensorSycl::internal::contraction_type ct,
bool skinny,
typename input_mapper_properties,
1453 typename Properties,
typename LhsMapper,
typename RhsMapper>
1454 void launchTT(EvaluatorPointerType buffer,
const LhsMapper &lhs,
const RhsMapper &rhs,
1455 const TripleDim &triple_dim)
const {
1456 const StorageIndex roundUpM = Eigen::TensorSycl::internal::roundUp(triple_dim.M, Properties::TileSizeDimM);
1457 const StorageIndex roundUpN = Eigen::TensorSycl::internal::roundUp(triple_dim.N, Properties::TileSizeDimN);
1458 const StorageIndex groupSizeM = roundUpM / Properties::TileSizeDimM;
1459 const StorageIndex groupSizeN = roundUpN / Properties::TileSizeDimN;
1461 const StorageIndex roundUpK = Eigen::TensorSycl::internal::roundUp(triple_dim.K, Properties::TileSizeDimK);
1462 StorageIndex totalTilesK = roundUpK / Properties::TileSizeDimK;
1463 StorageIndex groupSizeK =
1465 ? std::max(std::min(totalTilesK,
1466 (StorageIndex)(device().getPowerOfTwo(device().getNumSyclMultiProcessors(),
true) * 4) /
1467 (groupSizeM * groupSizeN)),
1471 const StorageIndex numTilesPerGroup = Eigen::TensorSycl::internal::roundUp(totalTilesK, groupSizeK) / groupSizeK;
1473 const StorageIndex totalGroupSize = groupSizeM * groupSizeN * groupSizeK;
1475 const StorageIndex localRange = Properties::LocalThreadSizeM * Properties::LocalThreadSizeN;
1476 const StorageIndex globalRange = totalGroupSize * localRange;
1478 const StorageIndex scratchSize = (ct == TensorSycl::internal::contraction_type::local)
1479 ? ((Properties::DoubleBuffer + 1) *
1480 (Properties::TileSizeDimM + Properties::BC) * (Properties::TileSizeDimK)) +
1481 ((Properties::DoubleBuffer + 1) * (Properties::TileSizeDimK) *
1482 (Properties::TileSizeDimN + Properties::BC))
1485 auto thread_range = cl::sycl::nd_range<1>(cl::sycl::range<1>(globalRange), cl::sycl::range<1>(localRange));
1486 if (groupSizeK == 1) {
1487 using ContractKernelName =
1488 TensorSycl::internal::TensorContractionKernel<CoeffReturnType, LhsScalar, RhsScalar, EvaluatorPointerType,
1489 LhsMapper, RhsMapper, StorageIndex, Properties, TripleDim,
1490 PacketAccess, input_mapper_properties,
true, ct>;
1492 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(
1493 lhs, rhs, buffer, thread_range, scratchSize, groupSizeM, groupSizeN, numTilesPerGroup, triple_dim)
1496 using ContractKernelName =
1497 TensorSycl::internal::TensorContractionKernel<CoeffReturnType, LhsScalar, RhsScalar, EvaluatorPointerType,
1498 LhsMapper, RhsMapper, StorageIndex, Properties, TripleDim,
1499 PacketAccess, input_mapper_properties,
false, ct>;
1500 CoeffReturnType *temp_pointer =
static_cast<CoeffReturnType *
>(
1501 device().allocate_temp(triple_dim.M * triple_dim.N * groupSizeK *
sizeof(CoeffReturnType)));
1502 EvaluatorPointerType tmp_global_accessor = device().get(temp_pointer);
1505 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(
1506 lhs, rhs, tmp_global_accessor, thread_range, scratchSize, groupSizeM, groupSizeN, numTilesPerGroup,
1510 using Op = Eigen::internal::SumReducer<CoeffReturnType>;
1512 using ReductionKernel =
1513 TensorSycl::internal::SecondStepPartialReduction<CoeffReturnType, StorageIndex, EvaluatorPointerType,
1514 EvaluatorPointerType, Op>;
1517 .template unary_kernel_launcher<CoeffReturnType, ReductionKernel>(
1518 tmp_global_accessor, buffer,
1519 cl::sycl::nd_range<1>(cl::sycl::range<1>(StorageIndex(
1520 Eigen::TensorSycl::internal::roundUp(triple_dim.M * triple_dim.N, localRange))),
1521 cl::sycl::range<1>(localRange)),
1522 StorageIndex(1), op, StorageIndex(triple_dim.M * triple_dim.N), groupSizeK)
1524 device().deallocate_temp(temp_pointer);
1528#ifndef EIGEN_SYCL_DISABLE_GEMV
1529 template <
bool is_lhs_vec,
typename VectorMapper,
typename TensorMapper,
typename StorageIndex>
1530 void EIGEN_ALWAYS_INLINE LaunchVT(EvaluatorPointerType buffer,
const VectorMapper &vec,
const TensorMapper &mat,
1531 StorageIndex NC, StorageIndex C)
const {
1532 const StorageIndex nonContractDim = NC;
1533 constexpr StorageIndex NCFactor = 1;
1534 constexpr StorageIndex CFactor = 1;
1535 constexpr StorageIndex NCWindow = 16;
1537 Eigen::TensorSycl::internal::TVPanelSize<CoeffReturnType, StorageIndex, NCWindow, CFactor, NCFactor>;
1538 const StorageIndex roundUpC = Eigen::TensorSycl::internal::roundUp(C, Properties::TileSizeDimC);
1539 const StorageIndex cNumGroups = roundUpC / (Properties::LocalThreadSizeC * Properties::WorkLoadPerThreadC);
1540 const StorageIndex roundUpNC = Eigen::TensorSycl::internal::roundUp(nonContractDim, Properties::TileSizeDimNC);
1541 const StorageIndex nCNumGroups = roundUpNC / (Properties::LocalThreadSizeNC * Properties::WorkLoadPerThreadNC);
1542 const StorageIndex globalRange =
1543 (roundUpNC / (Properties::WorkLoadPerThreadNC)) * (roundUpC / (Properties::WorkLoadPerThreadC));
1544 const StorageIndex localRange = Properties::LocalThreadSizeNC * Properties::LocalThreadSizeC;
1545 const StorageIndex scratchSize =
1546 (Properties::WorkLoadPerThreadNC + CFactor) * Properties::LocalThreadSizeC * Properties::LocalThreadSizeNC;
1547 auto thread_range = cl::sycl::nd_range<1>(cl::sycl::range<1>(globalRange), cl::sycl::range<1>(localRange));
1548 if (cNumGroups > 1) {
1549 using ContractKernelName =
1550 Eigen::TensorSycl::internal::GeneralVectorTensor<CoeffReturnType, EvaluatorPointerType, VectorMapper,
1551 TensorMapper, StorageIndex, Properties, CFactor,
false,
1553 CoeffReturnType *temp_pointer =
1554 static_cast<CoeffReturnType *
>(device().allocate_temp(nonContractDim * cNumGroups *
sizeof(CoeffReturnType)));
1555 EvaluatorPointerType tmp_global_accessor = device().get(temp_pointer);
1558 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(
1559 vec, mat, tmp_global_accessor, thread_range, scratchSize, nCNumGroups, nonContractDim, C)
1562 using Op = Eigen::internal::SumReducer<CoeffReturnType>;
1563 using ReductionKernel =
1564 TensorSycl::internal::SecondStepPartialReduction<CoeffReturnType, StorageIndex, EvaluatorPointerType,
1565 EvaluatorPointerType, Op>;
1568 .template unary_kernel_launcher<CoeffReturnType, ReductionKernel>(
1569 tmp_global_accessor, buffer,
1570 cl::sycl::nd_range<1>(
1571 cl::sycl::range<1>(Eigen::TensorSycl::internal::roundUp(nonContractDim, localRange)),
1572 cl::sycl::range<1>(localRange)),
1573 StorageIndex(1), Op(), nonContractDim, cNumGroups)
1575 device().deallocate_temp(temp_pointer);
1577 using ContractKernelName =
1578 Eigen::TensorSycl::internal::GeneralVectorTensor<CoeffReturnType, EvaluatorPointerType, VectorMapper,
1579 TensorMapper, StorageIndex, Properties, CFactor,
false,
1582 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(
1583 vec, mat, buffer, thread_range, scratchSize, nCNumGroups, nonContractDim, C)
1589#ifndef EIGEN_SYCL_DISABLE_SCALAR
1590 template <
typename LhsMapper,
typename RhsMapper>
1591 EIGEN_ALWAYS_INLINE
void launchSC(EvaluatorPointerType buffer,
const LhsMapper &lhs,
const RhsMapper &rhs,
1592 StorageIndex K)
const {
1593 EIGEN_STATIC_ASSERT(!((EIGEN_SYCL_LOCAL_THREAD_DIM0 * EIGEN_SYCL_LOCAL_THREAD_DIM1) &
1594 (EIGEN_SYCL_LOCAL_THREAD_DIM0 * EIGEN_SYCL_LOCAL_THREAD_DIM1 - 1)),
1595 "The Local thread size must be a power of 2 for the reduction "
1597 constexpr StorageIndex local_range = EIGEN_SYCL_LOCAL_THREAD_DIM0 * EIGEN_SYCL_LOCAL_THREAD_DIM1;
1601 const StorageIndex num_work_group = ((K + (512 * local_range - 1)) / (512 * local_range) > 1 ? local_range : 1);
1602 const StorageIndex global_range = num_work_group * local_range;
1604 using ContractKernelName = Eigen::TensorSycl::internal::GeneralScalarContraction<
1605 CoeffReturnType, LhsScalar, RhsScalar, EvaluatorPointerType, LhsMapper, RhsMapper, StorageIndex,
false>;
1606 auto thread_range = cl::sycl::nd_range<1>(cl::sycl::range<1>(global_range), cl::sycl::range<1>(local_range));
1607 if (num_work_group > 1) {
1608 CoeffReturnType *temp_pointer =
1609 static_cast<CoeffReturnType *
>(device().allocate_temp(num_work_group *
sizeof(CoeffReturnType)));
1610 EvaluatorPointerType tmp_global_accessor = device().get(temp_pointer);
1612 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(lhs, rhs, tmp_global_accessor,
1613 thread_range, local_range, K)
1615 using Op = Eigen::internal::SumReducer<CoeffReturnType>;
1616 using GenericRKernel =
1617 TensorSycl::internal::SecondStepFullReducer<CoeffReturnType, Op, EvaluatorPointerType, EvaluatorPointerType,
1618 StorageIndex, local_range>;
1620 .template unary_kernel_launcher<CoeffReturnType, GenericRKernel>(
1621 tmp_global_accessor, buffer,
1622 cl::sycl::nd_range<1>(cl::sycl::range<1>(local_range), cl::sycl::range<1>(local_range)), local_range,
1625 device().deallocate_temp(temp_pointer);
1628 .template binary_kernel_launcher<CoeffReturnType, ContractKernelName>(lhs, rhs, buffer, thread_range,
1635 EIGEN_STRONG_INLINE
void cleanup() {
1636 this->m_leftImpl.cleanup();
1637 this->m_rightImpl.cleanup();
1639 if (this->m_result) {
1640 this->m_device.deallocate_temp(this->m_result);
1641 this->m_result =
nullptr;
Definition TensorContraction.h:418
TensorContractionKernel is a template class that provides Tensor -Tensor contraction operation.
Definition TensorContractionSycl.h:457
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47
BlockProperties is a template class that provides different characteristic of a block of each Tensor ...
Definition TensorContractionSycl.h:322
TTPanelSize, a template class used for setting the panel size required for launching General Tensor T...
Definition TensorContractionSycl.h:84
TVPanelSize, a template class used for setting the panel size required for launching General TensorVe...
Definition TensorContractionSycl.h:48
MemHolder this is a place holder struct for creating memory hierarchy in SYCL. Inside SYCL kernel it ...
Definition TensorContractionSycl.h:502
TiledMemory: contains required memory pointer for loading each tile of the TensorContraction panel fr...
Definition TensorContractionSycl.h:535
ThreadProperties is a template class that provides each thread's properties within a workgroup....
Definition TensorContractionSycl.h:373