TensorReverse.h

// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
// Copyright (C) 2014 Navdeep Jaitly <ndjaitly@google.com>
//                    Benoit Steiner <benoit.steiner.goog@gmail.com>
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
// SPDX-License-Identifier: MPL-2.0
 
#ifndef EIGEN_TENSOR_TENSOR_REVERSE_H
#define EIGEN_TENSOR_TENSOR_REVERSE_H
// IWYU pragma: private
#include "./InternalHeaderCheck.h"
 
namespace Eigen {
 
namespace internal {
template <typename ReverseDimensions, typename XprType>
struct traits<TensorReverseOp<ReverseDimensions, XprType> > : public traits<XprType> {
  typedef typename XprType::Scalar Scalar;
  typedef traits<XprType> XprTraits;
  typedef typename XprTraits::StorageKind StorageKind;
  typedef typename XprTraits::Index Index;
  static constexpr int NumDimensions = XprTraits::NumDimensions;
  static constexpr int Layout = XprTraits::Layout;
  typedef typename XprTraits::PointerType PointerType;
};
 
template <typename ReverseDimensions, typename XprType>
struct eval<TensorReverseOp<ReverseDimensions, XprType>, Eigen::Dense> {
  typedef const TensorReverseOp<ReverseDimensions, XprType>& type;
};
 
}  // end namespace internal

template <typename ReverseDimensions, typename XprType>
class TensorReverseOp : public TensorBase<TensorReverseOp<ReverseDimensions, XprType>, WriteAccessors> {
 public:
  typedef TensorBase<TensorReverseOp<ReverseDimensions, XprType>, WriteAccessors> Base;
  typedef typename Eigen::internal::traits<TensorReverseOp>::Scalar Scalar;
  typedef typename Eigen::NumTraits<Scalar>::Real RealScalar;
  typedef typename XprType::CoeffReturnType CoeffReturnType;
  typedef typename Eigen::internal::ref_selector<TensorReverseOp>::type Nested;
  typedef typename Eigen::internal::traits<TensorReverseOp>::StorageKind StorageKind;
  typedef typename Eigen::internal::traits<TensorReverseOp>::Index Index;
 
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorReverseOp(const XprType& expr, const ReverseDimensions& reverse_dims)
      : m_xpr(expr), m_reverse_dims(reverse_dims) {}
 
  EIGEN_DEVICE_FUNC const ReverseDimensions& reverse() const { return m_reverse_dims; }
 
  EIGEN_DEVICE_FUNC const internal::remove_all_t<typename XprType::Nested>& expression() const { return m_xpr; }
 
  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(TensorReverseOp)
 
 protected:
  typename XprType::Nested m_xpr;
  const ReverseDimensions m_reverse_dims;
};
 
// Eval as rvalue
template <typename ReverseDimensions, typename ArgType, typename Device>
struct TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device> {
  typedef TensorReverseOp<ReverseDimensions, ArgType> XprType;
  typedef typename XprType::Index Index;
  static constexpr int NumDims = internal::array_size<ReverseDimensions>::value;
  typedef DSizes<Index, NumDims> Dimensions;
  typedef typename XprType::Scalar Scalar;
  typedef typename XprType::CoeffReturnType CoeffReturnType;
  typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType;
  static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
  typedef StorageMemory<CoeffReturnType, Device> Storage;
  typedef typename Storage::Type EvaluatorPointerType;
 
  static constexpr int Layout = TensorEvaluator<ArgType, Device>::Layout;
  enum {
    IsAligned = false,
    PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess,
    BlockAccess = NumDims > 0,
    PreferBlockAccess = true,
    CoordAccess = false,  // to be implemented
    RawAccess = false
  };
 
  typedef internal::TensorIntDivisor<Index> IndexDivisor;
 
  //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
  typedef internal::TensorBlockDescriptor<NumDims, Index> TensorBlockDesc;
  typedef internal::TensorBlockScratchAllocator<Device> TensorBlockScratch;
 
  typedef typename TensorEvaluator<const ArgType, Device>::TensorBlock ArgTensorBlock;
 
  typedef typename internal::TensorMaterializedBlock<CoeffReturnType, NumDims, Layout, Index> TensorBlock;
  //===--------------------------------------------------------------------===//
 
  EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device)
      : m_impl(op.expression(), device), m_reverse(op.reverse()), m_device(device) {
    // Reversing a scalar isn't supported yet. It would be a no-op anyway.
    EIGEN_STATIC_ASSERT((NumDims > 0), YOU_MADE_A_PROGRAMMING_MISTAKE);
 
    // Compute strides
    m_dimensions = m_impl.dimensions();
    EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
      m_strides[0] = 1;
      for (int i = 1; i < NumDims; ++i) {
        m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1];
        if (m_strides[i] > 0) m_fastStrides[i] = IndexDivisor(m_strides[i]);
      }
    } else {
      m_strides[NumDims - 1] = 1;
      for (int i = NumDims - 2; i >= 0; --i) {
        m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1];
        if (m_strides[i] > 0) m_fastStrides[i] = IndexDivisor(m_strides[i]);
      }
    }
  }
 
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; }
 
  EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType) {
    m_impl.evalSubExprsIfNeeded(nullptr);
    return true;
  }
 
#ifdef EIGEN_USE_THREADS
  template <typename EvalSubExprsCallback>
  EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(EvaluatorPointerType, EvalSubExprsCallback done) {
    m_impl.evalSubExprsIfNeededAsync(nullptr, [done](bool) { done(true); });
  }
#endif  // EIGEN_USE_THREADS
 
  EIGEN_STRONG_INLINE void cleanup() { m_impl.cleanup(); }
 
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index reverseIndex(Index index) const {
    eigen_assert(index < dimensions().TotalSize());
    Index inputIndex = 0;
    EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
      EIGEN_UNROLL_LOOP
      for (int i = NumDims - 1; i > 0; --i) {
        Index idx = index / m_fastStrides[i];
        index -= idx * m_strides[i];
        if (m_reverse[i]) {
          idx = m_dimensions[i] - idx - 1;
        }
        inputIndex += idx * m_strides[i];
      }
      if (m_reverse[0]) {
        inputIndex += m_dimensions[0] - index - 1;
      } else {
        inputIndex += index;
      }
    } else {
      EIGEN_UNROLL_LOOP
      for (int i = 0; i < NumDims - 1; ++i) {
        Index idx = index / m_fastStrides[i];
        index -= idx * m_strides[i];
        if (m_reverse[i]) {
          idx = m_dimensions[i] - idx - 1;
        }
        inputIndex += idx * m_strides[i];
      }
      if (m_reverse[NumDims - 1]) {
        inputIndex += m_dimensions[NumDims - 1] - index - 1;
      } else {
        inputIndex += index;
      }
    }
    return inputIndex;
  }
 
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
    return m_impl.coeff(reverseIndex(index));
  }
 
  template <int LoadMode>
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
    eigen_assert(index + PacketSize - 1 < dimensions().TotalSize());
 
    // Fast path: when the whole packet stays inside a single inner-most
    // slice of the input, replace PacketSize coeff() calls with one packet
    // load (plus a preverse when the inner dim is reversed).
    constexpr int inner_dim = (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? 0 : NumDims - 1;
    const Index inner_size = m_dimensions[inner_dim];
    const Index inner_pos = index % inner_size;
    if (inner_pos + PacketSize <= inner_size) {
      if (m_reverse[inner_dim]) {
        const Index input_index = reverseIndex(index + PacketSize - 1);
        return internal::preverse(m_impl.template packet<Unaligned>(input_index));
      }
      return m_impl.template packet<Unaligned>(reverseIndex(index));
    }
 
    // Slow path: the packet crosses an inner-slice boundary, so the
    // contiguous-load trick does not apply.
    EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
    std::remove_const_t<CoeffReturnType> values[PacketSize];
    EIGEN_UNROLL_LOOP
    for (int i = 0; i < PacketSize; ++i) {
      values[i] = coeff(index + i);
    }
    return internal::pload<PacketReturnType>(values);
  }
 
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
    const size_t target_size = m_device.lastLevelCacheSize();
    // Block evaluation reads underlying memory in reverse order, and default
    // cost model does not properly catch this in bytes stored/loaded.
    return internal::TensorBlockResourceRequirements::skewed<Scalar>(target_size).addCostPerCoeff({0, 0, 24});
  }
 
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
                                                          bool /*root_of_expr_ast*/ = false) const {
    // TODO(ezhulenev): If underlying tensor expression supports and prefers
    // block evaluation we must use it. Currently we use coeff and packet
    // access into the underlying tensor expression.
    static const bool isColMajor = static_cast<int>(Layout) == static_cast<int>(ColMajor);
 
    static constexpr Index inner_dim_idx = isColMajor ? 0 : NumDims - 1;
    const bool inner_dim_reversed = m_reverse[inner_dim_idx];
 
    // Offset in the output block.
    Index block_offset = 0;
 
    // Offset in the input Tensor.
    Index input_offset = reverseIndex(desc.offset());
 
    // Initialize output block iterator state. Dimensions in this array are
    // always in inner_most -> outer_most order (col major layout).
    array<BlockIteratorState, NumDims> it;
    for (int i = 0; i < NumDims; ++i) {
      const int dim = isColMajor ? i : NumDims - 1 - i;
      it[i].size = desc.dimension(dim);
      it[i].count = 0;
      it[i].reverse = m_reverse[dim];
 
      it[i].block_stride = i == 0 ? 1 : (it[i - 1].size * it[i - 1].block_stride);
      it[i].block_span = it[i].block_stride * (it[i].size - 1);
 
      it[i].input_stride = m_strides[dim];
      it[i].input_span = it[i].input_stride * (it[i].size - 1);
 
      if (it[i].reverse) {
        it[i].input_stride = -1 * it[i].input_stride;
        it[i].input_span = -1 * it[i].input_span;
      }
    }
 
    // If multiple inner dimensions have the same reverse flag, check if we can
    // merge them into a single virtual inner dimension.
    int effective_inner_dim = 0;
    for (int i = 1; i < NumDims; ++i) {
      if (it[i].reverse != it[effective_inner_dim].reverse) break;
      if (it[i].block_stride != it[effective_inner_dim].size) break;
      if (it[i].block_stride != numext::abs(it[i].input_stride)) break;
 
      it[i].size = it[effective_inner_dim].size * it[i].size;
 
      it[i].block_stride = 1;
      it[i].input_stride = (inner_dim_reversed ? -1 : 1);
 
      it[i].block_span = it[i].block_stride * (it[i].size - 1);
      it[i].input_span = it[i].input_stride * (it[i].size - 1);
 
      effective_inner_dim = i;
    }
 
    eigen_assert(it[effective_inner_dim].block_stride == 1);
    eigen_assert(it[effective_inner_dim].input_stride == (inner_dim_reversed ? -1 : 1));
 
    const Index inner_dim_size = it[effective_inner_dim].size;
 
    // Prepare storage for the materialized reverse result.
    const typename TensorBlock::Storage block_storage = TensorBlock::prepareStorage(desc, scratch);
    CoeffReturnType* block_buffer = block_storage.data();
 
    while (it[NumDims - 1].count < it[NumDims - 1].size) {
      // Copy inner-most dimension data from reversed location in input.
      Index dst = block_offset;
      Index src = input_offset;
 
      // NOTE(ezhulenev): Adding vectorized path with internal::preverse showed
      // worse results in benchmarks than a simple coefficient loop.
      if (inner_dim_reversed) {
        for (Index i = 0; i < inner_dim_size; ++i) {
          block_buffer[dst] = m_impl.coeff(src);
          ++dst;
          --src;
        }
      } else {
        for (Index i = 0; i < inner_dim_size; ++i) {
          block_buffer[dst] = m_impl.coeff(src);
          ++dst;
          ++src;
        }
      }
 
      // For the 1d tensor we need to generate only one inner-most dimension.
      if ((NumDims - effective_inner_dim) == 1) break;
 
      // Update offset.
      for (Index i = effective_inner_dim + 1; i < NumDims; ++i) {
        if (++it[i].count < it[i].size) {
          block_offset += it[i].block_stride;
          input_offset += it[i].input_stride;
          break;
        }
        if (i != NumDims - 1) it[i].count = 0;
        block_offset -= it[i].block_span;
        input_offset -= it[i].input_span;
      }
    }
 
    return block_storage.AsTensorMaterializedBlock();
  }
 
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
    double compute_cost = NumDims * (2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() +
                                     TensorOpCost::DivCost<Index>());
    for (int i = 0; i < NumDims; ++i) {
      if (m_reverse[i]) {
        compute_cost += 2 * TensorOpCost::AddCost<Index>();
      }
    }
    // The inner-slice fast path runs the per-coeff index math once per packet,
    // so the amortized compute cost matches the vectorized convention.
    return m_impl.costPerCoeff(vectorized) + TensorOpCost(0, 0, compute_cost, vectorized, PacketSize);
  }
 
  EIGEN_DEVICE_FUNC typename Storage::Type data() const { return nullptr; }
 
 protected:
  Dimensions m_dimensions;
  array<Index, NumDims> m_strides;
  array<IndexDivisor, NumDims> m_fastStrides;
  TensorEvaluator<ArgType, Device> m_impl;
  ReverseDimensions m_reverse;
  const Device EIGEN_DEVICE_REF m_device;
 
 private:
  struct BlockIteratorState {
    BlockIteratorState()
        : size(0), count(0), reverse(false), block_stride(0), block_span(0), input_stride(0), input_span(0) {}
 
    Index size;
    Index count;
    bool reverse;
    Index block_stride;
    Index block_span;
    Index input_stride;
    Index input_span;
  };
};
 
// Eval as lvalue
 
template <typename ReverseDimensions, typename ArgType, typename Device>
struct TensorEvaluator<TensorReverseOp<ReverseDimensions, ArgType>, Device>
    : public TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device> {
  typedef TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device> Base;
  typedef TensorReverseOp<ReverseDimensions, ArgType> XprType;
  typedef typename XprType::Index Index;
  static constexpr int NumDims = internal::array_size<ReverseDimensions>::value;
  typedef DSizes<Index, NumDims> Dimensions;
 
  static constexpr int Layout = TensorEvaluator<ArgType, Device>::Layout;
  enum {
    IsAligned = false,
    PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess,
    // writeBlock() assigns the re-reversed block expression straight into the
    // argument's buffer, so it needs raw storage underneath.
    BlockAccess = TensorEvaluator<ArgType, Device>::RawAccess,
    // Unlike the rvalue side there is no preference: the reversal cost moves
    // to the block-expression reads, so writing blocks only pays off when the
    // right-hand side prefers block evaluation anyway.
    PreferBlockAccess = false,
    CoordAccess = false,  // to be implemented
    RawAccess = false
  };
  EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : Base(op, device) {}
 
  typedef typename XprType::Scalar Scalar;
  typedef typename XprType::CoeffReturnType CoeffReturnType;
  typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType;
  static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
  typedef std::remove_const_t<Scalar> ScalarNoConst;
 
  //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
  typedef internal::TensorBlockDescriptor<NumDims, Index> TensorBlockDesc;
  //===--------------------------------------------------------------------===//
 
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return this->m_dimensions; }
 
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) const {
    return this->m_impl.coeffRef(this->reverseIndex(index));
  }
 
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
    // Deliberately not the rvalue evaluator's requirements. Reading a block
    // reverses memory as it materializes it straight into the output, which is
    // why that side asks for a last-level-cache sized, inner-dim-skewed block.
    // As a write destination we have no such preference: writeBlock() only
    // copies the block into a strided box. Since merge() lets kSkewedInnerDims
    // win over kUniformAllDims and keeps the larger size, inheriting them would
    // silently override the shape the right-hand side asked for -- a shuffle
    // that permutes the inner dimension requests small uniform tiles precisely
    // because that is what keeps a transpose cache-resident, and turning those
    // into one cache-sized skewed strip costs more than the block path wins.
    // Only impose a lower bound on the block size, so that a right-hand side
    // without any preference still gets sensibly sized blocks.
    return internal::TensorBlockResourceRequirements::merge(
        this->m_impl.getResourceRequirements(),
        internal::TensorBlockResourceRequirements::uniform<Scalar>(this->m_device.firstLevelCacheSize()));
  }
 
  template <int StoreMode>
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketReturnType& x) const {
    eigen_assert(index + PacketSize - 1 < dimensions().TotalSize());
 
    // Fast path, mirroring packet() in the rvalue evaluator: when the whole
    // packet stays inside a single inner-most slice of the input, replace
    // PacketSize coeffRef() calls (each paying a full reverseIndex walk) with
    // one packet store (plus a preverse when the inner dim is reversed).
    constexpr int inner_dim = (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? 0 : NumDims - 1;
    const Index inner_size = this->m_dimensions[inner_dim];
    const Index inner_pos = index % inner_size;
    if (inner_pos + PacketSize <= inner_size) {
      if (this->m_reverse[inner_dim]) {
        const Index input_index = this->reverseIndex(index + PacketSize - 1);
        this->m_impl.template writePacket<Unaligned>(input_index, internal::preverse(x));
      } else {
        this->m_impl.template writePacket<Unaligned>(this->reverseIndex(index), x);
      }
      return;
    }
 
    // Slow path: the packet crosses an inner-slice boundary.
    EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment) CoeffReturnType values[PacketSize];
    internal::pstore<CoeffReturnType, PacketReturnType>(values, x);
    EIGEN_UNROLL_LOOP
    for (int i = 0; i < PacketSize; ++i) {
      this->coeffRef(index + i) = values[i];
    }
  }
 
  template <typename TensorBlock>
  EIGEN_STRONG_INLINE void writeBlock(const TensorBlockDesc& desc, const TensorBlock& block) {
    eigen_assert(this->m_impl.data() != nullptr);
 
    // The destination of a block is a box in the underlying tensor: on a
    // reversed dimension the output range [o, o + e) maps to the input range
    // [n - o - e, n - o), whose corner sits (e - 1) strides below the image of
    // the block's origin.
    Index input_corner = this->reverseIndex(desc.offset());
    for (int i = 0; i < NumDims; ++i) {
      if (this->m_reverse[i]) input_corner -= (desc.dimension(i) - 1) * this->m_strides[i];
    }
 
    // Assigning the block expression reversed along the reversed dimensions
    // into that box cancels the reversal; the reversed reads vectorize via
    // the rvalue evaluator's inner-slice fast path while the stores stay
    // contiguous.
    typedef TensorReverseOp<const ReverseDimensions, const typename TensorBlock::XprType> RevBlockExpr;
    const RevBlockExpr reversed_block(block.expr(), this->m_reverse);
 
    typedef internal::TensorBlockAssignment<ScalarNoConst, NumDims, RevBlockExpr, Index> TensorBlockAssign;
    TensorBlockAssign::Run(TensorBlockAssign::target(desc.dimensions(), DSizes<Index, NumDims>(this->m_strides),
                                                     this->m_impl.data(), input_corner),
                           reversed_block);
  }
};
 
}  // end namespace Eigen
 
#endif  // EIGEN_TENSOR_TENSOR_REVERSE_H