151class ThreadedSparseProduct {
153 using SparseMatrixType = SparseMatrixType_;
154 using Scalar =
typename SparseMatrixType::Scalar;
155 using RealScalar =
typename SparseMatrixType::RealScalar;
156 using StorageIndex =
typename SparseMatrixType::StorageIndex;
158 enum { IsRowMajor =
static_cast<int>(SparseMatrixType::IsRowMajor) };
173 ThreadedSparseProduct() =
default;
175 explicit ThreadedSparseProduct(
const SparseMatrixType& mat, ThreadPool*
pool =
nullptr) : m_pool(
pool) {
179 ~ThreadedSparseProduct() {
delete m_mirror.load(std::memory_order_acquire); }
181 ThreadedSparseProduct(
const ThreadedSparseProduct&) =
delete;
182 ThreadedSparseProduct& operator=(
const ThreadedSparseProduct&) =
delete;
187 eigen_assert(mat.isCompressed() &&
"ThreadedSparseProduct requires a compressed SparseMatrix");
189 delete m_mirror.exchange(
nullptr, std::memory_order_acq_rel);
190 m_adj_partition.clear();
192 build_partition(mat.outerIndexPtr(), mat.outerSize(), mat.nonZeros(), m_native_partition);
203 delete m_mirror.exchange(
nullptr, std::memory_order_acq_rel);
207 Index rows()
const {
return m_mat ? m_mat->rows() : Index(0); }
208 Index cols()
const {
return m_mat ? m_mat->cols() : Index(0); }
211 void apply(
const ConstVectorRef& x, MutableVectorRef y)
const {
212 apply_impl<
false,
true>(x, y, Scalar(1));
216 apply_impl<
true,
true>(x, y, Scalar(1));
219 void applyAddTo(
const ConstVectorRef& x, MutableVectorRef y,
const Scalar& alpha)
const {
220 apply_impl<
false,
false>(x, y, alpha);
223 void applyAdjointAddTo(
const ConstVectorRef& x, MutableVectorRef y,
const Scalar& alpha)
const {
224 apply_impl<
true,
false>(x, y, alpha);
228 ThreadPool*
pool()
const {
return m_pool ? m_pool : &internal::default_threaded_sparse_pool(); }
231 bool hasMirror()
const {
return m_mirror.load(std::memory_order_acquire) !=
nullptr; }
234 template <
bool Adjo
int,
bool Overwrite>
235 void apply_impl(
const ConstVectorRef& x, MutableVectorRef& y,
const Scalar& alpha)
const {
236 eigen_assert(m_mat &&
"ThreadedSparseProduct: matrix not set; call analyzePattern() first");
237 EIGEN_INTERNAL_ASSERT_SIZE_MATCHES_AXIS(x, *m_mat, Adjoint);
238 EIGEN_INTERNAL_ASSERT_SIZE_MATCHES_AXIS(y, *m_mat, !Adjoint);
244 eigen_assert((x.size() == 0 || y.size() == 0 || std::uintptr_t(x.data() + x.size()) <= std::uintptr_t(y.data()) ||
245 std::uintptr_t(y.data() + y.size()) <= std::uintptr_t(x.data())) &&
246 "ThreadedSparseProduct: x and y must not overlap");
251 constexpr bool NeedRowMajorView = !Adjoint;
252 constexpr bool NativeIsRowMajor = IsRowMajor;
253 constexpr bool UseMirror = NeedRowMajorView != NativeIsRowMajor;
255 EIGEN_IF_CONSTEXPR (UseMirror) {
256 const MirrorType& m = ensure_mirror();
257 run<Adjoint, Overwrite>(m.valuePtr(), m.innerIndexPtr(), m.outerIndexPtr(), m.innerNonZeroPtr(),
258 m.outerSize(), m_adj_partition, x, y, alpha);
260 run<Adjoint, Overwrite>(m_mat->valuePtr(), m_mat->innerIndexPtr(), m_mat->outerIndexPtr(),
261 m_mat->innerNonZeroPtr(),
262 m_mat->outerSize(), m_native_partition, x, y, alpha);
267 static constexpr Index kThreadingThreshold = 20000;
269 template <
bool Conjugate,
bool Overwrite>
270 void run(
const Scalar* vals,
const StorageIndex* inner,
const StorageIndex* outer,
const StorageIndex* innerNnz,
271 Index outerSize,
const std::vector<Index>& part,
const ConstVectorRef& x, MutableVectorRef& y,
272 const Scalar& alpha)
const {
278#ifdef EIGEN_HAS_OPENMP
279 const int T = target_thread_count();
281 const int T =
static_cast<int>(part.size()) - 1;
283 const Index total_nnz = m_mat->nonZeros();
285 const Scalar* xp = x.data();
286 Scalar* yp = y.data();
287 if (T <= 1 || total_nnz < kThreadingThreshold) {
288 internal::run_dot_chunk<Conjugate, Overwrite, Scalar, StorageIndex>(vals, inner, outer, innerNnz, xp, yp, 0,
293#ifdef EIGEN_HAS_OPENMP
298 const Index chunk = numext::maxi<Index>(Index(1), (outerSize + Index(T) * 4 - 1) / (Index(T) * 4));
299#pragma omp parallel for schedule(dynamic, chunk) num_threads(T)
300 for (Index i = 0; i < outerSize; ++i) {
301 internal::run_dot_row<Conjugate, Overwrite, Scalar, StorageIndex>(vals, inner, outer, innerNnz, xp, yp, i, alpha);
307 Barrier barrier(
static_cast<unsigned>(T));
308 ThreadPool* p =
pool();
309 for (
int t = 1; t < T; ++t) {
310 const Index lo = part[t], hi = part[t + 1];
315 p->Schedule([=, &barrier]() {
316 internal::run_dot_chunk<Conjugate, Overwrite, Scalar, StorageIndex>(vals, inner, outer, innerNnz, xp, yp, lo,
321 internal::run_dot_chunk<Conjugate, Overwrite, Scalar, StorageIndex>(vals, inner, outer, innerNnz, xp, yp, part[0],
330 const MirrorType& ensure_mirror()
const {
331 MirrorType* m = m_mirror.load(std::memory_order_acquire);
333 std::lock_guard<std::mutex> lock(m_mirror_init_mu);
334 m = m_mirror.load(std::memory_order_relaxed);
345 std::unique_ptr<MirrorType> built(
new MirrorType(m_mat->rows(), m_mat->cols()));
347 built->makeCompressed();
348 build_partition(built->outerIndexPtr(), built->outerSize(), built->nonZeros(), m_adj_partition);
350 m_mirror.store(built.release(), std::memory_order_release);
358 int target_thread_count()
const {
359#ifdef EIGEN_HAS_OPENMP
360 return numext::maxi(1, Eigen::nbThreads());
362 return pool()->NumThreads();
371 void build_partition(
const StorageIndex* outer, Index outerSize, Index nnz, std::vector<Index>& part)
const {
372 const int T = target_thread_count();
373 internal::compute_nnz_balanced_partition(outer, outerSize, nnz, T, part);
375 for (std::size_t t = 0; t + 1 < part.size(); ++t)
376 if (part[t + 1] > part[t]) ++non_empty;
377 if (non_empty * 2 < T) {
384 const SparseMatrixType* m_mat =
nullptr;
385 ThreadPool* m_pool =
nullptr;
389 std::vector<Index> m_native_partition;
393 mutable std::atomic<MirrorType*> m_mirror{
nullptr};
394 mutable std::mutex m_mirror_init_mu;
395 mutable std::vector<Index> m_adj_partition;