Ginkgo Generated from branch based on master. Ginkgo version 1.7.0
A numerical linear algebra library targeting many-core architectures
Loading...
Searching...
No Matches
csr.hpp
1/*******************************<GINKGO LICENSE>******************************
2Copyright (c) 2017-2023, the Ginkgo authors
3All rights reserved.
4
5Redistribution and use in source and binary forms, with or without
6modification, are permitted provided that the following conditions
7are met:
8
91. Redistributions of source code must retain the above copyright
10notice, this list of conditions and the following disclaimer.
11
122. Redistributions in binary form must reproduce the above copyright
13notice, this list of conditions and the following disclaimer in the
14documentation and/or other materials provided with the distribution.
15
163. Neither the name of the copyright holder nor the names of its
17contributors may be used to endorse or promote products derived from
18this software without specific prior written permission.
19
20THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
21IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
23PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31******************************<GINKGO LICENSE>*******************************/
32
33#ifndef GKO_PUBLIC_CORE_MATRIX_CSR_HPP_
34#define GKO_PUBLIC_CORE_MATRIX_CSR_HPP_
35
36
37#include <ginkgo/core/base/array.hpp>
38#include <ginkgo/core/base/index_set.hpp>
39#include <ginkgo/core/base/lin_op.hpp>
40#include <ginkgo/core/base/math.hpp>
41#include <ginkgo/core/matrix/permutation.hpp>
42#include <ginkgo/core/matrix/scaled_permutation.hpp>
43
44
45namespace gko {
46namespace matrix {
47
48
49template <typename ValueType>
50class Dense;
51
52template <typename ValueType>
53class Diagonal;
54
55template <typename ValueType, typename IndexType>
56class Coo;
57
58template <typename ValueType, typename IndexType>
59class Ell;
60
61template <typename ValueType, typename IndexType>
62class Hybrid;
63
64template <typename ValueType, typename IndexType>
65class Sellp;
66
67template <typename ValueType, typename IndexType>
68class SparsityCsr;
69
70template <typename ValueType, typename IndexType>
71class Csr;
72
73template <typename ValueType, typename IndexType>
74class Fbcsr;
75
76template <typename ValueType, typename IndexType>
78
79
80namespace detail {
81
82
83template <typename ValueType = default_precision, typename IndexType = int32>
84void strategy_rebuild_helper(Csr<ValueType, IndexType>* result);
85
86
87} // namespace detail
88
89
128template <typename ValueType = default_precision, typename IndexType = int32>
129class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
130 public EnableCreateMethod<Csr<ValueType, IndexType>>,
131 public ConvertibleTo<Csr<next_precision<ValueType>, IndexType>>,
132 public ConvertibleTo<Dense<ValueType>>,
133 public ConvertibleTo<Coo<ValueType, IndexType>>,
134 public ConvertibleTo<Ell<ValueType, IndexType>>,
135 public ConvertibleTo<Fbcsr<ValueType, IndexType>>,
136 public ConvertibleTo<Hybrid<ValueType, IndexType>>,
137 public ConvertibleTo<Sellp<ValueType, IndexType>>,
138 public ConvertibleTo<SparsityCsr<ValueType, IndexType>>,
139 public DiagonalExtractable<ValueType>,
140 public ReadableFromMatrixData<ValueType, IndexType>,
141 public WritableToMatrixData<ValueType, IndexType>,
142 public Transposable,
143 public Permutable<IndexType>,
145 remove_complex<Csr<ValueType, IndexType>>>,
146 public ScaledIdentityAddable {
147 friend class EnableCreateMethod<Csr>;
148 friend class EnablePolymorphicObject<Csr, LinOp>;
149 friend class Coo<ValueType, IndexType>;
150 friend class Dense<ValueType>;
151 friend class Diagonal<ValueType>;
152 friend class Ell<ValueType, IndexType>;
153 friend class Hybrid<ValueType, IndexType>;
154 friend class Sellp<ValueType, IndexType>;
155 friend class SparsityCsr<ValueType, IndexType>;
156 friend class Fbcsr<ValueType, IndexType>;
157 friend class CsrBuilder<ValueType, IndexType>;
158 friend class Csr<to_complex<ValueType>, IndexType>;
159
160public:
161 using EnableLinOp<Csr>::convert_to;
162 using EnableLinOp<Csr>::move_to;
163 using ConvertibleTo<Csr<next_precision<ValueType>, IndexType>>::convert_to;
164 using ConvertibleTo<Csr<next_precision<ValueType>, IndexType>>::move_to;
165 using ConvertibleTo<Dense<ValueType>>::convert_to;
166 using ConvertibleTo<Dense<ValueType>>::move_to;
167 using ConvertibleTo<Coo<ValueType, IndexType>>::convert_to;
169 using ConvertibleTo<Ell<ValueType, IndexType>>::convert_to;
179 using ReadableFromMatrixData<ValueType, IndexType>::read;
180
181 using value_type = ValueType;
182 using index_type = IndexType;
186 using absolute_type = remove_complex<Csr>;
187
188 class automatical;
189
197 friend class automatical;
198
199 public:
205 strategy_type(std::string name) : name_(name) {}
206
207 virtual ~strategy_type() = default;
208
214 std::string get_name() { return name_; }
215
224
232 virtual int64_t clac_size(const int64_t nnz) = 0;
233
238 virtual std::shared_ptr<strategy_type> copy() = 0;
239
240 protected:
241 void set_name(std::string name) { name_ = name; }
242
243 private:
244 std::string name_;
245 };
246
253 class classical : public strategy_type {
254 public:
258 classical() : strategy_type("classical"), max_length_per_row_(0) {}
259
261 array<index_type>* mtx_srow) override
262 {
263 auto host_mtx_exec = mtx_row_ptrs.get_executor()->get_master();
265 const bool is_mtx_on_host{host_mtx_exec ==
266 mtx_row_ptrs.get_executor()};
267 const index_type* row_ptrs{};
268 if (is_mtx_on_host) {
269 row_ptrs = mtx_row_ptrs.get_const_data();
270 } else {
272 row_ptrs = row_ptrs_host.get_const_data();
273 }
274 auto num_rows = mtx_row_ptrs.get_num_elems() - 1;
275 max_length_per_row_ = 0;
276 for (size_type i = 0; i < num_rows; i++) {
277 max_length_per_row_ = std::max(max_length_per_row_,
278 row_ptrs[i + 1] - row_ptrs[i]);
279 }
280 }
281
282 int64_t clac_size(const int64_t nnz) override { return 0; }
283
284 index_type get_max_length_per_row() const noexcept
285 {
286 return max_length_per_row_;
287 }
288
289 std::shared_ptr<strategy_type> copy() override
290 {
291 return std::make_shared<classical>();
292 }
293
294 private:
295 index_type max_length_per_row_;
296 };
297
303 class merge_path : public strategy_type {
304 public:
308 merge_path() : strategy_type("merge_path") {}
309
313
314 int64_t clac_size(const int64_t nnz) override { return 0; }
315
316 std::shared_ptr<strategy_type> copy() override
317 {
318 return std::make_shared<merge_path>();
319 }
320 };
321
328 class cusparse : public strategy_type {
329 public:
333 cusparse() : strategy_type("cusparse") {}
334
338
339 int64_t clac_size(const int64_t nnz) override { return 0; }
340
341 std::shared_ptr<strategy_type> copy() override
342 {
343 return std::make_shared<cusparse>();
344 }
345 };
346
352 class sparselib : public strategy_type {
353 public:
357 sparselib() : strategy_type("sparselib") {}
358
362
363 int64_t clac_size(const int64_t nnz) override { return 0; }
364
365 std::shared_ptr<strategy_type> copy() override
366 {
367 return std::make_shared<sparselib>();
368 }
369 };
370
375 public:
383 : load_balance(std::move(
384 gko::CudaExecutor::create(0, gko::OmpExecutor::create())))
385 {}
386
392 load_balance(std::shared_ptr<const CudaExecutor> exec)
393 : load_balance(exec->get_num_warps(), exec->get_warp_size())
394 {}
395
401 load_balance(std::shared_ptr<const HipExecutor> exec)
402 : load_balance(exec->get_num_warps(), exec->get_warp_size(), false)
403 {}
404
412 load_balance(std::shared_ptr<const DpcppExecutor> exec)
413 : load_balance(exec->get_num_subgroups(), 32, false, "intel")
414 {}
415
428 bool cuda_strategy = true,
429 std::string strategy_name = "none")
430 : strategy_type("load_balance"),
431 nwarps_(nwarps),
432 warp_size_(warp_size),
433 cuda_strategy_(cuda_strategy),
434 strategy_name_(strategy_name)
435 {}
436
438 array<index_type>* mtx_srow) override
439 {
440 auto nwarps = mtx_srow->get_num_elems();
441
442 if (nwarps > 0) {
443 auto host_srow_exec = mtx_srow->get_executor()->get_master();
444 auto host_mtx_exec = mtx_row_ptrs.get_executor()->get_master();
445 const bool is_srow_on_host{host_srow_exec ==
446 mtx_srow->get_executor()};
447 const bool is_mtx_on_host{host_mtx_exec ==
448 mtx_row_ptrs.get_executor()};
451 const index_type* row_ptrs{};
452 index_type* srow{};
453 if (is_srow_on_host) {
454 srow = mtx_srow->get_data();
455 } else {
457 srow = srow_host.get_data();
458 }
459 if (is_mtx_on_host) {
460 row_ptrs = mtx_row_ptrs.get_const_data();
461 } else {
463 row_ptrs = row_ptrs_host.get_const_data();
464 }
465 for (size_type i = 0; i < nwarps; i++) {
466 srow[i] = 0;
467 }
468 const auto num_rows = mtx_row_ptrs.get_num_elems() - 1;
469 const auto num_elems = row_ptrs[num_rows];
470 const auto bucket_divider =
471 num_elems > 0 ? ceildiv(num_elems, warp_size_) : 1;
472 for (size_type i = 0; i < num_rows; i++) {
473 auto bucket =
474 ceildiv((ceildiv(row_ptrs[i + 1], warp_size_) * nwarps),
476 if (bucket < nwarps) {
477 srow[bucket]++;
478 }
479 }
480 // find starting row for thread i
481 for (size_type i = 1; i < nwarps; i++) {
482 srow[i] += srow[i - 1];
483 }
484 if (!is_srow_on_host) {
486 }
487 }
488 }
489
490 int64_t clac_size(const int64_t nnz) override
491 {
492 if (warp_size_ > 0) {
493 int multiple = 8;
494 if (nnz >= static_cast<int64_t>(2e8)) {
495 multiple = 2048;
496 } else if (nnz >= static_cast<int64_t>(2e7)) {
497 multiple = 512;
498 } else if (nnz >= static_cast<int64_t>(2e6)) {
499 multiple = 128;
500 } else if (nnz >= static_cast<int64_t>(2e5)) {
501 multiple = 32;
502 }
503 if (strategy_name_ == "intel") {
504 multiple = 8;
505 if (nnz >= static_cast<int64_t>(2e8)) {
506 multiple = 256;
507 } else if (nnz >= static_cast<int64_t>(2e7)) {
508 multiple = 32;
509 }
510 }
511#if GINKGO_HIP_PLATFORM_HCC
512 if (!cuda_strategy_) {
513 multiple = 8;
514 if (nnz >= static_cast<int64_t>(1e7)) {
515 multiple = 64;
516 } else if (nnz >= static_cast<int64_t>(1e6)) {
517 multiple = 16;
518 }
519 }
520#endif // GINKGO_HIP_PLATFORM_HCC
521
522 auto nwarps = nwarps_ * multiple;
523 return min(ceildiv(nnz, warp_size_), nwarps);
524 } else {
525 return 0;
526 }
527 }
528
529 std::shared_ptr<strategy_type> copy() override
530 {
531 return std::make_shared<load_balance>(
532 nwarps_, warp_size_, cuda_strategy_, strategy_name_);
533 }
534
535 private:
536 int64_t nwarps_;
537 int warp_size_;
538 bool cuda_strategy_;
539 std::string strategy_name_;
540 };
541
542 class automatical : public strategy_type {
543 public:
544 /* Use imbalance strategy when the maximum number of nonzero per row is
545 * more than 1024 on NVIDIA hardware */
546 const index_type nvidia_row_len_limit = 1024;
547 /* Use imbalance strategy when the matrix has more more than 1e6 on
548 * NVIDIA hardware */
549 const index_type nvidia_nnz_limit{static_cast<index_type>(1e6)};
550 /* Use imbalance strategy when the maximum number of nonzero per row is
551 * more than 768 on AMD hardware */
552 const index_type amd_row_len_limit = 768;
553 /* Use imbalance strategy when the matrix has more more than 1e8 on AMD
554 * hardware */
555 const index_type amd_nnz_limit{static_cast<index_type>(1e8)};
556 /* Use imbalance strategy when the maximum number of nonzero per row is
557 * more than 25600 on Intel hardware */
558 const index_type intel_row_len_limit = 25600;
559 /* Use imbalance strategy when the matrix has more more than 3e8 on
560 * Intel hardware */
561 const index_type intel_nnz_limit{static_cast<index_type>(3e8)};
562
563 public:
571 : automatical(std::move(
572 gko::CudaExecutor::create(0, gko::OmpExecutor::create())))
573 {}
574
580 automatical(std::shared_ptr<const CudaExecutor> exec)
581 : automatical(exec->get_num_warps(), exec->get_warp_size())
582 {}
583
589 automatical(std::shared_ptr<const HipExecutor> exec)
590 : automatical(exec->get_num_warps(), exec->get_warp_size(), false)
591 {}
592
600 automatical(std::shared_ptr<const DpcppExecutor> exec)
601 : automatical(exec->get_num_subgroups(), 32, false, "intel")
602 {}
603
616 bool cuda_strategy = true,
617 std::string strategy_name = "none")
618 : strategy_type("automatical"),
619 nwarps_(nwarps),
620 warp_size_(warp_size),
621 cuda_strategy_(cuda_strategy),
622 strategy_name_(strategy_name),
623 max_length_per_row_(0)
624 {}
625
627 array<index_type>* mtx_srow) override
628 {
629 // if the number of stored elements is larger than <nnz_limit> or
630 // the maximum number of stored elements per row is larger than
631 // <row_len_limit>, use load_balance otherwise use classical
632 index_type nnz_limit = nvidia_nnz_limit;
633 index_type row_len_limit = nvidia_row_len_limit;
634 if (strategy_name_ == "intel") {
635 nnz_limit = intel_nnz_limit;
636 row_len_limit = intel_row_len_limit;
637 }
638#if GINKGO_HIP_PLATFORM_HCC
639 if (!cuda_strategy_) {
640 nnz_limit = amd_nnz_limit;
641 row_len_limit = amd_row_len_limit;
642 }
643#endif // GINKGO_HIP_PLATFORM_HCC
644 auto host_mtx_exec = mtx_row_ptrs.get_executor()->get_master();
645 const bool is_mtx_on_host{host_mtx_exec ==
646 mtx_row_ptrs.get_executor()};
648 const index_type* row_ptrs{};
649 if (is_mtx_on_host) {
650 row_ptrs = mtx_row_ptrs.get_const_data();
651 } else {
653 row_ptrs = row_ptrs_host.get_const_data();
654 }
655 const auto num_rows = mtx_row_ptrs.get_num_elems() - 1;
656 if (row_ptrs[num_rows] > nnz_limit) {
657 load_balance actual_strategy(nwarps_, warp_size_,
658 cuda_strategy_, strategy_name_);
659 if (is_mtx_on_host) {
661 } else {
663 }
664 this->set_name(actual_strategy.get_name());
665 } else {
666 index_type maxnum = 0;
667 for (size_type i = 0; i < num_rows; i++) {
668 maxnum = std::max(maxnum, row_ptrs[i + 1] - row_ptrs[i]);
669 }
670 if (maxnum > row_len_limit) {
672 nwarps_, warp_size_, cuda_strategy_, strategy_name_);
673 if (is_mtx_on_host) {
675 } else {
677 }
678 this->set_name(actual_strategy.get_name());
679 } else {
681 if (is_mtx_on_host) {
683 max_length_per_row_ =
684 actual_strategy.get_max_length_per_row();
685 } else {
687 max_length_per_row_ =
688 actual_strategy.get_max_length_per_row();
689 }
690 this->set_name(actual_strategy.get_name());
691 }
692 }
693 }
694
695 int64_t clac_size(const int64_t nnz) override
696 {
697 return std::make_shared<load_balance>(
698 nwarps_, warp_size_, cuda_strategy_, strategy_name_)
699 ->clac_size(nnz);
700 }
701
702 index_type get_max_length_per_row() const noexcept
703 {
704 return max_length_per_row_;
705 }
706
707 std::shared_ptr<strategy_type> copy() override
708 {
709 return std::make_shared<automatical>(
710 nwarps_, warp_size_, cuda_strategy_, strategy_name_);
711 }
712
713 private:
714 int64_t nwarps_;
715 int warp_size_;
716 bool cuda_strategy_;
717 std::string strategy_name_;
718 index_type max_length_per_row_;
719 };
720
721 friend class Csr<next_precision<ValueType>, IndexType>;
722
723 void convert_to(
724 Csr<next_precision<ValueType>, IndexType>* result) const override;
725
726 void move_to(Csr<next_precision<ValueType>, IndexType>* result) override;
727
728 void convert_to(Dense<ValueType>* other) const override;
729
730 void move_to(Dense<ValueType>* other) override;
731
732 void convert_to(Coo<ValueType, IndexType>* result) const override;
733
734 void move_to(Coo<ValueType, IndexType>* result) override;
735
736 void convert_to(Ell<ValueType, IndexType>* result) const override;
737
738 void move_to(Ell<ValueType, IndexType>* result) override;
739
740 void convert_to(Fbcsr<ValueType, IndexType>* result) const override;
741
742 void move_to(Fbcsr<ValueType, IndexType>* result) override;
743
744 void convert_to(Hybrid<ValueType, IndexType>* result) const override;
745
746 void move_to(Hybrid<ValueType, IndexType>* result) override;
747
748 void convert_to(Sellp<ValueType, IndexType>* result) const override;
749
750 void move_to(Sellp<ValueType, IndexType>* result) override;
751
752 void convert_to(SparsityCsr<ValueType, IndexType>* result) const override;
753
754 void move_to(SparsityCsr<ValueType, IndexType>* result) override;
755
756 void read(const mat_data& data) override;
757
758 void read(const device_mat_data& data) override;
759
760 void read(device_mat_data&& data) override;
761
762 void write(mat_data& data) const override;
763
764 std::unique_ptr<LinOp> transpose() const override;
765
766 std::unique_ptr<LinOp> conj_transpose() const override;
767
782 std::unique_ptr<Csr> permute(
785
799 std::unique_ptr<Csr> permute(
802 bool invert = false) const;
803
813 std::unique_ptr<Csr> scale_permute(
816
829 std::unique_ptr<Csr> scale_permute(
834 bool invert = false) const;
835
836 std::unique_ptr<LinOp> permute(
837 const array<IndexType>* permutation_indices) const override;
838
839 std::unique_ptr<LinOp> inverse_permute(
840 const array<IndexType>* inverse_permutation_indices) const override;
841
842 std::unique_ptr<LinOp> row_permute(
843 const array<IndexType>* permutation_indices) const override;
844
845 std::unique_ptr<LinOp> column_permute(
846 const array<IndexType>* permutation_indices) const override;
847
848 std::unique_ptr<LinOp> inverse_row_permute(
849 const array<IndexType>* inverse_permutation_indices) const override;
850
851 std::unique_ptr<LinOp> inverse_column_permute(
852 const array<IndexType>* inverse_permutation_indices) const override;
853
854 std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
855
856 std::unique_ptr<absolute_type> compute_absolute() const override;
857
859
864
865 /*
866 * Tests if all row entry pairs (value, col_idx) are sorted by column index
867 *
868 * @returns True if all row entry pairs (value, col_idx) are sorted by
869 * column index
870 */
871 bool is_sorted_by_column_index() const;
872
878 value_type* get_values() noexcept { return values_.get_data(); }
879
887 const value_type* get_const_values() const noexcept
888 {
889 return values_.get_const_data();
890 }
891
897 index_type* get_col_idxs() noexcept { return col_idxs_.get_data(); }
898
907 {
908 return col_idxs_.get_const_data();
909 }
910
916 index_type* get_row_ptrs() noexcept { return row_ptrs_.get_data(); }
917
926 {
927 return row_ptrs_.get_const_data();
928 }
929
935 index_type* get_srow() noexcept { return srow_.get_data(); }
936
944 const index_type* get_const_srow() const noexcept
945 {
946 return srow_.get_const_data();
947 }
948
955 {
956 return srow_.get_num_elems();
957 }
958
965 {
966 return values_.get_num_elems();
967 }
968
973 std::shared_ptr<strategy_type> get_strategy() const noexcept
974 {
975 return strategy_;
976 }
977
983 void set_strategy(std::shared_ptr<strategy_type> strategy)
984 {
985 strategy_ = std::move(strategy->copy());
986 this->make_srow();
987 }
988
996 {
997 auto exec = this->get_executor();
998 GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1));
999 this->scale_impl(make_temporary_clone(exec, alpha).get());
1000 }
1001
1009 {
1010 auto exec = this->get_executor();
1011 GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1));
1012 this->inv_scale_impl(make_temporary_clone(exec, alpha).get());
1013 }
1014
1028 static std::unique_ptr<const Csr> create_const(
1029 std::shared_ptr<const Executor> exec, const dim<2>& size,
1030 gko::detail::const_array_view<ValueType>&& values,
1031 gko::detail::const_array_view<IndexType>&& col_idxs,
1032 gko::detail::const_array_view<IndexType>&& row_ptrs,
1033 std::shared_ptr<strategy_type> strategy)
1034 {
1035 // cast const-ness away, but return a const object afterwards,
1036 // so we can ensure that no modifications take place.
1037 return std::unique_ptr<const Csr>(new Csr{
1038 exec, size, gko::detail::array_const_cast(std::move(values)),
1039 gko::detail::array_const_cast(std::move(col_idxs)),
1040 gko::detail::array_const_cast(std::move(row_ptrs)), strategy});
1041 }
1042
1046 static std::unique_ptr<const Csr> create_const(
1047 std::shared_ptr<const Executor> exec, const dim<2>& size,
1048 gko::detail::const_array_view<ValueType>&& values,
1049 gko::detail::const_array_view<IndexType>&& col_idxs,
1050 gko::detail::const_array_view<IndexType>&& row_ptrs)
1051 {
1052 return Csr::create_const(exec, size, std::move(values),
1053 std::move(col_idxs), std::move(row_ptrs),
1054 Csr::make_default_strategy(exec));
1055 }
1056
1069 std::unique_ptr<Csr<ValueType, IndexType>> create_submatrix(
1072
1084 std::unique_ptr<Csr<ValueType, IndexType>> create_submatrix(
1085 const span& row_span, const span& column_span) const;
1086
1091
1098
1102 Csr(const Csr&);
1103
1110
1111protected:
1118 Csr(std::shared_ptr<const Executor> exec,
1119 std::shared_ptr<strategy_type> strategy)
1120 : Csr(std::move(exec), dim<2>{}, {}, std::move(strategy))
1121 {}
1122
1132 Csr(std::shared_ptr<const Executor> exec, const dim<2>& size,
1133 size_type num_nonzeros, std::shared_ptr<strategy_type> strategy)
1134 : EnableLinOp<Csr>(exec, size),
1135 values_(exec, num_nonzeros),
1136 col_idxs_(exec, num_nonzeros),
1137 row_ptrs_(exec, size[0] + 1),
1138 srow_(exec, strategy->clac_size(num_nonzeros)),
1139 strategy_(strategy->copy())
1140 {
1141 row_ptrs_.fill(0);
1142 this->make_srow();
1143 }
1144
1153 Csr(std::shared_ptr<const Executor> exec, const dim<2>& size = dim<2>{},
1155 : Csr{exec, size, num_nonzeros, Csr::make_default_strategy(exec)}
1156 {}
1157
1179 template <typename ValuesArray, typename ColIdxsArray,
1180 typename RowPtrsArray>
1181 Csr(std::shared_ptr<const Executor> exec, const dim<2>& size,
1182 ValuesArray&& values, ColIdxsArray&& col_idxs, RowPtrsArray&& row_ptrs,
1183 std::shared_ptr<strategy_type> strategy)
1184 : EnableLinOp<Csr>(exec, size),
1185 values_{exec, std::forward<ValuesArray>(values)},
1186 col_idxs_{exec, std::forward<ColIdxsArray>(col_idxs)},
1187 row_ptrs_{exec, std::forward<RowPtrsArray>(row_ptrs)},
1188 srow_(exec),
1189 strategy_(strategy->copy())
1190 {
1191 GKO_ASSERT_EQ(values_.get_num_elems(), col_idxs_.get_num_elems());
1192 GKO_ASSERT_EQ(this->get_size()[0] + 1, row_ptrs_.get_num_elems());
1193 this->make_srow();
1194 }
1195
1203 template <typename ValuesArray, typename ColIdxsArray,
1204 typename RowPtrsArray>
1205 Csr(std::shared_ptr<const Executor> exec, const dim<2>& size,
1206 ValuesArray&& values, ColIdxsArray&& col_idxs, RowPtrsArray&& row_ptrs)
1207 : Csr{exec,
1208 size,
1209 std::forward<ValuesArray>(values),
1210 std::forward<ColIdxsArray>(col_idxs),
1212 Csr::make_default_strategy(exec)}
1213 {}
1214
1215 void apply_impl(const LinOp* b, LinOp* x) const override;
1216
1217 void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta,
1218 LinOp* x) const override;
1219
1220 // TODO: This provides some more sane settings. Please fix this!
1221 static std::shared_ptr<strategy_type> make_default_strategy(
1222 std::shared_ptr<const Executor> exec)
1223 {
1224 auto cuda_exec = std::dynamic_pointer_cast<const CudaExecutor>(exec);
1225 auto hip_exec = std::dynamic_pointer_cast<const HipExecutor>(exec);
1226 auto dpcpp_exec = std::dynamic_pointer_cast<const DpcppExecutor>(exec);
1227 std::shared_ptr<strategy_type> new_strategy;
1228 if (cuda_exec) {
1229 new_strategy = std::make_shared<automatical>(cuda_exec);
1230 } else if (hip_exec) {
1231 new_strategy = std::make_shared<automatical>(hip_exec);
1232 } else if (dpcpp_exec) {
1233 new_strategy = std::make_shared<automatical>(dpcpp_exec);
1234 } else {
1235 new_strategy = std::make_shared<classical>();
1236 }
1237 return new_strategy;
1238 }
1239
1240 // TODO clean this up as soon as we improve strategy_type
1241 template <typename CsrType>
1242 void convert_strategy_helper(CsrType* result) const
1243 {
1244 auto strat = this->get_strategy().get();
1245 std::shared_ptr<typename CsrType::strategy_type> new_strat;
1246 if (dynamic_cast<classical*>(strat)) {
1247 new_strat = std::make_shared<typename CsrType::classical>();
1248 } else if (dynamic_cast<merge_path*>(strat)) {
1249 new_strat = std::make_shared<typename CsrType::merge_path>();
1250 } else if (dynamic_cast<cusparse*>(strat)) {
1251 new_strat = std::make_shared<typename CsrType::cusparse>();
1252 } else if (dynamic_cast<sparselib*>(strat)) {
1253 new_strat = std::make_shared<typename CsrType::sparselib>();
1254 } else {
1255 auto rexec = result->get_executor();
1256 auto cuda_exec =
1257 std::dynamic_pointer_cast<const CudaExecutor>(rexec);
1258 auto hip_exec = std::dynamic_pointer_cast<const HipExecutor>(rexec);
1259 auto dpcpp_exec =
1260 std::dynamic_pointer_cast<const DpcppExecutor>(rexec);
1261 auto lb = dynamic_cast<load_balance*>(strat);
1262 if (cuda_exec) {
1263 if (lb) {
1264 new_strat =
1265 std::make_shared<typename CsrType::load_balance>(
1266 cuda_exec);
1267 } else {
1268 new_strat = std::make_shared<typename CsrType::automatical>(
1269 cuda_exec);
1270 }
1271 } else if (hip_exec) {
1272 if (lb) {
1273 new_strat =
1274 std::make_shared<typename CsrType::load_balance>(
1275 hip_exec);
1276 } else {
1277 new_strat = std::make_shared<typename CsrType::automatical>(
1278 hip_exec);
1279 }
1280 } else if (dpcpp_exec) {
1281 if (lb) {
1282 new_strat =
1283 std::make_shared<typename CsrType::load_balance>(
1284 dpcpp_exec);
1285 } else {
1286 new_strat = std::make_shared<typename CsrType::automatical>(
1287 dpcpp_exec);
1288 }
1289 } else {
1290 // Try to preserve this executor's configuration
1291 auto this_cuda_exec =
1292 std::dynamic_pointer_cast<const CudaExecutor>(
1293 this->get_executor());
1294 auto this_hip_exec =
1295 std::dynamic_pointer_cast<const HipExecutor>(
1296 this->get_executor());
1297 auto this_dpcpp_exec =
1298 std::dynamic_pointer_cast<const DpcppExecutor>(
1299 this->get_executor());
1300 if (this_cuda_exec) {
1301 if (lb) {
1302 new_strat =
1303 std::make_shared<typename CsrType::load_balance>(
1305 } else {
1306 new_strat =
1307 std::make_shared<typename CsrType::automatical>(
1309 }
1310 } else if (this_hip_exec) {
1311 if (lb) {
1312 new_strat =
1313 std::make_shared<typename CsrType::load_balance>(
1315 } else {
1316 new_strat =
1317 std::make_shared<typename CsrType::automatical>(
1319 }
1320 } else if (this_dpcpp_exec) {
1321 if (lb) {
1322 new_strat =
1323 std::make_shared<typename CsrType::load_balance>(
1325 } else {
1326 new_strat =
1327 std::make_shared<typename CsrType::automatical>(
1329 }
1330 } else {
1331 // FIXME: this changes strategies.
1332 // We had a load balance or automatical strategy from a non
1333 // HIP or Cuda executor and are moving to a non HIP or Cuda
1334 // executor.
1335 new_strat = std::make_shared<typename CsrType::classical>();
1336 }
1337 }
1338 }
1339 result->set_strategy(new_strat);
1340 }
1341
1345 void make_srow()
1346 {
1347 srow_.resize_and_reset(strategy_->clac_size(values_.get_num_elems()));
1348 strategy_->process(row_ptrs_, &srow_);
1349 }
1350
1357 virtual void scale_impl(const LinOp* alpha);
1358
1365 virtual void inv_scale_impl(const LinOp* alpha);
1366
1367private:
1368 array<value_type> values_;
1369 array<index_type> col_idxs_;
1370 array<index_type> row_ptrs_;
1371 array<index_type> srow_;
1372 std::shared_ptr<strategy_type> strategy_;
1373
1374 void add_scaled_identity_impl(const LinOp* a, const LinOp* b) override;
1375};
1376
1377
1378namespace detail {
1379
1380
1387template <typename ValueType, typename IndexType>
1388void strategy_rebuild_helper(Csr<ValueType, IndexType>* result)
1389{
1390 using load_balance = typename Csr<ValueType, IndexType>::load_balance;
1391 using automatical = typename Csr<ValueType, IndexType>::automatical;
1392 auto strategy = result->get_strategy();
1393 auto executor = result->get_executor();
1394 if (std::dynamic_pointer_cast<load_balance>(strategy)) {
1395 if (auto exec =
1396 std::dynamic_pointer_cast<const HipExecutor>(executor)) {
1397 result->set_strategy(std::make_shared<load_balance>(exec));
1398 } else if (auto exec = std::dynamic_pointer_cast<const CudaExecutor>(
1399 executor)) {
1400 result->set_strategy(std::make_shared<load_balance>(exec));
1401 }
1402 } else if (std::dynamic_pointer_cast<automatical>(strategy)) {
1403 if (auto exec =
1404 std::dynamic_pointer_cast<const HipExecutor>(executor)) {
1405 result->set_strategy(std::make_shared<automatical>(exec));
1406 } else if (auto exec = std::dynamic_pointer_cast<const CudaExecutor>(
1407 executor)) {
1408 result->set_strategy(std::make_shared<automatical>(exec));
1409 }
1410 }
1411}
1412
1413
1414} // namespace detail
1415} // namespace matrix
1416} // namespace gko
1417
1418
1419#endif // GKO_PUBLIC_CORE_MATRIX_CSR_HPP_
ConvertibleTo interface is used to mark that the implementer can be converted to the object of Result...
Definition polymorphic_object.hpp:499
This is the Executor subclass which represents the CUDA device.
Definition executor.hpp:1513
The diagonal of a LinOp implementing this interface can be extracted.
Definition lin_op.hpp:772
The EnableAbsoluteComputation mixin provides the default implementations of compute_absolute_linop an...
Definition lin_op.hpp:823
This mixin implements a static create() method on ConcreteType that dynamically allocates the memory,...
Definition polymorphic_object.hpp:776
The EnableLinOp mixin can be used to provide sensible default implementations of the majority of the ...
Definition lin_op.hpp:908
This mixin inherits from (a subclass of) PolymorphicObject and provides a base implementation of a ne...
Definition polymorphic_object.hpp:691
Definition lin_op.hpp:146
This is the Executor subclass which represents the OpenMP device (typically CPU).
Definition executor.hpp:1366
Linear operators which support permutation should implement the Permutable interface.
Definition lin_op.hpp:513
std::shared_ptr< const Executor > get_executor() const noexcept
Returns the Executor of the object.
Definition polymorphic_object.hpp:263
A LinOp implementing this interface can read its data from a matrix_data structure.
Definition lin_op.hpp:634
Adds the operation M <- a I + b M for matrix M, identity operator I and scalars a and b,...
Definition lin_op.hpp:847
Linear operators which support transposition should implement the Transposable interface.
Definition lin_op.hpp:462
A LinOp implementing this interface can write its data to a matrix_data structure.
Definition lin_op.hpp:689
An array is a container which encapsulates fixed-sized arrays, stored on the Executor tied to the arr...
Definition array.hpp:187
value_type * get_data() noexcept
Returns a pointer to the block of memory used to store the elements of the array.
Definition array.hpp:646
const value_type * get_const_data() const noexcept
Returns a constant pointer to the block of memory used to store the elements of the array.
Definition array.hpp:655
void resize_and_reset(size_type num_elems)
Resizes the array so it is able to hold the specified number of elements.
Definition array.hpp:603
void fill(const value_type value)
Fill the array with the given value.
size_type get_num_elems() const noexcept
Returns the number of elements in the array.
Definition array.hpp:637
This type is a device-side equivalent to matrix_data.
Definition device_matrix_data.hpp:63
An index set class represents an ordered set of intervals.
Definition index_set.hpp:85
COO stores a matrix in the coordinate matrix format.
Definition coo.hpp:87
Definition csr.hpp:77
Definition csr.hpp:542
std::shared_ptr< strategy_type > copy() override
Copy a strategy.
Definition csr.hpp:707
automatical(int64_t nwarps, int warp_size=32, bool cuda_strategy=true, std::string strategy_name="none")
Creates an automatical strategy with specified parameters.
Definition csr.hpp:615
automatical()
Creates an automatical strategy.
Definition csr.hpp:570
int64_t clac_size(const int64_t nnz) override
Computes the srow size according to the number of nonzeros.
Definition csr.hpp:695
automatical(std::shared_ptr< const CudaExecutor > exec)
Creates an automatical strategy with CUDA executor.
Definition csr.hpp:580
void process(const array< index_type > &mtx_row_ptrs, array< index_type > *mtx_srow) override
Computes srow according to row pointers.
Definition csr.hpp:626
automatical(std::shared_ptr< const DpcppExecutor > exec)
Creates an automatical strategy with Dpcpp executor.
Definition csr.hpp:600
automatical(std::shared_ptr< const HipExecutor > exec)
Creates an automatical strategy with HIP executor.
Definition csr.hpp:589
classical is a strategy_type which uses the same number of threads on each row.
Definition csr.hpp:253
void process(const array< index_type > &mtx_row_ptrs, array< index_type > *mtx_srow) override
Computes srow according to row pointers.
Definition csr.hpp:260
std::shared_ptr< strategy_type > copy() override
Copy a strategy.
Definition csr.hpp:289
classical()
Creates a classical strategy.
Definition csr.hpp:258
int64_t clac_size(const int64_t nnz) override
Computes the srow size according to the number of nonzeros.
Definition csr.hpp:282
cusparse is a strategy_type which uses the sparselib csr.
Definition csr.hpp:328
int64_t clac_size(const int64_t nnz) override
Computes the srow size according to the number of nonzeros.
Definition csr.hpp:339
std::shared_ptr< strategy_type > copy() override
Copy a strategy.
Definition csr.hpp:341
cusparse()
Creates a cusparse strategy.
Definition csr.hpp:333
void process(const array< index_type > &mtx_row_ptrs, array< index_type > *mtx_srow) override
Computes srow according to row pointers.
Definition csr.hpp:335
load_balance is a strategy_type which uses the load balance algorithm.
Definition csr.hpp:374
void process(const array< index_type > &mtx_row_ptrs, array< index_type > *mtx_srow) override
Computes srow according to row pointers.
Definition csr.hpp:437
std::shared_ptr< strategy_type > copy() override
Copy a strategy.
Definition csr.hpp:529
load_balance(std::shared_ptr< const HipExecutor > exec)
Creates a load_balance strategy with HIP executor.
Definition csr.hpp:401
load_balance()
Creates a load_balance strategy.
Definition csr.hpp:382
int64_t clac_size(const int64_t nnz) override
Computes the srow size according to the number of nonzeros.
Definition csr.hpp:490
load_balance(int64_t nwarps, int warp_size=32, bool cuda_strategy=true, std::string strategy_name="none")
Creates a load_balance strategy with specified parameters.
Definition csr.hpp:427
load_balance(std::shared_ptr< const CudaExecutor > exec)
Creates a load_balance strategy with CUDA executor.
Definition csr.hpp:392
load_balance(std::shared_ptr< const DpcppExecutor > exec)
Creates a load_balance strategy with DPCPP executor.
Definition csr.hpp:412
merge_path is a strategy_type which uses the merge_path algorithm.
Definition csr.hpp:303
int64_t clac_size(const int64_t nnz) override
Computes the srow size according to the number of nonzeros.
Definition csr.hpp:314
std::shared_ptr< strategy_type > copy() override
Copy a strategy.
Definition csr.hpp:316
merge_path()
Creates a merge_path strategy.
Definition csr.hpp:308
void process(const array< index_type > &mtx_row_ptrs, array< index_type > *mtx_srow) override
Computes srow according to row pointers.
Definition csr.hpp:310
sparselib is a strategy_type which uses the sparselib csr.
Definition csr.hpp:352
int64_t clac_size(const int64_t nnz) override
Computes the srow size according to the number of nonzeros.
Definition csr.hpp:363
void process(const array< index_type > &mtx_row_ptrs, array< index_type > *mtx_srow) override
Computes srow according to row pointers.
Definition csr.hpp:359
sparselib()
Creates a sparselib strategy.
Definition csr.hpp:357
std::shared_ptr< strategy_type > copy() override
Copy a strategy.
Definition csr.hpp:365
strategy_type is to decide how to set the csr algorithm.
Definition csr.hpp:196
virtual int64_t clac_size(const int64_t nnz)=0
Computes the srow size according to the number of nonzeros.
std::string get_name()
Returns the name of strategy.
Definition csr.hpp:214
virtual std::shared_ptr< strategy_type > copy()=0
Copy a strategy.
virtual void process(const array< index_type > &mtx_row_ptrs, array< index_type > *mtx_srow)=0
Computes srow according to row pointers.
strategy_type(std::string name)
Creates a strategy_type.
Definition csr.hpp:205
CSR is a matrix format which stores only the nonzero coefficients by compressing each row of the matr...
Definition csr.hpp:146
std::unique_ptr< LinOp > column_permute(const array< IndexType > *permutation_indices) const override
Returns a LinOp representing the column permutation of the Permutable object.
Csr & operator=(const Csr &)
Copy-assigns a Csr matrix.
std::unique_ptr< Csr > scale_permute(ptr_param< const ScaledPermutation< value_type, index_type > > permutation, permute_mode=permute_mode::symmetric) const
Creates a scaled and permuted copy of this matrix.
void write(mat_data &data) const override
Writes a matrix to a matrix_data structure.
std::unique_ptr< absolute_type > compute_absolute() const override
Gets the AbsoluteLinOp.
const index_type * get_const_row_ptrs() const noexcept
Returns the row pointers of the matrix.
Definition csr.hpp:925
std::unique_ptr< Csr< ValueType, IndexType > > create_submatrix(const span &row_span, const span &column_span) const
Creates a submatrix from this Csr matrix given row and column spans.
void read(device_mat_data &&data) override
Reads a matrix from a device_matrix_data structure.
const index_type * get_const_srow() const noexcept
Returns the starting rows.
Definition csr.hpp:944
void set_strategy(std::shared_ptr< strategy_type > strategy)
Set the strategy.
Definition csr.hpp:983
void inv_scale(ptr_param< const LinOp > alpha)
Scales the matrix with the inverse of a scalar.
Definition csr.hpp:1008
void read(const device_mat_data &data) override
Reads a matrix from a device_matrix_data structure.
static std::unique_ptr< const Csr > create_const(std::shared_ptr< const Executor > exec, const dim< 2 > &size, gko::detail::const_array_view< ValueType > &&values, gko::detail::const_array_view< IndexType > &&col_idxs, gko::detail::const_array_view< IndexType > &&row_ptrs, std::shared_ptr< strategy_type > strategy)
Creates a constant (immutable) Csr matrix from a set of constant arrays.
Definition csr.hpp:1028
index_type * get_srow() noexcept
Returns the starting rows.
Definition csr.hpp:935
size_type get_num_srow_elements() const noexcept
Returns the number of the srow stored elements (involved warps)
Definition csr.hpp:954
std::unique_ptr< LinOp > inverse_permute(const array< IndexType > *inverse_permutation_indices) const override
Returns a LinOp representing the symmetric inverse row and column permutation of the Permutable objec...
std::unique_ptr< LinOp > row_permute(const array< IndexType > *permutation_indices) const override
Returns a LinOp representing the row permutation of the Permutable object.
std::unique_ptr< Csr< ValueType, IndexType > > create_submatrix(const index_set< IndexType > &row_index_set, const index_set< IndexType > &column_index_set) const
Creates a submatrix from this Csr matrix given row and column index_set objects.
std::unique_ptr< Diagonal< ValueType > > extract_diagonal() const override
Extracts the diagonal entries of the matrix into a vector.
index_type * get_row_ptrs() noexcept
Returns the row pointers of the matrix.
Definition csr.hpp:916
std::unique_ptr< Csr > permute(ptr_param< const Permutation< index_type > > permutation, permute_mode mode=permute_mode::symmetric) const
Creates a permuted copy of this matrix with the given permutation .
Csr(const Csr &)
Copy-constructs a Csr matrix.
Csr & operator=(Csr &&)
Move-assigns a Csr matrix.
std::unique_ptr< LinOp > transpose() const override
Returns a LinOp representing the transpose of the Transposable object.
const value_type * get_const_values() const noexcept
Returns the values of the matrix.
Definition csr.hpp:887
std::unique_ptr< LinOp > inverse_column_permute(const array< IndexType > *inverse_permutation_indices) const override
Returns a LinOp representing the row permutation of the inverse permuted object.
std::unique_ptr< LinOp > inverse_row_permute(const array< IndexType > *inverse_permutation_indices) const override
Returns a LinOp representing the row permutation of the inverse permuted object.
void compute_absolute_inplace() override
Compute absolute inplace on each element.
size_type get_num_stored_elements() const noexcept
Returns the number of elements explicitly stored in the matrix.
Definition csr.hpp:964
std::shared_ptr< strategy_type > get_strategy() const noexcept
Returns the strategy.
Definition csr.hpp:973
std::unique_ptr< LinOp > permute(const array< IndexType > *permutation_indices) const override
Returns a LinOp representing the symmetric row and column permutation of the Permutable object.
const index_type * get_const_col_idxs() const noexcept
Returns the column indexes of the matrix.
Definition csr.hpp:906
void read(const mat_data &data) override
Reads a matrix from a matrix_data structure.
void sort_by_column_index()
Sorts all (value, col_idx) pairs in each row by column index.
std::unique_ptr< Csr > scale_permute(ptr_param< const ScaledPermutation< value_type, index_type > > row_permutation, ptr_param< const ScaledPermutation< value_type, index_type > > column_permutation, bool invert=false) const
Creates a scaled and permuted copy of this matrix.
void scale(ptr_param< const LinOp > alpha)
Scales the matrix with a scalar.
Definition csr.hpp:995
value_type * get_values() noexcept
Returns the values of the matrix.
Definition csr.hpp:878
static std::unique_ptr< const Csr > create_const(std::shared_ptr< const Executor > exec, const dim< 2 > &size, gko::detail::const_array_view< ValueType > &&values, gko::detail::const_array_view< IndexType > &&col_idxs, gko::detail::const_array_view< IndexType > &&row_ptrs)
This is version of create_const with a default strategy.
Definition csr.hpp:1046
index_type * get_col_idxs() noexcept
Returns the column indexes of the matrix.
Definition csr.hpp:897
Csr(Csr &&)
Move-constructs a Csr matrix.
std::unique_ptr< Csr > permute(ptr_param< const Permutation< index_type > > row_permutation, ptr_param< const Permutation< index_type > > column_permutation, bool invert=false) const
Creates a non-symmetrically permuted copy of this matrix with the given row and column permutations...
std::unique_ptr< LinOp > conj_transpose() const override
Returns a LinOp representing the conjugate transpose of the Transposable object.
Dense is a matrix format which explicitly stores all values of the matrix.
Definition dense.hpp:136
This class is a utility which efficiently implements the diagonal matrix (a linear operator which sca...
Definition diagonal.hpp:79
ELL is a matrix format where stride with explicit zeros is used such that all rows have the same numb...
Definition ell.hpp:89
Fixed-block compressed sparse row storage matrix format.
Definition fbcsr.hpp:138
HYBRID is a matrix format which splits the matrix into ELLPACK and COO format.
Definition hybrid.hpp:81
Permutation is a matrix format that represents a permutation matrix, i.e.
Definition permutation.hpp:142
ScaledPermutation is a matrix combining a permutation with scaling factors.
Definition scaled_permutation.hpp:67
SELL-P is a matrix format similar to ELL format.
Definition sellp.hpp:80
SparsityCsr is a matrix format which stores only the sparsity pattern of a sparse matrix by compressi...
Definition sparsity_csr.hpp:87
This class is used for function parameters in the place of raw pointers.
Definition utils_helper.hpp:71
permute_mode
Specifies how a permutation will be applied to a matrix.
Definition permutation.hpp:71
@ symmetric
The rows and columns will be permuted.
The Ginkgo namespace.
Definition abstract_factory.hpp:48
constexpr T one()
Returns the multiplicative identity for T.
Definition math.hpp:803
typename detail::remove_complex_s< T >::type remove_complex
Obtain the type which removed the complex of complex/scalar type or the template parameter of class b...
Definition math.hpp:354
typename detail::next_precision_impl< T >::type next_precision
Obtains the next type in the singly-linked precision list.
Definition math.hpp:490
typename detail::to_complex_s< T >::type to_complex
Obtain the type which adds the complex of complex/scalar type or the template parameter of class by a...
Definition math.hpp:373
constexpr int64 ceildiv(int64 num, int64 den)
Performs integer division with rounding up.
Definition math.hpp:641
std::size_t size_type
Integral type used for allocation quantities.
Definition types.hpp:120
constexpr T min(const T &x, const T &y)
Returns the smaller of the arguments.
Definition math.hpp:891
detail::temporary_clone< detail::pointee< Ptr > > make_temporary_clone(std::shared_ptr< const Executor > exec, Ptr &&ptr)
Creates a temporary_clone.
Definition temporary_clone.hpp:207
A type representing the dimensions of a multidimensional object.
Definition dim.hpp:55
This structure is used as an intermediate data type to store a sparse matrix.
Definition matrix_data.hpp:155
A span is a lightweight structure used to create sub-ranges from other ranges.
Definition range.hpp:75