From 1f49b915db36866b9d91aa1082f71a47494cb53c Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Thu, 18 Jul 2024 16:21:32 +0000 Subject: [PATCH] [coll-comm] review update: - fix include guards - update docs - implement copy/move constructors/assignment with tests - add equality test for collective communicators (needed for testing) - always enable neighborhood comm, just throw if openmpi is too old Co-authored-by: Pratik Nayak --- core/CMakeLists.txt | 6 +- core/distributed/dense_communicator.cpp | 43 +++++ core/distributed/device_partition.hpp | 6 +- .../distributed/neighborhood_communicator.cpp | 59 +++++++ .../mpi/distributed/dense_communicator.cpp | 155 +++++++++++++----- .../distributed/neighborhood_communicator.cpp | 92 +++++++++-- include/ginkgo/core/base/types.hpp | 2 +- .../distributed/collective_communicator.hpp | 5 +- .../core/distributed/dense_communicator.hpp | 40 ++++- .../ginkgo/core/distributed/index_map_fwd.hpp | 6 +- .../distributed/neighborhood_communicator.hpp | 56 +++++-- 11 files changed, 382 insertions(+), 88 deletions(-) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 5e2a506f448..f7f1e00f17b 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -133,14 +133,10 @@ if(GINKGO_BUILD_MPI) distributed/collective_communicator.cpp distributed/dense_communicator.cpp distributed/matrix.cpp + distributed/neighborhood_communicator.cpp distributed/partition_helpers.cpp distributed/vector.cpp distributed/preconditioner/schwarz.cpp) - if(NOT GINKGO_HAVE_OPENMPI_PRE_4_1_X) - target_sources(${ginkgo_core} - PRIVATE - distributed/neighborhood_communicator.cpp) - endif() endif() # MSVC/shared: make ginkgo be the major library diff --git a/core/distributed/dense_communicator.cpp b/core/distributed/dense_communicator.cpp index f097e996aa5..ea1e3652f31 100644 --- a/core/distributed/dense_communicator.cpp +++ b/core/distributed/dense_communicator.cpp @@ -60,6 +60,27 @@ GKO_INSTANTIATE_FOR_EACH_LOCAL_GLOBAL_INDEX_TYPE(GKO_DECLARE_DENSE_CONSTRUCTOR); #undef GKO_DECLARE_DENSE_CONSTRUCTOR +DenseCommunicator::DenseCommunicator(DenseCommunicator&& other) noexcept + : DenseCommunicator(other.get_base_communicator()) +{ + *this = std::move(other); +} + + +DenseCommunicator& DenseCommunicator::operator=( + DenseCommunicator&& other) noexcept +{ + if (this != &other) { + *this = other; + std::fill(other.send_sizes_.begin(), other.send_sizes_.end(), 0); + std::fill(other.send_offsets_.begin(), other.send_offsets_.end(), 0); + std::fill(other.recv_sizes_.begin(), other.recv_sizes_.end(), 0); + std::fill(other.recv_offsets_.begin(), other.recv_offsets_.end(), 0); + } + return *this; +} + + DenseCommunicator::DenseCommunicator( communicator base, const std::vector& recv_sizes, const std::vector& recv_offsets, @@ -125,6 +146,28 @@ comm_index_type DenseCommunicator::get_send_size() const } +bool operator==(const DenseCommunicator& a, const DenseCommunicator& b) +{ + return (a.comm_.is_identical(b.comm_) || a.comm_.is_congruent(b.comm_)) && + a.send_sizes_.size() == b.send_sizes_.size() && + a.recv_sizes_.size() == b.recv_sizes_.size() && + std::equal(a.send_sizes_.begin(), a.send_sizes_.end(), + b.send_sizes_.begin()) && + std::equal(a.recv_sizes_.begin(), a.recv_sizes_.end(), + b.recv_sizes_.begin()) && + std::equal(a.send_offsets_.begin(), a.send_offsets_.end(), + b.send_offsets_.begin()) && + std::equal(a.recv_offsets_.begin(), a.recv_offsets_.end(), + b.recv_offsets_.begin()); +} + + +bool operator!=(const DenseCommunicator& a, const DenseCommunicator& b) +{ + return !(a == b); +} + + } // namespace mpi } // namespace experimental } // namespace gko diff --git a/core/distributed/device_partition.hpp b/core/distributed/device_partition.hpp index ba91019603f..e9c6cc255c6 100644 --- a/core/distributed/device_partition.hpp +++ b/core/distributed/device_partition.hpp @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: BSD-3-Clause -#ifndef GINKGO_PARTITION_HPP -#define GINKGO_PARTITION_HPP +#ifndef GKO_CORE_DISTRIBUTED_PARTITION_HPP +#define GKO_CORE_DISTRIBUTED_PARTITION_HPP #include @@ -89,4 +89,4 @@ to_device_const( } // namespace gko -#endif // GINKGO_PARTITION_HPP +#endif // GKO_CORE_DISTRIBUTED_PARTITION_HPP diff --git a/core/distributed/neighborhood_communicator.cpp b/core/distributed/neighborhood_communicator.cpp index 40e75b16ade..55af52d89f5 100644 --- a/core/distributed/neighborhood_communicator.cpp +++ b/core/distributed/neighborhood_communicator.cpp @@ -156,6 +156,9 @@ request NeighborhoodCommunicator::i_all_to_all_v( std::shared_ptr exec, const void* send_buffer, MPI_Datatype send_type, void* recv_buffer, MPI_Datatype recv_type) const { +#if GINKGO_HAVE_OPENMPI_PRE_4_1_X + GKO_NOT_IMPLEMENTED; +#else auto guard = exec->get_scoped_device_id_guard(); request req; GKO_ASSERT_NO_MPI_ERRORS(MPI_Ineighbor_alltoallv( @@ -163,6 +166,7 @@ request NeighborhoodCommunicator::i_all_to_all_v( recv_buffer, recv_sizes_.data(), recv_offsets_.data(), recv_type, comm_.get(), req.get())); return req; +#endif } @@ -179,6 +183,61 @@ NeighborhoodCommunicator::create_with_same_type( } +NeighborhoodCommunicator::NeighborhoodCommunicator( + NeighborhoodCommunicator&& other) noexcept + : NeighborhoodCommunicator(other.get_base_communicator()) +{ + *this = std::move(other); +} + + +NeighborhoodCommunicator& NeighborhoodCommunicator::operator=( + NeighborhoodCommunicator&& other) noexcept +{ + if (this != &other) { + comm_ = std::exchange(other.comm_, MPI_COMM_SELF); + // set topology for other comm + std::vector non_nullptr(1); + non_nullptr.resize(0); + other.comm_ = create_neighborhood_comm(this->get_base_communicator(), + non_nullptr, non_nullptr); + send_sizes_ = std::exchange( + other.send_sizes_, std::vector{}); + send_offsets_ = std::exchange( + other.send_offsets_, std::vector{0}); + recv_sizes_ = std::exchange( + other.recv_sizes_, std::vector{}); + recv_offsets_ = std::exchange( + other.recv_offsets_, std::vector{0}); + } + return *this; +} + + +bool operator==(const NeighborhoodCommunicator& a, + const NeighborhoodCommunicator& b) +{ + return (a.comm_.is_identical(b.comm_) || a.comm_.is_congruent(b.comm_)) && + a.send_sizes_.size() == b.send_sizes_.size() && + a.recv_sizes_.size() == b.recv_sizes_.size() && + std::equal(a.send_sizes_.begin(), a.send_sizes_.end(), + b.send_sizes_.begin()) && + std::equal(a.recv_sizes_.begin(), a.recv_sizes_.end(), + b.recv_sizes_.begin()) && + std::equal(a.send_offsets_.begin(), a.send_offsets_.end(), + b.send_offsets_.begin()) && + std::equal(a.recv_offsets_.begin(), a.recv_offsets_.end(), + b.recv_offsets_.begin()); +} + + +bool operator!=(const NeighborhoodCommunicator& a, + const NeighborhoodCommunicator& b) +{ + return !(a == b); +} + + template NeighborhoodCommunicator::NeighborhoodCommunicator( communicator base, diff --git a/core/test/mpi/distributed/dense_communicator.cpp b/core/test/mpi/distributed/dense_communicator.cpp index 1a868a047b9..8bb11b944d1 100644 --- a/core/test/mpi/distributed/dense_communicator.cpp +++ b/core/test/mpi/distributed/dense_communicator.cpp @@ -17,6 +17,22 @@ class DenseCommunicator : public ::testing::Test { void SetUp() override { ASSERT_EQ(comm.size(), 6); } + gko::experimental::mpi::DenseCommunicator create_default_comm() + { + auto part = gko::share(part_type::build_from_global_size_uniform( + ref, comm.size(), comm.size() * 3)); + gko::array recv_connections[] = {{ref, {3, 5, 10, 11}}, + {ref, {0, 1, 7, 12, 13}}, + {ref, {3, 4, 17}}, + {ref, {1, 2, 12, 14}}, + {ref, {4, 5, 9, 10, 16, 15}}, + {ref, {8, 12, 13, 14}}}; + auto imap = map_type{ref, part, comm.rank(), recv_connections[rank]}; + + + return {comm, imap}; + } + std::shared_ptr ref = gko::ReferenceExecutor::create(); gko::experimental::mpi::communicator comm = MPI_COMM_WORLD; int rank = comm.rank(); @@ -25,11 +41,11 @@ class DenseCommunicator : public ::testing::Test { TEST_F(DenseCommunicator, CanDefaultConstruct) { - gko::experimental::mpi::DenseCommunicator nhcomm{comm}; + gko::experimental::mpi::DenseCommunicator dcomm{comm}; - ASSERT_EQ(nhcomm.get_base_communicator(), comm); - ASSERT_EQ(nhcomm.get_send_size(), 0); - ASSERT_EQ(nhcomm.get_recv_size(), 0); + ASSERT_EQ(dcomm.get_base_communicator(), comm); + ASSERT_EQ(dcomm.get_send_size(), 0); + ASSERT_EQ(dcomm.get_recv_size(), 0); } @@ -45,11 +61,11 @@ TEST_F(DenseCommunicator, CanConstructFromIndexMap) {ref, {8, 12, 13, 14}}}; auto imap = map_type{ref, part, comm.rank(), recv_connections[rank]}; - gko::experimental::mpi::DenseCommunicator spcomm{comm, imap}; + gko::experimental::mpi::DenseCommunicator dcomm{comm, imap}; std::array send_sizes = {4, 6, 2, 4, 7, 3}; - ASSERT_EQ(spcomm.get_recv_size(), recv_connections[rank].get_size()); - ASSERT_EQ(spcomm.get_send_size(), send_sizes[rank]); + ASSERT_EQ(dcomm.get_recv_size(), recv_connections[rank].get_size()); + ASSERT_EQ(dcomm.get_send_size(), send_sizes[rank]); } @@ -90,12 +106,12 @@ TEST_F(DenseCommunicator, CanConstructFromEnvelopData) std::partial_sum(send_sizes[rank].begin(), send_sizes[rank].end(), send_offsets.begin() + 1); - gko::experimental::mpi::DenseCommunicator spcomm{ + gko::experimental::mpi::DenseCommunicator dcomm{ comm, recv_sizes[rank], recv_offsets, send_sizes[rank], send_offsets, }; - ASSERT_EQ(spcomm.get_recv_size(), recv_offsets.back()); - ASSERT_EQ(spcomm.get_send_size(), send_offsets.back()); + ASSERT_EQ(dcomm.get_recv_size(), recv_offsets.back()); + ASSERT_EQ(dcomm.get_send_size(), send_offsets.back()); } @@ -103,10 +119,10 @@ TEST_F(DenseCommunicator, CanConstructFromEmptyIndexMap) { auto imap = map_type{ref}; - gko::experimental::mpi::DenseCommunicator spcomm{comm, imap}; + gko::experimental::mpi::DenseCommunicator dcomm{comm, imap}; - ASSERT_EQ(spcomm.get_recv_size(), 0); - ASSERT_EQ(spcomm.get_send_size(), 0); + ASSERT_EQ(dcomm.get_recv_size(), 0); + ASSERT_EQ(dcomm.get_send_size(), 0); } @@ -116,10 +132,10 @@ TEST_F(DenseCommunicator, CanConstructFromIndexMapWithoutConnection) ref, comm.size(), comm.size() * 3)); auto imap = map_type{ref, part, comm.rank(), {ref, 0}}; - gko::experimental::mpi::DenseCommunicator spcomm{comm, imap}; + gko::experimental::mpi::DenseCommunicator dcomm{comm, imap}; - ASSERT_EQ(spcomm.get_recv_size(), 0); - ASSERT_EQ(spcomm.get_send_size(), 0); + ASSERT_EQ(dcomm.get_recv_size(), 0); + ASSERT_EQ(dcomm.get_send_size(), 0); } @@ -130,12 +146,78 @@ TEST_F(DenseCommunicator, CanConstructFromEmptyEnvelopData) std::vector recv_offsets{0}; std::vector send_offsets{0}; - gko::experimental::mpi::DenseCommunicator spcomm{ + gko::experimental::mpi::DenseCommunicator dcomm{ comm, recv_sizes, recv_offsets, send_sizes, send_offsets, }; - ASSERT_EQ(spcomm.get_recv_size(), 0); - ASSERT_EQ(spcomm.get_send_size(), 0); + ASSERT_EQ(dcomm.get_recv_size(), 0); + ASSERT_EQ(dcomm.get_send_size(), 0); +} + + +TEST_F(DenseCommunicator, CanTestEquality) +{ + auto comm_a = create_default_comm(); + auto comm_b = create_default_comm(); + + ASSERT_EQ(comm_a, comm_b); +} + + +TEST_F(DenseCommunicator, CanTestInequality) +{ + auto comm_a = create_default_comm(); + auto comm_b = gko::experimental::mpi::DenseCommunicator(comm); + + ASSERT_NE(comm_a, comm_b); +} + + +TEST_F(DenseCommunicator, CanCopyConstruct) +{ + auto dcomm = create_default_comm(); + + auto copy(dcomm); + + ASSERT_TRUE(copy == dcomm); +} + + +TEST_F(DenseCommunicator, CanCopyAssign) +{ + auto dcomm = create_default_comm(); + gko::experimental::mpi::DenseCommunicator copy{comm}; + + copy = dcomm; + + ASSERT_TRUE(copy == dcomm); +} + + +TEST_F(DenseCommunicator, CanMoveConstruct) +{ + auto dcomm = create_default_comm(); + auto moved_from = dcomm; + auto empty_comm = gko::experimental::mpi::DenseCommunicator{comm}; + + auto moved(std::move(moved_from)); + + ASSERT_TRUE(moved == dcomm); + ASSERT_TRUE(moved_from == empty_comm); +} + + +TEST_F(DenseCommunicator, CanMoveAssign) +{ + auto dcomm = create_default_comm(); + auto moved_from = dcomm; + auto empty_comm = gko::experimental::mpi::DenseCommunicator{comm}; + gko::experimental::mpi::DenseCommunicator moved{comm}; + + moved = std::move(moved_from); + + ASSERT_TRUE(moved == dcomm); + ASSERT_TRUE(moved_from == empty_comm); } @@ -150,7 +232,7 @@ TEST_F(DenseCommunicator, CanCommunicateIalltoall) {ref, {4, 5, 9, 10, 16, 15}}, {ref, {8, 12, 13, 14}}}; auto imap = map_type{ref, part, comm.rank(), recv_connections[rank]}; - gko::experimental::mpi::DenseCommunicator spcomm{comm, imap}; + gko::experimental::mpi::DenseCommunicator dcomm{comm, imap}; gko::array recv_buffer{ref, recv_connections[rank].get_size()}; gko::array send_buffers[] = {{ref, {0, 1, 1, 2}}, {ref, {3, 5, 3, 4, 4, 5}}, @@ -159,8 +241,8 @@ TEST_F(DenseCommunicator, CanCommunicateIalltoall) {ref, {12, 13, 12, 14, 12, 13, 14}}, {ref, {17, 16, 15}}}; - auto req = spcomm.i_all_to_all_v(ref, send_buffers[rank].get_const_data(), - recv_buffer.get_data()); + auto req = dcomm.i_all_to_all_v(ref, send_buffers[rank].get_const_data(), + recv_buffer.get_data()); req.wait(); GKO_ASSERT_ARRAY_EQ(recv_buffer, recv_connections[rank]); @@ -169,31 +251,22 @@ TEST_F(DenseCommunicator, CanCommunicateIalltoall) TEST_F(DenseCommunicator, CanCommunicateIalltoallWhenEmpty) { - gko::experimental::mpi::DenseCommunicator spcomm{comm}; + gko::experimental::mpi::DenseCommunicator dcomm{comm}; - auto req = spcomm.i_all_to_all_v(ref, static_cast(nullptr), - static_cast(nullptr)); + auto req = dcomm.i_all_to_all_v(ref, static_cast(nullptr), + static_cast(nullptr)); req.wait(); } TEST_F(DenseCommunicator, CanCreateInverse) { - auto part = gko::share(part_type::build_from_global_size_uniform( - ref, comm.size(), comm.size() * 3)); - gko::array recv_connections[] = {{ref, {3, 5, 10, 11}}, - {ref, {0, 1, 7, 12, 13}}, - {ref, {3, 4, 17}}, - {ref, {1, 2, 12, 14}}, - {ref, {4, 5, 9, 10, 16, 15}}, - {ref, {8, 12, 13, 14}}}; - auto imap = map_type{ref, part, comm.rank(), recv_connections[rank]}; - gko::experimental::mpi::DenseCommunicator spcomm{comm, imap}; + auto dcomm = create_default_comm(); - auto inverse = spcomm.create_inverse(); + auto inverse = dcomm.create_inverse(); - ASSERT_EQ(inverse->get_recv_size(), spcomm.get_send_size()); - ASSERT_EQ(inverse->get_send_size(), spcomm.get_recv_size()); + ASSERT_EQ(inverse->get_recv_size(), dcomm.get_send_size()); + ASSERT_EQ(inverse->get_send_size(), dcomm.get_recv_size()); } @@ -208,8 +281,8 @@ TEST_F(DenseCommunicator, CanCommunicateRoundTrip) {ref, {4, 5, 9, 10, 16, 15}}, {ref, {8, 12, 13, 14}}}; auto imap = map_type{ref, part, comm.rank(), recv_connections[rank]}; - gko::experimental::mpi::DenseCommunicator spcomm{comm, imap}; - auto inverse = spcomm.create_inverse(); + gko::experimental::mpi::DenseCommunicator dcomm{comm, imap}; + auto inverse = dcomm.create_inverse(); gko::array send_buffers[] = {{ref, {1, 2, 3, 4}}, {ref, {5, 6, 7, 8, 9, 10}}, {ref, {11, 12}}, @@ -219,7 +292,7 @@ TEST_F(DenseCommunicator, CanCommunicateRoundTrip) gko::array recv_buffer{ref, recv_connections[rank].get_size()}; gko::array round_trip{ref, send_buffers[rank].get_size()}; - spcomm + dcomm .i_all_to_all_v(ref, send_buffers[rank].get_const_data(), recv_buffer.get_data()) .wait(); diff --git a/core/test/mpi/distributed/neighborhood_communicator.cpp b/core/test/mpi/distributed/neighborhood_communicator.cpp index 07363a755eb..7a72e1206b1 100644 --- a/core/test/mpi/distributed/neighborhood_communicator.cpp +++ b/core/test/mpi/distributed/neighborhood_communicator.cpp @@ -17,6 +17,21 @@ class NeighborhoodCommunicator : public ::testing::Test { void SetUp() override { ASSERT_EQ(comm.size(), 6); } + gko::experimental::mpi::NeighborhoodCommunicator create_default_comm() + { + auto part = gko::share(part_type::build_from_global_size_uniform( + ref, comm.size(), comm.size() * 3)); + gko::array recv_connections[] = {{ref, {3, 5, 10, 11}}, + {ref, {0, 1, 7, 12, 13}}, + {ref, {3, 4, 17}}, + {ref, {1, 2, 12, 14}}, + {ref, {4, 5, 9, 10, 16, 15}}, + {ref, {8, 12, 13, 14}}}; + auto imap = map_type{ref, part, comm.rank(), recv_connections[rank]}; + + return {comm, imap}; + } + std::shared_ptr ref = gko::ReferenceExecutor::create(); gko::experimental::mpi::communicator comm = MPI_COMM_WORLD; int rank = comm.rank(); @@ -120,6 +135,72 @@ TEST_F(NeighborhoodCommunicator, CanConstructFromEmptyEnvelopData) } +TEST_F(NeighborhoodCommunicator, CanTestEquality) +{ + auto comm_a = create_default_comm(); + auto comm_b = create_default_comm(); + + ASSERT_EQ(comm_a, comm_b); +} + + +TEST_F(NeighborhoodCommunicator, CanTestInequality) +{ + auto comm_a = create_default_comm(); + auto comm_b = gko::experimental::mpi::NeighborhoodCommunicator(comm); + + ASSERT_NE(comm_a, comm_b); +} + + +TEST_F(NeighborhoodCommunicator, CanCopyConstruct) +{ + auto spcomm = create_default_comm(); + + auto copy(spcomm); + + ASSERT_TRUE(copy == spcomm); +} + + +TEST_F(NeighborhoodCommunicator, CanCopyAssign) +{ + auto spcomm = create_default_comm(); + gko::experimental::mpi::NeighborhoodCommunicator copy{comm}; + + copy = spcomm; + + ASSERT_TRUE(copy == spcomm); +} + + +TEST_F(NeighborhoodCommunicator, CanMoveConstruct) +{ + auto spcomm = create_default_comm(); + auto moved_from = spcomm; + auto empty_comm = gko::experimental::mpi::NeighborhoodCommunicator{comm}; + + auto moved(std::move(moved_from)); + + ASSERT_TRUE(moved == spcomm); + ASSERT_TRUE(moved_from == empty_comm); +} + + +TEST_F(NeighborhoodCommunicator, CanMoveAssign) +{ + auto spcomm = create_default_comm(); + auto moved_from = spcomm; + auto empty_comm = gko::experimental::mpi::NeighborhoodCommunicator{comm}; + gko::experimental::mpi::NeighborhoodCommunicator moved{comm}; + + moved = std::move(moved_from); + + ASSERT_TRUE(moved == spcomm); + ASSERT_TRUE(moved_from == empty_comm); +} + + TEST_F(NeighborhoodCommunicator, CanCommunicateIalltoall) { auto part = gko::share(part_type::build_from_global_size_uniform( @@ -160,16 +241,7 @@ TEST_F(NeighborhoodCommunicator, CanCommunicateIalltoallWhenEmpty) TEST_F(NeighborhoodCommunicator, CanCreateInverse) { - auto part = gko::share(part_type::build_from_global_size_uniform( - ref, comm.size(), comm.size() * 3)); - gko::array recv_connections[] = {{ref, {3, 5, 10, 11}}, - {ref, {0, 1, 7, 12, 13}}, - {ref, {3, 4, 17}}, - {ref, {1, 2, 12, 14}}, - {ref, {4, 5, 9, 10, 16, 15}}, - {ref, {8, 12, 13, 14}}}; - auto imap = map_type{ref, part, comm.rank(), recv_connections[rank]}; - gko::experimental::mpi::NeighborhoodCommunicator spcomm{comm, imap}; + auto spcomm = create_default_comm(); auto inverse = spcomm.create_inverse(); diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp index 6d8e2f3e928..3783d2c4b41 100644 --- a/include/ginkgo/core/base/types.hpp +++ b/include/ginkgo/core/base/types.hpp @@ -813,7 +813,7 @@ namespace distributed { /** - * Make mpi::comm_index_type avaiable in this namespace + * Make mpi::comm_index_type available in this namespace */ using mpi::comm_index_type; diff --git a/include/ginkgo/core/distributed/collective_communicator.hpp b/include/ginkgo/core/distributed/collective_communicator.hpp index f325283d42b..bdae36faadc 100644 --- a/include/ginkgo/core/distributed/collective_communicator.hpp +++ b/include/ginkgo/core/distributed/collective_communicator.hpp @@ -36,8 +36,9 @@ class CollectiveCommunicator { /** * Non-blocking all-to-all communication. * - * The send_buffer must have size get_send_size, and the recv_buffer - * must have size get_recv_size. + * The send_buffer must have allocated at least get_send_size number of + * elements, and the recv_buffer must have allocated at least get_recv_size + * number of elements. * * @tparam SendType the type of the elements to send * @tparam RecvType the type of the elements to receive diff --git a/include/ginkgo/core/distributed/dense_communicator.hpp b/include/ginkgo/core/distributed/dense_communicator.hpp index 20a8ad99114..d0d902f9792 100644 --- a/include/ginkgo/core/distributed/dense_communicator.hpp +++ b/include/ginkgo/core/distributed/dense_communicator.hpp @@ -24,15 +24,20 @@ namespace mpi { /** * A collective_communicator that uses a dense communication. * - * The neighborhood communicator is defined by a list of neighbors this - * rank sends data to and a list of neighbors this rank receives data from. - * No communication with any ranks that is not in one of those lists will - * take place. + * The dense communicator uses the MPI_Alltoall function for its communication. */ class DenseCommunicator final : public CollectiveCommunicator { public: using CollectiveCommunicator::i_all_to_all_v; + DenseCommunicator(const DenseCommunicator& other) = default; + + DenseCommunicator(DenseCommunicator&& other) noexcept; + + DenseCommunicator& operator=(const DenseCommunicator& other) = default; + + DenseCommunicator& operator=(DenseCommunicator&& other) noexcept; + /** * Default constructor with empty communication pattern * @param base the base communicator @@ -40,7 +45,7 @@ class DenseCommunicator final : public CollectiveCommunicator { explicit DenseCommunicator(communicator base); /** - * Create a neighborhood_communicator from an index map. + * Create a DenseCommunicator from an index map. * * The receive neighbors are defined by the remote indices and their * owning ranks of the index map. The send neighbors are deduced @@ -57,7 +62,7 @@ class DenseCommunicator final : public CollectiveCommunicator { const distributed::index_map& imap); /** - * Create a neighborhood_communicator by explicitly defining the + * Create a DenseCommunicator by explicitly defining the * neighborhood lists and sizes/offsets. * * @param base the base communicator @@ -90,7 +95,7 @@ class DenseCommunicator final : public CollectiveCommunicator { const distributed::index_map_variant& imap) const override; /** - * Creates the inverse neighborhood_communicator by switching sources + * Creates the inverse DenseCommunicator by switching sources * and destinations. * * @return collective_communicator with the inverse communication pattern @@ -108,6 +113,27 @@ class DenseCommunicator final : public CollectiveCommunicator { */ [[nodiscard]] comm_index_type get_send_size() const override; + + /** + * Compares two communicators for equality. + * + * Equality is defined as having identical or congruent communicators and + * their communication pattern is equal. No communication is done, i.e. + * there is no reduction over the local equality check results. + * + * @return true if both communicators are equal. + */ + friend bool operator==(const DenseCommunicator& a, + const DenseCommunicator& b); + + /** + * Compares two communicators for inequality. + * + * @see operator== + */ + friend bool operator!=(const DenseCommunicator& a, + const DenseCommunicator& b); + private: communicator comm_; diff --git a/include/ginkgo/core/distributed/index_map_fwd.hpp b/include/ginkgo/core/distributed/index_map_fwd.hpp index 1d40a0008ec..425e06ca16a 100644 --- a/include/ginkgo/core/distributed/index_map_fwd.hpp +++ b/include/ginkgo/core/distributed/index_map_fwd.hpp @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: BSD-3-Clause -#ifndef GINKGO_INDEX_MAP_FWD_HPP -#define GINKGO_INDEX_MAP_FWD_HPP +#ifndef GKO_PUBLIC_CORE_INDEX_MAP_FWD_HPP +#define GKO_PUBLIC_CORE_INDEX_MAP_FWD_HPP #include @@ -27,4 +27,4 @@ using index_map_variant = } // namespace experimental } // namespace gko -#endif // GINKGO_INDEX_MAP_FWD_HPP +#endif // GKO_PUBLIC_CORE_INDEX_MAP_FWD_HPP diff --git a/include/ginkgo/core/distributed/neighborhood_communicator.hpp b/include/ginkgo/core/distributed/neighborhood_communicator.hpp index 3a02780eb49..6b77ffba0c3 100644 --- a/include/ginkgo/core/distributed/neighborhood_communicator.hpp +++ b/include/ginkgo/core/distributed/neighborhood_communicator.hpp @@ -9,7 +9,7 @@ #include -#if GINKGO_BUILD_MPI && !GINKGO_HAVE_OPENMPI_PRE_4_1_X +#if GINKGO_BUILD_MPI #include #include @@ -33,6 +33,16 @@ class NeighborhoodCommunicator final : public CollectiveCommunicator { public: using CollectiveCommunicator::i_all_to_all_v; + NeighborhoodCommunicator(const NeighborhoodCommunicator& other) = default; + + NeighborhoodCommunicator(NeighborhoodCommunicator&& other) noexcept; + + NeighborhoodCommunicator& operator=(const NeighborhoodCommunicator& other) = + default; + + NeighborhoodCommunicator& operator=( + NeighborhoodCommunicator&& other) noexcept; + /** * Default constructor with empty communication pattern * @param base the base communicator @@ -40,7 +50,7 @@ class NeighborhoodCommunicator final : public CollectiveCommunicator { explicit NeighborhoodCommunicator(communicator base); /** - * Create a neighborhood_communicator from an index map. + * Create a NeighborhoodCommunicator from an index map. * * The receive neighbors are defined by the remote indices and their * owning ranks of the index map. The send neighbors are deduced @@ -57,7 +67,7 @@ class NeighborhoodCommunicator final : public CollectiveCommunicator { const distributed::index_map& imap); /** - * Create a neighborhood_communicator by explicitly defining the + * Create a NeighborhoodCommunicator by explicitly defining the * neighborhood lists and sizes/offsets. * * @param base the base communicator @@ -78,17 +88,10 @@ class NeighborhoodCommunicator final : public CollectiveCommunicator { const std::vector& send_offsets); /** - * Communicate data from all ranks to all other ranks using the - * neighboorhood communication MPI_Ineighbor_alltoallv. See MPI - * documentation for more details + * @copydoc CollectiveCommunicator::i_all_to_all_v * - * @param exec The executor, on which the message buffers are located. - * @param send_buffer the buffer to send - * @param send_type the MPI_Datatype for the send buffer - * @param recv_buffer the buffer to gather into - * @param recv_type the MPI_Datatype for the recv buffer - * - * @return the request handle for the call + * This implementation uses the neighborhood communication + * MPI_Ineighbor_alltoallv. See MPI documentation for more details. */ request i_all_to_all_v(std::shared_ptr exec, const void* send_buffer, MPI_Datatype send_type, @@ -98,8 +101,9 @@ class NeighborhoodCommunicator final : public CollectiveCommunicator { std::unique_ptr create_with_same_type( communicator base, const distributed::index_map_variant& imap) const override; + /** - * Creates the inverse neighborhood_communicator by switching sources + * Creates the inverse NeighborhoodCommunicator by switching sources * and destinations. * * @return collective_communicator with the inverse communication pattern @@ -110,12 +114,32 @@ class NeighborhoodCommunicator final : public CollectiveCommunicator { /** * @copydoc collective_communicator::get_recv_size */ - comm_index_type get_recv_size() const override; + [[nodiscard]] comm_index_type get_recv_size() const override; /** * @copydoc collective_communicator::get_recv_size */ - comm_index_type get_send_size() const override; + [[nodiscard]] comm_index_type get_send_size() const override; + + /** + * Compares two communicators for equality locally. + * + * Equality is defined as having identical or congruent communicators and + * their communication pattern is equal. No communication is done, i.e. + * there is no reduction over the local equality check results. + * + * @return true if both communicators are equal. + */ + friend bool operator==(const NeighborhoodCommunicator& a, + const NeighborhoodCommunicator& b); + + /** + * Compares two communicators for inequality. + * + * @see operator== + */ + friend bool operator!=(const NeighborhoodCommunicator& a, + const NeighborhoodCommunicator& b); private: communicator comm_;