[coll-comm] review update:

- fix include guards
- update docs
- implement copy/move constructors/assignment with tests
- add equality test for collective communicators (needed for testing)
- always enable neighborhood comm, just throw if openmpi is too old

Co-authored-by: Pratik Nayak <pratik.nayak@kit.edu>

core/CMakeLists.txt

@@ -133,14 +133,10 @@ if(GINKGO_BUILD_MPI)
         distributed/collective_communicator.cpp
         distributed/dense_communicator.cpp
         distributed/matrix.cpp
+        distributed/neighborhood_communicator.cpp
         distributed/partition_helpers.cpp
         distributed/vector.cpp
         distributed/preconditioner/schwarz.cpp)
-    if(NOT GINKGO_HAVE_OPENMPI_PRE_4_1_X)
-        target_sources(${ginkgo_core}
-            PRIVATE
-            distributed/neighborhood_communicator.cpp)
-    endif()
 endif()
 
 # MSVC/shared: make ginkgo be the major library

core/distributed/dense_communicator.cpp

@@ -60,6 +60,27 @@ GKO_INSTANTIATE_FOR_EACH_LOCAL_GLOBAL_INDEX_TYPE(GKO_DECLARE_DENSE_CONSTRUCTOR);
 #undef GKO_DECLARE_DENSE_CONSTRUCTOR
 
 
+DenseCommunicator::DenseCommunicator(DenseCommunicator&& other) noexcept
+    : DenseCommunicator(other.get_base_communicator())
+{
+    *this = std::move(other);
+}
+
+
+DenseCommunicator& DenseCommunicator::operator=(
+    DenseCommunicator&& other) noexcept
+{
+    if (this != &other) {
+        *this = other;
+        std::fill(other.send_sizes_.begin(), other.send_sizes_.end(), 0);
+        std::fill(other.send_offsets_.begin(), other.send_offsets_.end(), 0);
+        std::fill(other.recv_sizes_.begin(), other.recv_sizes_.end(), 0);
+        std::fill(other.recv_offsets_.begin(), other.recv_offsets_.end(), 0);
+    }
+    return *this;
+}
+
+
 DenseCommunicator::DenseCommunicator(
     communicator base, const std::vector<comm_index_type>& recv_sizes,
     const std::vector<comm_index_type>& recv_offsets,
@@ -125,6 +146,28 @@ comm_index_type DenseCommunicator::get_send_size() const
 }
 
 
+bool operator==(const DenseCommunicator& a, const DenseCommunicator& b)
+{
+    return (a.comm_.is_identical(b.comm_) || a.comm_.is_congruent(b.comm_)) &&
+           a.send_sizes_.size() == b.send_sizes_.size() &&
+           a.recv_sizes_.size() == b.recv_sizes_.size() &&
+           std::equal(a.send_sizes_.begin(), a.send_sizes_.end(),
+                      b.send_sizes_.begin()) &&
+           std::equal(a.recv_sizes_.begin(), a.recv_sizes_.end(),
+                      b.recv_sizes_.begin()) &&
+           std::equal(a.send_offsets_.begin(), a.send_offsets_.end(),
+                      b.send_offsets_.begin()) &&
+           std::equal(a.recv_offsets_.begin(), a.recv_offsets_.end(),
+                      b.recv_offsets_.begin());
+}
+
+
+bool operator!=(const DenseCommunicator& a, const DenseCommunicator& b)
+{
+    return !(a == b);
+}
+
+
 }  // namespace mpi
 }  // namespace experimental
 }  // namespace gko

core/distributed/device_partition.hpp

@@ -2,8 +2,8 @@
 //
 // SPDX-License-Identifier: BSD-3-Clause
 
-#ifndef GINKGO_PARTITION_HPP
-#define GINKGO_PARTITION_HPP
+#ifndef GKO_CORE_DISTRIBUTED_PARTITION_HPP
+#define GKO_CORE_DISTRIBUTED_PARTITION_HPP
 
 #include <ginkgo/core/distributed/partition.hpp>
 
@@ -89,4 +89,4 @@ to_device_const(
 }  // namespace gko
 
 
-#endif  // GINKGO_PARTITION_HPP
+#endif  // GKO_CORE_DISTRIBUTED_PARTITION_HPP

core/distributed/neighborhood_communicator.cpp

@@ -156,13 +156,17 @@ request NeighborhoodCommunicator::i_all_to_all_v(
     std::shared_ptr<const Executor> exec, const void* send_buffer,
     MPI_Datatype send_type, void* recv_buffer, MPI_Datatype recv_type) const
 {
+#if GINKGO_HAVE_OPENMPI_PRE_4_1_X
+    GKO_NOT_IMPLEMENTED;
+#else
     auto guard = exec->get_scoped_device_id_guard();
     request req;
     GKO_ASSERT_NO_MPI_ERRORS(MPI_Ineighbor_alltoallv(
         send_buffer, send_sizes_.data(), send_offsets_.data(), send_type,
         recv_buffer, recv_sizes_.data(), recv_offsets_.data(), recv_type,
         comm_.get(), req.get()));
     return req;
+#endif
 }
 
 
@@ -179,6 +183,61 @@ NeighborhoodCommunicator::create_with_same_type(
 }
 
 
+NeighborhoodCommunicator::NeighborhoodCommunicator(
+    NeighborhoodCommunicator&& other) noexcept
+    : NeighborhoodCommunicator(other.get_base_communicator())
+{
+    *this = std::move(other);
+}
+
+
+NeighborhoodCommunicator& NeighborhoodCommunicator::operator=(
+    NeighborhoodCommunicator&& other) noexcept
+{
+    if (this != &other) {
+        comm_ = std::exchange(other.comm_, MPI_COMM_SELF);
+        // set topology for other comm
+        std::vector<comm_index_type> non_nullptr(1);
+        non_nullptr.resize(0);
+        other.comm_ = create_neighborhood_comm(this->get_base_communicator(),
+                                               non_nullptr, non_nullptr);
+        send_sizes_ = std::exchange(
+            other.send_sizes_, std::vector<distributed::comm_index_type>{});
+        send_offsets_ = std::exchange(
+            other.send_offsets_, std::vector<distributed::comm_index_type>{0});
+        recv_sizes_ = std::exchange(
+            other.recv_sizes_, std::vector<distributed::comm_index_type>{});
+        recv_offsets_ = std::exchange(
+            other.recv_offsets_, std::vector<distributed::comm_index_type>{0});
+    }
+    return *this;
+}
+
+
+bool operator==(const NeighborhoodCommunicator& a,
+                const NeighborhoodCommunicator& b)
+{
+    return (a.comm_.is_identical(b.comm_) || a.comm_.is_congruent(b.comm_)) &&
+           a.send_sizes_.size() == b.send_sizes_.size() &&
+           a.recv_sizes_.size() == b.recv_sizes_.size() &&
+           std::equal(a.send_sizes_.begin(), a.send_sizes_.end(),
+                      b.send_sizes_.begin()) &&
+           std::equal(a.recv_sizes_.begin(), a.recv_sizes_.end(),
+                      b.recv_sizes_.begin()) &&
+           std::equal(a.send_offsets_.begin(), a.send_offsets_.end(),
+                      b.send_offsets_.begin()) &&
+           std::equal(a.recv_offsets_.begin(), a.recv_offsets_.end(),
+                      b.recv_offsets_.begin());
+}
+
+
+bool operator!=(const NeighborhoodCommunicator& a,
+                const NeighborhoodCommunicator& b)
+{
+    return !(a == b);
+}
+
+
 template <typename LocalIndexType, typename GlobalIndexType>
 NeighborhoodCommunicator::NeighborhoodCommunicator(
     communicator base,