review updates:

- adds helper to create oneDPL policy
- re-enable distributed matrix test with dpcpp

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>

dpcpp/base/device_matrix_data_kernels.dp.cpp

@@ -33,7 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 // force-top: on
 // oneDPL needs to be first to avoid issues with libstdc++ TBB impl
 #include <oneapi/dpl/algorithm>
-#include <oneapi/dpl/execution>
 // force-top: off
 
 
@@ -43,6 +42,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 
 
+#include "dpcpp/base/onedpl.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace dpcpp {
@@ -56,8 +58,7 @@ void remove_zeros(std::shared_ptr<const DefaultExecutor> exec,
 {
     using nonzero_type = matrix_data_entry<ValueType, IndexType>;
     auto size = values.get_num_elems();
-    auto policy =
-        oneapi::dpl::execution::make_device_policy(*exec->get_queue());
+    auto policy = onedpl_policy(exec);
     auto nnz = std::count_if(
         policy, values.get_const_data(), values.get_const_data() + size,
         [](ValueType val) { return is_nonzero<ValueType>(val); });
@@ -96,8 +97,7 @@ void sum_duplicates(std::shared_ptr<const DefaultExecutor> exec, size_type,
     if (size == 0) {
         return;
     }
-    auto policy =
-        oneapi::dpl::execution::make_device_policy(*exec->get_queue());
+    auto policy = onedpl_policy(exec);
     auto in_loc_it = oneapi::dpl::make_zip_iterator(row_idxs.get_const_data(),
                                                     col_idxs.get_const_data());
     auto adj_in_loc_it =
@@ -136,8 +136,7 @@ template <typename ValueType, typename IndexType>
 void sort_row_major(std::shared_ptr<const DefaultExecutor> exec,
                     device_matrix_data<ValueType, IndexType>& data)
 {
-    auto policy =
-        oneapi::dpl::execution::make_device_policy(*exec->get_queue());
+    auto policy = onedpl_policy(exec);
     auto input_it = oneapi::dpl::make_zip_iterator(
         data.get_row_idxs(), data.get_col_idxs(), data.get_values());
     std::sort(policy, input_it, input_it + data.get_num_elems(),

dpcpp/base/onedpl.hpp

@@ -0,0 +1,61 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2023, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_BASE_ONEDPL_HPP_
+#define GKO_DPCPP_BASE_ONEDPL_HPP_
+
+
+// force-top: on
+#include <oneapi/dpl/execution>
+// force-top: off
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+inline auto onedpl_policy(std::shared_ptr<const DpcppExecutor> exec)
+{
+    return oneapi::dpl::execution::make_device_policy(*exec->get_queue());
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_BASE_ONEDPL_HPP_

dpcpp/distributed/partition_kernels.dp.cpp

@@ -32,7 +32,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 // force-top: on
 #include <oneapi/dpl/algorithm>
-#include <oneapi/dpl/execution>
 #include <oneapi/dpl/iterator>
 // force-top: off
 
@@ -42,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common/unified/base/kernel_launch.hpp"
 #include "core/components/fill_array_kernels.hpp"
+#include "dpcpp/base/onedpl.hpp"
 
 
 namespace gko {
@@ -72,7 +72,7 @@ void setup_sizes_ids_permutation(
             permutation[i] = static_cast<GlobalIndexType>(i);
         },
         num_ranges, num_ranges, num_parts, range_offsets, range_parts,
-        range_sizes.get_data(), part_ids.get_data(), permutation.get_data());
+        range_sizes, part_ids, permutation);
 }
 
 
@@ -102,8 +102,8 @@ void compute_part_sizes_and_starting_indices(
                 prev_part == cur_part ? grouped_starting_indices[i - 1]
                                       : LocalIndexType{};
         },
-        num_ranges, range_sizes.get_const_data(), part_ids.get_const_data(),
-        permutation.get_const_data(), starting_indices, part_sizes);
+        num_ranges, range_sizes, part_ids, permutation, starting_indices,
+        part_sizes);
 }
 
 
@@ -120,8 +120,7 @@ void build_starting_indices(std::shared_ptr<const DefaultExecutor> exec,
                             LocalIndexType* part_sizes)
 {
     if (num_ranges > 0) {
-        auto policy =
-            oneapi::dpl::execution::make_device_policy(*exec->get_queue());
+        auto policy = onedpl_policy(exec);
 
         Array<LocalIndexType> range_sizes{exec, num_ranges};
         // num_parts sentinel at the end

dpcpp/multigrid/pgm_kernels.dp.cpp

@@ -33,7 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 // force-top: on
 // oneDPL needs to be first to avoid issues with libstdc++ TBB impl
 #include <oneapi/dpl/algorithm>
-#include <oneapi/dpl/execution>
 // force-top: off
 
 
@@ -48,6 +47,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/multigrid/pgm.hpp>
 
 
+#include "dpcpp/base/onedpl.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace dpcpp {
@@ -63,8 +65,7 @@ template <typename IndexType>
 void sort_agg(std::shared_ptr<const DefaultExecutor> exec, IndexType num,
               IndexType* row_idxs, IndexType* col_idxs)
 {
-    auto policy =
-        oneapi::dpl::execution::make_device_policy(*exec->get_queue());
+    auto policy = onedpl_policy(exec);
     auto it = oneapi::dpl::make_zip_iterator(row_idxs, col_idxs);
     std::sort(policy, it, it + num, [](auto a, auto b) {
         return std::tie(std::get<0>(a), std::get<1>(a)) <
@@ -79,8 +80,7 @@ template <typename ValueType, typename IndexType>
 void sort_row_major(std::shared_ptr<const DefaultExecutor> exec, size_type nnz,
                     IndexType* row_idxs, IndexType* col_idxs, ValueType* vals)
 {
-    auto policy =
-        oneapi::dpl::execution::make_device_policy(*exec->get_queue());
+    auto policy = onedpl_policy(exec);
     auto it = oneapi::dpl::make_zip_iterator(row_idxs, col_idxs, vals);
     // Because reduce_by_segment is not determinstic, so we do not need
     // stable_sort

test/mpi/CMakeLists.txt

@@ -1,4 +1,4 @@
-ginkgo_create_common_and_reference_test(matrix MPI_SIZE 3 DISABLE_EXECUTORS dpcpp)
+ginkgo_create_common_and_reference_test(matrix MPI_SIZE 3)
 ginkgo_create_common_and_reference_test(vector MPI_SIZE 3)
 
 add_subdirectory(preconditioner)