ensure ops affected by Reset after increase_exec

ginkgo-project · Jul 9, 2021 · ad7746d · ad7746d
1 parent f707dc9
commit ad7746d
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 34 deletions.
diff --git a/cuda/test/base/CMakeLists.txt b/cuda/test/base/CMakeLists.txt
@@ -1,6 +1,6 @@
 ginkgo_create_cuda_test(array)
 ginkgo_create_cuda_test(cuda_executor)
-ginkgo_create_cuda_test(cuda_executor_reset)
+ginkgo_create_test(cuda_executor_reset)
 if(GINKGO_HAVE_HWLOC)
     find_package(NUMA REQUIRED)
     ginkgo_create_cuda_test(cuda_executor_topology NUMA::NUMA)

diff --git a/cuda/test/base/cuda_executor_reset.cu → cuda/test/base/cuda_executor_reset.cpp b/cuda/test/base/cuda_executor_reset.cu → cuda/test/base/cuda_executor_reset.cpp
@@ -45,6 +45,7 @@ namespace {
 #define GTEST_ASSERT_NO_EXIT(statement) \
     ASSERT_EXIT({ {statement} exit(0); }, ::testing::ExitedWithCode(0), "")
 
+
 TEST(DeviceReset, HipCuda)
 {
     GTEST_ASSERT_NO_EXIT({
@@ -65,30 +66,18 @@ TEST(DeviceReset, CudaHip)
 }
 
 
-template <typename Executor>
 void func()
 {
     auto ref = gko::ReferenceExecutor::create();
-    auto exec = Executor::create(0, ref, true);
+    auto exec = gko::CudaExecutor::create(0, ref, true);
 }
 
 
 TEST(DeviceReset, CudaCuda)
 {
     GTEST_ASSERT_NO_EXIT({
-        std::thread t1(func<gko::CudaExecutor>);
-        std::thread t2(func<gko::CudaExecutor>);
-        t1.join();
-        t2.join();
-    });
-}
-
-
-TEST(DeviceReset, HipHip)
-{
-    GTEST_ASSERT_NO_EXIT({
-        std::thread t1(func<gko::CudaExecutor>);
-        std::thread t2(func<gko::CudaExecutor>);
+        std::thread t1(func);
+        std::thread t2(func);
         t1.join();
         t2.join();
     });

diff --git a/hip/test/base/CMakeLists.txt b/hip/test/base/CMakeLists.txt
@@ -1,5 +1,5 @@
 ginkgo_create_hip_test(hip_executor)
-ginkgo_create_hip_test(hip_executor_reset)
+ginkgo_create_test(hip_executor_reset)
 if(GINKGO_HAVE_HWLOC)
     find_package(NUMA REQUIRED)
     ginkgo_create_hip_test(hip_executor_topology NUMA::NUMA)

diff --git a/hip/test/base/hip_executor_reset.hip.cpp → hip/test/base/hip_executor_reset.cpp b/hip/test/base/hip_executor_reset.hip.cpp → hip/test/base/hip_executor_reset.cpp
@@ -45,6 +45,7 @@ namespace {
 #define GTEST_ASSERT_NO_EXIT(statement) \
     ASSERT_EXIT({ {statement} exit(0); }, ::testing::ExitedWithCode(0), "")
 
+
 TEST(DeviceReset, HipCuda)
 {
     GTEST_ASSERT_NO_EXIT({
@@ -65,30 +66,18 @@ TEST(DeviceReset, CudaHip)
 }
 
 
-template <typename Executor>
 void func()
 {
     auto ref = gko::ReferenceExecutor::create();
-    auto exec = Executor::create(0, ref, true);
-}
-
-
-TEST(DeviceReset, CudaCuda)
-{
-    GTEST_ASSERT_NO_EXIT({
-        std::thread t1(func<gko::CudaExecutor>);
-        std::thread t2(func<gko::CudaExecutor>);
-        t1.join();
-        t2.join();
-    });
+    auto exec = gko::HipExecutor::create(0, ref, true);
 }
 
 
 TEST(DeviceReset, HipHip)
 {
     GTEST_ASSERT_NO_EXIT({
-        std::thread t1(func<gko::CudaExecutor>);
-        std::thread t2(func<gko::CudaExecutor>);
+        std::thread t1(func);
+        std::thread t2(func);
         t1.join();
         t2.join();
     });

diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
@@ -1424,9 +1424,13 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
             MachineTopology::get_instance()->bind_to_pus(
                 this->get_closest_pus());
         }
+        // it only gets attribute from device, so it should not be affected by
+        // DeviceReset.
         this->set_gpu_property();
-        this->init_handles();
+        // increase the number of executor before any operations may be affected
+        // by DeviceReset.
         increase_num_execs(this->get_exec_info().device_id);
+        this->init_handles();
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1647,9 +1651,13 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
             MachineTopology::get_instance()->bind_to_pus(
                 this->get_closest_pus());
         }
+        // it only gets attribute from device, so it should not be affected by
+        // DeviceReset.
         this->set_gpu_property();
-        this->init_handles();
+        // increase the number of executor before any operations may be affected
+        // by DeviceReset.
         increase_num_execs(this->get_exec_info().device_id);
+        this->init_handles();
     }
 
     void *raw_alloc(size_type size) const override;