handle deleter in create not destructor of class

it makes us keep mutex not recursive_mutex Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
ginkgo-project · Jul 29, 2021 · 2da69cd · 2da69cd
1 parent 23d3197
commit 2da69cd
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 26 deletions.
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
@@ -65,10 +65,12 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
                          alloc_mode),
         [device_id](CudaExecutor *exec) {
             auto device_reset = exec->get_device_reset();
-            std::lock_guard<std::recursive_mutex> guard(
+            std::lock_guard<std::mutex> guard(
                 device_class::get_mutex(device_id));
             delete exec;
-            if (!CudaExecutor::get_num_execs(device_id) && device_reset) {
+            auto &num_execs = device_class::get_num_execs(device_id);
+            num_execs--;
+            if (!num_execs && device_reset) {
                 cuda::device_guard g(device_id);
                 cudaDeviceReset();
             }

diff --git a/devices/device.cpp b/devices/device.cpp
@@ -40,9 +40,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace gko {
 
 
-std::recursive_mutex &NvidiaDevice::get_mutex(int i)
+std::mutex &NvidiaDevice::get_mutex(int i)
 {
-    static std::recursive_mutex mutex[max_devices];
+    static std::mutex mutex[max_devices];
     return mutex[i];
 }
 
@@ -52,9 +52,9 @@ int &NvidiaDevice::get_num_execs(int i)
     return num_execs[i];
 }
 
-std::recursive_mutex &AmdDevice::get_mutex(int i)
+std::mutex &AmdDevice::get_mutex(int i)
 {
-    static std::recursive_mutex mutex[max_devices];
+    static std::mutex mutex[max_devices];
     return mutex[i];
 }
 

diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
@@ -64,10 +64,12 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
         new HipExecutor(device_id, std::move(master), device_reset, alloc_mode),
         [device_id](HipExecutor *exec) {
             auto device_reset = exec->get_device_reset();
-            std::lock_guard<std::recursive_mutex> guard(
+            std::lock_guard<std::mutex> guard(
                 device_class::get_mutex(device_id));
             delete exec;
-            if (!HipExecutor::get_num_execs(device_id) && device_reset) {
+            auto &num_execs = device_class::get_num_execs(device_id);
+            num_execs--;
+            if (!num_execs && device_reset) {
                 hip::device_guard g(device_id);
                 hipDeviceReset();
             }

diff --git a/include/ginkgo/core/base/device.hpp b/include/ginkgo/core/base/device.hpp
@@ -70,7 +70,7 @@ class NvidiaDevice {
      *
      * @return recursive_mutex reference
      */
-    static std::recursive_mutex &get_mutex(int i);
+    static std::mutex &get_mutex(int i);
 
     /**
      * get_num_execs gets the static num_execs reference at i.
@@ -104,7 +104,7 @@ class AmdDevice {
      *
      * @return recursive_mutex reference
      */
-    static std::recursive_mutex &get_mutex(int i);
+    static std::mutex &get_mutex(int i);
 
     /**
      * get_num_execs gets the static num_execs reference at i.

diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
@@ -1302,8 +1302,6 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         bool device_reset = false,
         allocation_mode alloc_mode = default_cuda_alloc_mode);
 
-    ~CudaExecutor() { decrease_num_execs(this->get_device_id()); }
-
     std::shared_ptr<Executor> get_master() noexcept override;
 
     std::shared_ptr<const Executor> get_master() const noexcept override;
@@ -1457,22 +1455,19 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     static void increase_num_execs(unsigned device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)++;
     }
 
     static void decrease_num_execs(unsigned device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)--;
     }
 
     static unsigned get_num_execs(unsigned device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         return device_class::get_num_execs(device_id);
     }
 
@@ -1525,8 +1520,6 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         bool device_reset = false,
         allocation_mode alloc_mode = default_hip_alloc_mode);
 
-    ~HipExecutor() { decrease_num_execs(this->get_device_id()); }
-
     std::shared_ptr<Executor> get_master() noexcept override;
 
     std::shared_ptr<const Executor> get_master() const noexcept override;
@@ -1684,22 +1677,19 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     static void increase_num_execs(int device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)++;
     }
 
     static void decrease_num_execs(int device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)--;
     }
 
     static int get_num_execs(int device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         return device_class::get_num_execs(device_id);
     }