Skip to content

Commit

Permalink
Add device_guard anywhere a CUDA function is called.
Browse files Browse the repository at this point in the history
  • Loading branch information
@tcojean
tcojean committed May 10, 2019
1 parent 12f0287 commit d37a8cb
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 30 deletions.
37 changes: 30 additions & 7 deletions benchmark/spmv/cuda_linops.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <ginkgo/ginkgo.hpp>


#include "cuda/base/cusparse_bindings.hpp"


#include <cuda_runtime.h>
#include <cusparse.h>
#include <memory>


#include "cuda/base/cusparse_bindings.hpp"
#include "cuda/base/device_guard.hpp"


namespace detail {


Expand Down Expand Up @@ -79,14 +80,12 @@ class CuspBase : public gko::LinOp {
void initialize_descr()
{
const auto id = this->gpu_exec->get_device_id();
gko::device_guard g{id};
this->descr_ = handle_manager<cusparseMatDescr>(
gko::kernels::cuda::cusparse::create_mat_descr(),
[id](cusparseMatDescr_t descr) {
int original_device_id{};
GKO_ASSERT_NO_CUDA_ERRORS(cudaGetDevice(&original_device_id));
GKO_ASSERT_NO_CUDA_ERRORS(cudaSetDevice(id));
gko::device_guard g{id};
gko::kernels::cuda::cusparse::destroy(descr);
GKO_ASSERT_NO_CUDA_ERRORS(cudaSetDevice(original_device_id));
});
}

Expand Down Expand Up @@ -124,6 +123,9 @@ class CuspCsrmp
auto dx = dense_x->get_values();
auto alpha = gko::one<ValueType>();
auto beta = gko::zero<ValueType>();

const auto id = this->get_gpu_exec()->get_device_id();
gko::device_guard g{id};
GKO_ASSERT_NO_CUSPARSE_ERRORS(cusparseDcsrmv_mp(
this->get_gpu_exec()->get_cusparse_handle(), trans_,
this->get_size()[0], this->get_size()[1],
Expand Down Expand Up @@ -178,6 +180,9 @@ class CuspCsr
auto dx = dense_x->get_values();
auto alpha = gko::one<ValueType>();
auto beta = gko::zero<ValueType>();

const auto id = this->get_gpu_exec()->get_device_id();
gko::device_guard g{id};
GKO_ASSERT_NO_CUSPARSE_ERRORS(cusparseDcsrmv(
this->get_gpu_exec()->get_cusparse_handle(), trans_,
this->get_size()[0], this->get_size()[1],
Expand Down Expand Up @@ -232,6 +237,8 @@ class CuspCsrmm
auto alpha = gko::one<ValueType>();
auto beta = gko::zero<ValueType>();

const auto id = this->get_gpu_exec()->get_device_id();
gko::device_guard g{id};
GKO_ASSERT_NO_CUSPARSE_ERRORS(cusparseDcsrmm(
this->get_gpu_exec()->get_cusparse_handle(), trans_,
this->get_size()[0], dense_b->get_size()[1], this->get_size()[1],
Expand Down Expand Up @@ -280,6 +287,8 @@ class CuspCsrEx
auto beta = gko::zero<ValueType>();
this->set_size(gko::dim<2>{csr_->get_size()});

const auto id = this->get_gpu_exec()->get_device_id();
gko::device_guard g{id};
GKO_ASSERT_NO_CUSPARSE_ERRORS(cusparseCsrmvEx_bufferSize(
this->get_gpu_exec()->get_cusparse_handle(), algmode_, trans_,
this->get_size()[0], this->get_size()[1],
Expand All @@ -300,6 +309,9 @@ class CuspCsrEx
auto dx = dense_x->get_values();
auto alpha = gko::one<ValueType>();
auto beta = gko::zero<ValueType>();

const auto id = this->get_gpu_exec()->get_device_id();
gko::device_guard g{id};
GKO_ASSERT_NO_CUSPARSE_ERRORS(cusparseCsrmvEx(
this->get_gpu_exec()->get_cusparse_handle(), algmode_, trans_,
this->get_size()[0], this->get_size()[1],
Expand Down Expand Up @@ -337,6 +349,8 @@ class CuspCsrEx

~CuspCsrEx() override
{
const auto id = this->get_gpu_exec()->get_device_id();
gko::device_guard g{id};
if (set_buffer_) {
try {
GKO_ASSERT_NO_CUDA_ERRORS(cudaFree(buffer_));
Expand Down Expand Up @@ -379,6 +393,9 @@ class CuspHybrid
std::make_shared<typename csr::classical>());
t_csr->read(data);
this->set_size(gko::dim<2>{t_csr->get_size()});

const auto id = this->get_gpu_exec()->get_device_id();
gko::device_guard g{id};
GKO_ASSERT_NO_CUSPARSE_ERRORS(cusparseDcsr2hyb(
this->get_gpu_exec()->get_cusparse_handle(), this->get_size()[0],
this->get_size()[1], this->get_descr(), t_csr->get_const_values(),
Expand All @@ -394,6 +411,8 @@ class CuspHybrid
auto dx = dense_x->get_values();
auto alpha = gko::one<ValueType>();
auto beta = gko::zero<ValueType>();
const auto id = this->get_gpu_exec()->get_device_id();
gko::device_guard g{id};
GKO_ASSERT_NO_CUSPARSE_ERRORS(
cusparseDhybmv(this->get_gpu_exec()->get_cusparse_handle(), trans_,
&alpha, this->get_descr(), hyb_, db, &beta, dx));
Expand All @@ -404,11 +423,15 @@ class CuspHybrid
: gko::EnableLinOp<CuspHybrid, CuspBase>(exec, size),
trans_(CUSPARSE_OPERATION_NON_TRANSPOSE)
{
const auto id = this->get_gpu_exec()->get_device_id();
gko::device_guard g{id};
GKO_ASSERT_NO_CUSPARSE_ERRORS(cusparseCreateHybMat(&hyb_));
}

~CuspHybrid() override
{
const auto id = this->get_gpu_exec()->get_device_id();
gko::device_guard g{id};
try {
GKO_ASSERT_NO_CUSPARSE_ERRORS(cusparseDestroyHybMat(hyb_));
} catch (std::exception &e) {
Expand Down
34 changes: 34 additions & 0 deletions cuda/base/device_guard.hpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@

#include <cuda_runtime.h>


#include <ginkgo/core/base/exception_helpers.hpp>


namespace gko {


class device_guard {
public:
device_guard(int device_id)
{
GKO_ASSERT_NO_CUDA_ERRORS(cudaGetDevice(&original_device_id));
GKO_ASSERT_NO_CUDA_ERRORS(cudaSetDevice(device_id));
}

~device_guard() noexcept(false)
{
/* Ignore the error during stack unwinding for this call */
if (std::uncaught_exception()) {
cudaSetDevice(original_device_id);
} else {
GKO_ASSERT_NO_CUDA_ERRORS(cudaSetDevice(original_device_id));
}
}

private:
int original_device_id{};
};


} // namespace gko
24 changes: 1 addition & 23 deletions cuda/base/executor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,35 +45,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "cuda/base/cublas_bindings.hpp"
#include "cuda/base/cusparse_bindings.hpp"
#include "cuda/base/device_guard.hpp"


namespace gko {
namespace {


class device_guard {
public:
device_guard(int device_id)
{
GKO_ASSERT_NO_CUDA_ERRORS(cudaGetDevice(&original_device_id));
GKO_ASSERT_NO_CUDA_ERRORS(cudaSetDevice(device_id));
}

~device_guard() noexcept(false)
{
/* Ignore the error during stack unwinding for this call */
if (std::uncaught_exception()) {
cudaSetDevice(original_device_id);
} else {
GKO_ASSERT_NO_CUDA_ERRORS(cudaSetDevice(original_device_id));
}
}

private:
int original_device_id{};
};


// The function is copied from _ConvertSMVer2Cores of
// cuda-9.2/samples/common/inc/helper_cuda.h
inline int convert_sm_ver_to_cores(int major, int minor)
Expand Down

0 comments on commit d37a8cb

Please sign in to comment.