Skip to content

Commit

Permalink
review update
Browse files Browse the repository at this point in the history
- move some kernels to common
- use GKO_ENABLE_DEFAULT_HOST when possible
- use expilict type when intial declare
- delete unused function

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
  • Loading branch information
5 people committed Jul 30, 2021
1 parent 3e9ed3a commit 00075c0
Show file tree
Hide file tree
Showing 53 changed files with 482 additions and 2,119 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -30,63 +30,48 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
******************************<GINKGO LICENSE>*******************************/

#ifndef GKO_DPCPP_COMPONENTS_INTRINSICS_DP_HPP_
#define GKO_DPCPP_COMPONENTS_INTRINSICS_DP_HPP_
#include "core/matrix/coo_kernels.hpp"


#include <CL/sycl.hpp>
#include <ginkgo/core/base/math.hpp>


#include <ginkgo/core/base/types.hpp>


#include "dpcpp/base/dpct.hpp"
#include "common/base/kernel_launch.hpp"


namespace gko {
namespace kernels {
namespace dpcpp {


namespace GKO_DEVICE_NAMESPACE {
/**
* @internal
* Returns the number of set bits in the given mask.
* @brief The Coo matrix format namespace.
*
* @ingroup coo
*/
__dpct_inline__ int popcnt(uint32 mask) { return sycl::popcount(mask); }
namespace coo {

/** @copydoc popcnt */
__dpct_inline__ int popcnt(uint64 mask) { return sycl::popcount(mask); }


/**
* @internal
* Returns the (1-based!) index of the first set bit in the given mask,
* starting from the least significant bit.
*/
__dpct_inline__ int ffs(uint32 mask) { return __builtin_ffs(mask); }

/** @copydoc ffs */
__dpct_inline__ int ffs(uint64 mask)
template <typename ValueType, typename IndexType>
void extract_diagonal(std::shared_ptr<const DefaultExecutor> exec,
const matrix::Coo<ValueType, IndexType> *orig,
matrix::Diagonal<ValueType> *diag)
{
// the cast is necessary, as the overloads defined by HIP are ambiguous
return __builtin_ffsll(static_cast<unsigned long long int>(mask));
run_kernel(
exec,
[] GKO_KERNEL(auto tidx, auto orig_values, auto orig_row_idxs,
auto orig_col_idxs, auto diag) {
if (orig_row_idxs[tidx] == orig_col_idxs[tidx]) {
diag[orig_row_idxs[tidx]] = orig_values[tidx];
}
},
orig->get_num_stored_elements(), orig->get_const_values(),
orig->get_const_row_idxs(), orig->get_const_col_idxs(),
diag->get_values());
}
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_COO_EXTRACT_DIAGONAL_KERNEL);


/**
* @internal
* Returns the number of zero bits before the first set bit in the given mask,
* starting from the most significant bit.
*/
__dpct_inline__ int clz(uint32 mask) { return __builtin_clz(mask); }

/** @copydoc clz */
__dpct_inline__ int clz(uint64 mask) { return __builtin_clzll(mask); }


} // namespace dpcpp
} // namespace coo
} // namespace GKO_DEVICE_NAMESPACE
} // namespace kernels
} // namespace gko


#endif // GKO_DPCPP_COMPONENTS_INTRINSICS_DP_HPP_
16 changes: 0 additions & 16 deletions common/matrix/coo_kernels.hpp.inc
Original file line number Diff line number Diff line change
Expand Up @@ -272,20 +272,4 @@ __global__ __launch_bounds__(default_block_size) void fill_in_dense(
}


template <typename ValueType, typename IndexType>
__global__ __launch_bounds__(default_block_size) void extract_diagonal(
size_type nnz, const ValueType *__restrict__ orig_values,
const IndexType *__restrict__ orig_row_idxs,
const IndexType *__restrict__ orig_col_idxs, ValueType *__restrict__ diag)
{
const auto tidx = thread::get_thread_id_flat();

if (tidx < nnz) {
if (orig_row_idxs[tidx] == orig_col_idxs[tidx]) {
diag[orig_row_idxs[tidx]] = orig_values[tidx];
}
}
}


} // namespace kernel
26 changes: 13 additions & 13 deletions common/matrix/csr_kernels.hpp.inc
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,9 @@ __device__ __forceinline__ void find_next_row(
if (ind >= *row_end) {
*row = row_predict;
*row_end = row_predict_end;
for (; ind >= *row_end; *row_end = row_ptr[++*row + 1])
;
while (ind >= *row_end) {
*row_end = row_ptr[++*row + 1];
}
}

} else {
Expand Down Expand Up @@ -140,8 +141,8 @@ template <typename IndexType>
__device__ __forceinline__ IndexType get_warp_start_idx(
const IndexType nwarps, const IndexType nnz, const IndexType warp_idx)
{
const long long cache_lines = ceildivT<IndexType>(nnz, wsize);
return (warp_idx * cache_lines / nwarps) * wsize;
const long long cache_lines = ceildivT<IndexType>(nnz, config::warp_size);
return (warp_idx * cache_lines / nwarps) * config::warp_size;
}


Expand All @@ -160,9 +161,9 @@ __device__ __forceinline__ void spmv_kernel(
}
const IndexType data_size = row_ptrs[num_rows];
const IndexType start = get_warp_start_idx(nwarps, data_size, warp_idx);
const IndexType end =
min(get_warp_start_idx(nwarps, data_size, warp_idx + 1),
ceildivT<IndexType>(data_size, wsize) * wsize);
const IndexType end = min(
get_warp_start_idx(nwarps, data_size, warp_idx + 1),
ceildivT<IndexType>(data_size, config::warp_size) * config::warp_size);
auto row = srow[warp_idx];
auto row_end = row_ptrs[row + 1];
auto nrow = row;
Expand All @@ -171,10 +172,10 @@ __device__ __forceinline__ void spmv_kernel(
IndexType ind = start + threadIdx.x;
find_next_row<true>(num_rows, data_size, ind, &row, &row_end, nrow,
nrow_end, row_ptrs);
const IndexType ind_end = end - wsize;
const IndexType ind_end = end - config::warp_size;
const auto tile_block =
group::tiled_partition<wsize>(group::this_thread_block());
for (; ind < ind_end; ind += wsize) {
group::tiled_partition<config::warp_size>(group::this_thread_block());
for (; ind < ind_end; ind += config::warp_size) {
process_window<false>(tile_block, num_rows, data_size, ind, &row,
&row_end, &nrow, &nrow_end, &temp_val, val,
col_idxs, row_ptrs, b, b_stride, c, c_stride,
Expand Down Expand Up @@ -359,8 +360,7 @@ __device__ void merge_path_spmv(
tmp_val[threadIdx.x] = value;
tmp_ind[threadIdx.x] = row_i;
group::this_thread_block().sync();
bool last = block_segment_scan_reverse(static_cast<IndexType *>(tmp_ind),
static_cast<ValueType *>(tmp_val));
bool last = block_segment_scan_reverse(tmp_ind, tmp_val);
if (threadIdx.x == spmv_block_size - 1) {
row_out[blockIdx.x] = min(end_x, num_rows - 1);
val_out[blockIdx.x] = tmp_val[threadIdx.x];
Expand Down Expand Up @@ -1088,4 +1088,4 @@ __global__ __launch_bounds__(default_block_size) void inv_symm_permute_kernel(
out_cols[out_begin + i] = permutation[in_cols[in_begin + i]];
out_vals[out_begin + i] = in_vals[in_begin + i];
}
}
}
153 changes: 153 additions & 0 deletions common/matrix/diagonal_kernels.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
/*******************************<GINKGO LICENSE>******************************
Copyright (c) 2017-2021, the Ginkgo authors
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
******************************<GINKGO LICENSE>*******************************/

#include "core/matrix/diagonal_kernels.hpp"


#include <ginkgo/core/base/math.hpp>


#include "common/base/kernel_launch.hpp"


namespace gko {
namespace kernels {
namespace GKO_DEVICE_NAMESPACE {
/**
* @brief The Diagonal matrix format namespace.
*
* @ingroup diagonal
*/
namespace diagonal {


template <typename ValueType>
void apply_to_dense(std::shared_ptr<const DefaultExecutor> exec,
const matrix::Diagonal<ValueType> *a,
const matrix::Dense<ValueType> *b,
matrix::Dense<ValueType> *c)
{
run_kernel(
exec,
[] GKO_KERNEL(auto row, auto col, auto diag, auto source, auto result) {
result(row, col) = source(row, col) * diag[row];
},
b->get_size(), a->get_const_values(), b, c);
}

GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_APPLY_TO_DENSE_KERNEL);


template <typename ValueType>
void right_apply_to_dense(std::shared_ptr<const DefaultExecutor> exec,
const matrix::Diagonal<ValueType> *a,
const matrix::Dense<ValueType> *b,
matrix::Dense<ValueType> *c)
{
run_kernel(
exec,
[] GKO_KERNEL(auto row, auto col, auto diag, auto source, auto result) {
result(row, col) = source(row, col) * diag[col];
},
b->get_size(), a->get_const_values(), b, c);
}

GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_DENSE_KERNEL);


template <typename ValueType, typename IndexType>
void right_apply_to_csr(std::shared_ptr<const DefaultExecutor> exec,
const matrix::Diagonal<ValueType> *a,
const matrix::Csr<ValueType, IndexType> *b,
matrix::Csr<ValueType, IndexType> *c)
{
// TODO: combine copy and diag apply together
c->copy_from(b);
run_kernel(
exec,
[] GKO_KERNEL(auto tidx, auto diag, auto result_values, auto col_idxs) {
result_values[tidx] *= diag[col_idxs[tidx]];
},
c->get_num_stored_elements(), a->get_const_values(), c->get_values(),
c->get_const_col_idxs());
}

GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_CSR_KERNEL);


template <typename ValueType, typename IndexType>
void convert_to_csr(std::shared_ptr<const DefaultExecutor> exec,
const matrix::Diagonal<ValueType> *source,
matrix::Csr<ValueType, IndexType> *result)
{
run_kernel(
exec,
[] GKO_KERNEL(auto tidx, auto size, auto diag_values, auto row_ptrs,
auto col_idxs, auto csr_values) {
row_ptrs[tidx] = tidx;
col_idxs[tidx] = tidx;
csr_values[tidx] = diag_values[tidx];
if (tidx == size - 1) {
row_ptrs[size] = size;
}
},
source->get_size()[0], source->get_size()[0],
source->get_const_values(), result->get_row_ptrs(),
result->get_col_idxs(), result->get_values());
}

GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DIAGONAL_CONVERT_TO_CSR_KERNEL);


template <typename ValueType>
void conj_transpose(std::shared_ptr<const DefaultExecutor> exec,
const matrix::Diagonal<ValueType> *orig,
matrix::Diagonal<ValueType> *trans)
{
run_kernel(
exec,
[] GKO_KERNEL(auto tidx, auto orig_values, auto trans_values) {
trans_values[tidx] = conj(orig_values[tidx]);
},
orig->get_size()[0], orig->get_const_values(), trans->get_values());
}

GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_CONJ_TRANSPOSE_KERNEL);


} // namespace diagonal
} // namespace GKO_DEVICE_NAMESPACE
} // namespace kernels
} // namespace gko
Loading

0 comments on commit 00075c0

Please sign in to comment.