Skip to content

HIP support #340

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 16 commits into
base: develop
Choose a base branch
from
Open

HIP support #340

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
efb2a21
Add tioga-cce-18-rocm-6.2.1 host config
victorapm Dec 15, 2024
0d032f8
tioga-base: enable HIP on hypre by default
victorapm Dec 15, 2024
8121214
tioga-base: force CUDA off
victorapm Dec 15, 2024
0f01019
Add MPI_INCLUDE_DIR
victorapm Dec 15, 2024
65c0ec6
Merge branch 'feature/paludettomag1/hip' of github.com:GEOS-DEV/LvArr…
victorapm Dec 15, 2024
7e624fb
tioga-base -> amdgpu-base
victorapm Dec 15, 2024
a6d24fc
Remove fgpu-rdc
victorapm Dec 15, 2024
c0b7da8
Add tuo host configs
victorapm Dec 19, 2024
6daf4b0
Merge branch 'develop' into feature/paludettomag1/hip
victorapm May 7, 2025
d310686
Add tuo-cce-19-rocm-6.4.0 host config
victorapm May 8, 2025
404f1d7
Add matrix host configs
victorapm May 31, 2025
dd43693
Fix CXX std 14 -> 17
victorapm Jun 1, 2025
5ef6481
Update HCs
victorapm Jun 2, 2025
71ce0db
Fix HCs
victorapm Jun 2, 2025
8cf92d5
Fix tuo HC
victorapm Jun 2, 2025
d2bcd13
Enabling back ENABLE_FIND_MPI
victorapm Jun 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions host-configs/LLNL/amdgpu-base.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
set( CMAKE_CXX_STANDARD 17 CACHE STRING "" )

set( ENABLE_MPI ON CACHE BOOL "" FORCE )
#set( ENABLE_FIND_MPI ON CACHE BOOL "" FORCE )
set( ENABLE_CUDA OFF CACHE BOOL "" FORCE )
set( ENABLE_HYPRE ON CACHE BOOL "" FORCE )
set( ENABLE_HYPRE_DEVICE "HIP" CACHE STRING "" FORCE )

# HIP Options
set( ENABLE_HIP ON CACHE BOOL "" FORCE )

# suppress -Werror for now
set( ENABLE_WARNINGS_AS_ERRORS FALSE CACHE BOOL "" FORCE )

# GTEST
set( ENABLE_GTEST_DEATH_TESTS OFF CACHE BOOL "" )
set( gtest_disable_pthreads ON CACHE BOOL "" )

# disable most binaries and doc generation
set( ENABLE_TESTS OFF CACHE BOOL "" FORCE )
set( DISABLE_UNIT_TESTS ON CACHE BOOL "" FORCE )
set( ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE )
set( ENABLE_BENCHMARKS OFF CACHE BOOL "" FORCE )
set( ENABLE_DOCS OFF CACHE BOOL "" FORCE )

# BLT trying to find MPI fails on cray with cce
#set( ENABLE_FIND_MPI OFF CACHE BOOL "" )
16 changes: 16 additions & 0 deletions host-configs/LLNL/matrix-cuda-12-base.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
include(${CMAKE_CURRENT_LIST_DIR}/lassen-base.cmake)

# Cuda options
set(ENABLE_CUDA ON CACHE BOOL "")
set(CUDA_TOOLKIT_ROOT_DIR /usr/tce/packages/cuda/cuda-12.2.2 CACHE STRING "")
set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "")
set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")
set(CUDA_ARCH sm_90 CACHE STRING "")
set(CMAKE_CUDA_ARCHITECTURES 90 CACHE STRING "")
set(CMAKE_CUDA_STANDARD 17 CACHE STRING "")
set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "")
set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 " CACHE STRING "")
set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -g " CACHE STRING "")
set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")

set(CHAI_CUDA_FLAGS "-arch ${CUDA_ARCH}" CACHE STRING "" FORCE)
13 changes: 13 additions & 0 deletions host-configs/LLNL/matrix-gcc-12-cuda-12.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
set(CONFIG_NAME "matrix-gcc-12-cuda-12" CACHE PATH "")

# Set compiler info
set(COMPILER_DIR /usr/tce/packages/gcc/gcc-12.1.1)
set(CMAKE_C_COMPILER ${COMPILER_DIR}/bin/gcc CACHE PATH "")
set(CMAKE_CXX_COMPILER ${COMPILER_DIR}/bin/g++ CACHE PATH "")

# C++ options
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG" CACHE STRING "")
set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g" CACHE STRING "")

include(${CMAKE_CURRENT_LIST_DIR}/matrix-cuda-12-base.cmake)
24 changes: 0 additions & 24 deletions host-configs/LLNL/tioga-base.cmake
View file
Open in desktop

This file was deleted.

46 changes: 46 additions & 0 deletions host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
set( CCE_VERSION_STRING "18.0.1" )
set( HIP_VERSION_STRING "6.2.1" )
set( CONFIG_NAME "tioga-cce-${CCE_VERSION_STRING}-rocm-${HIP_VERSION_STRING}" CACHE PATH "" )
include( ${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake )

# TODO: Set up the tpls
set( ENABLE_UMPIRE ON CACHE BOOL "" )
set( ENABLE_CHAI ON CACHE BOOL "" )
set( ENABLE_CALIPER ON CACHE BOOL "" )
set( ENABLE_ADIAK ON CACHE BOOL "" )

set( CAMP_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" )
set( RAJA_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" )
set( UMPIRE_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" )
set( CHAI_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" )
set( CALIPER_DIR ${GEOS_TPL_DIR}/caliper CACHE PATH "" )

# MPI options
set( MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "" )
set( MPI_INCLUDE_DIR ${MPI_HOME}/include CACHE PATH "" ) # Needed by hypre

# C++ options
set( CRAYPE_VERSION "2.7.33")
set( CMAKE_C_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/cc" CACHE PATH "" )
set( CMAKE_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH "" )
set( CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "" )

if( ENABLE_HIP )
set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE )

set( HIP_ROOT "/opt/rocm-${HIP_VERSION_STRING}" CACHE PATH "" )
set( ROCM_PATH ${HIP_ROOT} CACHE PATH "" )

set( CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "" FORCE )
set( CMAKE_HIP_FLAGS "-munsafe-fp-atomics -fno-gpu-rdc" CACHE STRING "" FORCE )
set( CMAKE_HIP_LINK_FLAGS "-fno-gpu-rdc --hip-link -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE )

##############################################################################
# The flag "fgpu-rdc" causes link issues when using cce-18
# lld: /workspace/llvm/lib/Analysis/LoopAccessAnalysis.cpp:430:
# bool llvm::RuntimeCheckingPtrGroup::addPointer(unsigned int, const llvm::SCEV*, const llvm::SCEV*, unsigned int, bool, llvm::ScalarEvolution&):
# Assertion `AddressSpace == AS && "all pointers in a checking group must be in the same address space"' failed.
##############################################################################
#set( CMAKE_CXX_FLAGS "-munsafe-fp-atomics -fgpu-rdc" CACHE STRING "" FORCE )
#set( CMAKE_CXX_LINK_FLAGS "-fgpu-rdc --hip-link" CACHE STRING "" FORCE )
endif()
38 changes: 38 additions & 0 deletions host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
set( CCE_VERSION_STRING "19.0.0" )
set( HIP_VERSION_STRING "6.4.0" )
set( CONFIG_NAME "tioga-cce-${CCE_VERSION_STRING}-rocm-${HIP_VERSION_STRING}" CACHE PATH "" )
include( ${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake )

# TODO: Set up GEOS_TPL_DIR
set( ENABLE_UMPIRE ON CACHE BOOL "" )
set( ENABLE_CHAI ON CACHE BOOL "" )
set( ENABLE_CALIPER ON CACHE BOOL "" )
set( ENABLE_ADIAK ON CACHE BOOL "" )

set( CAMP_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" )
set( RAJA_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" )
set( UMPIRE_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" )
set( CHAI_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" )
set( CALIPER_DIR ${GEOS_TPL_DIR}/caliper CACHE PATH "" )

# MPI options
set( MPI_HOME /opt/cray/pe/mpich/8.1.33.1/ofi/crayclang/18.0 CACHE PATH "" )
set( MPI_INCLUDE_DIR ${MPI_HOME}/include CACHE PATH "" ) # Needed by hypre

# C++ options
set( CRAYPE_VERSION "2.7.34")
set( CMAKE_C_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/cc" CACHE PATH "" )
set( CMAKE_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH "" )
set( CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "" )

if( ENABLE_HIP )
set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE )

set( HIP_ROOT "/opt/rocm-${HIP_VERSION_STRING}" CACHE PATH "" )
set( HIP_ROOT_DIR ${HIP_ROOT} CACHE PATH "" )
set( ROCM_PATH ${HIP_ROOT} CACHE PATH "" )

set( CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "" FORCE )
set( CMAKE_HIP_FLAGS "-munsafe-fp-atomics -fno-gpu-rdc -I${MPI_INCLUDE_DIR}" CACHE STRING "" FORCE )
set( CMAKE_HIP_LINK_FLAGS "-fno-gpu-rdc --hip-link -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE )
endif()
37 changes: 37 additions & 0 deletions host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
set( CCE_VERSION_STRING "18.0.1" )
set( HIP_VERSION_STRING "6.2.1" )
set( CONFIG_NAME "tuo-cce-${CCE_VERSION_STRING}-rocm-${HIP_VERSION_STRING}" CACHE PATH "" )
include( ${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake )

# TODO: Set up GEOS_TPL_DIR
set( ENABLE_UMPIRE ON CACHE BOOL "" )
set( ENABLE_CHAI ON CACHE BOOL "" )
set( ENABLE_CALIPER ON CACHE BOOL "" )
set( ENABLE_ADIAK ON CACHE BOOL "" )

set( CAMP_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" )
set( RAJA_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" )
set( UMPIRE_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" )
set( CHAI_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" )
set( CALIPER_DIR ${GEOS_TPL_DIR}/caliper CACHE PATH "" )

# MPI options
set( MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "" )
set( MPI_INCLUDE_DIR ${MPI_HOME}/include CACHE PATH "" ) # Needed by hypre

# C++ options
set( CRAYPE_VERSION "2.7.33")
set( CMAKE_C_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/cc" CACHE PATH "" )
set( CMAKE_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH "" )
set( CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "" )

if( ENABLE_HIP )
set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE )

set( HIP_ROOT "/opt/rocm-${HIP_VERSION_STRING}" CACHE PATH "" )
set( ROCM_PATH ${HIP_ROOT} CACHE PATH "" )

set( CMAKE_HIP_ARCHITECTURES "gfx942" CACHE STRING "" FORCE )
set( CMAKE_HIP_FLAGS "-munsafe-fp-atomics -fno-gpu-rdc" CACHE STRING "" FORCE )
set( CMAKE_HIP_LINK_FLAGS "-fno-gpu-rdc --hip-link -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE )
endif()
41 changes: 41 additions & 0 deletions host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
set( CCE_VERSION_STRING "19.0.0" )
set( HIP_VERSION_STRING "6.4.0" )
set( CONFIG_NAME "tuo-cce-${CCE_VERSION_STRING}-rocm-${HIP_VERSION_STRING}" CACHE PATH "" )
include( ${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake )

# TODO: Set up GEOS_TPL_DIR
set( ENABLE_UMPIRE ON CACHE BOOL "" )
set( ENABLE_CHAI ON CACHE BOOL "" )
set( ENABLE_CALIPER ON CACHE BOOL "" )
set( ENABLE_ADIAK ON CACHE BOOL "" )

set( CAMP_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" )
set( RAJA_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" )
set( UMPIRE_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" )
set( CHAI_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" )
set( CALIPER_DIR ${GEOS_TPL_DIR}/caliper CACHE PATH "" )

# C++ options
set( CRAYPE_VERSION "2.7.34")
set( CMAKE_C_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/cc" CACHE PATH "" )
set( CMAKE_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH "" )
set( CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "" )

# MPI options
set( MPI_HOME /opt/cray/pe/mpich/8.1.33.1/ofi/crayclang/18.0 CACHE PATH "" )
set( MPI_C_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/cc" CACHE PATH "" )
set( MPI_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH "" )
set( MPI_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "" )
set( MPI_INCLUDE_DIR ${MPI_HOME}/include CACHE PATH "" ) # Needed by hypre

if( ENABLE_HIP )
set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE )

set( HIP_ROOT "/opt/rocm-${HIP_VERSION_STRING}" CACHE PATH "" )
set( HIP_ROOT_DIR ${HIP_ROOT} CACHE PATH "" )
set( ROCM_PATH ${HIP_ROOT} CACHE PATH "" )

set( CMAKE_HIP_ARCHITECTURES "gfx942" CACHE STRING "" FORCE )
set( CMAKE_HIP_FLAGS "-munsafe-fp-atomics -fno-gpu-rdc -I${MPI_INCLUDE_DIR}" CACHE STRING "" FORCE )
set( CMAKE_HIP_LINK_FLAGS "-fno-gpu-rdc --hip-link -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE )
endif()