diff --git a/host-configs/LLNL/amdgpu-base.cmake b/host-configs/LLNL/amdgpu-base.cmake new file mode 100644 index 00000000..1aee82cd --- /dev/null +++ b/host-configs/LLNL/amdgpu-base.cmake @@ -0,0 +1,27 @@ +set( CMAKE_CXX_STANDARD 17 CACHE STRING "" ) + +set( ENABLE_MPI ON CACHE BOOL "" FORCE ) +#set( ENABLE_FIND_MPI ON CACHE BOOL "" FORCE ) +set( ENABLE_CUDA OFF CACHE BOOL "" FORCE ) +set( ENABLE_HYPRE ON CACHE BOOL "" FORCE ) +set( ENABLE_HYPRE_DEVICE "HIP" CACHE STRING "" FORCE ) + +# HIP Options +set( ENABLE_HIP ON CACHE BOOL "" FORCE ) + +# suppress -Werror for now +set( ENABLE_WARNINGS_AS_ERRORS FALSE CACHE BOOL "" FORCE ) + +# GTEST +set( ENABLE_GTEST_DEATH_TESTS OFF CACHE BOOL "" ) +set( gtest_disable_pthreads ON CACHE BOOL "" ) + +# disable most binaries and doc generation +set( ENABLE_TESTS OFF CACHE BOOL "" FORCE ) +set( DISABLE_UNIT_TESTS ON CACHE BOOL "" FORCE ) +set( ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE ) +set( ENABLE_BENCHMARKS OFF CACHE BOOL "" FORCE ) +set( ENABLE_DOCS OFF CACHE BOOL "" FORCE ) + +# BLT trying to find MPI fails on cray with cce +#set( ENABLE_FIND_MPI OFF CACHE BOOL "" ) diff --git a/host-configs/LLNL/matrix-cuda-12-base.cmake b/host-configs/LLNL/matrix-cuda-12-base.cmake new file mode 100644 index 00000000..d614ed05 --- /dev/null +++ b/host-configs/LLNL/matrix-cuda-12-base.cmake @@ -0,0 +1,16 @@ +include(${CMAKE_CURRENT_LIST_DIR}/lassen-base.cmake) + +# Cuda options +set(ENABLE_CUDA ON CACHE BOOL "") +set(CUDA_TOOLKIT_ROOT_DIR /usr/tce/packages/cuda/cuda-12.2.2 CACHE STRING "") +set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "") +set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "") +set(CUDA_ARCH sm_90 CACHE STRING "") +set(CMAKE_CUDA_ARCHITECTURES 90 CACHE STRING "") +set(CMAKE_CUDA_STANDARD 17 CACHE STRING "") +set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "") +set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 " CACHE STRING "") +set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -g " CACHE STRING "") +set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "") + +set(CHAI_CUDA_FLAGS "-arch ${CUDA_ARCH}" CACHE STRING "" FORCE) diff --git a/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake b/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake new file mode 100644 index 00000000..048acd08 --- /dev/null +++ b/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake @@ -0,0 +1,13 @@ +set(CONFIG_NAME "matrix-gcc-12-cuda-12" CACHE PATH "") + +# Set compiler info +set(COMPILER_DIR /usr/tce/packages/gcc/gcc-12.1.1) +set(CMAKE_C_COMPILER ${COMPILER_DIR}/bin/gcc CACHE PATH "") +set(CMAKE_CXX_COMPILER ${COMPILER_DIR}/bin/g++ CACHE PATH "") + +# C++ options +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG" CACHE STRING "") +set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g" CACHE STRING "") + +include(${CMAKE_CURRENT_LIST_DIR}/matrix-cuda-12-base.cmake) diff --git a/host-configs/LLNL/tioga-base.cmake b/host-configs/LLNL/tioga-base.cmake deleted file mode 100644 index 411cc676..00000000 --- a/host-configs/LLNL/tioga-base.cmake +++ /dev/null @@ -1,24 +0,0 @@ -set(CMAKE_CXX_STANDARD 14 CACHE STRING "") - -set( ENABLE_MPI ON CACHE BOOL "" FORCE ) -set( ENABLE_FIND_MPI ON CACHE BOOL "" FORCE ) - -# HIP Options -set( ENABLE_HIP ON CACHE BOOL "" FORCE ) - -# suppress -Werror for now -set( ENABLE_WARNINGS_AS_ERRORS FALSE CACHE BOOL "" FORCE ) - -# GTEST -set(ENABLE_GTEST_DEATH_TESTS OFF CACHE BOOL "") -set(gtest_disable_pthreads ON CACHE BOOL "") - -# disable most binaries and doc generation -set(ENABLE_TESTS OFF CACHE BOOL "" FORCE) -set(DISABLE_UNIT_TESTS ON CACHE BOOL "" FORCE) -set(ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE) -set(ENABLE_BENCHMARKS OFF CACHE BOOL "" FORCE) -set(ENABLE_DOCS OFF CACHE BOOL "" FORCE) - -# BLT trying to find MPI fails on cray with cce -set(ENABLE_FIND_MPI FALSE CACHE BOOL "") diff --git a/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake b/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake new file mode 100644 index 00000000..2636f0fd --- /dev/null +++ b/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake @@ -0,0 +1,46 @@ +set( CCE_VERSION_STRING "18.0.1" ) +set( HIP_VERSION_STRING "6.2.1" ) +set( CONFIG_NAME "tioga-cce-${CCE_VERSION_STRING}-rocm-${HIP_VERSION_STRING}" CACHE PATH "" ) +include( ${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake ) + +# TODO: Set up the tpls +set( ENABLE_UMPIRE ON CACHE BOOL "" ) +set( ENABLE_CHAI ON CACHE BOOL "" ) +set( ENABLE_CALIPER ON CACHE BOOL "" ) +set( ENABLE_ADIAK ON CACHE BOOL "" ) + +set( CAMP_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" ) +set( RAJA_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" ) +set( UMPIRE_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" ) +set( CHAI_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" ) +set( CALIPER_DIR ${GEOS_TPL_DIR}/caliper CACHE PATH "" ) + +# MPI options +set( MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "" ) +set( MPI_INCLUDE_DIR ${MPI_HOME}/include CACHE PATH "" ) # Needed by hypre + +# C++ options +set( CRAYPE_VERSION "2.7.33") +set( CMAKE_C_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/cc" CACHE PATH "" ) +set( CMAKE_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH "" ) +set( CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "" ) + +if( ENABLE_HIP ) + set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE ) + + set( HIP_ROOT "/opt/rocm-${HIP_VERSION_STRING}" CACHE PATH "" ) + set( ROCM_PATH ${HIP_ROOT} CACHE PATH "" ) + + set( CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "" FORCE ) + set( CMAKE_HIP_FLAGS "-munsafe-fp-atomics -fno-gpu-rdc" CACHE STRING "" FORCE ) + set( CMAKE_HIP_LINK_FLAGS "-fno-gpu-rdc --hip-link -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE ) + + ############################################################################## + # The flag "fgpu-rdc" causes link issues when using cce-18 + # lld: /workspace/llvm/lib/Analysis/LoopAccessAnalysis.cpp:430: + # bool llvm::RuntimeCheckingPtrGroup::addPointer(unsigned int, const llvm::SCEV*, const llvm::SCEV*, unsigned int, bool, llvm::ScalarEvolution&): + # Assertion `AddressSpace == AS && "all pointers in a checking group must be in the same address space"' failed. + ############################################################################## + #set( CMAKE_CXX_FLAGS "-munsafe-fp-atomics -fgpu-rdc" CACHE STRING "" FORCE ) + #set( CMAKE_CXX_LINK_FLAGS "-fgpu-rdc --hip-link" CACHE STRING "" FORCE ) +endif() diff --git a/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake b/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake new file mode 100644 index 00000000..2cc283b0 --- /dev/null +++ b/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake @@ -0,0 +1,38 @@ +set( CCE_VERSION_STRING "19.0.0" ) +set( HIP_VERSION_STRING "6.4.0" ) +set( CONFIG_NAME "tioga-cce-${CCE_VERSION_STRING}-rocm-${HIP_VERSION_STRING}" CACHE PATH "" ) +include( ${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake ) + +# TODO: Set up GEOS_TPL_DIR +set( ENABLE_UMPIRE ON CACHE BOOL "" ) +set( ENABLE_CHAI ON CACHE BOOL "" ) +set( ENABLE_CALIPER ON CACHE BOOL "" ) +set( ENABLE_ADIAK ON CACHE BOOL "" ) + +set( CAMP_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" ) +set( RAJA_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" ) +set( UMPIRE_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" ) +set( CHAI_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" ) +set( CALIPER_DIR ${GEOS_TPL_DIR}/caliper CACHE PATH "" ) + +# MPI options +set( MPI_HOME /opt/cray/pe/mpich/8.1.33.1/ofi/crayclang/18.0 CACHE PATH "" ) +set( MPI_INCLUDE_DIR ${MPI_HOME}/include CACHE PATH "" ) # Needed by hypre + +# C++ options +set( CRAYPE_VERSION "2.7.34") +set( CMAKE_C_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/cc" CACHE PATH "" ) +set( CMAKE_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH "" ) +set( CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "" ) + +if( ENABLE_HIP ) + set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE ) + + set( HIP_ROOT "/opt/rocm-${HIP_VERSION_STRING}" CACHE PATH "" ) + set( HIP_ROOT_DIR ${HIP_ROOT} CACHE PATH "" ) + set( ROCM_PATH ${HIP_ROOT} CACHE PATH "" ) + + set( CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "" FORCE ) + set( CMAKE_HIP_FLAGS "-munsafe-fp-atomics -fno-gpu-rdc -I${MPI_INCLUDE_DIR}" CACHE STRING "" FORCE ) + set( CMAKE_HIP_LINK_FLAGS "-fno-gpu-rdc --hip-link -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE ) +endif() diff --git a/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake b/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake new file mode 100644 index 00000000..18201010 --- /dev/null +++ b/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake @@ -0,0 +1,37 @@ +set( CCE_VERSION_STRING "18.0.1" ) +set( HIP_VERSION_STRING "6.2.1" ) +set( CONFIG_NAME "tuo-cce-${CCE_VERSION_STRING}-rocm-${HIP_VERSION_STRING}" CACHE PATH "" ) +include( ${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake ) + +# TODO: Set up GEOS_TPL_DIR +set( ENABLE_UMPIRE ON CACHE BOOL "" ) +set( ENABLE_CHAI ON CACHE BOOL "" ) +set( ENABLE_CALIPER ON CACHE BOOL "" ) +set( ENABLE_ADIAK ON CACHE BOOL "" ) + +set( CAMP_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" ) +set( RAJA_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" ) +set( UMPIRE_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" ) +set( CHAI_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" ) +set( CALIPER_DIR ${GEOS_TPL_DIR}/caliper CACHE PATH "" ) + +# MPI options +set( MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "" ) +set( MPI_INCLUDE_DIR ${MPI_HOME}/include CACHE PATH "" ) # Needed by hypre + +# C++ options +set( CRAYPE_VERSION "2.7.33") +set( CMAKE_C_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/cc" CACHE PATH "" ) +set( CMAKE_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH "" ) +set( CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "" ) + +if( ENABLE_HIP ) + set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE ) + + set( HIP_ROOT "/opt/rocm-${HIP_VERSION_STRING}" CACHE PATH "" ) + set( ROCM_PATH ${HIP_ROOT} CACHE PATH "" ) + + set( CMAKE_HIP_ARCHITECTURES "gfx942" CACHE STRING "" FORCE ) + set( CMAKE_HIP_FLAGS "-munsafe-fp-atomics -fno-gpu-rdc" CACHE STRING "" FORCE ) + set( CMAKE_HIP_LINK_FLAGS "-fno-gpu-rdc --hip-link -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE ) +endif() diff --git a/host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake b/host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake new file mode 100644 index 00000000..e7fca85a --- /dev/null +++ b/host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake @@ -0,0 +1,41 @@ +set( CCE_VERSION_STRING "19.0.0" ) +set( HIP_VERSION_STRING "6.4.0" ) +set( CONFIG_NAME "tuo-cce-${CCE_VERSION_STRING}-rocm-${HIP_VERSION_STRING}" CACHE PATH "" ) +include( ${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake ) + +# TODO: Set up GEOS_TPL_DIR +set( ENABLE_UMPIRE ON CACHE BOOL "" ) +set( ENABLE_CHAI ON CACHE BOOL "" ) +set( ENABLE_CALIPER ON CACHE BOOL "" ) +set( ENABLE_ADIAK ON CACHE BOOL "" ) + +set( CAMP_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" ) +set( RAJA_DIR ${GEOS_TPL_DIR}/raja CACHE PATH "" ) +set( UMPIRE_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" ) +set( CHAI_DIR ${GEOS_TPL_DIR}/chai CACHE PATH "" ) +set( CALIPER_DIR ${GEOS_TPL_DIR}/caliper CACHE PATH "" ) + +# C++ options +set( CRAYPE_VERSION "2.7.34") +set( CMAKE_C_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/cc" CACHE PATH "" ) +set( CMAKE_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH "" ) +set( CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "" ) + +# MPI options +set( MPI_HOME /opt/cray/pe/mpich/8.1.33.1/ofi/crayclang/18.0 CACHE PATH "" ) +set( MPI_C_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/cc" CACHE PATH "" ) +set( MPI_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH "" ) +set( MPI_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "" ) +set( MPI_INCLUDE_DIR ${MPI_HOME}/include CACHE PATH "" ) # Needed by hypre + +if( ENABLE_HIP ) + set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE ) + + set( HIP_ROOT "/opt/rocm-${HIP_VERSION_STRING}" CACHE PATH "" ) + set( HIP_ROOT_DIR ${HIP_ROOT} CACHE PATH "" ) + set( ROCM_PATH ${HIP_ROOT} CACHE PATH "" ) + + set( CMAKE_HIP_ARCHITECTURES "gfx942" CACHE STRING "" FORCE ) + set( CMAKE_HIP_FLAGS "-munsafe-fp-atomics -fno-gpu-rdc -I${MPI_INCLUDE_DIR}" CACHE STRING "" FORCE ) + set( CMAKE_HIP_LINK_FLAGS "-fno-gpu-rdc --hip-link -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE ) +endif()