diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 867b9a93ab..9853a691e0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ "build": { "dockerfile": "Dockerfile", "args": { - "GEOS_TPL_TAG": "307-681" + "GEOS_TPL_TAG": "312-717" } }, "runArgs": [ diff --git a/host-configs/LLNL/tioga-base.cmake b/host-configs/LLNL/amdgpu-base.cmake similarity index 51% rename from host-configs/LLNL/tioga-base.cmake rename to host-configs/LLNL/amdgpu-base.cmake index 02a9317b68..59bb2a2757 100644 --- a/host-configs/LLNL/tioga-base.cmake +++ b/host-configs/LLNL/amdgpu-base.cmake @@ -1,17 +1,15 @@ -set( ENABLE_MATHPRESSO OFF CACHE BOOL "" ) +set( GEOS_ENABLE_VEM OFF CACHE BOOL "" FORCE ) -set( ENABLE_PAMELA ON CACHE BOOL "" ) +set( ENABLE_MATHPRESSO OFF CACHE BOOL "" ) +set( ENABLE_PAMELA OFF CACHE BOOL "" ) set( ENABLE_PVTPackage ON CACHE BOOL "" ) - set( ENABLE_PETSC OFF CACHE BOOL "" FORCE ) - -set( ENABLE_CALIPER OFF CACHE BOOL "" ) +set( ENABLE_CALIPER ON CACHE BOOL "" ) set( ENABLE_PAPI OFF CACHE BOOL "" ) - set( ENABLE_ESSL OFF CACHE BOOL "" ) - set( ENABLE_TRILINOS OFF CACHE BOOL "" ) -set( ENABLE_VTK OFF CACHE BOOL "" ) +set( ENABLE_VTK ON CACHE BOOL "" ) +set( ENABLE_OPENMP OFF CACHE BOOL "" FORCE ) set( CAMP_STANDALONE TRUE CACHE BOOL "" ) @@ -30,3 +28,17 @@ set( ENABLE_DOCS OFF CACHE BOOL "" FORCE ) set( ENABLE_SCOTCH OFF CACHE BOOL "" FORCE ) set( ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE ) +set( ENABLE_SUITESPARSE OFF CACHE BOOL "" FORCE ) + +# HYPRE options +set( ENABLE_HYPRE_DEVICE "HIP" CACHE STRING "Enable HIP backend for hypre" ) +set( ENABLE_HYPRE_MIXINT ON CACHE BOOL "Enable 64-bit integers for hypre" ) +set( ENABLE_HYPRE_GPU_AWARE_MPI OFF CACHE BOOL "Enable GPU-aware MPI for hypre" ) + +# TPLs +include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake) + +# ATS +set(USER $ENV{USER} CACHE STRING "") +set(ATS_WORKING_DIR "/p/lustre2/${USER}/integratedTestsGEOS/${CONFIG_NAME}" CACHE PATH "") +set(ATS_BASELINE_DIR "/p/lustre2/${USER}/integratedTestsGEOS/baselines" CACHE PATH "") diff --git a/host-configs/LLNL/matrix-base.cmake b/host-configs/LLNL/matrix-base.cmake new file mode 100644 index 0000000000..a27ef79f9b --- /dev/null +++ b/host-configs/LLNL/matrix-base.cmake @@ -0,0 +1,67 @@ +############################################################################### +# +# Base configuration for LC Matrix builds +# Calling configuration file must define the following CMAKE variables: +# +# MPI_HOME +# +############################################################################### + +set(GEOS_BUILD_OBJ_LIBS OFF CACHE BOOL "") + +# Fortran +set(ENABLE_FORTRAN OFF CACHE BOOL "") + +# MPI +set(ENABLE_MPI ON CACHE BOOL "") +set(MPI_HOME "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1" CACHE PATH "") +set(MPI_C_COMPILER ${MPI_HOME}/bin/mpicc CACHE PATH "") +set(MPI_CXX_COMPILER ${MPI_HOME}/bin/mpicxx CACHE PATH "") +set(MPIEXEC srun CACHE STRING "") +set(MPIEXEC_NUMPROC_FLAG -n CACHE STRING "") +set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "") + +# OpenMP +set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE) + +# CUDA +# LvArray sets this to the CMAKE_CXX_COMPILER. +set(CMAKE_CUDA_HOST_COMPILER ${MPI_CXX_COMPILER} CACHE STRING "") +set(ENABLE_CUDA_NVTOOLSEXT OFF CACHE BOOL "") + +# ESSL +set(ENABLE_ESSL OFF CACHE BOOL "" FORCE ) + +# TPL +set(ENABLE_PAPI OFF CACHE BOOL "") + +# GEOS specific options +set(ENABLE_PVTPackage ON CACHE BOOL "") +set(ENABLE_MATHPRESSO OFF CACHE BOOL "") +set(ENABLE_PETSC OFF CACHE BOOL "" FORCE ) +set(ENABLE_TRILINOS OFF CACHE BOOL "" FORCE ) +set(ENABLE_SUITESPARSE OFF CACHE BOOL "" FORCE ) +set(ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE ) +set(ENABLE_HYPRE_DEVICE "CUDA" CACHE STRING "" FORCE) +set(ENABLE_HYPRE_MIXINT ON CACHE BOOL "" FORCE) + +# Documentation +set(ENABLE_UNCRUSTIFY OFF CACHE BOOL "" FORCE) +set(ENABLE_DOXYGEN OFF CACHE BOOL "" FORCE) + +# YAPF python formatting +#set(YAPF_EXECUTABLE /usr/gapps/GEOSX/thirdPartyLibs/python/lassen-gcc-python/python/bin/yapf CACHE PATH "" FORCE) + +# PYGEOSX +set(ENABLE_PYGEOSX ON CACHE BOOL "") +set(PYTHON_EXECUTABLE /usr/tce/packages/python/python-3.9.12/bin/python CACHE PATH "") +set(Python3_ROOT_DIR /usr/tce/packages/python/python-3.9.12 CACHE PATH "") +set(Python3_EXECUTABLE /usr/tce/packages/python/python-3.9.12/bin/python3 CACHE PATH "") + +# ATS +set(ATS_ARGUMENTS "--ats jsrun_omp --ats jsrun_bind=packed" CACHE STRING "") +set(USER $ENV{USER} CACHE STRING "") +set(ATS_WORKING_DIR "/p/lustre2/${USER}/integratedTestsGEOS/${CONFIG_NAME}" CACHE PATH "") +set(ATS_BASELINE_DIR "/p/lustre2/${USER}/integratedTestsGEOS/baselines" CACHE PATH "") + +include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake) diff --git a/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake b/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake new file mode 100644 index 0000000000..df82f1531f --- /dev/null +++ b/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake @@ -0,0 +1,17 @@ +include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/matrix-base.cmake) + +# C++ +# The "-march=native -mtune=native" which LvArray adds breaks the PVT package. +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" CACHE STRING "" FORCE) + +# Fortran +set(CMAKE_Fortran_COMPILER /usr/tce/packages/gcc/gcc-12.1.1/bin/gfortran CACHE PATH "") +set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "") +set(FORTRAN_MANGLE_NO_UNDERSCORE OFF CACHE BOOL "") + +# MPI +set(MPI_Fortran_COMPILER ${MPI_HOME}/bin/mpifort CACHE PATH "") + +set(ENABLE_CUDA_NVTOOLSEXT OFF CACHE BOOL "") diff --git a/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake b/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake new file mode 100644 index 0000000000..dd4dd9b25f --- /dev/null +++ b/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake @@ -0,0 +1,9 @@ +include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake) + +# MPI +set(MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "") + +# Workaround for linking errors involving missing explicit instantiations of certain functions. +# TODO: we should add the missing explicit instantiations +set( CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE ) diff --git a/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake b/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake new file mode 100644 index 0000000000..204d3d6821 --- /dev/null +++ b/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake @@ -0,0 +1,10 @@ +include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake) + +# MPI +set(MPI_HOME /opt/cray/pe/mpich/8.1.33.1/ofi/crayclang/18.0 CACHE PATH "") + +# GPU-aware MPI option +if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( CMAKE_HIP_LINK_FLAGS "${CMAKE_HIP_LINK_FLAGS} $ENV{PE_MPICH_GTL_DIR_amd_gfx90a} $ENV{PE_MPICH_GTL_LIBS_amd_gfx90a}" CACHE STRING "" FORCE ) +endif() diff --git a/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake b/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake new file mode 100644 index 0000000000..a366f21f3d --- /dev/null +++ b/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake @@ -0,0 +1,9 @@ +include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake) + +# MPI +set(MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "") + +# Workaround for linking errors involving missing explicit instantiations of certain functions. +# TODO: we should add the missing explicit instantiations +set( CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE ) diff --git a/host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake b/host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake new file mode 100644 index 0000000000..54da2a818f --- /dev/null +++ b/host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake @@ -0,0 +1,9 @@ +include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake) + +# MPI +set(MPI_HOME /opt/cray/pe/mpich/8.1.32/ofi/crayclang/19.0 CACHE PATH "") + +# Workaround for linking errors involving missing explicit instantiations of certain functions. +# TODO: we should add the missing explicit instantiations +set( CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE ) diff --git a/scripts/config-build.py b/scripts/config-build.py index 505d213b4e..2aefa4d0f6 100755 --- a/scripts/config-build.py +++ b/scripts/config-build.py @@ -214,6 +214,12 @@ def main(calling_script, args, unknown_args): # Append cache file at the end of the command line to make previous argument visible to the cache. cmake_line.append("-C%s" % cache_file) + # Add them back + for unknown_arg in unknown_args: + if not unknown_arg.startswith('-D'): + logging.warning("Additional argument '%s' does not start with '-D'. Keeping it nevertheless." % unknown_arg) + cmake_line.append(unknown_arg) + cmake_line.append(os.path.normpath(os.path.join(scripts_dir, "..", "src"))) # Dump the cmake command to file for convenience diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d9c175fec2..10a11fc8a9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,6 +6,9 @@ cmake_policy(SET CMP0104 OLD) # when using nvcc populate CMAKE_CUDA_ARCHITECTURE cmake_policy(SET CMP0074 NEW) # dont ignore _ROOT env vars when searching for packages via find_package() cmake_policy(SET CMP0066 NEW) # use CMAKE__FLAGS_ for try_compile() instead of only CMAKE__FLAGS cmake_policy(SET CMP0056 NEW) # use CMAKE_EXE_LINKER_FLAGS in try_compile() in addition to CMAKE__FLAGS +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.27") + cmake_policy(SET CMP0144 NEW) # find_package uses upper-case _ROOT variables. +endif() ################################ # GEOSX @@ -85,7 +88,6 @@ add_subdirectory( coreComponents ) add_subdirectory( externalComponents ) add_subdirectory( main ) - ################################ # Add docs ################################ @@ -104,7 +106,7 @@ install( DIRECTORY ${PROJECT_BINARY_DIR}/include DESTINATION . ) # Generate version information ################################ include( cmake/GeosxVersion.cmake ) -message( STATUS "Configuring GEOSX version ${GEOS_VERSION_FULL}" ) +message( STATUS "Configuring GEOS version ${GEOS_VERSION_FULL}" ) ################################ @@ -123,10 +125,16 @@ endif() if( ENABLE_CUDA ) list( APPEND extraComponentsLinkList cuda ) -endif() -if( ENABLE_CUDA AND ENABLE_CUDA_NVTOOLSEXT ) - list( APPEND extraComponentsLinkList CUDA::nvToolsExt ) + if (ENABLE_CUDA_NVTOOLSEXT) + list( APPEND extraComponentsLinkList CUDA::nvToolsExt ) + message( STATUS "Enabling CUDA::nvToolsExt" ) + endif() + + if (ENABLE_CUDA_NVML) + list( APPEND extraComponentsLinkList CUDA::nvml ) + message( STATUS "Enabling CUDA::nvml" ) + endif() endif() if( ENABLE_HIP ) @@ -135,7 +143,7 @@ endif() blt_add_executable( NAME geosx SOURCES main/main.cpp - DEPENDS_ON geosx_core + DEPENDS_ON geosx_core ${extraComponentsLinkList} ${externalComponentsLinkList} ) @@ -309,7 +317,7 @@ if ( ENABLE_ATS ) if (NOT Python3_EXECUTABLE) message( FATAL_ERROR "An appropriate version of python was not found (required for integrated testing system). Try setting Python3_ROOT_DIR and/or Python3_EXECUTABLE in your host config." ) - endif() + endif() # Setup testing set( ATS_SCRIPT diff --git a/src/cmake/GeosxOptions.cmake b/src/cmake/GeosxOptions.cmake index c9072b081a..21c7bf3876 100644 --- a/src/cmake/GeosxOptions.cmake +++ b/src/cmake/GeosxOptions.cmake @@ -120,10 +120,22 @@ endif() # Physics packages option( GEOS_ENABLE_CONTACT "Enables contact physics package" ON ) option( GEOS_ENABLE_FLUIDFLOW "Enables fluid flow physics package" ON ) +option( GEOS_ENABLE_FLUIDFLOW_ISOTHERMAL_FVM "Enables isothermal FVM kernels in fluid flow physics package" ON ) +option( GEOS_ENABLE_FLUIDFLOW_THERMAL_FVM "Enables thermal FVM kernels in fluid flow physics package" ON ) +option( GEOS_ENABLE_FLUIDFLOW_REACTIVE "Enables reactive kernels in fluid flow physics package" ON ) +option( GEOS_ENABLE_FLUIDFLOW_HYBRIDFVM "Enables Hybrid FVM kernels in fluid flow physics package" ON ) option( GEOS_ENABLE_INDUCEDSEISMICITY "Enables induced seismicity physics package" ON ) option( GEOS_ENABLE_MULTIPHYSICS "Enables multiphysics physics package" ON ) +option( GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS "Enables poromechanics kernels in multiphysics package" ON ) +option( GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS_EFEM "Enables poromechanics EFEM kernels in multiphysics package" ON ) +option( GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS_DAMAGE "Enables poromechanics damage kernels in multiphysics package" ON ) +option( GEOS_ENABLE_MULTIPHYSICS_THERMOPOROMECHANICS "Enables thermoporomechanics kernels in multiphysics package" ON ) option( GEOS_ENABLE_SIMPLEPDE "Enables simple PDE physics package" ON ) option( GEOS_ENABLE_SOLIDMECHANICS "Enables solid mechanics physics package" ON ) +option( GEOS_ENABLE_SOLIDMECHANICS_ELASTIC "Enables elastic kernels in solid mechanics physics package" ON ) +option( GEOS_ENABLE_SOLIDMECHANICS_EXTRA "Enables extra kernels in solid mechanics physics package" ON ) +option( GEOS_ENABLE_SOLIDMECHANICS_THERMOPOROELASTIC "Enables thermoporoelastic kernels in solid mechanics physics package" OFF ) +option( GEOS_ENABLE_SOLIDMECHANICS_THERMOPOROMECHANICS "Enables thermoporomechanics kernels in solid mechanics physics package" ON ) option( GEOS_ENABLE_SURFACEGENERATION "Enables surface generation physics package" ON ) option( GEOS_ENABLE_WAVEPROPAGATION "Enables wave propagation physics package" ON ) diff --git a/src/cmake/blt b/src/cmake/blt index 9ff77344f0..b01fb10015 160000 --- a/src/cmake/blt +++ b/src/cmake/blt @@ -1 +1 @@ -Subproject commit 9ff77344f0b2a6ee345e452bddd6bfd46cbbfa35 +Subproject commit b01fb10015c1aacf484536a0bc22e4cfc6d22845 diff --git a/src/cmake/thirdparty/SetupGeosxThirdParty.cmake b/src/cmake/thirdparty/SetupGeosxThirdParty.cmake index 6736e46062..88765d3415 100644 --- a/src/cmake/thirdparty/SetupGeosxThirdParty.cmake +++ b/src/cmake/thirdparty/SetupGeosxThirdParty.cmake @@ -211,7 +211,7 @@ else() endif() ################################ -# ZLIB +# ZLIB ################################ if(DEFINED ZLIB_DIR) list(PREPEND CMAKE_PREFIX_PATH "${ZLIB_DIR}") @@ -719,6 +719,14 @@ if(DEFINED HYPRE_DIR AND ENABLE_HYPRE) message( " ----> hypre_VERSION = ${hypre_VERSION}" ) endif() + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + if( "${header_file}" MATCHES "HYPRE_USING_GPU_AWARE_MPI 1" ) + message( " ----> GPU-aware MPI support activated" ) + else () + message(FATAL_ERROR "GEOS configured with ENABLE_HYPRE_GPU_AWARE_MPI=ON, however hypre was not built with GPU-aware MPI support") + endif () + endif () + # Prepend Hypre to link flags, fix for Umpire appearing before Hypre on the link line # if (NOT CMAKE_HOST_APPLE) # blt_add_target_link_flags (TO hypre FLAGS "-Wl,--whole-archive ${HYPRE_DIR}/lib/libHYPRE.a -Wl,--no-whole-archive") diff --git a/src/coreComponents/CMakeLists.txt b/src/coreComponents/CMakeLists.txt index 7cf2532422..4e3c224247 100644 --- a/src/coreComponents/CMakeLists.txt +++ b/src/coreComponents/CMakeLists.txt @@ -1,3 +1,5 @@ +set(TOTAL_GENERATED_KERNELS 0 CACHE INTERNAL "Total number of generated kernels") + set( subdirs math common @@ -83,5 +85,6 @@ endif() if( GEOS_ENABLE_TESTS ) add_subdirectory( unitTests ) -endif( ) +endif() +message( STATUS "Total number of generated kernels: ${TOTAL_GENERATED_KERNELS}" ) \ No newline at end of file diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index e848125162..71ce0db516 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit e848125162b5b6af76d2dd40c8cfa7fd6ee18cbe +Subproject commit 71ce0db51605a37681a3f3b76efcac067014f487 diff --git a/src/coreComponents/common/GeosxConfig.hpp.in b/src/coreComponents/common/GeosxConfig.hpp.in index bc20648df6..544429f267 100644 --- a/src/coreComponents/common/GeosxConfig.hpp.in +++ b/src/coreComponents/common/GeosxConfig.hpp.in @@ -5,7 +5,7 @@ * Copyright (c) 2016-2024 Lawrence Livermore National Security LLC * Copyright (c) 2018-2024 TotalEnergies * Copyright (c) 2018-2024 The Board of Trustees of the Leland Stanford Junior University - * Copyright (c) 2023-2024 Chevron + * Copyright (c) 2023-2024 Chevron * Copyright (c) 2019- GEOS/GEOSX Contributors * All rights reserved * @@ -95,6 +95,9 @@ /// Enables use of SuperLU_dist library through HYPRE (CMake option ENABLE_SUPERLU_DIST) #cmakedefine GEOS_USE_SUPERLU_DIST +/// Enables use of SuiteSparse library (CMake option ENABLE_SUITESPARSE) +#cmakedefine GEOS_USE_SUITESPARSE + /// Enables use of PETSc library (CMake option ENABLE_PETSC) #cmakedefine GEOS_USE_PETSC @@ -200,4 +203,3 @@ #endif #endif /* GEOS_CONFIG_HPP */ - diff --git a/src/coreComponents/common/LifoStorage.hpp b/src/coreComponents/common/LifoStorage.hpp index c887dacb22..2b73e5e64f 100644 --- a/src/coreComponents/common/LifoStorage.hpp +++ b/src/coreComponents/common/LifoStorage.hpp @@ -64,7 +64,7 @@ class LifoStorage m_bufferCount( 0 ) { LIFO_LOG_RANK( " LIFO : maximum size "<< m_maxNumberOfBuffers << " buffers " ); - LIFO_LOG_RANK( " LIFO : buffer size " << m_bufferSize / ( 1024.0 * 1024.0 ) << "MB" ); + LIFO_LOG_RANK( " LIFO : buffer size " << ( ( double ) m_bufferSize ) / ( 1024.0 * 1024.0 ) << "MB" ); if( numberOfBuffersToStoreOnDevice < 0 ) { #ifdef GEOS_USE_CUDA diff --git a/src/coreComponents/common/LifoStorageCommon.hpp b/src/coreComponents/common/LifoStorageCommon.hpp index 3183cde65b..7d18978aa7 100644 --- a/src/coreComponents/common/LifoStorageCommon.hpp +++ b/src/coreComponents/common/LifoStorageCommon.hpp @@ -164,7 +164,7 @@ class LifoStorageCommon GEOS_ERROR( "To use LifoStorage, both _SC_AVPHYS_PAGES and _SC_PAGESIZE must be defined." ); #endif int numberOfBuffersToStoreOnHost = std::max( 1, std::min( ( int )( 0.01 * percent * free / bufferSize ), maxNumberOfBuffers - numberOfBuffersToStoreOnDevice ) ); - double freeGB = ( ( double ) free ) / ( 1024.0 * 1024.0 * 1024.0 ) / MpiWrapper::nodeCommSize(); + GEOS_MAYBE_UNUSED double freeGB = ( ( double ) free ) / ( 1024.0 * 1024.0 * 1024.0 ) / MpiWrapper::nodeCommSize(); LIFO_LOG_RANK( " LIFO : available memory on host " << freeGB << " GB" ); return numberOfBuffersToStoreOnHost; } diff --git a/src/coreComponents/common/MemoryInfos.cpp b/src/coreComponents/common/MemoryInfos.cpp index 1e31cdb6be..d31a0f3e44 100644 --- a/src/coreComponents/common/MemoryInfos.cpp +++ b/src/coreComponents/common/MemoryInfos.cpp @@ -44,6 +44,11 @@ MemoryInfos::MemoryInfos( umpire::MemoryResourceTraits::resource_type resourceTy case umpire::MemoryResourceTraits::resource_type::um: #if defined( GEOS_USE_CUDA ) cudaMemGetInfo( &m_availableMemory, &m_totalMemory ); + #elif defined( GEOS_USE_HIP ) + { + hipError_t const err = hipMemGetInfo( &m_availableMemory, &m_totalMemory ); + GEOS_WARNING_IF( err != hipSuccess, GEOS_FMT( "HIP error found: {} at {}:{}", hipGetErrorString( err ), __FILE__, __LINE__ ) ); + } #else GEOS_WARNING( "Unknown device physical memory size getter for this compiler." ); m_physicalMemoryHandled = 0; diff --git a/src/coreComponents/common/MpiWrapper.hpp b/src/coreComponents/common/MpiWrapper.hpp index b884e15a8b..6efc6f0e6d 100644 --- a/src/coreComponents/common/MpiWrapper.hpp +++ b/src/coreComponents/common/MpiWrapper.hpp @@ -1280,6 +1280,7 @@ U MpiWrapper::prefixSum( T const value, MPI_Comm comm ) U const convertedValue = value; int const error = MPI_Exscan( &convertedValue, &localResult, 1, internal::getMpiType< U >(), MPI_SUM, comm ); MPI_CHECK_ERROR( error ); + MPI_Barrier( comm ); #endif if( commRank() == 0 ) { diff --git a/src/coreComponents/common/TypeDispatch.hpp b/src/coreComponents/common/TypeDispatch.hpp index b38b585806..70a207b45f 100644 --- a/src/coreComponents/common/TypeDispatch.hpp +++ b/src/coreComponents/common/TypeDispatch.hpp @@ -376,7 +376,7 @@ bool dispatch( LIST const combinations, if( !success ) { auto typePrinter = []( auto t ){ return LvArray::system::demangle( typeid( typename decltype(t)::type ).name() ); }; - auto typeListPrinter = [typePrinter]( auto tlist ){ return internal::listToString( typename decltype( tlist )::type{}, "\n ", "", typePrinter ); }; + GEOS_MAYBE_UNUSED auto typeListPrinter = [typePrinter]( auto tlist ){ return internal::listToString( typename decltype( tlist )::type{}, "\n ", "", typePrinter ); }; GEOS_ERROR( "Types were not dispatched to the lambda of type\n" << LvArray::system::demangleType< LAMBDA >() << "\n" diff --git a/src/coreComponents/constitutive/PVTPackage b/src/coreComponents/constitutive/PVTPackage index 374b72962f..59d81b149e 160000 --- a/src/coreComponents/constitutive/PVTPackage +++ b/src/coreComponents/constitutive/PVTPackage @@ -1 +1 @@ -Subproject commit 374b72962f7e605e74a1943c541e94d7b3f492a3 +Subproject commit 59d81b149e35ea5db0d715e608217d8bd56e9291 diff --git a/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/CubicEOSPhaseModel.hpp b/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/CubicEOSPhaseModel.hpp index 1b9be2bf38..0961134ae1 100644 --- a/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/CubicEOSPhaseModel.hpp +++ b/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/CubicEOSPhaseModel.hpp @@ -874,8 +874,8 @@ computeCompressibilityFactor( integer const numComps, } } - stackArray1d< real64, MultiFluidConstants::MAX_NUM_COMPONENTS > logFugacityCoefficientsMax( numComps ); - stackArray1d< real64, MultiFluidConstants::MAX_NUM_COMPONENTS > logFugacityCoefficientsMin( numComps ); + StackArray< real64, 1, MultiFluidConstants::MAX_NUM_COMPONENTS > logFugacityCoefficientsMax( numComps ); + StackArray< real64, 1, MultiFluidConstants::MAX_NUM_COMPONENTS > logFugacityCoefficientsMin( numComps ); computeLogFugacityCoefficients( numComps, composition, binaryInteractionCoefficients, zMin, aPureCoefficient, bPureCoefficient, aMixtureCoefficient, bMixtureCoefficient, logFugacityCoefficientsMin.toSlice() ); diff --git a/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/NegativeTwoPhaseFlash.hpp b/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/NegativeTwoPhaseFlash.hpp index 3027f0a2ec..942f1475fe 100644 --- a/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/NegativeTwoPhaseFlash.hpp +++ b/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/NegativeTwoPhaseFlash.hpp @@ -376,10 +376,14 @@ void NegativeTwoPhaseFlash::computeDerivatives( else { // Calculate the liquid and vapour fugacities and derivatives - stackArray1d< real64, maxNumComps > logLiquidFugacity( numComps ); - stackArray1d< real64, maxNumComps > logVapourFugacity( numComps ); - stackArray2d< real64, maxNumComps * maxNumDofs > logLiquidFugacityDerivs( numComps, numDofs ); - stackArray2d< real64, maxNumComps * maxNumDofs > logVapourFugacityDerivs( numComps, numDofs ); + //stackArray1d< real64, maxNumComps > logLiquidFugacity( numComps ); + //stackArray1d< real64, maxNumComps > logVapourFugacity( numComps ); + //stackArray2d< real64, maxNumComps * maxNumDofs > logLiquidFugacityDerivs( numComps, numDofs ); + //stackArray2d< real64, maxNumComps * maxNumDofs > logVapourFugacityDerivs( numComps, numDofs ); + StackArray< real64, 1, maxNumComps > logLiquidFugacity( numComps ); + StackArray< real64, 1, maxNumComps > logVapourFugacity( numComps ); + StackArray< real64, 2, maxNumComps * maxNumDofs, MatrixLayout::ROW_MAJOR_PERM > logLiquidFugacityDerivs( numComps, numDofs ); + StackArray< real64, 2, maxNumComps * maxNumDofs, MatrixLayout::ROW_MAJOR_PERM > logVapourFugacityDerivs( numComps, numDofs ); FugacityCalculator::computeLogFugacity( numComps, pressure, diff --git a/src/coreComponents/constitutive/fluid/multifluid/compositional/models/ImmiscibleWaterFlashModel.hpp b/src/coreComponents/constitutive/fluid/multifluid/compositional/models/ImmiscibleWaterFlashModel.hpp index 59a32c87a4..cc338739c4 100644 --- a/src/coreComponents/constitutive/fluid/multifluid/compositional/models/ImmiscibleWaterFlashModel.hpp +++ b/src/coreComponents/constitutive/fluid/multifluid/compositional/models/ImmiscibleWaterFlashModel.hpp @@ -172,7 +172,7 @@ void ImmiscibleWaterFlashModelUpdate::compute( ComponentProperties::KernelWrappe // Hydrocarbon phases // Calculate normalised hyrdocarbon composition - stackArray1d< real64, maxNumComps > composition( m_numComponents ); + StackArray< real64, 1, maxNumComps > composition( m_numComponents ); for( integer ic = 0; ic < m_numComponents; ++ic ) { composition[ic] = compFraction[ic] / z_hc; diff --git a/src/coreComponents/constitutive/fluid/multifluid/compositional/models/PhillipsBrineDensity.cpp b/src/coreComponents/constitutive/fluid/multifluid/compositional/models/PhillipsBrineDensity.cpp index 5094c7f9b8..a66bdade08 100644 --- a/src/coreComponents/constitutive/fluid/multifluid/compositional/models/PhillipsBrineDensity.cpp +++ b/src/coreComponents/constitutive/fluid/multifluid/compositional/models/PhillipsBrineDensity.cpp @@ -233,7 +233,9 @@ void PhillipsBrineDensity::calculateEosWaterMolarVolume( arraySlice1d< real64 co // Create pure water composition stackArray1d< real64, MultiFluidConstants::MAX_NUM_COMPONENTS > waterComposition( numComps ); - stackArray1d< real64, 2 + MultiFluidConstants::MAX_NUM_COMPONENTS > tempDerivs( numDofs ); + //stackArray1d< real64, 2 + MultiFluidConstants::MAX_NUM_COMPONENTS > tempDerivs( numDofs ); + //stackArray1d< real64, 2 + MultiFluidConstants::MAX_NUM_COMPONENTS > tempDerivs; + StackArray< real64, 1, 2 + MultiFluidConstants::MAX_NUM_COMPONENTS > tempDerivs( numDofs ); for( integer ic = 0; ic < numComps; ++ic ) { waterComposition[ic] = 0.0; diff --git a/src/coreComponents/linearAlgebra/CMakeLists.txt b/src/coreComponents/linearAlgebra/CMakeLists.txt index 1837effe1f..49ac36ab0b 100644 --- a/src/coreComponents/linearAlgebra/CMakeLists.txt +++ b/src/coreComponents/linearAlgebra/CMakeLists.txt @@ -38,7 +38,6 @@ set( linearAlgebra_headers solvers/BicgstabSolver.hpp solvers/BlockPreconditioner.hpp solvers/CgSolver.hpp - solvers/GmresSolver.hpp solvers/KrylovSolver.hpp solvers/KrylovUtils.hpp solvers/PreconditionerBlockJacobi.hpp @@ -60,6 +59,16 @@ set( linearAlgebra_headers utilities/NormalOperator.hpp utilities/ReverseCutHillMcKeeOrdering.hpp utilities/TransposeOperator.hpp ) + +# TODO: workaround for issue with undefined hidden symbols +# LvArray::internal::toChaiExecutionSpace(camp::resources::v1::Platform) +# LvArray::internal::getArrayManager() +# chai::ArrayManager::move(void*, chai::PointerRecord*, chai::ExecutionSpace) +# geos::HypreVector::~HypreVector() +if( NOT ${ENABLE_HIP} ) + list( APPEND linearAlgebra_headers solvers/GmresSolver.hpp ) +endif() + # # Specify all sources # @@ -68,26 +77,35 @@ set( linearAlgebra_sources solvers/BicgstabSolver.cpp solvers/BlockPreconditioner.cpp solvers/CgSolver.cpp - solvers/GmresSolver.cpp solvers/KrylovSolver.cpp solvers/SeparateComponentPreconditioner.cpp utilities/ReverseCutHillMcKeeOrdering.cpp ) +# TODO: workaround for issue with undefined hidden symbols +# LvArray::internal::toChaiExecutionSpace(camp::resources::v1::Platform) +# LvArray::internal::getArrayManager() +# chai::ArrayManager::move(void*, chai::PointerRecord*, chai::ExecutionSpace) +# geos::HypreVector::~HypreVector() +if( NOT ${ENABLE_HIP} ) + list( APPEND linearAlgebra_sources solvers/GmresSolver.cpp ) +endif() + set( dependencyList ${parallelDeps} mesh denseLinearAlgebra finiteVolume ) set( tplDependencyList "" ) -list( APPEND linearAlgebra_headers interfaces/direct/SuiteSparse.hpp ) -list( APPEND linearAlgebra_sources interfaces/direct/SuiteSparse.cpp ) -list( APPEND tplDependencyList suitesparse ) +if( ENABLE_SUITESPARSE ) + list( APPEND linearAlgebra_headers interfaces/direct/SuiteSparse.hpp ) + list( APPEND linearAlgebra_sources interfaces/direct/SuiteSparse.cpp ) + list( APPEND tplDependencyList suitesparse ) +endif( ) if( ENABLE_SUPERLU_DIST ) - list( APPEND linearAlgebra_headers interfaces/direct/SuperLUDist.hpp ) - list( APPEND linearAlgebra_sources interfaces/direct/SuperLUDist.cpp ) - list( APPEND tplDependencyList superlu_dist ) + list( APPEND linearAlgebra_headers interfaces/direct/SuperLUDist.hpp ) + list( APPEND linearAlgebra_sources interfaces/direct/SuperLUDist.cpp ) + list( APPEND tplDependencyList superlu_dist ) endif( ) if( ENABLE_TRILINOS ) - list( APPEND linearAlgebra_headers interfaces/trilinos/EpetraExport.hpp interfaces/trilinos/EpetraMatrix.hpp @@ -106,11 +124,9 @@ if( ENABLE_TRILINOS ) interfaces/trilinos/TrilinosSolver.cpp ) list( APPEND tplDependencyList trilinos ) - endif() if( ENABLE_HYPRE ) - list( APPEND linearAlgebra_headers interfaces/hypre/HypreExport.hpp interfaces/hypre/HypreInterface.hpp @@ -154,11 +170,9 @@ if( ENABLE_HYPRE ) interfaces/hypre/HypreVector.cpp ) list( APPEND tplDependencyList hypre umpire ) - endif() if( ENABLE_PETSC ) - list( APPEND linearAlgebra_headers interfaces/petsc/PetscExport.hpp interfaces/petsc/PetscInterface.hpp @@ -177,7 +191,6 @@ if( ENABLE_PETSC ) interfaces/petsc/PetscVector.cpp ) list( APPEND tplDependencyList petsc ) - endif() geos_decorate_link_dependencies( LIST decoratedDependencies @@ -198,4 +211,3 @@ install( TARGETS linearAlgebra LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib ) if( GEOS_ENABLE_TESTS ) add_subdirectory( unitTests ) endif( ) - diff --git a/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp b/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp index 302e220b21..42a1b8db35 100644 --- a/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp @@ -704,6 +704,12 @@ class MatrixBase : public virtual LinearOperator< VECTOR > virtual void leftRightScale( Vector const & vecLeft, Vector const & vecRight ) = 0; + /** + * @brief Compute left and right scaling vectors for diagonal scaling. + * @param scaling Vector to be filled with scaling factors. + */ + virtual void computeScalingVector( Vector & scaling ) const = 0; + /** * @brief Rescales selected rows of matrix using row sum reciprocal as a factor. * @param rowIndices global indicies of rows to scale (all must be locally owned) diff --git a/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp b/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp index 2efb0725fa..cb56748fbc 100644 --- a/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp @@ -242,12 +242,18 @@ class VectorBase real64 const beta ) = 0; /** - * @brief Compute the component-wise multiplication y = v * x. - * @param x first vector (input) - * @param y second vector (output) + * @brief Element-wise multiplication of this vector with another vector. + * @param x Vector to multiply with. + * @note This modifies the current vector in place: this[i] *= x[i] */ - virtual void pointwiseProduct( Vector const & x, - Vector & y ) const = 0; + virtual void pointwiseProduct( Vector const & x ) = 0; + + /** + * @brief Element-wise division of this vector by another vector. + * @param x Vector to divide by. + * @note This modifies the current vector in place: this[i] /= x[i] + */ + virtual void pointwiseDivide( Vector const & x ) = 0; /** * @brief 1-norm of the vector. diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp index 783c2e6b48..c7b5630e93 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp @@ -19,16 +19,17 @@ #include "HypreInterface.hpp" +#if defined(GEOS_USE_SUITESPARSE) #include "linearAlgebra/interfaces/direct/SuiteSparse.hpp" +#endif +#if defined(GEOS_USE_SUPERLU_DIST) +#include "linearAlgebra/interfaces/direct/SuperLUDist.hpp" +#endif #include "linearAlgebra/interfaces/hypre/HypreMatrix.hpp" #include "linearAlgebra/interfaces/hypre/HyprePreconditioner.hpp" #include "linearAlgebra/interfaces/hypre/HypreSolver.hpp" #include "linearAlgebra/interfaces/hypre/HypreUtils.hpp" -#if defined(GEOS_USE_SUPERLU_DIST) -#include "linearAlgebra/interfaces/direct/SuperLUDist.hpp" -#endif - #include "HYPRE_utilities.h" #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP #include "_hypre_utilities.h" @@ -55,7 +56,13 @@ void HypreInterface::initialize() HYPRE_Initialize(); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP HYPRE_SetExecutionPolicy( HYPRE_EXEC_DEVICE ); +#if defined(HYPRE_USING_HIP) + HYPRE_SetSpGemmUseVendor( 1 ); +#else + // TODO: Is it better to use vendor's SpGEMM with CUDA? HYPRE_SetSpGemmUseVendor( 0 ); +#endif + HYPRE_DeviceInitialize(); #endif HYPRE_SetMemoryLocation( hypre::memoryLocation ); @@ -69,6 +76,7 @@ void HypreInterface::initialize() #endif HYPRE_SetLogLevel( getenv( "HYPRE_LOG_LEVEL" ) ? atoi( getenv( "HYPRE_LOG_LEVEL" ) ) : 0 ); + HYPRE_SetGpuAwareMPI( getenv( "MPICH_GPU_SUPPORT_ENABLED" ) ? atoi( getenv( "MPICH_GPU_SUPPORT_ENABLED" ) ) : 0 ); } void HypreInterface::finalize() @@ -86,13 +94,18 @@ HypreInterface::createSolver( LinearSolverParameters params ) #if defined(GEOS_USE_SUPERLU_DIST) return std::make_unique< SuperLUDist< HypreInterface > >( std::move( params ) ); #else - GEOS_ERROR( "GEOSX is configured without support for SuperLU_dist." ); + GEOS_ERROR( "GEOS is configured without support for SuperLU_dist." ); return std::unique_ptr< LinearSolverBase< HypreInterface > >( nullptr ); #endif } else { +#if defined(GEOS_USE_SUITESPARSE) return std::make_unique< SuiteSparse< HypreInterface > >( std::move( params ) ); +#else + GEOS_ERROR( "GEOS is configured without support for SuiteSparse." ); + return std::unique_ptr< LinearSolverBase< HypreInterface > >( nullptr ); +#endif } } else diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp index 2224ffbc41..9a41a225ea 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp @@ -180,7 +180,8 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxRowSum( solver.ptr, 1.0 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetStrongThreshold( solver.ptr, 0.6 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType( solver.ptr, hypre::getAMGInterpolationType( LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); + //GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType( solver.ptr, hypre::getAMGInterpolationType( + // LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, hypre::getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::chebyshev ) ) ); @@ -211,7 +212,7 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::multipass ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::modifiedExtendedE ) ) ); + //GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::modifiedExtendedE ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::l1jacobi ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumSweeps( solver.ptr, 2 ) ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp index 5e68ad1887..bf74d3d632 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp @@ -803,8 +803,29 @@ void HypreMatrix::leftScale( HypreVector const & vec ) GEOS_LAI_ASSERT( ready() ); GEOS_LAI_ASSERT( vec.ready() ); GEOS_LAI_ASSERT_EQ( vec.localSize(), numLocalRows() ); - hypre::scaleMatrixRows( hypre_ParCSRMatrixDiag( m_parcsr_mat ), hypre_ParVectorLocalVector( vec.unwrapped() ) ); - hypre::scaleMatrixRows( hypre_ParCSRMatrixOffd( m_parcsr_mat ), hypre_ParVectorLocalVector( vec.unwrapped() ) ); + + GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixDiagScale( m_parcsr_mat, vec.unwrapped(), nullptr ) ); +} + +void HypreMatrix::rightScale( HypreVector const & vec ) +{ + GEOS_LAI_ASSERT( ready() ); + GEOS_LAI_ASSERT( vec.ready() ); + GEOS_LAI_ASSERT_EQ( vec.localSize(), numLocalRows() ); + + GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixDiagScale( m_parcsr_mat, nullptr, vec.unwrapped() ) ); +} + +void HypreMatrix::leftRightScale( HypreVector const & vecLeft, + HypreVector const & vecRight ) +{ + GEOS_LAI_ASSERT( ready() ); + GEOS_LAI_ASSERT( vecLeft.ready() ); + GEOS_LAI_ASSERT( vecRight.ready() ); + GEOS_LAI_ASSERT_EQ( vecLeft.localSize(), numLocalRows() ); + GEOS_LAI_ASSERT_EQ( vecRight.localSize(), numLocalRows() ); + + GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixDiagScale( m_parcsr_mat, vecLeft.unwrapped(), vecRight.unwrapped() ) ); } void HypreMatrix::rescaleRows( arrayView1d< globalIndex const > const & rowIndices, @@ -1425,22 +1446,6 @@ real64 HypreMatrix::normMax( arrayView1d< globalIndex const > const & rowIndices return MpiWrapper::max( maxNorm, comm() ); } -void HypreMatrix::rightScale( HypreVector const & vec ) -{ - GEOS_LAI_ASSERT( ready() ); - HypreMatrix t; - transpose( t ); - t.leftScale( vec ); - t.transpose( *this ); -} - -void HypreMatrix::leftRightScale( HypreVector const & vecLeft, - HypreVector const & vecRight ) -{ - leftScale( vecLeft ); - rightScale( vecRight ); -} - void HypreMatrix::transpose( HypreMatrix & dst ) const { GEOS_LAI_ASSERT( ready() ); @@ -1459,4 +1464,27 @@ MPI_Comm HypreMatrix::comm() const return hypre_IJMatrixComm( m_ij_mat ); } +void HypreMatrix::computeScalingVector( HypreVector & scaling ) const +{ + GEOS_LAI_ASSERT( ready() ); + + // Get number of components + HYPRE_Int num_tags = LvArray::integerConversion< HYPRE_Int >( + (m_dofManager->*static_cast< integer (geos::DofManager::*)() const >(&geos::DofManager::numComponents))() + ); + + // Get local dof component labels + array1d< HYPRE_Int > pointMarkers( numLocalRows() ); + m_dofManager->getLocalDofComponentLabels( pointMarkers ); + + // Create scaling vector + scaling.create( numLocalRows(), comm() ); + HYPRE_ParVector hypre_vec = scaling.unwrapped(); + HYPRE_ParVector *hypre_vec_ptr = &hypre_vec; + + // Compute scaling vector + GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixComputeScalingTagged( m_parcsr_mat, 1, HYPRE_MEMORY_HOST, num_tags, + pointMarkers.data(), hypre_vec_ptr ) ); +} + }// end namespace geos diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp index c3dc2055bc..7a268dfda0 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp @@ -21,6 +21,7 @@ #define GEOS_LINEARALGEBRA_INTERFACES_HYPREMATRIX_HPP_ #include "common/DataTypes.hpp" +#include "linearAlgebra/DofManager.hpp" #include "linearAlgebra/interfaces/hypre/HypreVector.hpp" #include "linearAlgebra/interfaces/hypre/HypreExport.hpp" #include "linearAlgebra/common/LinearOperator.hpp" @@ -269,6 +270,15 @@ class HypreMatrix final : public virtual LinearOperator< HypreVector >, virtual void rightScale( HypreVector const & vec ) override; + /** + * @copydoc MatrixBase::computeScalingVector + */ + virtual void computeScalingVector( HypreVector & scaling ) const override; + + /** + * @copydoc MatrixBase::leftRightScale + */ + virtual void leftRightScale( HypreVector const & vecLeft, HypreVector const & vecRight ) override; diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp index 3526345823..59e227b7c2 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp @@ -78,7 +78,7 @@ void createHypreGMRES( LinearSolverParameters const & params, GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRGMRESSetTol( solver.ptr, params.krylov.relTolerance ) ); // Default for now - HYPRE_Int logLevel = (params.logLevel >= 3) ? 2 : 0; + HYPRE_Int logLevel = (params.logLevel >= 3) ? 3 : 0; GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRGMRESSetPrintLevel( solver.ptr, logLevel ) ); // print iteration info GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRGMRESSetLogging( solver.ptr, 1 ) ); /* needed to get run info later */ diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp index 5b6f7b3079..048a299c3b 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp @@ -102,7 +102,6 @@ void HypreVector::create( localIndex const localSize, // Set up the parallel and local vector data structures m_vec = hypre_ParVectorCreate( comm, globalSize, partitioning ); - hypre_ParVectorOwnsData( m_vec ) = false; hypre_Vector * const localVector = hypre_ParVectorLocalVector( m_vec ); hypre_VectorOwnsData( localVector ) = false; @@ -161,11 +160,9 @@ void HypreVector::scale( real64 const scalingFactor ) void HypreVector::reciprocal() { GEOS_LAI_ASSERT( ready() ); - arrayView1d< real64 > values = m_values.toView(); - forAll< hypre::execPolicy >( localSize(), [values] GEOS_HYPRE_DEVICE ( localIndex const i ) - { - values[i] = 1.0 / values[i]; - } ); + + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorPointwiseInverse( m_vec, &m_vec ) ); + touch(); } real64 HypreVector::dot( HypreVector const & vec ) const @@ -225,22 +222,24 @@ void HypreVector::axpby( real64 const alpha, } } -void HypreVector::pointwiseProduct( HypreVector const & x, - HypreVector & y ) const +void HypreVector::pointwiseProduct( HypreVector const & x ) { GEOS_LAI_ASSERT( ready() ); GEOS_LAI_ASSERT( x.ready() ); - GEOS_LAI_ASSERT( y.ready() ); GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); - GEOS_LAI_ASSERT_EQ( localSize(), y.localSize() ); - arrayView1d< real64 const > const my_values = m_values.toViewConst(); - arrayView1d< real64 const > const x_values = x.m_values.toViewConst(); - arrayView1d< real64 > const y_values = y.m_values.toView(); - forAll< hypre::execPolicy >( localSize(), [y_values, my_values, x_values] GEOS_HYPRE_DEVICE ( localIndex const i ) - { - y_values[i] = my_values[i] * x_values[i]; - } ); + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorPointwiseProduct( x.m_vec, m_vec, &m_vec ) ); + touch(); +} + +void HypreVector::pointwiseDivide( HypreVector const & x ) +{ + GEOS_LAI_ASSERT( ready() ); + GEOS_LAI_ASSERT( x.ready() ); + GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); + + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorPointwiseDivision( x.m_vec, m_vec, &m_vec ) ); + touch(); } real64 HypreVector::norm1() const diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp index 771c1cfe30..36df6e3c40 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp @@ -140,8 +140,15 @@ class HypreVector final : private VectorBase< HypreVector > HypreVector const & x, real64 const beta ) override; - virtual void pointwiseProduct( HypreVector const & x, - HypreVector & y ) const override; + /** + * @copydoc VectorBase::pointwiseProduct + */ + virtual void pointwiseProduct( HypreVector const & x ) override; + + /** + * @copydoc VectorBase::pointwiseDivide + */ + virtual void pointwiseDivide( HypreVector const & x ) override; /** * @copydoc VectorBase::norm1 diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp index 3555d7d824..023ef212ab 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp @@ -64,7 +64,7 @@ class Hydrofracture : public MGRStrategyBase< 1 > m_levelFRelaxIters[0] = 1; m_levelInterpType[0] = MGRInterpolationType::jacobi; m_levelRestrictType[0] = MGRRestrictionType::injection; - m_levelCoarseGridMethod[0] = MGRCoarseGridMethod::galerkin; + m_levelCoarseGridMethod[0] = MGRCoarseGridMethod::nonGalerkin; m_levelGlobalSmootherType[0] = MGRGlobalSmootherType::none; } @@ -85,7 +85,7 @@ class Hydrofracture : public MGRStrategyBase< 1 > // Configure the BoomerAMG solver used as mgr coarse solver for the pressure reduced system setPressureAMG( mgrData.coarseSolver ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMinCoarseSize( mgrData.coarseSolver.ptr, 1000 ) ); + //GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMinCoarseSize( mgrData.coarseSolver.ptr, 1000 ) ); } }; diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscInterface.cpp index bb404c744e..d5b7de097d 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscInterface.cpp @@ -19,8 +19,12 @@ #include "PetscInterface.hpp" +#if defined(GEOS_USE_SUITESPARSE) #include "linearAlgebra/interfaces/direct/SuiteSparse.hpp" +#endif +#if defined(GEOS_USE_SUPERLU_DIST) #include "linearAlgebra/interfaces/direct/SuperLUDist.hpp" +#endif #include "linearAlgebra/interfaces/petsc/PetscPreconditioner.hpp" #include "linearAlgebra/interfaces/petsc/PetscSolver.hpp" @@ -47,13 +51,20 @@ PetscInterface::createSolver( LinearSolverParameters params ) { if( params.solverType == LinearSolverParameters::SolverType::direct ) { +#if defined(GEOS_USE_SUPERLU_DIST) if( params.direct.parallel ) { return std::make_unique< SuperLUDist< PetscInterface > >( std::move( params ) ); } else +#endif { +#if defined(GEOS_USE_SUITESPARSE) return std::make_unique< SuiteSparse< PetscInterface > >( std::move( params ) ); +#else + GEOS_ERROR( "GEOS is configured without support for SuiteSparse." ); + return std::unique_ptr< LinearSolverBase< TrilinosInterface > >( nullptr ); +#endif } } else diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp index 1ddf73808e..dac9a208a7 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp @@ -653,6 +653,12 @@ void PetscMatrix::leftRightScale( PetscVector const & vecLeft, GEOS_LAI_CHECK_ERROR( MatDiagonalScale( m_mat, vecLeft.unwrapped(), vecRight.unwrapped() ) ); } +void PetscMatrix::computeScalingVector( PetscVector & scaling ) const +{ + GEOS_UNUSED_VAR( scaling ); + GEOS_ERROR( "Not implemented!!!" ); +} + void PetscMatrix::multiplyRAP( PetscMatrix const & R, PetscMatrix const & P, PetscMatrix & dst ) const diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp index 8c1666d4c4..847ed89cc0 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp @@ -255,6 +255,8 @@ class PetscMatrix final : public virtual LinearOperator< PetscVector >, virtual void leftRightScale( PetscVector const & vecLeft, PetscVector const & vecRight ) override; + virtual void computeScalingVector( PetscVector & scaling ) const override; + virtual void rescaleRows( arrayView1d< globalIndex const > const & rowIndices, RowSumType const rowSumType ) override; diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.cpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.cpp index 5e2106127b..52f9bb6ee1 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.cpp @@ -214,17 +214,26 @@ void PetscVector::axpby( real64 const alpha, } } -void PetscVector::pointwiseProduct( PetscVector const & x, - PetscVector & y ) const +void PetscVector::pointwiseProduct( PetscVector const & x ) { GEOS_LAI_ASSERT( ready() ); GEOS_LAI_ASSERT( x.ready() ); - GEOS_LAI_ASSERT( y.ready() ); + GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); GEOS_LAI_ASSERT_EQ( globalSize(), x.globalSize() ); - GEOS_LAI_ASSERT_EQ( globalSize(), y.globalSize() ); - GEOS_LAI_CHECK_ERROR( VecPointwiseMult( y.m_vec, m_vec, x.m_vec ) ); - y.touch(); + GEOS_LAI_CHECK_ERROR( VecPointwiseMult( m_vec, m_vec, x.m_vec ) ); + touch(); +} + +void PetscVector::pointwiseDivide( PetscVector const & x ) +{ + GEOS_LAI_ASSERT( ready() ); + GEOS_LAI_ASSERT( x.ready() ); + GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); + GEOS_LAI_ASSERT_EQ( globalSize(), x.globalSize() ); + + GEOS_LAI_CHECK_ERROR( VecPointwiseDivide( m_vec, m_vec, x.m_vec ) ); + touch(); } real64 PetscVector::norm1() const diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp index 16a325323f..e75d3b3088 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp @@ -134,8 +134,9 @@ class PetscVector final : private VectorBase< PetscVector > PetscVector const & x, real64 const beta ) override; - virtual void pointwiseProduct( PetscVector const & x, - PetscVector & y ) const override; + virtual void pointwiseProduct( PetscVector const & x ) override; + + virtual void pointwiseDivide( PetscVector const & x ) override; /** * @copydoc VectorBase::norm1 diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp index bfd54eef9e..295714afa9 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp @@ -531,6 +531,12 @@ void EpetraMatrix::leftRightScale( EpetraVector const & vecLeft, rightScale( vecRight ); } +void EpetraMatrix::computeScalingVector( EpetraVector & scaling ) const +{ + GEOS_UNUSED_VAR( scaling ); + GEOS_ERROR( "Not implemented!!!" ); +} + void EpetraMatrix::transpose( EpetraMatrix & dst ) const { GEOS_LAI_ASSERT( ready() ); diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp index 501e5d0391..49b30321df 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp @@ -244,6 +244,8 @@ class EpetraMatrix final : public virtual LinearOperator< EpetraVector >, virtual void leftRightScale( EpetraVector const & vecLeft, EpetraVector const & vecRight ) override; + virtual void computeScalingVector( EpetraVector & scaling ) const override; + virtual void rescaleRows( arrayView1d< globalIndex const > const & rowIndices, RowSumType const rowSumType ) override; diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp index a928559710..f5e0b213d2 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp @@ -187,17 +187,21 @@ void EpetraVector::axpby( real64 const alpha, touch(); } -void EpetraVector::pointwiseProduct( EpetraVector const & x, - EpetraVector & y ) const +void EpetraVector::pointwiseProduct( EpetraVector const & x ) { GEOS_LAI_ASSERT( ready() ); GEOS_LAI_ASSERT( x.ready() ); - GEOS_LAI_ASSERT( y.ready() ); + GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); GEOS_LAI_ASSERT_EQ( globalSize(), x.globalSize() ); - GEOS_LAI_ASSERT_EQ( globalSize(), y.globalSize() ); - GEOS_LAI_CHECK_ERROR( ( y.unwrapped() ).Multiply( 1.0, unwrapped(), x.unwrapped(), 0.0 ) ); - y.touch(); + GEOS_LAI_CHECK_ERROR( ( unwrapped() ).Multiply( 1.0, unwrapped(), x.unwrapped(), 0.0 ) ); + touch(); +} + +void EpetraVector::pointwiseDivide( EpetraVector const & x ) +{ + GEOS_UNUSED_VAR( x ); + GEOS_ERROR( "Not implemented!!!" ); } real64 EpetraVector::norm1() const diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp index 0d1b394a00..92df009dac 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp @@ -123,8 +123,9 @@ class EpetraVector final : private VectorBase< EpetraVector > EpetraVector const & x, real64 const beta ) override; - virtual void pointwiseProduct( EpetraVector const & x, - EpetraVector & y ) const override; + virtual void pointwiseProduct( EpetraVector const & x ) override; + + virtual void pointwiseDivide( EpetraVector const & x ) override; /** * @copydoc VectorBase::norm1 diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp index 39c5883876..4f7cf52aff 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp @@ -19,8 +19,12 @@ #include "TrilinosInterface.hpp" +#if defined(GEOS_USE_SUITESPARSE) #include "linearAlgebra/interfaces/direct/SuiteSparse.hpp" +#endif +#if defined(GEOS_USE_SUPERLU_DIST) #include "linearAlgebra/interfaces/direct/SuperLUDist.hpp" +#endif #include "linearAlgebra/interfaces/trilinos/TrilinosPreconditioner.hpp" #include "linearAlgebra/interfaces/trilinos/TrilinosSolver.hpp" @@ -38,13 +42,20 @@ TrilinosInterface::createSolver( LinearSolverParameters params ) { if( params.solverType == LinearSolverParameters::SolverType::direct ) { +#if defined(GEOS_USE_SUPERLU_DIST) if( params.direct.parallel ) { return std::make_unique< SuperLUDist< TrilinosInterface > >( std::move( params ) ); } else +#endif { +#if defined(GEOS_USE_SUITESPARSE) return std::make_unique< SuiteSparse< TrilinosInterface > >( std::move( params ) ); +#else + GEOS_ERROR( "GEOS is configured without support for SuiteSparse." ); + return std::unique_ptr< LinearSolverBase< TrilinosInterface > >( nullptr ); +#endif } } else diff --git a/src/coreComponents/mainInterface/version.cpp b/src/coreComponents/mainInterface/version.cpp index 0ed60fc6b1..7999ede8c5 100644 --- a/src/coreComponents/mainInterface/version.cpp +++ b/src/coreComponents/mainInterface/version.cpp @@ -60,10 +60,14 @@ static std::string getGpuCompilerIdString() #if defined( GEOS_USE_CUDA ) oss << " - CUDA compiler version: " << CUDA_VERSION/10/100 << "." << CUDA_VERSION/10%100; + +#elif defined( GEOS_USE_HIP ) + oss << " - HIP compiler version: " + << HIP_VERSION_MAJOR << "." + << HIP_VERSION_MINOR << "." + << HIP_VERSION_PATCH; #endif -#if defined( GEOS_USE_HIP ) - oss << " - ROCm compiler version: " << ROCM_VERSION/100/100 << "." << ROCM_VERSION/100%100; -#endif + return oss.str(); } @@ -72,13 +76,13 @@ void outputVersionInfo() GEOS_LOG_RANK_0( "GEOS version: " << getVersion() ); - GEOS_LOG_RANK_0( " - c++ compiler: " << getCppCompilerIdString() ); + GEOS_LOG_RANK_0( " - C++ compiler: " << getCppCompilerIdString() ); std::string const gpuCompilerIdString = getGpuCompilerIdString(); GEOS_LOG_RANK_0_IF( !gpuCompilerIdString.empty(), gpuCompilerIdString ); #if defined(_OPENMP) - GEOS_LOG_RANK_0( " - openmp version: " << _OPENMP ); + GEOS_LOG_RANK_0( " - OpenMP version: " << _OPENMP ); #endif #if defined(GEOS_USE_MPI) @@ -102,56 +106,56 @@ void outputVersionInfo() GEOS_LOG_RANK_0( " - VTK version: " << STRINGIZE( VTK_VERSION ) ); #endif -#if defined(RAJA_VERSION) +#if defined(GEOS_USE_RAJA) && defined(RAJA_VERSION) GEOS_LOG_RANK_0( " - RAJA version: " << STRINGIZE( RAJA_VERSION ) ); #endif -#if defined(umpire_VERSION) - GEOS_LOG_RANK_0( " - umpire version: " << STRINGIZE( umpire_VERSION ) ); +#if defined(GEOS_USE_UMPIRE) && defined(umpire_VERSION) + GEOS_LOG_RANK_0( " - Umpire version: " << STRINGIZE( umpire_VERSION ) ); #endif -#if defined(chai_VERSION) - GEOS_LOG_RANK_0( " - chai version: " << STRINGIZE( chai_VERSION ) ); +#if defined(GEOS_USE_CHAI) && defined(chai_VERSION) + GEOS_LOG_RANK_0( " - CHAI version: " << STRINGIZE( chai_VERSION ) ); #endif -#if defined(adiak_VERSION) +#if defined(GEOS_USE_ADIAK) && defined(adiak_VERSION) GEOS_LOG_RANK_0( " - adiak version: " << STRINGIZE( adiak_VERSION ) ); #endif -#if defined(caliper_VERSION) - GEOS_LOG_RANK_0( " - caliper version: " << STRINGIZE( caliper_VERSION ) ); +#if defined(GEOS_USE_CALIPER) && defined(caliper_VERSION) + GEOS_LOG_RANK_0( " - Caliper version: " << STRINGIZE( caliper_VERSION ) ); #endif -#if defined(metis_VERSION) +#if defined(GEOS_USE_METIS) && defined(metis_VERSION) GEOS_LOG_RANK_0( " - METIS version: " << STRINGIZE( metis_VERSION ) ); #endif -#if defined(parmetis_VERSION) - GEOS_LOG_RANK_0( " - PARAMETIS version: " << STRINGIZE( parmetis_VERSION ) ); +#if defined(GEOS_USE_PARMETIS) && defined(parmetis_VERSION) + GEOS_LOG_RANK_0( " - PARMETIS version: " << STRINGIZE( parmetis_VERSION ) ); #endif -#if defined(scotch_VERSION) - GEOS_LOG_RANK_0( " - scotch version: " << STRINGIZE( scotch_VERSION ) ); +#if defined(GEOS_USE_SCOTCH) && defined(scotch_VERSION) + GEOS_LOG_RANK_0( " - Scotch version: " << STRINGIZE( scotch_VERSION ) ); #endif -#if defined(superlu_dist_VERSION) - GEOS_LOG_RANK_0( " - superlu_dist version: " << STRINGIZE( superlu_dist_VERSION ) ); +#if defined(GEOS_USE_SUPERLU_DIST) && defined(superlu_dist_VERSION) + GEOS_LOG_RANK_0( " - SuperLU_Dist version: " << STRINGIZE( superlu_dist_VERSION ) ); #endif -#if defined(suitesparse_VERSION) - GEOS_LOG_RANK_0( " - suitesparse version: " << STRINGIZE( suitesparse_VERSION ) ); +#if defined(GEOS_USE_SUITESPARSE) && defined(suitesparse_VERSION) + GEOS_LOG_RANK_0( " - SuiteSparse version: " << STRINGIZE( suitesparse_VERSION ) ); #endif -#if defined(hypre_VERSION) - GEOS_LOG_RANK_0( " - hypre version: " << STRINGIZE( hypre_VERSION ) ); +#if defined(GEOS_USE_HYPRE) && defined(hypre_VERSION) + GEOS_LOG_RANK_0( " - HYPRE version: " << STRINGIZE( hypre_VERSION ) ); #endif -#if defined(trilinos_VERSION) +#if defined(GEOS_USE_TRILINOS) && defined(trilinos_VERSION) GEOS_LOG_RANK_0( " - trilinos version: " << STRINGIZE( trilinos_VERSION ) ); #endif -#if defined(petsc_VERSION) - GEOS_LOG_RANK_0( " - petsc version: " << STRINGIZE( petsc_VERSION ) ); +#if defined(GEOS_USE_PETSC) && defined(petsc_VERSION) + GEOS_LOG_RANK_0( " - PETSc version: " << STRINGIZE( petsc_VERSION ) ); #endif #if defined(Python3_VERSION) @@ -167,11 +171,11 @@ void outputVersionInfo() defined(GEOS_USE_HYPRE) && \ ( GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CPU ) GEOS_LOG_RANK_0( "" ); - GEOS_LOG_RANK_0( "**************************************************" ); - GEOS_LOG_RANK_0( "* WARNING!!! *" ); - GEOS_LOG_RANK_0( "* *" ); - GEOS_LOG_RANK_0( "* GEOS has GPU support enabled, but not HYPRE! *" ); - GEOS_LOG_RANK_0( "**************************************************" ); + GEOS_LOG_RANK_0( "*******************************************************" ); + GEOS_LOG_RANK_0( "* WARNING!! WARNING!! WARNING!! *" ); + GEOS_LOG_RANK_0( "* *" ); + GEOS_LOG_RANK_0( "* GEOS has GPU support enabled, but HYPRE does not! *" ); + GEOS_LOG_RANK_0( "*******************************************************" ); GEOS_LOG_RANK_0( "" ); #endif diff --git a/src/coreComponents/mesh/FaceElementSubRegion.cpp b/src/coreComponents/mesh/FaceElementSubRegion.cpp index 52b19f0953..242bd32cb0 100644 --- a/src/coreComponents/mesh/FaceElementSubRegion.cpp +++ b/src/coreComponents/mesh/FaceElementSubRegion.cpp @@ -449,10 +449,11 @@ localIndex FaceElementSubRegion::unpackToFaceRelation( buffer_unit_type const * * @param[in] elem2dToElems3d A mapping. * @param[in,out] elem2dToFaces This mapping will be corrected if needed to match @p elem2dToElems3d. */ -void fixNeighborMappingsInconsistency( string const & fractureName, +void fixNeighborMappingsInconsistency( GEOS_MAYBE_UNUSED string const & fractureName, FixedToManyElementRelation const & elem2dToElems3d, FaceElementSubRegion::FaceMapType & elem2dToFaces ) { + GEOS_MAYBE_UNUSED static constexpr std::string_view mappingInconsistency= "Mapping neighbor inconsistency detected for fracture {}."; { localIndex const num2dElems = elem2dToFaces.size( 0 ); for( int e2d = 0; e2d < num2dElems; ++e2d ) @@ -486,9 +487,9 @@ void fixNeighborMappingsInconsistency( string const & fractureName, { std::swap( elem2dToFaces[e2d][0], elem2dToFaces[e2d][1] ); } - else if( !matchStraight ) + else { - GEOS_ERROR( "Mapping neighbor inconsistency detected for fracture " << fractureName ); + GEOS_ERROR_IF( !matchStraight, GEOS_FMT( mappingInconsistency, fractureName ) ); } } } @@ -608,8 +609,7 @@ buildCollocatedEdgeBuckets( std::map< globalIndex, globalIndex > const & referen std::map< std::pair< globalIndex, globalIndex >, std::set< localIndex > > collocatedEdgeBuckets; for( auto const & p: edgesIds ) { - static constexpr auto nodeNotFound = "Internal error when trying to access the reference collocated node for global node {}."; - GEOS_UNUSED_VAR( nodeNotFound ); // Not used in GPU builds. + GEOS_MAYBE_UNUSED static constexpr auto nodeNotFound = "Internal error when trying to access the reference collocated node for global node {}."; std::pair< globalIndex, globalIndex > const & nodes = p.first; localIndex const & edge = p.second; diff --git a/src/coreComponents/mesh/FaceManager.cpp b/src/coreComponents/mesh/FaceManager.cpp index ba170dce58..b36e5da1f7 100644 --- a/src/coreComponents/mesh/FaceManager.cpp +++ b/src/coreComponents/mesh/FaceManager.cpp @@ -174,8 +174,7 @@ void FaceManager::setGeometricalRelations( CellBlockManagerABC const & cellBlock return; } - static constexpr auto err = "Internal error when trying to connect matrix mapping and fracture mapping. Face {} seems wrongly connected."; - GEOS_UNUSED_VAR( err ); // Not used in GPU builds. + GEOS_MAYBE_UNUSED static constexpr auto err = "Internal error when trying to connect matrix mapping and fracture mapping. Face {} seems wrongly connected."; FaceElementSubRegion const & subRegion = region.getUniqueSubRegion< FaceElementSubRegion >(); int const esr = 0; // Since there's only on unique subregion, the index is always 0. diff --git a/src/coreComponents/mesh/generators/CollocatedNodes.cpp b/src/coreComponents/mesh/generators/CollocatedNodes.cpp index 349c397300..55ace596e6 100644 --- a/src/coreComponents/mesh/generators/CollocatedNodes.cpp +++ b/src/coreComponents/mesh/generators/CollocatedNodes.cpp @@ -27,9 +27,9 @@ CollocatedNodes::CollocatedNodes( string const & faceBlockName, vtkSmartPointer< vtkDataSet > faceMesh ) { // The vtk field to the collocated nodes for fractures. - string const COLLOCATED_NODES = "collocated_nodes"; + string const collocatedNodesField = "collocated_nodes"; - vtkIdTypeArray const * collocatedNodes = vtkIdTypeArray::FastDownCast( faceMesh->GetPointData()->GetArray( COLLOCATED_NODES.c_str() ) ); + vtkIdTypeArray const * collocatedNodes = vtkIdTypeArray::FastDownCast( faceMesh->GetPointData()->GetArray( collocatedNodesField.c_str() ) ); // Depending on the parallel split, the vtk face mesh may be empty on a rank. // In that case, vtk will not provide any field for the emtpy mesh. @@ -44,7 +44,7 @@ CollocatedNodes::CollocatedNodes( string const & faceBlockName, { GEOS_LOG_RANK_0( " - " << faceMesh->GetPointData()->GetArrayName( i ) << " of type '" << faceMesh->GetPointData()->GetArray( i )->GetDataTypeAsString() << "'" ); } - GEOS_ERROR( "Could not find valid field \"" << COLLOCATED_NODES << "\" for fracture \"" << faceBlockName << "\"." ); + GEOS_ERROR( "Could not find valid field \"" << collocatedNodesField << "\" for fracture \"" << faceBlockName << "\"." ); } if( collocatedNodes ) diff --git a/src/coreComponents/mesh/generators/ParMETISInterface.hpp b/src/coreComponents/mesh/generators/ParMETISInterface.hpp index fa1bb2c854..8e948f762e 100644 --- a/src/coreComponents/mesh/generators/ParMETISInterface.hpp +++ b/src/coreComponents/mesh/generators/ParMETISInterface.hpp @@ -26,13 +26,8 @@ namespace geos { -#if defined(GEOS_USE_HIP) // still need int32 hypre for the current hip-capable build -/// Typedef to allow us to specify required parmetis integer type. -using pmet_idx_t = int32_t; -#else /// Typedef to allow us to specify required parmetis integer type. using pmet_idx_t = int64_t; -#endif namespace parmetis { diff --git a/src/coreComponents/mesh/generators/PrismUtilities.hpp b/src/coreComponents/mesh/generators/PrismUtilities.hpp index 8dd8e983e3..1e489de4e1 100644 --- a/src/coreComponents/mesh/generators/PrismUtilities.hpp +++ b/src/coreComponents/mesh/generators/PrismUtilities.hpp @@ -40,8 +40,7 @@ localIndex getFaceNodesPrism( localIndex const faceNum, { static_assert( N > 4, "Function getFaceNodePrism can be called for a prism with N-sided polygon base where N > 5." ); - static constexpr auto nodeCountError = "Not enough nodes for {} element (face index = {}).\n"; - GEOS_UNUSED_VAR( nodeCountError ); // Not used in GPU builds. + GEOS_MAYBE_UNUSED static constexpr auto nodeCountError = "Not enough nodes for {} element (face index = {}).\n"; if( faceNum == 0 ) { diff --git a/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp b/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp index 0f07f00f02..ee2b7b19a7 100644 --- a/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp +++ b/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp @@ -976,11 +976,15 @@ void CommunicationTools::asyncSendRecv( stdVector< NeighborCommunicator > & neig parallelDeviceEvents & events ) { GEOS_MARK_FUNCTION; +#if !defined( GEOS_USE_HIP ) if( onDevice ) { waitAllDeviceEvents( events ); } - +#else + GEOS_UNUSED_VAR( events ); + GEOS_UNUSED_VAR( onDevice ); +#endif // could swap this to test and make this function call async as well, only launch the sends/recvs for // those we've already recv'd sizing for, go back to some usefule compute / launch some other compute, then @@ -1068,10 +1072,15 @@ void CommunicationTools::finalizeUnpack( MeshLevel & mesh, // poll mpi for completion then wait 10 nanoseconds 6,000,000,000 times (60 sec timeout) GEOS_ASYNC_WAIT( 6000000000, 10, asyncUnpack( mesh, neighbors, icomm, onDevice, events, op ) ); +#if !defined( GEOS_USE_HIP ) if( onDevice ) { waitAllDeviceEvents( events ); } +#else + GEOS_UNUSED_VAR( events ); + GEOS_UNUSED_VAR( onDevice ); +#endif MpiWrapper::waitAll( icomm.size(), icomm.mpiSendBufferSizeRequest(), diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index a6809a5343..384ad5e87a 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -43,11 +43,19 @@ PhysicsSolverBase::PhysicsSolverBase( string const & name, m_nextDt( 1e99 ), m_numTimestepsSinceLastDtCut( -1 ), m_dofManager( name ), + m_usePhysicsScaling(), m_linearSolverParameters( groupKeyStruct::linearSolverParametersString(), this ), m_nonlinearSolverParameters( groupKeyStruct::nonlinearSolverParametersString(), this ), m_solverStatistics( groupKeyStruct::solverStatisticsString(), this ), m_systemSetupTimestamp( 0 ) { + // Physics-scaling is enabled by default only with hypre builds +#ifdef GEOS_USE_HYPRE + integer usePhysicsScaling = 1; +#else + integer usePhysicsScaling = 0; +#endif + setInputFlags( InputFlags::OPTIONAL_NONUNIQUE ); // This sets a flag to indicate that this object is going to select the time step size @@ -93,6 +101,11 @@ PhysicsSolverBase::PhysicsSolverBase( string const & name, setRestartFlags( RestartFlags::WRITE_AND_READ ). setDescription( "Cut time step if linear solution fail without going until max nonlinear iterations." ); + registerWrapper( viewKeyStruct::usePhysicsScalingString(), &m_usePhysicsScaling ) + .setApplyDefaultValue( usePhysicsScaling ) + .setInputFlag( InputFlags::OPTIONAL ) + .setDescription( "Enable physics-based scaling of the linear system. Default: true." ); + addLogLevel< logInfo::Fields >(); addLogLevel< logInfo::LinearSolver >(); addLogLevel< logInfo::Solution >(); @@ -786,6 +799,32 @@ real64 PhysicsSolverBase::eisenstatWalker( real64 const newNewtonNorm, return krylovTol; } + +void MemoryUsageOutput( std::string const & message ) +{ + GEOS_UNUSED_VAR(message); +#if 0 +#if defined( GEOS_USE_HIP ) + size_t free, total; + + for( int rank=0; rank < MpiWrapper::commSize(); ++rank ) + { + MpiWrapper::barrier(); + if( rank == MpiWrapper::commRank() ) + { + hipError_t const err = hipMemGetInfo( &free, &total ); + std::cout << std::fixed << std::setprecision(3) + << "Rank " << rank + << " - " << message + << " - allocated memory: " << static_cast(total - free) / (1 << 30) + << " GiB" << std::endl; + } + } +#endif +#endif +} + + real64 PhysicsSolverBase::nonlinearImplicitStep( real64 const & time_n, real64 const & dt, integer const cycleNumber, @@ -796,6 +835,10 @@ real64 PhysicsSolverBase::nonlinearImplicitStep( real64 const & time_n, // value to track the achieved dt for this step. real64 stepDt = dt; + + MemoryUsageOutput("Breakpoint 01"); + + integer const maxNumberDtCuts = m_nonlinearSolverParameters.m_maxTimeStepCuts; real64 const dtCutFactor = m_nonlinearSolverParameters.m_timeStepCutFactor; @@ -813,6 +856,9 @@ real64 PhysicsSolverBase::nonlinearImplicitStep( real64 const & time_n, // required. for( dtAttempt = 0; dtAttempt < maxNumberDtCuts; ++dtAttempt ) { + + MemoryUsageOutput("Breakpoint 02-" + std::to_string(dtAttempt)); + // reset the solver state, since we are restarting the time step if( dtAttempt > 0 ) { @@ -924,6 +970,7 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, for( newtonIter = 0; newtonIter < maxNewtonIter; ++newtonIter ) { + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-01 newtonIter:" + std::to_string(newtonIter)); GEOS_LOG_LEVEL_RANK_0( logInfo::NonlinearSolver, GEOS_FMT( " Attempt: {:2}, ConfigurationIter: {:2}, NewtonIter: {:2}", dtAttempt, configurationLoopIter, newtonIter )); @@ -950,6 +997,8 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, m_localMatrix.toViewConstSizes(), localRhs ); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-02 newtonIter:" + std::to_string(newtonIter)); + // apply boundary conditions to system applyBoundaryConditions( time_n, stepDt, @@ -960,6 +1009,8 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, m_rhs.close(); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-03 newtonIter:" + std::to_string(newtonIter)); + if( m_assemblyCallback ) { // Make a copy of LA objects and ship off to the callback @@ -975,6 +1026,9 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, // get residual norm residualNorm = calculateResidualNorm( time_n, stepDt, domain, m_dofManager, m_rhs.values() ); + + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-04 newtonIter:" + std::to_string(newtonIter)); + GEOS_LOG_LEVEL_RANK_0( logInfo::Convergence, GEOS_FMT( " ( R ) = ( {:4.2e} )", residualNorm ) ); } @@ -1053,6 +1107,8 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, } { + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-05 newtonIter:" + std::to_string(newtonIter)); + Timer timer( m_timers["linear solver total"] ); // if using adaptive Krylov tolerance scheme, update tolerance. @@ -1074,16 +1130,21 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, // Compose parallel LA matrix/rhs out of local LA matrix/rhs // m_matrix.create( m_localMatrix.toViewConst(), m_dofManager.numLocalDofs(), MPI_COMM_GEOS ); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-06 newtonIter:" + std::to_string(newtonIter)); + } // Output the linear system matrix/rhs for debugging purposes debugOutputSystem( time_n, cycleNumber, newtonIter, m_matrix, m_rhs ); // Solve the linear system + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-07 newtonIter:" + std::to_string(newtonIter)); solveLinearSystem( m_dofManager, m_matrix, m_rhs, m_solution ); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-08 newtonIter:" + std::to_string(newtonIter)); // Increment the solver statistics for reporting purposes m_solverStatistics.logNonlinearIteration( m_linearSolverResult.numIterations ); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-09 newtonIter:" + std::to_string(newtonIter)); // Output the linear system solution for debugging purposes debugOutputSolution( time_n, cycleNumber, newtonIter, m_solution ); @@ -1111,6 +1172,7 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, // apply the system solution to the fields/variables applySystemSolution( m_dofManager, m_solution.values(), scaleFactor, stepDt, domain ); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-10 newtonIter:" + std::to_string(newtonIter)); } { @@ -1123,6 +1185,8 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, lastResidual = residualNorm; } + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-11: exit"); + return isNewtonConverged; } @@ -1310,6 +1374,17 @@ void PhysicsSolverBase::solveLinearSystem( DofManager const & dofManager, LinearSolverParameters const & params = m_linearSolverParameters.get(); matrix.setDofManager( &dofManager ); + // Apply physics-based scaling to the linear system if enabled + if( m_usePhysicsScaling ) + { + Timer timer_setup( m_timers["linear solver scaling"] ); + + matrix.computeScalingVector( m_scaling ); + matrix.leftRightScale( m_scaling, m_scaling ); + rhs.pointwiseProduct( m_scaling ); + // Assume the solution is zeroed out, thus no need to scale it + } + if( params.solverType == LinearSolverParameters::SolverType::direct || !m_precond ) { std::unique_ptr< LinearSolverBase< LAInterface > > solver = LAInterface::createSolver( params ); @@ -1350,6 +1425,14 @@ void PhysicsSolverBase::solveLinearSystem( DofManager const & dofManager, { GEOS_WARNING_IF( !m_linearSolverResult.success(), getDataContext() << ": Linear solution failed" ); } + + // Unscale the solution vector if physics-based scaling was applied + if( m_usePhysicsScaling ) + { + Timer timer_setup( m_timers["linear solver scaling"] ); + + solution.pointwiseProduct( m_scaling ); + } } bool PhysicsSolverBase::checkSystemSolution( DomainPartition & GEOS_UNUSED_PARAM( domain ), diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp index d05b54366d..0e981ce804 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp @@ -691,6 +691,9 @@ class PhysicsSolverBase : public ExecutableGroup /// @return string for the writeLinearSystem wrapper static constexpr char const * writeLinearSystemString() { return "writeLinearSystem"; } + /// @return string for the usePhysicsScaling wrapper + static constexpr char const * usePhysicsScalingString() { return "usePhysicsScaling"; } + /// @return string for the allowNonConvergedLinearSolverSolution wrapper static constexpr char const * allowNonConvergedLinearSolverSolutionString() { return "allowNonConvergedLinearSolverSolution"; } }; @@ -1027,6 +1030,12 @@ class PhysicsSolverBase : public ExecutableGroup /// System solution vector ParallelVector m_solution; + /// Diagonal scaling vector D (Ahat = D * A * D, bhat = D * b, x = D * xhat) + ParallelVector m_scaling; + + /// Flag to decide whether to apply physics-based scaling to the linear system + integer m_usePhysicsScaling; + /// Local system matrix and rhs CRSMatrix< real64, globalIndex > m_localMatrix; diff --git a/src/coreComponents/physicsSolvers/fluidFlow/CMakeLists.txt b/src/coreComponents/physicsSolvers/fluidFlow/CMakeLists.txt index f0ce814700..db3d984563 100644 --- a/src/coreComponents/physicsSolvers/fluidFlow/CMakeLists.txt +++ b/src/coreComponents/physicsSolvers/fluidFlow/CMakeLists.txt @@ -20,6 +20,7 @@ Contains: set( fluidFlowSolvers_headers FlowSolverBase.hpp FlowSolverBaseFields.hpp + SinglePhaseHybridFVM.hpp CompositionalMultiphaseBase.hpp CompositionalMultiphaseBaseFields.hpp CompositionalMultiphaseStatistics.hpp @@ -36,14 +37,12 @@ set( fluidFlowSolvers_headers SinglePhaseBaseFields.hpp SinglePhaseStatistics.hpp SinglePhaseFVM.hpp - SinglePhaseHybridFVM.hpp SinglePhaseProppantBase.hpp StencilAccessors.hpp StencilDataCollection.hpp LogLevelsInfo.hpp kernels/MinPoreVolumeMaxPorosityKernel.hpp kernels/StencilWeightsUpdateKernel.hpp - kernels/HybridFVMHelperKernels.hpp kernels/singlePhase/AccumulationKernels.hpp kernels/singlePhase/AquiferBCKernel.hpp kernels/singlePhase/DirichletFluxComputeKernel.hpp @@ -54,7 +53,6 @@ set( fluidFlowSolvers_headers kernels/singlePhase/HydrostaticPressureKernel.hpp kernels/singlePhase/MobilityKernel.hpp kernels/singlePhase/ResidualNormKernel.hpp - kernels/singlePhase/SinglePhaseHybridFVMKernels.hpp kernels/singlePhase/SolidInternalEnergyUpdateKernel.hpp kernels/singlePhase/SolutionCheckKernel.hpp kernels/singlePhase/SolutionScalingKernel.hpp @@ -71,7 +69,6 @@ set( fluidFlowSolvers_headers kernels/compositional/C1PPUPhaseFlux.hpp kernels/compositional/CapillaryPressureUpdateKernel.hpp kernels/compositional/CFLKernel.hpp - kernels/compositional/CompositionalMultiphaseHybridFVMKernels.hpp kernels/compositional/DiffusionDispersionFluxComputeKernel.hpp kernels/compositional/DirichletFluxComputeKernel.hpp kernels/compositional/DissipationFluxComputeKernel.hpp @@ -126,31 +123,30 @@ set( fluidFlowSolvers_headers # Specify solver sources set( fluidFlowSolvers_sources - CompositionalMultiphaseBase.cpp - CompositionalMultiphaseFVM.cpp - CompositionalMultiphaseStatistics.cpp - CompositionalMultiphaseHybridFVM.cpp - ImmiscibleMultiphaseFlow.cpp - ReactiveCompositionalMultiphaseOBL.cpp FlowSolverBase.cpp SinglePhaseBase.cpp SinglePhaseStatistics.cpp SinglePhaseFVM.cpp SinglePhaseHybridFVM.cpp SinglePhaseProppantBase.cpp + CompositionalMultiphaseBase.cpp + CompositionalMultiphaseFVM.cpp + CompositionalMultiphaseHybridFVM.cpp + CompositionalMultiphaseStatistics.cpp + ImmiscibleMultiphaseFlow.cpp + ReactiveCompositionalMultiphaseOBL.cpp SourceFluxStatistics.cpp StencilDataCollection.cpp kernels/singlePhase/proppant/ProppantFluxKernels.cpp kernels/compositional/AquiferBCKernel.cpp kernels/compositional/CFLKernel.cpp kernels/compositional/FluxComputeKernelBase.cpp - kernels/compositional/CompositionalMultiphaseHybridFVMKernels.cpp wells/CompositionalMultiphaseWell.cpp wells/kernels/CompositionalMultiphaseWellKernels.cpp wells/SinglePhaseWell.cpp wells/kernels/SinglePhaseWellKernels.cpp wells/WellControls.cpp - wells/WellSolverBase.cpp + wells/WellSolverBase.cpp proppantTransport/ProppantTransport.cpp proppantTransport/ProppantTransportKernels.cpp ) @@ -158,54 +154,100 @@ set( dependencyList ${parallelDeps} physicsSolversBase ) geos_decorate_link_dependencies( LIST decoratedDependencies DEPENDENCIES ${dependencyList} ) +# Add isothermal FVM fluid flow kernels +if( GEOS_ENABLE_FLUIDFLOW_ISOTHERMAL_FVM ) + file( READ "${CMAKE_CURRENT_LIST_DIR}/IsothermalFVM_KernelSpecs.json" kernelSpecs ) + # TODO: The kernel below has non-matching file names and JSON keys. + # Either fix them to follow pattern, or come up with another mechanism. + generateKernels( TEMPLATE "CompositionalMultiphaseFVMKernel_isothermal.cpp.template" + JSON kernelSpecs + KEY "IsothermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels" + HEADERS headerFiles + SOURCES sourceFiles ) + list(APPEND fluidFlowSolvers_headers ${headerFiles}) + list(APPEND fluidFlowSolvers_sources ${sourceFiles}) + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") +endif() -file( READ "${CMAKE_CURRENT_LIST_DIR}/kernelSpecs.json" kernelSpecs ) -set( kernelTemplateFileList "" ) - -list( APPEND kernelTemplateFileList - SinglePhaseHybridFVMKernels.cpp.template - CompositionalMultiphaseHybridFVMKernels_upwinding.cpp.template - CompositionalMultiphaseHybridFVMKernels_assembly.cpp.template - CompositionalMultiphaseHybridFVMKernels_flux.cpp.template - ReactiveCompositionalMultiphaseOBLKernels.cpp.template ) - +# Add thermal FVM fluid flow kernels +if( GEOS_ENABLE_FLUIDFLOW_THERMAL_FVM ) + file( READ "${CMAKE_CURRENT_LIST_DIR}/ThermalFVM_KernelSpecs.json" kernelSpecs ) -foreach( kernelTemplateFile ${kernelTemplateFileList} ) - get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) - generateKernels( TEMPLATE ${kernelTemplateFile} + # TODO: The kernel below has non-matching file names and JSON keys. + # Either fix them to follow pattern, or come up with another mechanism. + generateKernels( TEMPLATE "CompositionalMultiphaseFVMKernel_thermal.cpp.template" JSON kernelSpecs - KEY ${jsonKey} + KEY "ThermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels" HEADERS headerFiles SOURCES sourceFiles ) + list(APPEND fluidFlowSolvers_headers ${headerFiles}) + list(APPEND fluidFlowSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") +endif() + +# Add reactive fluid flow kernels +if( GEOS_ENABLE_FLUIDFLOW_REACTIVE ) + file( READ "${CMAKE_CURRENT_LIST_DIR}/Reactive_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList + ReactiveCompositionalMultiphaseOBLKernels.cpp.template ) -list(APPEND fluidFlowSolvers_headers ${headerFiles}) -list(APPEND fluidFlowSolvers_sources ${sourceFiles}) -endforeach() + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) -# TODO: The two kernels below have non-matching file names and JSON keys. -# Either fix them to follow pattern, or come up with another mechanism. + list(APPEND fluidFlowSolvers_headers ${headerFiles}) + list(APPEND fluidFlowSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() -generateKernels( TEMPLATE "CompositionalMultiphaseFVMKernel_isothermal.cpp.template" - JSON kernelSpecs - KEY "IsothermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels" - HEADERS headerFiles - SOURCES sourceFiles ) -list(APPEND fluidFlowSolvers_headers ${headerFiles}) -list(APPEND fluidFlowSolvers_sources ${sourceFiles}) +# Add Hybrid FVM fluid flow code +if( GEOS_ENABLE_FLUIDFLOW_HYBRIDFVM ) + list( APPEND fluidFlowSolvers_headers + CompositionalMultiphaseHybridFVM.hpp + kernels/HybridFVMHelperKernels.hpp + kernels/singlePhase/SinglePhaseHybridFVMKernels.hpp + kernels/compositional/CompositionalMultiphaseHybridFVMKernels.hpp ) + list( APPEND fluidFlowSolvers_sources + SinglePhaseHybridFVM.cpp + CompositionalMultiphaseHybridFVM.cpp + kernels/compositional/CompositionalMultiphaseHybridFVMKernels.cpp ) -generateKernels( TEMPLATE "CompositionalMultiphaseFVMKernel_thermal.cpp.template" - JSON kernelSpecs - KEY "ThermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels" - HEADERS headerFiles - SOURCES sourceFiles ) -list(APPEND fluidFlowSolvers_headers ${headerFiles}) -list(APPEND fluidFlowSolvers_sources ${sourceFiles}) + file( READ "${CMAKE_CURRENT_LIST_DIR}/HybridFVM_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList + SinglePhaseHybridFVMKernels.cpp.template + CompositionalMultiphaseHybridFVMKernels_upwinding.cpp.template + CompositionalMultiphaseHybridFVMKernels_assembly.cpp.template + CompositionalMultiphaseHybridFVMKernels_flux.cpp.template ) + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + list(APPEND fluidFlowSolvers_headers ${headerFiles}) + list(APPEND fluidFlowSolvers_sources ${sourceFiles}) + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() +set(TOTAL_GENERATED_KERNELS ${TOTAL_GENERATED_KERNELS} CACHE INTERNAL "Total number of generated kernels") blt_add_library( NAME fluidFlowSolvers SOURCES ${fluidFlowSolvers_sources} diff --git a/src/coreComponents/physicsSolvers/fluidFlow/kernelSpecs.json b/src/coreComponents/physicsSolvers/fluidFlow/HybridFVM_KernelSpecs.json similarity index 58% rename from src/coreComponents/physicsSolvers/fluidFlow/kernelSpecs.json rename to src/coreComponents/physicsSolvers/fluidFlow/HybridFVM_KernelSpecs.json index e2c3ed6173..a0cf819222 100644 --- a/src/coreComponents/physicsSolvers/fluidFlow/kernelSpecs.json +++ b/src/coreComponents/physicsSolvers/fluidFlow/HybridFVM_KernelSpecs.json @@ -141,108 +141,5 @@ ] }, "explicit": [] - }, - - "ReactiveCompositionalMultiphaseOBLKernels": { - "vars": [ - "NAME", - "NDIM", - "NCOMP", - "ENABLE_ENERGY", - "STENCIL" - ], - "constants": [ - [ "FluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "NAME": [ - "FluxComputeKernel" - ], - "NDIM": [ - 1, - 2, - 3 - ], - "NCOMP": [ - 1, - 2, - 3, - 4, - 5 - ], - "ENABLE_ENERGY": [ - "true", - "false" - ], - "STENCIL": [ - "CellElementStencilTPFA", - "SurfaceElementStencil", - "EmbeddedSurfaceToCellStencil", - "FaceElementToCellStencil" - ] - }, - "explicit": [] - }, - - "IsothermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels": { - "vars": [ - "NAME", - "NCOMP_NDOF", - "KERNEL_TYPE" - ], - "constants": [ - [ "DirichletFluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "NAME": [ - "DirichletFluxComputeKernel" - ], - "NCOMP_NDOF": [ - "1, 2", - "2, 3", - "3, 4", - "4, 5", - "5, 6" - ], - "KERNEL_TYPE": [ - "constitutive::DeadOilFluid", - "constitutive::BlackOilFluid", - "constitutive::CO2BrinePhillipsFluid", - "constitutive::CO2BrineEzrokhiFluid", - "constitutive::CO2BrinePhillipsThermalFluid" - ] - }, - "explicit": [] - }, - - "ThermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels": { - "vars": [ - "NAME", - "NCOMP_NDOF", - "KERNEL_TYPE" - ], - "constants": [ - [ "DirichletFluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "NAME": [ - "DirichletFluxComputeKernel" - ], - "NCOMP_NDOF": [ - "1, 2", - "2, 3", - "3, 4", - "4, 5", - "5, 6" - ], - "KERNEL_TYPE": [ - "constitutive::DeadOilFluid", - "constitutive::BlackOilFluid", - "constitutive::CO2BrinePhillipsFluid", - "constitutive::CO2BrineEzrokhiFluid", - "constitutive::CO2BrinePhillipsThermalFluid" - ] - }, - "explicit": [] } } diff --git a/src/coreComponents/physicsSolvers/fluidFlow/IsothermalFVM_KernelSpecs.json b/src/coreComponents/physicsSolvers/fluidFlow/IsothermalFVM_KernelSpecs.json new file mode 100644 index 0000000000..0ba359c5b4 --- /dev/null +++ b/src/coreComponents/physicsSolvers/fluidFlow/IsothermalFVM_KernelSpecs.json @@ -0,0 +1,57 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "IsothermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels": { + "vars": [ + "NAME", + "NCOMP_NDOF", + "KERNEL_TYPE" + ], + "constants": [ + [ "DirichletFluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "NAME": [ + "DirichletFluxComputeKernel" + ], + "NCOMP_NDOF": [ + "1, 2", + "2, 3", + "3, 4", + "4, 5", + "5, 6" + ], + "KERNEL_TYPE": [ + "constitutive::DeadOilFluid", + "constitutive::BlackOilFluid", + "constitutive::CO2BrinePhillipsFluid", + "constitutive::CO2BrineEzrokhiFluid", + "constitutive::CO2BrinePhillipsThermalFluid" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/fluidFlow/Reactive_KernelSpecs.json b/src/coreComponents/physicsSolvers/fluidFlow/Reactive_KernelSpecs.json new file mode 100644 index 0000000000..1cfcc94784 --- /dev/null +++ b/src/coreComponents/physicsSolvers/fluidFlow/Reactive_KernelSpecs.json @@ -0,0 +1,67 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "ReactiveCompositionalMultiphaseOBLKernels": { + "vars": [ + "NAME", + "NDIM", + "NCOMP", + "ENABLE_ENERGY", + "STENCIL" + ], + "constants": [ + [ "FluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "NAME": [ + "FluxComputeKernel" + ], + "NDIM": [ + 1, + 2, + 3 + ], + "NCOMP": [ + 1, + 2, + 3, + 4, + 5 + ], + "ENABLE_ENERGY": [ + "true", + "false" + ], + "STENCIL": [ + "CellElementStencilTPFA", + "SurfaceElementStencil", + "EmbeddedSurfaceToCellStencil", + "FaceElementToCellStencil" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/fluidFlow/SinglePhaseBase.cpp b/src/coreComponents/physicsSolvers/fluidFlow/SinglePhaseBase.cpp index 76db9327e8..a70e323c87 100644 --- a/src/coreComponents/physicsSolvers/fluidFlow/SinglePhaseBase.cpp +++ b/src/coreComponents/physicsSolvers/fluidFlow/SinglePhaseBase.cpp @@ -987,7 +987,7 @@ void SinglePhaseBase::applySourceFluxBC( real64 const time_n, mesh, SourceFluxBoundaryCondition::catalogName(), [&, isThermal]( SourceFluxBoundaryCondition const & fs, - string const & setName, + GEOS_MAYBE_UNUSED string const & setName, SortedArrayView< localIndex const > const & targetSet, ElementSubRegionBase & subRegion, string const & ) diff --git a/src/coreComponents/physicsSolvers/fluidFlow/ThermalFVM_KernelSpecs.json b/src/coreComponents/physicsSolvers/fluidFlow/ThermalFVM_KernelSpecs.json new file mode 100644 index 0000000000..c311085c9a --- /dev/null +++ b/src/coreComponents/physicsSolvers/fluidFlow/ThermalFVM_KernelSpecs.json @@ -0,0 +1,57 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "ThermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels": { + "vars": [ + "NAME", + "NCOMP_NDOF", + "KERNEL_TYPE" + ], + "constants": [ + [ "DirichletFluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "NAME": [ + "DirichletFluxComputeKernel" + ], + "NCOMP_NDOF": [ + "1, 2", + "2, 3", + "3, 4", + "4, 5", + "5, 6" + ], + "KERNEL_TYPE": [ + "constitutive::DeadOilFluid", + "constitutive::BlackOilFluid", + "constitutive::CO2BrinePhillipsFluid", + "constitutive::CO2BrineEzrokhiFluid", + "constitutive::CO2BrinePhillipsThermalFluid" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt b/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt index be6090a67e..53fc33e08d 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt +++ b/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt @@ -24,16 +24,9 @@ set( multiPhysicsSolvers_headers CoupledSolver.hpp PoromechanicsSolver.hpp FlowProppantTransportSolver.hpp - HydrofractureSolver.hpp - HydrofractureSolverKernels.hpp - MultiphasePoromechanics.hpp OneWayCoupledFractureFlowContactMechanics.hpp - MultiphasePoromechanicsConformingFractures.hpp PhaseFieldFractureSolver.hpp - PhaseFieldPoromechanicsSolver.hpp - PoromechanicsInitialization.hpp PoromechanicsFields.hpp - PoromechanicsInitialization.hpp poromechanicsKernels/MultiphasePoromechanics.hpp poromechanicsKernels/MultiphasePoromechanics_impl.hpp poromechanicsKernels/MultiphasePoromechanicsConformingFractures.hpp @@ -55,9 +48,6 @@ set( multiPhysicsSolvers_headers poromechanicsKernels/ThermalSinglePhasePoromechanicsEFEM_impl.hpp poromechanicsKernels/ThermalSinglePhasePoromechanicsConformingFractures.hpp poromechanicsKernels/ThermalSinglePhasePoromechanicsEmbeddedFractures.hpp - SinglePhasePoromechanics.hpp - SinglePhasePoromechanicsEmbeddedFractures.hpp - SinglePhasePoromechanicsConformingFractures.hpp SinglePhaseReservoirAndWells.hpp ) # Specify solver sources @@ -65,16 +55,8 @@ set( multiPhysicsSolvers_sources CompositionalMultiphaseReservoirAndWells.cpp CoupledReservoirAndWellsBase.cpp FlowProppantTransportSolver.cpp - HydrofractureSolver.cpp - MultiphasePoromechanics.cpp OneWayCoupledFractureFlowContactMechanics.cpp - MultiphasePoromechanicsConformingFractures.cpp PhaseFieldFractureSolver.cpp - PhaseFieldPoromechanicsSolver.cpp - PoromechanicsInitialization.cpp - SinglePhasePoromechanics.cpp - SinglePhasePoromechanicsEmbeddedFractures.cpp - SinglePhasePoromechanicsConformingFractures.cpp SinglePhaseReservoirAndWells.cpp ) #include(poromechanicsKernels/PoromechanicsKernels.cmake) @@ -83,28 +65,110 @@ set( dependencyList ${parallelDeps} fluidFlowSolvers simplePDESolvers solidMecha geos_decorate_link_dependencies( LIST decoratedDependencies DEPENDENCIES ${dependencyList} ) +if (GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS) + list( APPEND multiPhysicsSolvers_headers + PoromechanicsInitialization.hpp + SinglePhasePoromechanics.hpp + SinglePhasePoromechanicsConformingFractures.hpp + MultiphasePoromechanics.hpp + MultiphasePoromechanicsConformingFractures.hpp + PhaseFieldPoromechanicsSolver.hpp + HydrofractureSolver.hpp + HydrofractureSolverKernels.hpp ) + list( APPEND multiPhysicsSolvers_sources + PoromechanicsInitialization.cpp + SinglePhasePoromechanics.cpp + SinglePhasePoromechanicsConformingFractures.cpp + MultiphasePoromechanics.cpp + MultiphasePoromechanicsConformingFractures.cpp + PhaseFieldPoromechanicsSolver.cpp + HydrofractureSolver.cpp ) + + file( READ "${CMAKE_CURRENT_LIST_DIR}/Poromechanics_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList poromechanicsKernels/PoromechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND multiPhysicsSolvers_headers ${headerFiles}) + list(APPEND multiPhysicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() + +if (GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS_EFEM) + list( APPEND multiPhysicsSolvers_headers + SinglePhasePoromechanicsEmbeddedFractures.hpp ) + list( APPEND multiPhysicsSolvers_sources + SinglePhasePoromechanicsEmbeddedFractures.cpp ) + + file( READ "${CMAKE_CURRENT_LIST_DIR}/PoromechanicsEFEM_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList poromechanicsKernels/PoromechanicsEFEMKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND multiPhysicsSolvers_headers ${headerFiles}) + list(APPEND multiPhysicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() -file( READ "${CMAKE_CURRENT_LIST_DIR}/kernelSpecs.json" kernelSpecs ) -set( kernelTemplateFileList "" ) +if (GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS_DAMAGE) + file( READ "${CMAKE_CURRENT_LIST_DIR}/PoromechanicsDamage_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList poromechanicsKernels/PoromechanicsDamageKernels.cpp.template ) - list( APPEND kernelTemplateFileList - poromechanicsKernels/PoromechanicsKernels.cpp.template - poromechanicsKernels/PoromechanicsEFEMKernels.cpp.template - poromechanicsKernels/ThermoPoromechanicsKernels.cpp.template - poromechanicsKernels/PoromechanicsDamageKernels.cpp.template ) + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + list(APPEND multiPhysicsSolvers_headers ${headerFiles}) + list(APPEND multiPhysicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() -foreach( kernelTemplateFile ${kernelTemplateFileList} ) - get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) - generateKernels( TEMPLATE ${kernelTemplateFile} - JSON kernelSpecs - KEY ${jsonKey} - HEADERS headerFiles - SOURCES sourceFiles ) +if (GEOS_ENABLE_MULTIPHYSICS_THERMOPOROMECHANICS) + file( READ "${CMAKE_CURRENT_LIST_DIR}/ThermoPoromechanics_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList poromechanicsKernels/ThermoPoromechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND multiPhysicsSolvers_headers ${headerFiles}) + list(APPEND multiPhysicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() -list(APPEND multiPhysicsSolvers_headers ${headerFiles}) -list(APPEND multiPhysicsSolvers_sources ${sourceFiles}) -endforeach() +set(TOTAL_GENERATED_KERNELS ${TOTAL_GENERATED_KERNELS} CACHE INTERNAL "Total number of generated kernels") blt_add_library( NAME multiPhysicsSolvers SOURCES ${multiPhysicsSolvers_sources} diff --git a/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsDamage_KernelSpecs.json b/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsDamage_KernelSpecs.json new file mode 100644 index 0000000000..4201abb6b0 --- /dev/null +++ b/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsDamage_KernelSpecs.json @@ -0,0 +1,55 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "PoromechanicsDamageKernels": { + "vars": [ + "SUBREGION_TYPE", + "CONSTITUTIVE_TYPE", + "FE_TYPE" + ], + "constants": [ + [ "SinglePhasePoromechanicsDamagePolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "PorousDamageSolid>", + "PorousDamageSolid>", + "PorousDamageSolid>" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Tetrahedron_Lagrange1_Gauss14", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsEFEM_KernelSpecs.json b/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsEFEM_KernelSpecs.json new file mode 100644 index 0000000000..8fe3b139af --- /dev/null +++ b/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsEFEM_KernelSpecs.json @@ -0,0 +1,53 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "PoromechanicsEFEMKernels": { + "vars": [ + "SUBREGION_TYPE", + "CONSTITUTIVE_TYPE", + "FE_TYPE" + ], + "constants": [ + [ "SinglePhasePoromechanicsEFEMPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "PorousSolid" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Tetrahedron_Lagrange1_Gauss14", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/multiphysics/Poromechanics_KernelSpecs.json b/src/coreComponents/physicsSolvers/multiphysics/Poromechanics_KernelSpecs.json new file mode 100644 index 0000000000..a28953231c --- /dev/null +++ b/src/coreComponents/physicsSolvers/multiphysics/Poromechanics_KernelSpecs.json @@ -0,0 +1,67 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "PoromechanicsKernels": { + "vars": [ + "SUBREGION_TYPE", + "CONSTITUTIVE_TYPE", + "FE_TYPE" + ], + "constants": [ + [ "SinglePhasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "MultiphasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "PorousSolid", + "PorousSolid", + "PorousSolid", + "PorousSolid", + "PorousSolid", + "PorousSolid", + "PorousSolid", + "PorousSolid", + "PorousSolid>", + "PorousSolid>", + "PorousSolid>", + "PorousDamageSolid>", + "PorousDamageSolid>", + "PorousDamageSolid>" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Tetrahedron_Lagrange1_Gauss14", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/multiphysics/ThermoPoromechanics_KernelSpecs.json b/src/coreComponents/physicsSolvers/multiphysics/ThermoPoromechanics_KernelSpecs.json new file mode 100644 index 0000000000..db759c295d --- /dev/null +++ b/src/coreComponents/physicsSolvers/multiphysics/ThermoPoromechanics_KernelSpecs.json @@ -0,0 +1,58 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "ThermoPoromechanicsKernels": { + "vars": [ + "SUBREGION_TYPE", + "CONSTITUTIVE_TYPE", + "FE_TYPE" + ], + "constants": [ + [ "ThermalMultiphasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ThermalSinglePhasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ThermalSinglePhasePoromechanicsEFEMPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "PorousSolid", + "PorousSolid", + "PorousSolid", + "PorousSolid" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Pyramid_Lagrange1_Gauss5", + "H1_Tetrahedron_Lagrange1_Gauss14" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/multiphysics/kernelSpecs.json b/src/coreComponents/physicsSolvers/multiphysics/kernelSpecs.json deleted file mode 100644 index 7acbc5cb99..0000000000 --- a/src/coreComponents/physicsSolvers/multiphysics/kernelSpecs.json +++ /dev/null @@ -1,164 +0,0 @@ -{ - "doc" : - { - "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", - "vars" : - [ - "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", - "they are also the names of the CMAKE variables which will be set to one of the values specified", - "in the combinations dict just prior to cmake calling configure_file() on the template file" - ], - "constants" : - [ - [ "key", "value" ], - [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] - ], - "combinations" : - { - "vars" : [ "each value in this list will be used to generate kernel template files", - "combinatorally with all other value lists" ] - }, - "explicit" : [ "specify a specific combination of the variables to generate", - "specify by choosing one value from each variable you want generated", - "and concating them with '#' (this can by modified internally at the", - "generation call). Duplicate entries are unified internally" ] - }, - - "PoromechanicsKernels": { - "vars": [ - "SUBREGION_TYPE", - "CONSTITUTIVE_TYPE", - "FE_TYPE" - ], - "constants": [ - [ "SinglePhasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "MultiphasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "PorousSolid", - "PorousSolid", - "PorousSolid", - "PorousSolid", - "PorousSolid", - "PorousSolid", - "PorousSolid", - "PorousSolid", - "PorousSolid>", - "PorousSolid>", - "PorousSolid>", - "PorousDamageSolid>", - "PorousDamageSolid>", - "PorousDamageSolid>" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Tetrahedron_Lagrange1_Gauss14", - "H1_Pyramid_Lagrange1_Gauss5" - ] - }, - "explicit": [ "CellElementSubRegion#PorousSolid#H1_Tetrahedron_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism5_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism6_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism7_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism8_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism9_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism10_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Hexahedron_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Wedge_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism11_VEM_Gauss1" ] - }, - - "PoromechanicsEFEMKernels": { - "vars": [ - "SUBREGION_TYPE", - "CONSTITUTIVE_TYPE", - "FE_TYPE" - ], - "constants": [ - [ "SinglePhasePoromechanicsEFEMPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "PorousSolid" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Tetrahedron_Lagrange1_Gauss14", - "H1_Pyramid_Lagrange1_Gauss5" - ] - }, - "explicit": [] - }, - - "ThermoPoromechanicsKernels": { - "vars": [ - "SUBREGION_TYPE", - "CONSTITUTIVE_TYPE", - "FE_TYPE" - ], - "constants": [ - [ "ThermalMultiphasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ThermalSinglePhasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ThermalSinglePhasePoromechanicsEFEMPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "PorousSolid", - "PorousSolid", - "PorousSolid", - "PorousSolid" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Pyramid_Lagrange1_Gauss5", - "H1_Tetrahedron_Lagrange1_Gauss14" - ] - }, - "explicit": [] - }, - - "PoromechanicsDamageKernels": { - "vars": [ - "SUBREGION_TYPE", - "CONSTITUTIVE_TYPE", - "FE_TYPE" - ], - "constants": [ - [ "SinglePhasePoromechanicsDamagePolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "PorousDamageSolid>", - "PorousDamageSolid>", - "PorousDamageSolid>" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Tetrahedron_Lagrange1_Gauss14", - "H1_Pyramid_Lagrange1_Gauss5" - ] - }, - "explicit": [] - } -} diff --git a/src/coreComponents/physicsSolvers/solidMechanics/CMakeLists.txt b/src/coreComponents/physicsSolvers/solidMechanics/CMakeLists.txt index ced15c47ba..da37d961c2 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/CMakeLists.txt +++ b/src/coreComponents/physicsSolvers/solidMechanics/CMakeLists.txt @@ -55,7 +55,7 @@ set( solidMechanicsSolvers_headers contact/kernels/SolidMechanicsALMKernels.hpp contact/kernels/SolidMechanicsConformingContactKernelsHelper.hpp contact/kernels/SolidMechanicsContactFaceBubbleKernels.hpp - contact/kernels/SolidMechanicsLagrangeContactKernels.hpp + contact/kernels/SolidMechanicsLagrangeContactKernels.hpp contact/LogLevelsInfo.hpp ) # Specify solver sources @@ -76,27 +76,89 @@ set( dependencyList ${parallelDeps} physicsSolversBase ) geos_decorate_link_dependencies( LIST decoratedDependencies DEPENDENCIES ${dependencyList} ) - -file( READ "${CMAKE_CURRENT_LIST_DIR}/kernelSpecs.json" kernelSpecs ) -set( kernelTemplateFileList "" ) - - list( APPEND kernelTemplateFileList - kernels/SolidMechanicsKernels.cpp.template - kernels/SolidMechanicsFixedStressThermoPoromechanicsKernels.cpp.template ) - - -foreach( kernelTemplateFile ${kernelTemplateFileList} ) - get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) - generateKernels( TEMPLATE ${kernelTemplateFile} - JSON kernelSpecs - KEY ${jsonKey} - HEADERS headerFiles - SOURCES sourceFiles ) - -list(APPEND solidMechanicsSolvers_headers ${headerFiles}) -list(APPEND solidMechanicsSolvers_sources ${sourceFiles}) -endforeach() - +if (GEOS_ENABLE_SOLIDMECHANICS_ELASTIC) + file( READ "${CMAKE_CURRENT_LIST_DIR}/SolidMechanicsElastic_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList kernels/SolidMechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND solidMechanicsSolvers_headers ${headerFiles}) + list(APPEND solidMechanicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() + +if (GEOS_ENABLE_SOLIDMECHANICS_EXTRA) + file( READ "${CMAKE_CURRENT_LIST_DIR}/SolidMechanicsExtra_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList kernels/SolidMechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND solidMechanicsSolvers_headers ${headerFiles}) + list(APPEND solidMechanicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() + +# TODO: There is an error if we turn this on. Are we missing SolidMechanicsFixedStressThermoPoroElasticKernels.cpp.template? +if (GEOS_ENABLE_SOLIDMECHANICS_THERMOPOROELASTIC) + file( READ "${CMAKE_CURRENT_LIST_DIR}/SolidMechanicsFixedStressThermoPoroElastic_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList + kernels/SolidMechanicsFixedStressThermoPoromechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND solidMechanicsSolvers_headers ${headerFiles}) + list(APPEND solidMechanicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() + +if (GEOS_ENABLE_SOLIDMECHANICS_THERMOPOROMECHANICS) + file( READ "${CMAKE_CURRENT_LIST_DIR}/SolidMechanicsFixedStressThermoPoromechanics_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList kernels/SolidMechanicsFixedStressThermoPoromechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND solidMechanicsSolvers_headers ${headerFiles}) + list(APPEND solidMechanicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() + +set(TOTAL_GENERATED_KERNELS ${TOTAL_GENERATED_KERNELS} CACHE INTERNAL "Total number of generated kernels") blt_add_library( NAME solidMechanicsSolvers SOURCES ${solidMechanicsSolvers_sources} @@ -112,4 +174,4 @@ install( TARGETS solidMechanicsSolvers LIBRARY DESTINATION ${CMAKE_INSTALL_PREFI if( externalComponentDeps ) target_include_directories( solidMechanicsSolvers PUBLIC ${CMAKE_SOURCE_DIR}/externalComponents ) -endif() \ No newline at end of file +endif() diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsElastic_KernelSpecs.json b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsElastic_KernelSpecs.json new file mode 100644 index 0000000000..970544949c --- /dev/null +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsElastic_KernelSpecs.json @@ -0,0 +1,57 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "SolidMechanicsKernels": { + "vars": [ "SUBREGION_TYPE", "CONSTITUTIVE_TYPE", "FE_TYPE" ], + "constants": [ + [ "ExplicitSmallStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ExplicitFiniteStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ImplicitSmallStrainNewmarkPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ImplicitSmallStrainQuasiStaticPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "ElasticIsotropic", + "ElasticTransverseIsotropic", + "ElasticIsotropicPressureDependent", + "ElasticOrthotropic" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Tetrahedron_Lagrange1_Gauss5", + "H1_Tetrahedron_Lagrange1_Gauss14", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsExtra_KernelSpecs.json b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsExtra_KernelSpecs.json new file mode 100644 index 0000000000..53ecc7653f --- /dev/null +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsExtra_KernelSpecs.json @@ -0,0 +1,63 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "SolidMechanicsKernels": { + "vars": [ "SUBREGION_TYPE", "CONSTITUTIVE_TYPE", "FE_TYPE" ], + "constants": [ + [ "ExplicitSmallStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ExplicitFiniteStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ImplicitSmallStrainNewmarkPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ImplicitSmallStrainQuasiStaticPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "DamageSpectral", + "DamageVolDev", + "Damage", + "DuvautLionsSolid", + "DuvautLionsSolid", + "DuvautLionsSolid", + "DruckerPragerExtended", + "ModifiedCamClay", + "DelftEgg", + "DruckerPrager" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Tetrahedron_Lagrange1_Gauss5", + "H1_Tetrahedron_Lagrange1_Gauss14", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoroElastic_KernelSpecs.json b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoroElastic_KernelSpecs.json new file mode 100644 index 0000000000..d26e8604f9 --- /dev/null +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoroElastic_KernelSpecs.json @@ -0,0 +1,52 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "SolidMechanicsFixedStressThermoPoroElasticKernels": { + "vars": [ + "SUBREGION_TYPE", + "CONSTITUTIVE_TYPE", + "FE_TYPE" + ], + "constants": [ + [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "PorousSolid" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoromechanics_KernelSpecs.json b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoromechanics_KernelSpecs.json new file mode 100644 index 0000000000..009c478464 --- /dev/null +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoromechanics_KernelSpecs.json @@ -0,0 +1,56 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "SolidMechanicsFixedStressThermoPoromechanicsKernels": { + "vars": [ + "SUBREGION_TYPE", + "CONSTITUTIVE_TYPE", + "FE_TYPE" + ], + "constants": [ + [ "ExplicitSmallStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ExplicitFiniteStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ImplicitSmallStrainNewmarkPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ImplicitSmallStrainQuasiStaticPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "PorousSolid" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/solidMechanics/kernelSpecs.json b/src/coreComponents/physicsSolvers/solidMechanics/kernelSpecs.json deleted file mode 100644 index e511828633..0000000000 --- a/src/coreComponents/physicsSolvers/solidMechanics/kernelSpecs.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "doc" : - { - "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", - "vars" : - [ - "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", - "they are also the names of the CMAKE variables which will be set to one of the values specified", - "in the combinations dict just prior to cmake calling configure_file() on the template file" - ], - "constants" : - [ - [ "key", "value" ], - [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] - ], - "combinations" : - { - "vars" : [ "each value in this list will be used to generate kernel template files", - "combinatorally with all other value lists" ] - }, - "explicit" : [ "specify a specific combination of the variables to generate", - "specify by choosing one value from each variable you want generated", - "and concating them with '#' (this can by modified internally at the", - "generation call). Duplicate entries are unified internally" ] - }, - - "SolidMechanicsKernels": { - "vars": [ "SUBREGION_TYPE", "CONSTITUTIVE_TYPE", "FE_TYPE" ], - "constants": [ - [ "ExplicitSmallStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ExplicitFiniteStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ImplicitSmallStrainNewmarkPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ImplicitSmallStrainQuasiStaticPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "DamageSpectral", - "DamageVolDev", - "Damage", - "DuvautLionsSolid", - "DuvautLionsSolid", - "DuvautLionsSolid", - "DruckerPragerExtended", - "ModifiedCamClay", - "DelftEgg", - "DruckerPrager", - "ElasticIsotropic", - "ElasticTransverseIsotropic", - "ElasticIsotropicPressureDependent", - "ElasticOrthotropic" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Tetrahedron_Lagrange1_Gauss5", - "H1_Tetrahedron_Lagrange1_Gauss14", - "H1_Pyramid_Lagrange1_Gauss5" - ] - }, - "explicit": [ "CellElementSubRegion#ElasticIsotropic#H1_Tetrahedron_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism5_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism6_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism7_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism8_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism9_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism10_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Hexahedron_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Wedge_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism11_VEM_Gauss1" ] - }, - - "SolidMechanicsFixedStressThermoPoromechanicsKernels": { - "vars": [ - "SUBREGION_TYPE", - "CONSTITUTIVE_TYPE", - "FE_TYPE" - ], - "constants": [ - [ "ExplicitSmallStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ExplicitFiniteStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ImplicitSmallStrainNewmarkPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ImplicitSmallStrainQuasiStaticPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "PorousSolid" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Pyramid_Lagrange1_Gauss5", - "H1_Tetrahedron_VEM_Gauss1", - "H1_Prism5_VEM_Gauss1", - "H1_Prism6_VEM_Gauss1", - "H1_Prism7_VEM_Gauss1", - "H1_Prism8_VEM_Gauss1", - "H1_Prism9_VEM_Gauss1", - "H1_Prism10_VEM_Gauss1", - "H1_Hexahedron_VEM_Gauss1", - "H1_Wedge_VEM_Gauss1", - "H1_Prism11_VEM_Gauss1" - ] - }, - "explicit": [] - }, - - "SolidMechanicsFixedStressThermoPoroElasticKernels": { - "vars": [ - "SUBREGION_TYPE", - "CONSTITUTIVE_TYPE", - "FE_TYPE" - ], - "constants": [ - [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "PorousSolid" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Pyramid_Lagrange1_Gauss5" - ] - }, - "explicit": [] - } -}