From 73eab60855066a3b8ae14d338b954f151579c327 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 18:50:17 -0500 Subject: [PATCH 01/67] Fix runtime issue with MemoryInfo --- src/coreComponents/common/MemoryInfos.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/coreComponents/common/MemoryInfos.cpp b/src/coreComponents/common/MemoryInfos.cpp index 1e31cdb6bed..d31a0f3e44a 100644 --- a/src/coreComponents/common/MemoryInfos.cpp +++ b/src/coreComponents/common/MemoryInfos.cpp @@ -44,6 +44,11 @@ MemoryInfos::MemoryInfos( umpire::MemoryResourceTraits::resource_type resourceTy case umpire::MemoryResourceTraits::resource_type::um: #if defined( GEOS_USE_CUDA ) cudaMemGetInfo( &m_availableMemory, &m_totalMemory ); + #elif defined( GEOS_USE_HIP ) + { + hipError_t const err = hipMemGetInfo( &m_availableMemory, &m_totalMemory ); + GEOS_WARNING_IF( err != hipSuccess, GEOS_FMT( "HIP error found: {} at {}:{}", hipGetErrorString( err ), __FILE__, __LINE__ ) ); + } #else GEOS_WARNING( "Unknown device physical memory size getter for this compiler." ); m_physicalMemoryHandled = 0; From 6618e7b83531e5db5a91f17e84fb51b90bcf2871 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 18:52:22 -0500 Subject: [PATCH 02/67] Fix runtime issue with CommunicationTools --- .../mesh/mpiCommunications/CommunicationTools.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp b/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp index 0abb2fc2f53..7cda510bfa8 100644 --- a/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp +++ b/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp @@ -976,11 +976,14 @@ void CommunicationTools::asyncSendRecv( std::vector< NeighborCommunicator > & ne parallelDeviceEvents & events ) { GEOS_MARK_FUNCTION; +#if !defined( GEOS_USE_HIP ) if( onDevice ) { waitAllDeviceEvents( events ); } - +#else + GEOS_UNUSED_VAR( onDevice ); +#endif // could swap this to test and make this function call async as well, only launch the sends/recvs for // those we've already recv'd sizing for, go back to some usefule compute / launch some other compute, then @@ -1068,10 +1071,14 @@ void CommunicationTools::finalizeUnpack( MeshLevel & mesh, // poll mpi for completion then wait 10 nanoseconds 6,000,000,000 times (60 sec timeout) GEOS_ASYNC_WAIT( 6000000000, 10, asyncUnpack( mesh, neighbors, icomm, onDevice, events, op ) ); +#if !defined( GEOS_USE_HIP ) if( onDevice ) { waitAllDeviceEvents( events ); } +#else + GEOS_UNUSED_VAR( onDevice ); +#endif MpiWrapper::waitAll( icomm.size(), icomm.mpiSendBufferSizeRequest(), From f6fb4b95d1693ee80b9150938d3e776a007a7062 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 18:52:57 -0500 Subject: [PATCH 03/67] Fix usage of version macros --- src/coreComponents/mainInterface/version.cpp | 42 +++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/coreComponents/mainInterface/version.cpp b/src/coreComponents/mainInterface/version.cpp index 0ed60fc6b18..57470db31f3 100644 --- a/src/coreComponents/mainInterface/version.cpp +++ b/src/coreComponents/mainInterface/version.cpp @@ -60,10 +60,14 @@ static std::string getGpuCompilerIdString() #if defined( GEOS_USE_CUDA ) oss << " - CUDA compiler version: " << CUDA_VERSION/10/100 << "." << CUDA_VERSION/10%100; + +#elif defined( GEOS_USE_HIP ) + oss << " - HIP compiler version: " + << HIP_VERSION_MAJOR << "." + << HIP_VERSION_MINOR << "." + << HIP_VERSION_PATCH; #endif -#if defined( GEOS_USE_HIP ) - oss << " - ROCm compiler version: " << ROCM_VERSION/100/100 << "." << ROCM_VERSION/100%100; -#endif + return oss.str(); } @@ -72,13 +76,13 @@ void outputVersionInfo() GEOS_LOG_RANK_0( "GEOS version: " << getVersion() ); - GEOS_LOG_RANK_0( " - c++ compiler: " << getCppCompilerIdString() ); + GEOS_LOG_RANK_0( " - C++ compiler: " << getCppCompilerIdString() ); std::string const gpuCompilerIdString = getGpuCompilerIdString(); GEOS_LOG_RANK_0_IF( !gpuCompilerIdString.empty(), gpuCompilerIdString ); #if defined(_OPENMP) - GEOS_LOG_RANK_0( " - openmp version: " << _OPENMP ); + GEOS_LOG_RANK_0( " - OpenMP version: " << _OPENMP ); #endif #if defined(GEOS_USE_MPI) @@ -107,11 +111,11 @@ void outputVersionInfo() #endif #if defined(umpire_VERSION) - GEOS_LOG_RANK_0( " - umpire version: " << STRINGIZE( umpire_VERSION ) ); + GEOS_LOG_RANK_0( " - Umpire version: " << STRINGIZE( umpire_VERSION ) ); #endif #if defined(chai_VERSION) - GEOS_LOG_RANK_0( " - chai version: " << STRINGIZE( chai_VERSION ) ); + GEOS_LOG_RANK_0( " - CHAI version: " << STRINGIZE( chai_VERSION ) ); #endif #if defined(adiak_VERSION) @@ -119,7 +123,7 @@ void outputVersionInfo() #endif #if defined(caliper_VERSION) - GEOS_LOG_RANK_0( " - caliper version: " << STRINGIZE( caliper_VERSION ) ); + GEOS_LOG_RANK_0( " - Caliper version: " << STRINGIZE( caliper_VERSION ) ); #endif #if defined(metis_VERSION) @@ -127,23 +131,23 @@ void outputVersionInfo() #endif #if defined(parmetis_VERSION) - GEOS_LOG_RANK_0( " - PARAMETIS version: " << STRINGIZE( parmetis_VERSION ) ); + GEOS_LOG_RANK_0( " - PARMETIS version: " << STRINGIZE( parmetis_VERSION ) ); #endif #if defined(scotch_VERSION) - GEOS_LOG_RANK_0( " - scotch version: " << STRINGIZE( scotch_VERSION ) ); + GEOS_LOG_RANK_0( " - Scotch version: " << STRINGIZE( scotch_VERSION ) ); #endif #if defined(superlu_dist_VERSION) - GEOS_LOG_RANK_0( " - superlu_dist version: " << STRINGIZE( superlu_dist_VERSION ) ); + GEOS_LOG_RANK_0( " - SuperLU_Dist version: " << STRINGIZE( superlu_dist_VERSION ) ); #endif #if defined(suitesparse_VERSION) - GEOS_LOG_RANK_0( " - suitesparse version: " << STRINGIZE( suitesparse_VERSION ) ); + GEOS_LOG_RANK_0( " - SuiteSparse version: " << STRINGIZE( suitesparse_VERSION ) ); #endif #if defined(hypre_VERSION) - GEOS_LOG_RANK_0( " - hypre version: " << STRINGIZE( hypre_VERSION ) ); + GEOS_LOG_RANK_0( " - HYPRE version: " << STRINGIZE( hypre_VERSION ) ); #endif #if defined(trilinos_VERSION) @@ -151,7 +155,7 @@ void outputVersionInfo() #endif #if defined(petsc_VERSION) - GEOS_LOG_RANK_0( " - petsc version: " << STRINGIZE( petsc_VERSION ) ); + GEOS_LOG_RANK_0( " - PETSc version: " << STRINGIZE( petsc_VERSION ) ); #endif #if defined(Python3_VERSION) @@ -167,11 +171,11 @@ void outputVersionInfo() defined(GEOS_USE_HYPRE) && \ ( GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CPU ) GEOS_LOG_RANK_0( "" ); - GEOS_LOG_RANK_0( "**************************************************" ); - GEOS_LOG_RANK_0( "* WARNING!!! *" ); - GEOS_LOG_RANK_0( "* *" ); - GEOS_LOG_RANK_0( "* GEOS has GPU support enabled, but not HYPRE! *" ); - GEOS_LOG_RANK_0( "**************************************************" ); + GEOS_LOG_RANK_0( "*******************************************************" ); + GEOS_LOG_RANK_0( "* WARNING!!! *" ); + GEOS_LOG_RANK_0( "* *" ); + GEOS_LOG_RANK_0( "* GEOS has GPU support enabled, but HYPRE does not! *" ); + GEOS_LOG_RANK_0( "*******************************************************" ); GEOS_LOG_RANK_0( "" ); #endif From 2d2dd64a55e99c0463547bb2770e6e101d237451 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 18:54:38 -0500 Subject: [PATCH 04/67] Use rocSPARSE's SpGEMM with HIP --- .../linearAlgebra/interfaces/hypre/HypreInterface.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp index 783c2e6b483..8f4080c916a 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp @@ -55,7 +55,13 @@ void HypreInterface::initialize() HYPRE_Initialize(); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP HYPRE_SetExecutionPolicy( HYPRE_EXEC_DEVICE ); +#if defined(HYPRE_USING_HIP) + HYPRE_SetSpGemmUseVendor( 1 ); +#else + // TODO: Is it better to use vendor's SpGEMM with CUDA? HYPRE_SetSpGemmUseVendor( 0 ); +#endif + HYPRE_DeviceInitialize(); #endif HYPRE_SetMemoryLocation( hypre::memoryLocation ); From 104baff0313e642fe08bd9488bcf90f0e8202e6b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 18:04:51 -0800 Subject: [PATCH 05/67] Update LvArray --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index 00e3ead6706..efb2a21c406 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit 00e3ead67060d3c3d4b8291d8e4abbf680b87eb3 +Subproject commit efb2a21c406997005dac5b9d57776c913a9b732c From de3d256fc76385ac9a3caa6b6512449ad368d183 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 19:03:46 -0800 Subject: [PATCH 06/67] Update LvArray --- host-configs/LLNL/tioga-base.cmake | 18 ++++++++++-------- .../LLNL/tioga-cce-18-rocm-6.2.1.cmake | 5 +++++ src/coreComponents/LvArray | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake diff --git a/host-configs/LLNL/tioga-base.cmake b/host-configs/LLNL/tioga-base.cmake index 02a9317b688..27233836265 100644 --- a/host-configs/LLNL/tioga-base.cmake +++ b/host-configs/LLNL/tioga-base.cmake @@ -1,17 +1,15 @@ -set( ENABLE_MATHPRESSO OFF CACHE BOOL "" ) +set( GEOS_ENABLE_VEM OFF CACHE BOOL "" FORCE ) -set( ENABLE_PAMELA ON CACHE BOOL "" ) +set( ENABLE_MATHPRESSO OFF CACHE BOOL "" ) +set( ENABLE_PAMELA OFF CACHE BOOL "" ) set( ENABLE_PVTPackage ON CACHE BOOL "" ) - set( ENABLE_PETSC OFF CACHE BOOL "" FORCE ) - -set( ENABLE_CALIPER OFF CACHE BOOL "" ) +set( ENABLE_CALIPER ON CACHE BOOL "" ) set( ENABLE_PAPI OFF CACHE BOOL "" ) - set( ENABLE_ESSL OFF CACHE BOOL "" ) - set( ENABLE_TRILINOS OFF CACHE BOOL "" ) -set( ENABLE_VTK OFF CACHE BOOL "" ) +set( ENABLE_VTK ON CACHE BOOL "" ) +set( ENABLE_OPENMP OFF CACHE BOOL "" FORCE ) set( CAMP_STANDALONE TRUE CACHE BOOL "" ) @@ -30,3 +28,7 @@ set( ENABLE_DOCS OFF CACHE BOOL "" FORCE ) set( ENABLE_SCOTCH OFF CACHE BOOL "" FORCE ) set( ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE ) + +# HYPRE options +set( ENABLE_HYPRE_DEVICE "HIP" CACHE STRING "" ) +set( ENABLE_HYPRE_MIXINT ON CACHE STRING "" ) diff --git a/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake b/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake new file mode 100644 index 00000000000..f43a701511a --- /dev/null +++ b/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake @@ -0,0 +1,5 @@ +include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/tioga-base.cmake) + +# MPI +set(MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "") diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index efb2a21c406..0d032f89f27 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit efb2a21c406997005dac5b9d57776c913a9b732c +Subproject commit 0d032f89f27de9d5b4077ac24a4cbeaeee983f47 From caff8543bdfabc2751a1a525955f810b1aa8b1d3 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 19:08:07 -0800 Subject: [PATCH 07/67] LvArray --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index 0d032f89f27..ae2a4642e41 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit 0d032f89f27de9d5b4077ac24a4cbeaeee983f47 +Subproject commit ae2a4642e41a3896e63ca443247a4993a83b5938 From db5b03ad1227c1ade22ed150b1fdeea554d11288 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 19:10:39 -0800 Subject: [PATCH 08/67] LvArray --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index ae2a4642e41..8121214ab27 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit ae2a4642e41a3896e63ca443247a4993a83b5938 +Subproject commit 8121214ab27f2386b20da55deda57514a0965f7e From bf1734eebef1dbd2dbc3b56ada42394c8e1117f3 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 19:49:19 -0800 Subject: [PATCH 09/67] LvArray --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index 8121214ab27..65c0ec6c779 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit 8121214ab27f2386b20da55deda57514a0965f7e +Subproject commit 65c0ec6c779b9080e920d00704e41ee4cc324b81 From f22603128e610d931c7e6b1c65d8119173bebdce Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 20:56:38 -0800 Subject: [PATCH 10/67] Add TPL include to tioga-base --- host-configs/LLNL/tioga-base.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/host-configs/LLNL/tioga-base.cmake b/host-configs/LLNL/tioga-base.cmake index 27233836265..08191c58f4f 100644 --- a/host-configs/LLNL/tioga-base.cmake +++ b/host-configs/LLNL/tioga-base.cmake @@ -32,3 +32,6 @@ set( ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE ) # HYPRE options set( ENABLE_HYPRE_DEVICE "HIP" CACHE STRING "" ) set( ENABLE_HYPRE_MIXINT ON CACHE STRING "" ) + +# TPLs +include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake) From 1f56d51d3ac538fd3062bf3fc14270bfce44792b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 20:59:30 -0800 Subject: [PATCH 11/67] Create amdgpu-base.cmake --- host-configs/LLNL/{tioga-base.cmake => amdgpu-base.cmake} | 5 +++++ 1 file changed, 5 insertions(+) rename host-configs/LLNL/{tioga-base.cmake => amdgpu-base.cmake} (84%) diff --git a/host-configs/LLNL/tioga-base.cmake b/host-configs/LLNL/amdgpu-base.cmake similarity index 84% rename from host-configs/LLNL/tioga-base.cmake rename to host-configs/LLNL/amdgpu-base.cmake index 08191c58f4f..ab92058b034 100644 --- a/host-configs/LLNL/tioga-base.cmake +++ b/host-configs/LLNL/amdgpu-base.cmake @@ -35,3 +35,8 @@ set( ENABLE_HYPRE_MIXINT ON CACHE STRING "" ) # TPLs include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake) + +# ATS +set(USER $ENV{USER} CACHE STRING "") +set(ATS_WORKING_DIR "/p/lustre2/${USER}/integratedTestsGEOS/${CONFIG_NAME}" CACHE PATH "") +set(ATS_BASELINE_DIR "/p/lustre2/${USER}/integratedTestsGEOS/baselines" CACHE PATH "") From e5521f3786459616af5dd03dfd8bb363df05f56d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 21:04:06 -0800 Subject: [PATCH 12/67] LvArray --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index 65c0ec6c779..7e624fb1823 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit 65c0ec6c779b9080e920d00704e41ee4cc324b81 +Subproject commit 7e624fb182362b719c93d1bfef41261c150a0c04 From 3ff69c8aff2783136121f93cf214bdb3f5a416c1 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 22:31:37 -0800 Subject: [PATCH 13/67] Remove old limitation --- src/coreComponents/mesh/generators/ParMETISInterface.hpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/coreComponents/mesh/generators/ParMETISInterface.hpp b/src/coreComponents/mesh/generators/ParMETISInterface.hpp index fa1bb2c8545..8e948f762e1 100644 --- a/src/coreComponents/mesh/generators/ParMETISInterface.hpp +++ b/src/coreComponents/mesh/generators/ParMETISInterface.hpp @@ -26,13 +26,8 @@ namespace geos { -#if defined(GEOS_USE_HIP) // still need int32 hypre for the current hip-capable build -/// Typedef to allow us to specify required parmetis integer type. -using pmet_idx_t = int32_t; -#else /// Typedef to allow us to specify required parmetis integer type. using pmet_idx_t = int64_t; -#endif namespace parmetis { From 890ce4219be8ada278bc0c9ba18d77137915b665 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 23:41:52 -0800 Subject: [PATCH 14/67] LvArray --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index 7e624fb1823..a6d24fcc82f 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit 7e624fb182362b719c93d1bfef41261c150a0c04 +Subproject commit a6d24fcc82f2f7ed6b2e79cad5e9ac8813088732 From 292eac7f530a9c0e87b024d30517376f1a757428 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Dec 2024 23:48:03 -0800 Subject: [PATCH 15/67] Add workaround for link issue with cce-18 --- .../linearAlgebra/CMakeLists.txt | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/coreComponents/linearAlgebra/CMakeLists.txt b/src/coreComponents/linearAlgebra/CMakeLists.txt index 99cc0bc77d4..687239a8169 100644 --- a/src/coreComponents/linearAlgebra/CMakeLists.txt +++ b/src/coreComponents/linearAlgebra/CMakeLists.txt @@ -38,7 +38,6 @@ set( linearAlgebra_headers solvers/BicgstabSolver.hpp solvers/BlockPreconditioner.hpp solvers/CgSolver.hpp - solvers/GmresSolver.hpp solvers/KrylovSolver.hpp solvers/KrylovUtils.hpp solvers/PreconditionerBlockJacobi.hpp @@ -60,6 +59,12 @@ set( linearAlgebra_headers utilities/NormalOperator.hpp utilities/ReverseCutHillMcKeeOrdering.hpp utilities/TransposeOperator.hpp ) + +# Workaround for link issue with cce-18 +if( NOT ${ENABLE_HIP} ) + list( APPEND linearAlgebra_headers solvers/GmresSolver.hpp ) +endif() + # # Specify all sources # @@ -68,26 +73,31 @@ set( linearAlgebra_sources solvers/BicgstabSolver.cpp solvers/BlockPreconditioner.cpp solvers/CgSolver.cpp - solvers/GmresSolver.cpp solvers/KrylovSolver.cpp solvers/SeparateComponentPreconditioner.cpp utilities/ReverseCutHillMcKeeOrdering.cpp ) +# Workaround for link issue with cce-18 +if( NOT ${ENABLE_HIP} ) + list( APPEND linearAlgebra_sources solvers/GmresSolver.hpp ) +endif() + set( dependencyList ${parallelDeps} mesh denseLinearAlgebra finiteVolume ) set( tplDependencyList "" ) -list( APPEND linearAlgebra_headers interfaces/direct/SuiteSparse.hpp ) -list( APPEND linearAlgebra_sources interfaces/direct/SuiteSparse.cpp ) -list( APPEND tplDependencyList suitesparse ) +if( ENABLE_SUITESPARSE ) + list( APPEND linearAlgebra_headers interfaces/direct/SuiteSparse.hpp ) + list( APPEND linearAlgebra_sources interfaces/direct/SuiteSparse.cpp ) + list( APPEND tplDependencyList suitesparse ) +endif( ) if( ENABLE_SUPERLU_DIST ) - list( APPEND linearAlgebra_headers interfaces/direct/SuperLUDist.hpp ) - list( APPEND linearAlgebra_sources interfaces/direct/SuperLUDist.cpp ) - list( APPEND tplDependencyList superlu_dist ) + list( APPEND linearAlgebra_headers interfaces/direct/SuperLUDist.hpp ) + list( APPEND linearAlgebra_sources interfaces/direct/SuperLUDist.cpp ) + list( APPEND tplDependencyList superlu_dist ) endif( ) if( ENABLE_TRILINOS ) - list( APPEND linearAlgebra_headers interfaces/trilinos/EpetraExport.hpp interfaces/trilinos/EpetraMatrix.hpp @@ -106,11 +116,9 @@ if( ENABLE_TRILINOS ) interfaces/trilinos/TrilinosSolver.cpp ) list( APPEND tplDependencyList trilinos ) - endif() if( ENABLE_HYPRE ) - list( APPEND linearAlgebra_headers interfaces/hypre/HypreExport.hpp interfaces/hypre/HypreInterface.hpp @@ -151,11 +159,9 @@ if( ENABLE_HYPRE ) interfaces/hypre/HypreVector.cpp ) list( APPEND tplDependencyList hypre umpire ) - endif() if( ENABLE_PETSC ) - list( APPEND linearAlgebra_headers interfaces/petsc/PetscExport.hpp interfaces/petsc/PetscInterface.hpp @@ -174,7 +180,6 @@ if( ENABLE_PETSC ) interfaces/petsc/PetscVector.cpp ) list( APPEND tplDependencyList petsc ) - endif() geos_decorate_link_dependencies( LIST decoratedDependencies @@ -195,4 +200,3 @@ install( TARGETS linearAlgebra LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib ) if( GEOS_ENABLE_TESTS ) add_subdirectory( unitTests ) endif( ) - From 5c93fc90d17177fa6adfbfb6bd9d436b2d371a71 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 18 Dec 2024 22:40:56 -0800 Subject: [PATCH 16/67] Resolve compilation warnings --- src/coreComponents/common/LifoStorage.hpp | 2 +- src/coreComponents/common/LifoStorageCommon.hpp | 2 +- src/coreComponents/common/TypeDispatch.hpp | 2 +- src/coreComponents/mesh/FaceElementSubRegion.cpp | 9 +++++---- src/coreComponents/mesh/FaceManager.cpp | 2 +- src/coreComponents/mesh/generators/PrismUtilities.hpp | 2 +- .../mesh/mpiCommunications/CommunicationTools.cpp | 2 ++ .../physicsSolvers/fluidFlow/SinglePhaseBase.cpp | 2 +- .../physicsSolvers/fluidFlow/SourceFluxStatistics.cpp | 4 ++-- 9 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/coreComponents/common/LifoStorage.hpp b/src/coreComponents/common/LifoStorage.hpp index a8e68640fc5..b82ed43afc2 100644 --- a/src/coreComponents/common/LifoStorage.hpp +++ b/src/coreComponents/common/LifoStorage.hpp @@ -64,7 +64,7 @@ class LifoStorage m_bufferCount( 0 ) { LIFO_LOG_RANK( " LIFO : maximum size "<< m_maxNumberOfBuffers << " buffers " ); - double bufferSize = ( ( double ) m_bufferSize ) / ( 1024.0 * 1024.0 ); + GEOS_MAYBE_UNUSED double bufferSize = ( ( double ) m_bufferSize ) / ( 1024.0 * 1024.0 ); LIFO_LOG_RANK( " LIFO : buffer size "<< bufferSize << "MB" ); if( numberOfBuffersToStoreOnDevice < 0 ) { diff --git a/src/coreComponents/common/LifoStorageCommon.hpp b/src/coreComponents/common/LifoStorageCommon.hpp index 3183cde65b8..7d18978aa70 100644 --- a/src/coreComponents/common/LifoStorageCommon.hpp +++ b/src/coreComponents/common/LifoStorageCommon.hpp @@ -164,7 +164,7 @@ class LifoStorageCommon GEOS_ERROR( "To use LifoStorage, both _SC_AVPHYS_PAGES and _SC_PAGESIZE must be defined." ); #endif int numberOfBuffersToStoreOnHost = std::max( 1, std::min( ( int )( 0.01 * percent * free / bufferSize ), maxNumberOfBuffers - numberOfBuffersToStoreOnDevice ) ); - double freeGB = ( ( double ) free ) / ( 1024.0 * 1024.0 * 1024.0 ) / MpiWrapper::nodeCommSize(); + GEOS_MAYBE_UNUSED double freeGB = ( ( double ) free ) / ( 1024.0 * 1024.0 * 1024.0 ) / MpiWrapper::nodeCommSize(); LIFO_LOG_RANK( " LIFO : available memory on host " << freeGB << " GB" ); return numberOfBuffersToStoreOnHost; } diff --git a/src/coreComponents/common/TypeDispatch.hpp b/src/coreComponents/common/TypeDispatch.hpp index a1881944de2..9ed2603d89d 100644 --- a/src/coreComponents/common/TypeDispatch.hpp +++ b/src/coreComponents/common/TypeDispatch.hpp @@ -376,7 +376,7 @@ bool dispatch( LIST const combinations, if( !success ) { auto typePrinter = []( auto t ){ return LvArray::system::demangle( typeid( typename decltype(t)::type ).name() ); }; - auto typeListPrinter = [typePrinter]( auto tlist ){ return internal::listToString( typename decltype( tlist )::type{}, "\n ", "", typePrinter ); }; + GEOS_MAYBE_UNUSED auto typeListPrinter = [typePrinter]( auto tlist ){ return internal::listToString( typename decltype( tlist )::type{}, "\n ", "", typePrinter ); }; GEOS_ERROR( "Types were not dispatched. The types of the input objects are:\n" << "( "<<( ( "\n " + LvArray::system::demangle( internal::typeIdWrapper( objects ).name() ) ) + ... )<<" \n)\n"<< diff --git a/src/coreComponents/mesh/FaceElementSubRegion.cpp b/src/coreComponents/mesh/FaceElementSubRegion.cpp index f1e1162a060..2bd06f27db6 100644 --- a/src/coreComponents/mesh/FaceElementSubRegion.cpp +++ b/src/coreComponents/mesh/FaceElementSubRegion.cpp @@ -449,10 +449,11 @@ localIndex FaceElementSubRegion::unpackToFaceRelation( buffer_unit_type const * * @param[in] elem2dToElems3d A mapping. * @param[in,out] elem2dToFaces This mapping will be corrected if needed to match @p elem2dToElems3d. */ -void fixNeighborMappingsInconsistency( string const & fractureName, +void fixNeighborMappingsInconsistency( GEOS_MAYBE_UNUSED string const & fractureName, FixedToManyElementRelation const & elem2dToElems3d, FaceElementSubRegion::FaceMapType & elem2dToFaces ) { + GEOS_MAYBE_UNUSED static constexpr std::string_view mappingInconsistency= "Mapping neighbor inconsistency detected for fracture {}."; { localIndex const num2dElems = elem2dToFaces.size( 0 ); for( int e2d = 0; e2d < num2dElems; ++e2d ) @@ -486,9 +487,9 @@ void fixNeighborMappingsInconsistency( string const & fractureName, { std::swap( elem2dToFaces[e2d][0], elem2dToFaces[e2d][1] ); } - else if( !matchStraight ) + else { - GEOS_ERROR( "Mapping neighbor inconsistency detected for fracture " << fractureName ); + GEOS_ERROR_IF( !matchStraight, GEOS_FMT( mappingInconsistency, fractureName ) ); } } } @@ -608,7 +609,7 @@ buildCollocatedEdgeBuckets( std::map< globalIndex, globalIndex > const & referen std::map< std::pair< globalIndex, globalIndex >, std::set< localIndex > > collocatedEdgeBuckets; for( auto const & p: edgesIds ) { - static constexpr std::string_view nodeNotFound = "Internal error when trying to access the reference collocated node for global node {}."; + GEOS_MAYBE_UNUSED static constexpr std::string_view nodeNotFound = "Internal error when trying to access the reference collocated node for global node {}."; std::pair< globalIndex, globalIndex > const & nodes = p.first; localIndex const & edge = p.second; diff --git a/src/coreComponents/mesh/FaceManager.cpp b/src/coreComponents/mesh/FaceManager.cpp index 044cd1beb47..985bc21456b 100644 --- a/src/coreComponents/mesh/FaceManager.cpp +++ b/src/coreComponents/mesh/FaceManager.cpp @@ -174,7 +174,7 @@ void FaceManager::setGeometricalRelations( CellBlockManagerABC const & cellBlock return; } - constexpr char err[] = "Internal error when trying to connect matrix mapping and fracture mapping. Face {} seems wrongly connected."; + GEOS_MAYBE_UNUSED constexpr char err[] = "Internal error when trying to connect matrix mapping and fracture mapping. Face {} seems wrongly connected."; FaceElementSubRegion const & subRegion = region.getUniqueSubRegion< FaceElementSubRegion >(); int const esr = 0; // Since there's only on unique subregion, the index is always 0. diff --git a/src/coreComponents/mesh/generators/PrismUtilities.hpp b/src/coreComponents/mesh/generators/PrismUtilities.hpp index f6e70aef310..1e489de4e1a 100644 --- a/src/coreComponents/mesh/generators/PrismUtilities.hpp +++ b/src/coreComponents/mesh/generators/PrismUtilities.hpp @@ -40,7 +40,7 @@ localIndex getFaceNodesPrism( localIndex const faceNum, { static_assert( N > 4, "Function getFaceNodePrism can be called for a prism with N-sided polygon base where N > 5." ); - static constexpr auto nodeCountError = "Not enough nodes for {} element (face index = {}).\n"; + GEOS_MAYBE_UNUSED static constexpr auto nodeCountError = "Not enough nodes for {} element (face index = {}).\n"; if( faceNum == 0 ) { diff --git a/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp b/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp index 7cda510bfa8..1bc4bb4738c 100644 --- a/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp +++ b/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp @@ -982,6 +982,7 @@ void CommunicationTools::asyncSendRecv( std::vector< NeighborCommunicator > & ne waitAllDeviceEvents( events ); } #else + GEOS_UNUSED_VAR( events ); GEOS_UNUSED_VAR( onDevice ); #endif @@ -1077,6 +1078,7 @@ void CommunicationTools::finalizeUnpack( MeshLevel & mesh, waitAllDeviceEvents( events ); } #else + GEOS_UNUSED_VAR( events ); GEOS_UNUSED_VAR( onDevice ); #endif diff --git a/src/coreComponents/physicsSolvers/fluidFlow/SinglePhaseBase.cpp b/src/coreComponents/physicsSolvers/fluidFlow/SinglePhaseBase.cpp index 98934988d8d..bcd75897f1e 100644 --- a/src/coreComponents/physicsSolvers/fluidFlow/SinglePhaseBase.cpp +++ b/src/coreComponents/physicsSolvers/fluidFlow/SinglePhaseBase.cpp @@ -996,7 +996,7 @@ void SinglePhaseBase::applySourceFluxBC( real64 const time_n, mesh, SourceFluxBoundaryCondition::catalogName(), [&, isThermal]( SourceFluxBoundaryCondition const & fs, - string const & setName, + GEOS_MAYBE_UNUSED string const & setName, SortedArrayView< localIndex const > const & targetSet, ElementSubRegionBase & subRegion, string const & ) diff --git a/src/coreComponents/physicsSolvers/fluidFlow/SourceFluxStatistics.cpp b/src/coreComponents/physicsSolvers/fluidFlow/SourceFluxStatistics.cpp index 6c8a1e36cb3..cea08b08f5e 100644 --- a/src/coreComponents/physicsSolvers/fluidFlow/SourceFluxStatistics.cpp +++ b/src/coreComponents/physicsSolvers/fluidFlow/SourceFluxStatistics.cpp @@ -125,7 +125,7 @@ void SourceFluxStatsAggregator::registerDataOnMesh( Group & meshBodies ) } void SourceFluxStatsAggregator::writeStatsToLog( integer minLogLevel, - string_view elementSetName, + GEOS_MAYBE_UNUSED string_view elementSetName, WrappedStats const & wrappedStats ) { if( getLogLevel() >= minLogLevel && logger::internal::rank == 0 ) @@ -135,7 +135,7 @@ void SourceFluxStatsAggregator::writeStatsToLog( integer minLogLevel, wrappedStats.stats().m_elementCount ) ); // we want to format differently if we have got multiple phases or not - string_view massUnit = units::getSymbol( m_solver->getMassUnit() ); + GEOS_MAYBE_UNUSED string_view massUnit = units::getSymbol( m_solver->getMassUnit() ); if( wrappedStats.stats().m_producedMass.size() == 1 ) { GEOS_LOG_RANK( GEOS_FMT( "{} {} (of {}, in {}): Produced mass = {} {}", From 4737637b5c7412cb1e9d30788b6b8ff7cadda6c1 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 18 Dec 2024 22:41:38 -0800 Subject: [PATCH 17/67] Check SuiteSparse presence --- .../linearAlgebra/interfaces/direct/SuiteSparse.cpp | 3 +++ .../linearAlgebra/interfaces/direct/SuiteSparse.hpp | 2 ++ .../linearAlgebra/interfaces/hypre/HypreInterface.cpp | 8 +++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.cpp b/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.cpp index 53a45e2ce7f..30cc3a9fc63 100644 --- a/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.cpp @@ -19,6 +19,7 @@ #include "SuiteSparse.hpp" +#if defined(suitesparse_VERSION) #include "codingUtilities/Utilities.hpp" #include "common/Stopwatch.hpp" #include "linearAlgebra/common/common.hpp" @@ -384,3 +385,5 @@ template class SuiteSparse< PetscInterface >; #endif } + +#endif /* if defined(suitesparse_VERSION) */ diff --git a/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.hpp b/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.hpp index 77534242052..501b2befc40 100644 --- a/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.hpp @@ -20,6 +20,7 @@ #ifndef GEOS_LINEARALGEBRA_INTERFACES_SUITESPARSE_HPP_ #define GEOS_LINEARALGEBRA_INTERFACES_SUITESPARSE_HPP_ +#if defined(suitesparse_VERSION) #include "common/DataTypes.hpp" #include "common/LinearSolverBase.hpp" #include "common/PreconditionerBase.hpp" @@ -142,4 +143,5 @@ class SuiteSparse final : public LinearSolverBase< LAI > } +#endif /* if defined(suitesparse_VERSION) */ #endif /*GEOS_LINEARALGEBRA_INTERFACES_SUITESPARSE_HPP_*/ diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp index 8f4080c916a..339262da651 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp @@ -92,13 +92,19 @@ HypreInterface::createSolver( LinearSolverParameters params ) #if defined(GEOS_USE_SUPERLU_DIST) return std::make_unique< SuperLUDist< HypreInterface > >( std::move( params ) ); #else - GEOS_ERROR( "GEOSX is configured without support for SuperLU_dist." ); + GEOS_ERROR( "GEOS is configured without support for SuperLU_dist." ); return std::unique_ptr< LinearSolverBase< HypreInterface > >( nullptr ); #endif } else { + /* TODO: add GEOS_USE_SUITESPARSE and use it below */ +#if defined(suitesparse_VERSION) return std::make_unique< SuiteSparse< HypreInterface > >( std::move( params ) ); +#else + GEOS_ERROR( "GEOS is configured without support for SuiteSparse." ); + return std::unique_ptr< LinearSolverBase< HypreInterface > >( nullptr ); +#endif } } else From dc805b1499023fb1d4df00fb64f964693df9af2d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 18 Dec 2024 22:42:21 -0800 Subject: [PATCH 18/67] Link flag workaround --- host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake b/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake index f43a701511a..71ba7cef44c 100644 --- a/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake +++ b/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake @@ -3,3 +3,7 @@ include(${CMAKE_CURRENT_LIST_DIR}/tioga-base.cmake) # MPI set(MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "") + +# Workaround for linking errors involving missing explicit instantiations of certain functions. +# TODO: we should add the missing explicit instantiations +set( CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE ) From 555973cbc92fca598925225f3d430235f4a1eb8b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 18 Dec 2024 22:43:39 -0800 Subject: [PATCH 19/67] Add tuo host configs --- host-configs/LLNL/tuo-base.cmake | 41 +++++++++++++++++++ host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake | 9 ++++ 2 files changed, 50 insertions(+) create mode 100644 host-configs/LLNL/tuo-base.cmake create mode 100644 host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake diff --git a/host-configs/LLNL/tuo-base.cmake b/host-configs/LLNL/tuo-base.cmake new file mode 100644 index 00000000000..207236dc9eb --- /dev/null +++ b/host-configs/LLNL/tuo-base.cmake @@ -0,0 +1,41 @@ +set( GEOS_ENABLE_VEM OFF CACHE BOOL "" FORCE ) + +set( ENABLE_MATHPRESSO OFF CACHE BOOL "" ) +set( ENABLE_PAMELA OFF CACHE BOOL "" ) +set( ENABLE_PVTPackage ON CACHE BOOL "" ) +set( ENABLE_PETSC OFF CACHE BOOL "" FORCE ) +set( ENABLE_CALIPER ON CACHE BOOL "" ) +set( ENABLE_PAPI OFF CACHE BOOL "" ) +set( ENABLE_ESSL OFF CACHE BOOL "" ) +set( ENABLE_TRILINOS OFF CACHE BOOL "" ) +set( ENABLE_VTK ON CACHE BOOL "" ) +set( ENABLE_OPENMP OFF CACHE BOOL "" FORCE ) + +set( CAMP_STANDALONE TRUE CACHE BOOL "" ) + +# ROCM options +set( ENABLE_ROCM ON CACHE BOOL "" FORCE ) +set( ROCM_ROOT "${HIP_ROOT}" CACHE PATH "" ) + +set( GEOS_BUILD_OBJ_LIBS OFF CACHE BOOL "" FORCE ) +set( ENABLE_GTEST_DEATH_TESTS OFF CACHE BOOL "" ) +set( gtest_disable_pthreads ON CACHE BOOL "" ) + +set( ENABLE_TESTS ON CACHE BOOL "" FORCE ) +set( ENABLE_EXAMPLES ON CACHE BOOL "" FORCE ) +set( ENABLE_BENCHMARKS ON CACHE BOOL "" FORCE ) +set( ENABLE_DOCS OFF CACHE BOOL "" FORCE ) + +set( ENABLE_SCOTCH OFF CACHE BOOL "" FORCE ) +set( ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE ) + +# HYPRE options +set( ENABLE_HYPRE_MIXINT ON CACHE STRING "" ) + +# TPLs +include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake) + +# ATS +set(USER $ENV{USER} CACHE STRING "") +set(ATS_WORKING_DIR "/p/lustre2/${USER}/integratedTestsGEOS/${CONFIG_NAME}" CACHE PATH "") +set(ATS_BASELINE_DIR "/p/lustre2/${USER}/integratedTestsGEOS/baselines" CACHE PATH "") diff --git a/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake b/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake new file mode 100644 index 00000000000..a366f21f3d1 --- /dev/null +++ b/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake @@ -0,0 +1,9 @@ +include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake) + +# MPI +set(MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "") + +# Workaround for linking errors involving missing explicit instantiations of certain functions. +# TODO: we should add the missing explicit instantiations +set( CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE ) From 38698bfd72df50533d42499935c42b4064a4f13d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 18 Dec 2024 22:44:41 -0800 Subject: [PATCH 20/67] LvArray hash --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index a6d24fcc82f..c0b7da835b0 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit a6d24fcc82f2f7ed6b2e79cad5e9ac8813088732 +Subproject commit c0b7da835b04080305a20c114224f0c8710ff983 From 8323d86f8ba7b504b95bf14eebcc52c2e5529017 Mon Sep 17 00:00:00 2001 From: Randolph Settgast Date: Fri, 20 Dec 2024 10:39:04 -0800 Subject: [PATCH 21/67] add some memory output info in the newton loop --- src/coreComponents/LvArray | 2 +- .../physicsSolvers/PhysicsSolverBase.cpp | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index c0b7da835b0..00e3ead6706 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit c0b7da835b04080305a20c114224f0c8710ff983 +Subproject commit 00e3ead67060d3c3d4b8291d8e4abbf680b87eb3 diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index 14ec3cbfd45..07a8163e7f5 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -760,6 +760,26 @@ real64 PhysicsSolverBase::eisenstatWalker( real64 const newNewtonNorm, return krylovTol; } + +void MemoryUsageOutput( std::string const & message ) +{ +#if defined( GEOS_USE_HIP ) + + std::cout< 0 ) { @@ -896,6 +923,8 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, for( newtonIter = 0; newtonIter < maxNewtonIter; ++newtonIter ) { + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-01 newtonIter:" + std::to_string(newtonIter)); + GEOS_LOG_LEVEL_INFO_RANK_0( logInfo::NonlinearSolver, GEOS_FMT( " Attempt: {:2}, ConfigurationIter: {:2}, NewtonIter: {:2}", dtAttempt, configurationLoopIter, newtonIter ) ); @@ -921,6 +950,8 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, m_localMatrix.toViewConstSizes(), localRhs ); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-02 newtonIter:" + std::to_string(newtonIter)); + // apply boundary conditions to system applyBoundaryConditions( time_n, stepDt, @@ -931,6 +962,8 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, m_rhs.close(); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-03 newtonIter:" + std::to_string(newtonIter)); + if( m_assemblyCallback ) { // Make a copy of LA objects and ship off to the callback @@ -946,6 +979,8 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, // get residual norm residualNorm = calculateResidualNorm( time_n, stepDt, domain, m_dofManager, m_rhs.values() ); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-04 newtonIter:" + std::to_string(newtonIter)); + GEOS_LOG_LEVEL_INFO_RANK_0( logInfo::Convergence, GEOS_FMT( " ( R ) = ( {:4.2e} )", residualNorm ) ); } @@ -1020,6 +1055,8 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, } { + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-05 newtonIter:" + std::to_string(newtonIter)); + Timer timer( m_timers["linear solver total"] ); // if using adaptive Krylov tolerance scheme, update tolerance. @@ -1041,16 +1078,21 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, // Compose parallel LA matrix/rhs out of local LA matrix/rhs // m_matrix.create( m_localMatrix.toViewConst(), m_dofManager.numLocalDofs(), MPI_COMM_GEOS ); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-06 newtonIter:" + std::to_string(newtonIter)); + } // Output the linear system matrix/rhs for debugging purposes debugOutputSystem( time_n, cycleNumber, newtonIter, m_matrix, m_rhs ); // Solve the linear system + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-07 newtonIter:" + std::to_string(newtonIter)); solveLinearSystem( m_dofManager, m_matrix, m_rhs, m_solution ); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-08 newtonIter:" + std::to_string(newtonIter)); // Increment the solver statistics for reporting purposes m_solverStatistics.logNonlinearIteration( m_linearSolverResult.numIterations ); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-09 newtonIter:" + std::to_string(newtonIter)); // Output the linear system solution for debugging purposes debugOutputSolution( time_n, cycleNumber, newtonIter, m_solution ); @@ -1073,6 +1115,7 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, // apply the system solution to the fields/variables applySystemSolution( m_dofManager, m_solution.values(), scaleFactor, stepDt, domain ); + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-10 newtonIter:" + std::to_string(newtonIter)); } { @@ -1085,6 +1128,8 @@ bool PhysicsSolverBase::solveNonlinearSystem( real64 const & time_n, lastResidual = residualNorm; } + MemoryUsageOutput("PhysicsSolverBase::solveNonlinearSystem-11: exit"); + return isNewtonConverged; } From 010fdc0ef4942b13feecb7a8e46698698f6dd5e7 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 20 Dec 2024 19:43:16 -0800 Subject: [PATCH 22/67] LvArray hash --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index 00e3ead6706..c0b7da835b0 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit 00e3ead67060d3c3d4b8291d8e4abbf680b87eb3 +Subproject commit c0b7da835b04080305a20c114224f0c8710ff983 From b0bd42d7b2f8a1f503d862d52f32eaf7313dad5f Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 20 Dec 2024 19:44:26 -0800 Subject: [PATCH 23/67] Minor --- host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake | 2 +- .../linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake b/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake index 71ba7cef44c..dd4dd9b25fd 100644 --- a/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake +++ b/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake @@ -1,5 +1,5 @@ include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake) -include(${CMAKE_CURRENT_LIST_DIR}/tioga-base.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake) # MPI set(MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "") diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp index 39c5883876f..eb1244bf687 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp @@ -38,11 +38,13 @@ TrilinosInterface::createSolver( LinearSolverParameters params ) { if( params.solverType == LinearSolverParameters::SolverType::direct ) { +#if defined(GEOS_USE_SUPERLU_DIST) if( params.direct.parallel ) { return std::make_unique< SuperLUDist< TrilinosInterface > >( std::move( params ) ); } else +#endif { return std::make_unique< SuiteSparse< TrilinosInterface > >( std::move( params ) ); } From e8c03bf38abee4d9d74d5a13c43a7735ee593b99 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 30 Dec 2024 19:25:33 -0800 Subject: [PATCH 24/67] Fix MemoryUsageOutput --- .../physicsSolvers/PhysicsSolverBase.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index 07a8163e7f5..5679a4797f2 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -764,16 +764,19 @@ real64 PhysicsSolverBase::eisenstatWalker( real64 const newNewtonNorm, void MemoryUsageOutput( std::string const & message ) { #if defined( GEOS_USE_HIP ) + size_t free, total; - std::cout<(total - free) / (1e9) + << " GiB" << std::endl; } } #endif From 6bfc7af0fbf0d48926d716eff8fe2deacbc4c7e7 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 30 Dec 2024 19:26:31 -0800 Subject: [PATCH 25/67] Fix unused parameter warning --- src/coreComponents/mesh/generators/CollocatedNodes.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/coreComponents/mesh/generators/CollocatedNodes.cpp b/src/coreComponents/mesh/generators/CollocatedNodes.cpp index 89f6fdcd484..6dcded468b1 100644 --- a/src/coreComponents/mesh/generators/CollocatedNodes.cpp +++ b/src/coreComponents/mesh/generators/CollocatedNodes.cpp @@ -28,9 +28,12 @@ CollocatedNodes::CollocatedNodes( string const & faceBlockName, bool isParallel ) { // The vtk field to the collocated nodes for fractures. - string const COLLOCATED_NODES = "collocated_nodes"; + string const collocatedNodesField = "collocated_nodes"; - vtkIdTypeArray const * collocatedNodes = vtkIdTypeArray::FastDownCast( faceMesh->GetPointData()->GetArray( COLLOCATED_NODES.c_str() ) ); + // Error message in case field is not found + GEOS_MAYBE_UNUSED constexpr char err[] = "Could not find valid field {} for fracture {}."; + + vtkIdTypeArray const * collocatedNodes = vtkIdTypeArray::FastDownCast( faceMesh->GetPointData()->GetArray( collocatedNodesField.c_str() ) ); if( isParallel ) { // Depending on the parallel split, the vtk face mesh may be empty on a rank. @@ -41,7 +44,7 @@ CollocatedNodes::CollocatedNodes( string const & faceBlockName, std::uintptr_t const address = MpiWrapper::max( reinterpret_cast< std::uintptr_t >(collocatedNodes) ); if( address == 0 ) { - GEOS_ERROR_IF( collocatedNodes == nullptr, "Could not find valid field \"" << COLLOCATED_NODES << "\" for fracture \"" << faceBlockName << "\"." ); + GEOS_ERROR_IF_EQ_MSG( collocatedNodes, nullptr, GEOS_FMT( err, collocatedNodesField, faceBlockName ) ); } } From fad14d0b1d425525a5de7491b8c3954683c4c1e1 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 30 Dec 2024 19:28:14 -0800 Subject: [PATCH 26/67] Minor fix --- src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index 5679a4797f2..8afe7fb5b17 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -775,7 +775,7 @@ void MemoryUsageOutput( std::string const & message ) std::cout << std::fixed << std::setprecision(3) << "Rank " << rank << " - " << message - << " - allocated memory: " << static_cast(total - free) / (1e9) + << " - allocated memory: " << static_cast(total - free) / (1 << 30) << " GiB" << std::endl; } } From 537b873caadfb2cd7253e2e528350a2a8a64c4f9 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 7 May 2025 21:59:17 -0400 Subject: [PATCH 27/67] Update blt --- src/cmake/blt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmake/blt b/src/cmake/blt index 9ff77344f0b..b01fb10015c 160000 --- a/src/cmake/blt +++ b/src/cmake/blt @@ -1 +1 @@ -Subproject commit 9ff77344f0b2a6ee345e452bddd6bfd46cbbfa35 +Subproject commit b01fb10015c1aacf484536a0bc22e4cfc6d22845 From 9d041b2ea94a1cbd5dbd6df81393a3bfdd613126 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 8 May 2025 14:04:33 -0700 Subject: [PATCH 28/67] Add tuo-cce-19-rocm-6.4.0 host config --- host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake | 9 +++++++++ src/coreComponents/LvArray | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake diff --git a/host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake b/host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake new file mode 100644 index 00000000000..54da2a818fb --- /dev/null +++ b/host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake @@ -0,0 +1,9 @@ +include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/tuo-cce-19-rocm-6.4.0.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake) + +# MPI +set(MPI_HOME /opt/cray/pe/mpich/8.1.32/ofi/crayclang/19.0 CACHE PATH "") + +# Workaround for linking errors involving missing explicit instantiations of certain functions. +# TODO: we should add the missing explicit instantiations +set( CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE ) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index 6daf4b05d7c..d3106866b2b 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit 6daf4b05d7c8da5a34e7e5aeabc5040cb381b4e8 +Subproject commit d3106866b2b7785bb0b5d6d1cf1a88ebe5364b96 From 56f3daa64b21a42d8fbb2e8e422108b5d15b8579 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 8 May 2025 14:04:46 -0700 Subject: [PATCH 29/67] Comment out VEM kernels --- .../multiphysics/kernelSpecs.json | 11 +-------- .../solidMechanics/kernelSpecs.json | 23 ++----------------- 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/src/coreComponents/physicsSolvers/multiphysics/kernelSpecs.json b/src/coreComponents/physicsSolvers/multiphysics/kernelSpecs.json index 7acbc5cb996..875f6ff442f 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/kernelSpecs.json +++ b/src/coreComponents/physicsSolvers/multiphysics/kernelSpecs.json @@ -62,16 +62,7 @@ "H1_Pyramid_Lagrange1_Gauss5" ] }, - "explicit": [ "CellElementSubRegion#PorousSolid#H1_Tetrahedron_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism5_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism6_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism7_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism8_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism9_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism10_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Hexahedron_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Wedge_VEM_Gauss1", - "CellElementSubRegion#PorousSolid#H1_Prism11_VEM_Gauss1" ] + "explicit": [] }, "PoromechanicsEFEMKernels": { diff --git a/src/coreComponents/physicsSolvers/solidMechanics/kernelSpecs.json b/src/coreComponents/physicsSolvers/solidMechanics/kernelSpecs.json index e5118286337..9781aa9b7c2 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/kernelSpecs.json +++ b/src/coreComponents/physicsSolvers/solidMechanics/kernelSpecs.json @@ -62,16 +62,7 @@ "H1_Pyramid_Lagrange1_Gauss5" ] }, - "explicit": [ "CellElementSubRegion#ElasticIsotropic#H1_Tetrahedron_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism5_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism6_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism7_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism8_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism9_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism10_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Hexahedron_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Wedge_VEM_Gauss1", - "CellElementSubRegion#ElasticIsotropic#H1_Prism11_VEM_Gauss1" ] + "explicit": [] }, "SolidMechanicsFixedStressThermoPoromechanicsKernels": { @@ -98,17 +89,7 @@ "H1_Hexahedron_Lagrange1_GaussLegendre2", "H1_Wedge_Lagrange1_Gauss6", "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Pyramid_Lagrange1_Gauss5", - "H1_Tetrahedron_VEM_Gauss1", - "H1_Prism5_VEM_Gauss1", - "H1_Prism6_VEM_Gauss1", - "H1_Prism7_VEM_Gauss1", - "H1_Prism8_VEM_Gauss1", - "H1_Prism9_VEM_Gauss1", - "H1_Prism10_VEM_Gauss1", - "H1_Hexahedron_VEM_Gauss1", - "H1_Wedge_VEM_Gauss1", - "H1_Prism11_VEM_Gauss1" + "H1_Pyramid_Lagrange1_Gauss5" ] }, "explicit": [] From c6687fa4ef5e1fe33353bbe0d04e221b00c8ff95 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 26 May 2025 11:42:14 -0400 Subject: [PATCH 30/67] Add physics-based scaling option --- .../linearAlgebra/interfaces/MatrixBase.hpp | 6 ++ .../linearAlgebra/interfaces/VectorBase.hpp | 16 +++-- .../interfaces/hypre/HypreMatrix.cpp | 63 +++++++++++++------ .../interfaces/hypre/HypreMatrix.hpp | 10 +++ .../interfaces/hypre/HypreSolver.cpp | 2 +- .../interfaces/hypre/HypreVector.cpp | 32 +++++----- .../interfaces/hypre/HypreVector.hpp | 11 +++- .../physicsSolvers/PhysicsSolverBase.cpp | 21 +++++++ .../physicsSolvers/PhysicsSolverBase.hpp | 9 +++ 9 files changed, 128 insertions(+), 42 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp b/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp index 302e220b213..ebdc0d096a9 100644 --- a/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp @@ -167,6 +167,12 @@ class MatrixBase : public virtual LinearOperator< VECTOR > return m_dofManager; } + /** + * @brief Compute left and right scaling vectors for diagonal scaling. + * @param scaling Vector to be filled with scaling factors. + */ + virtual void computeScalingVector(Vector & scaling) const = 0; + ///@} /** diff --git a/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp b/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp index 2efb0725fa0..a821cecd4cc 100644 --- a/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp @@ -242,12 +242,18 @@ class VectorBase real64 const beta ) = 0; /** - * @brief Compute the component-wise multiplication y = v * x. - * @param x first vector (input) - * @param y second vector (output) + * @brief Element-wise multiplication of this vector with another vector. + * @param x Vector to multiply with. + * @note This modifies the current vector in place: this[i] *= x[i] */ - virtual void pointwiseProduct( Vector const & x, - Vector & y ) const = 0; + virtual void pointwiseScale( Vector const & x ) = 0; + + /** + * @brief Element-wise division of this vector by another vector. + * @param x Vector to divide by. + * @note This modifies the current vector in place: this[i] /= x[i] + */ + virtual void pointwiseDivide( Vector const & x ) = 0; /** * @brief 1-norm of the vector. diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp index 5e68ad1887d..76329e865c4 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp @@ -803,8 +803,29 @@ void HypreMatrix::leftScale( HypreVector const & vec ) GEOS_LAI_ASSERT( ready() ); GEOS_LAI_ASSERT( vec.ready() ); GEOS_LAI_ASSERT_EQ( vec.localSize(), numLocalRows() ); - hypre::scaleMatrixRows( hypre_ParCSRMatrixDiag( m_parcsr_mat ), hypre_ParVectorLocalVector( vec.unwrapped() ) ); - hypre::scaleMatrixRows( hypre_ParCSRMatrixOffd( m_parcsr_mat ), hypre_ParVectorLocalVector( vec.unwrapped() ) ); + + GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixDiagScale( m_parcsr_mat, vec.unwrapped(), nullptr ) ); +} + +void HypreMatrix::rightScale( HypreVector const & vec ) +{ + GEOS_LAI_ASSERT( ready() ); + GEOS_LAI_ASSERT( vec.ready() ); + GEOS_LAI_ASSERT_EQ( vec.localSize(), numLocalRows() ); + + GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixDiagScale( m_parcsr_mat, nullptr, vec.unwrapped() ) ); +} + +void HypreMatrix::leftRightScale( HypreVector const & vecLeft, + HypreVector const & vecRight ) +{ + GEOS_LAI_ASSERT( ready() ); + GEOS_LAI_ASSERT( vecLeft.ready() ); + GEOS_LAI_ASSERT( vecRight.ready() ); + GEOS_LAI_ASSERT_EQ( vecLeft.localSize(), numLocalRows() ); + GEOS_LAI_ASSERT_EQ( vecRight.localSize(), numLocalRows() ); + + GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixDiagScale( m_parcsr_mat, vecLeft.unwrapped(), vecRight.unwrapped() ) ); } void HypreMatrix::rescaleRows( arrayView1d< globalIndex const > const & rowIndices, @@ -1425,22 +1446,6 @@ real64 HypreMatrix::normMax( arrayView1d< globalIndex const > const & rowIndices return MpiWrapper::max( maxNorm, comm() ); } -void HypreMatrix::rightScale( HypreVector const & vec ) -{ - GEOS_LAI_ASSERT( ready() ); - HypreMatrix t; - transpose( t ); - t.leftScale( vec ); - t.transpose( *this ); -} - -void HypreMatrix::leftRightScale( HypreVector const & vecLeft, - HypreVector const & vecRight ) -{ - leftScale( vecLeft ); - rightScale( vecRight ); -} - void HypreMatrix::transpose( HypreMatrix & dst ) const { GEOS_LAI_ASSERT( ready() ); @@ -1459,4 +1464,26 @@ MPI_Comm HypreMatrix::comm() const return hypre_IJMatrixComm( m_ij_mat ); } +void HypreMatrix::computeScalingVector(HypreVector & scaling) const +{ + GEOS_LAI_ASSERT( ready() ); + + // Get number of components + HYPRE_Int num_tags = LvArray::integerConversion< HYPRE_Int >( + (m_dofManager->*static_cast(&geos::DofManager::numComponents))() + ); + + // Get local dof component labels + array1d< HYPRE_Int > pointMarkers( numLocalRows() ); + m_dofManager->getLocalDofComponentLabels( pointMarkers ); + + // Create scaling vector + scaling.create( numLocalRows(), comm() ); + HYPRE_ParVector hypre_vec = scaling.unwrapped(); + HYPRE_ParVector *hypre_vec_ptr = &hypre_vec; + + // Compute scaling vector + GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixCompScalingTagged( m_parcsr_mat, 0, num_tags, pointMarkers.data(), hypre_vec_ptr ) ); +} + }// end namespace geos diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp index c3dc2055bcc..672fbae498b 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp @@ -21,6 +21,7 @@ #define GEOS_LINEARALGEBRA_INTERFACES_HYPREMATRIX_HPP_ #include "common/DataTypes.hpp" +#include "linearAlgebra/DofManager.hpp" #include "linearAlgebra/interfaces/hypre/HypreVector.hpp" #include "linearAlgebra/interfaces/hypre/HypreExport.hpp" #include "linearAlgebra/common/LinearOperator.hpp" @@ -269,6 +270,15 @@ class HypreMatrix final : public virtual LinearOperator< HypreVector >, virtual void rightScale( HypreVector const & vec ) override; + /** + * @copydoc MatrixBase::computeScalingVector + */ + virtual void computeScalingVector(HypreVector & scaling) const override; + + /** + * @copydoc MatrixBase::leftRightScale + */ + virtual void leftRightScale( HypreVector const & vecLeft, HypreVector const & vecRight ) override; diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp index 35263458231..59e227b7c2a 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp @@ -78,7 +78,7 @@ void createHypreGMRES( LinearSolverParameters const & params, GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRGMRESSetTol( solver.ptr, params.krylov.relTolerance ) ); // Default for now - HYPRE_Int logLevel = (params.logLevel >= 3) ? 2 : 0; + HYPRE_Int logLevel = (params.logLevel >= 3) ? 3 : 0; GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRGMRESSetPrintLevel( solver.ptr, logLevel ) ); // print iteration info GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRGMRESSetLogging( solver.ptr, 1 ) ); /* needed to get run info later */ diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp index 5b6f7b30790..38813c0c021 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp @@ -161,11 +161,9 @@ void HypreVector::scale( real64 const scalingFactor ) void HypreVector::reciprocal() { GEOS_LAI_ASSERT( ready() ); - arrayView1d< real64 > values = m_values.toView(); - forAll< hypre::execPolicy >( localSize(), [values] GEOS_HYPRE_DEVICE ( localIndex const i ) - { - values[i] = 1.0 / values[i]; - } ); + + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorElmInverse( m_vec, &m_vec ) ); + touch(); } real64 HypreVector::dot( HypreVector const & vec ) const @@ -225,22 +223,24 @@ void HypreVector::axpby( real64 const alpha, } } -void HypreVector::pointwiseProduct( HypreVector const & x, - HypreVector & y ) const +void HypreVector::pointwiseScale( HypreVector const & x ) { GEOS_LAI_ASSERT( ready() ); GEOS_LAI_ASSERT( x.ready() ); - GEOS_LAI_ASSERT( y.ready() ); GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); - GEOS_LAI_ASSERT_EQ( localSize(), y.localSize() ); - arrayView1d< real64 const > const my_values = m_values.toViewConst(); - arrayView1d< real64 const > const x_values = x.m_values.toViewConst(); - arrayView1d< real64 > const y_values = y.m_values.toView(); - forAll< hypre::execPolicy >( localSize(), [y_values, my_values, x_values] GEOS_HYPRE_DEVICE ( localIndex const i ) - { - y_values[i] = my_values[i] * x_values[i]; - } ); + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorElmProduct( x.m_vec, m_vec, &m_vec ) ); + touch(); +} + +void HypreVector::pointwiseDivide( HypreVector const & x ) +{ + GEOS_LAI_ASSERT( ready() ); + GEOS_LAI_ASSERT( x.ready() ); + GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); + + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorElmDivision( x.m_vec, m_vec, &m_vec ) ); + touch(); } real64 HypreVector::norm1() const diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp index 771c1cfe30a..66d6fe7d89a 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp @@ -140,8 +140,15 @@ class HypreVector final : private VectorBase< HypreVector > HypreVector const & x, real64 const beta ) override; - virtual void pointwiseProduct( HypreVector const & x, - HypreVector & y ) const override; + /** + * @copydoc VectorBase::pointwiseScale + */ + virtual void pointwiseScale( HypreVector const & x ) override; + + /** + * @copydoc VectorBase::pointwiseDivide + */ + virtual void pointwiseDivide( HypreVector const & x ) override; /** * @copydoc VectorBase::norm1 diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index 84eb8190bf0..4d28c420090 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -44,6 +44,7 @@ PhysicsSolverBase::PhysicsSolverBase( string const & name, m_nextDt( 1e99 ), m_numTimestepsSinceLastDtCut( -1 ), m_dofManager( name ), + m_usePhysicsScaling( 1 ), m_linearSolverParameters( groupKeyStruct::linearSolverParametersString(), this ), m_nonlinearSolverParameters( groupKeyStruct::nonlinearSolverParametersString(), this ), m_solverStatistics( groupKeyStruct::solverStatisticsString(), this ), @@ -104,6 +105,11 @@ PhysicsSolverBase::PhysicsSolverBase( string const & name, setRestartFlags( RestartFlags::WRITE_AND_READ ). setDescription( "Cut time step if linear solution fail without going until max nonlinear iterations." ); + registerWrapper( viewKeyStruct::usePhysicsScalingString(), &m_usePhysicsScaling ) + .setApplyDefaultValue( 1 ) + .setInputFlag( InputFlags::OPTIONAL ) + .setDescription( "Enable physics-based scaling of the linear system. Default: true." ); + addLogLevel< logInfo::Fields >(); addLogLevel< logInfo::LinearSolver >(); addLogLevel< logInfo::Solution >(); @@ -1313,6 +1319,15 @@ void PhysicsSolverBase::solveLinearSystem( DofManager const & dofManager, LinearSolverParameters const & params = m_linearSolverParameters.get(); matrix.setDofManager( &dofManager ); + // Apply physics-based scaling to the linear system if enabled + if( m_usePhysicsScaling ) + { + matrix.computeScalingVector( m_scaling ); + matrix.leftRightScale( m_scaling, m_scaling ); + rhs.pointwiseScale( m_scaling ); + // Assume the solution is zeroed out, thus no need to scale it + } + if( params.solverType == LinearSolverParameters::SolverType::direct || !m_precond ) { std::unique_ptr< LinearSolverBase< LAInterface > > solver = LAInterface::createSolver( params ); @@ -1353,6 +1368,12 @@ void PhysicsSolverBase::solveLinearSystem( DofManager const & dofManager, { GEOS_WARNING_IF( !m_linearSolverResult.success(), getDataContext() << ": Linear solution failed" ); } + + // Unscale the solution vector if physics-based scaling was applied + if( m_usePhysicsScaling ) + { + solution.pointwiseScale( m_scaling ); + } } bool PhysicsSolverBase::checkSystemSolution( DomainPartition & GEOS_UNUSED_PARAM( domain ), diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp index a0d6d7f26b3..98eaea0f468 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp @@ -696,6 +696,9 @@ class PhysicsSolverBase : public ExecutableGroup /// @return string for the writeLinearSystem wrapper static constexpr char const * writeLinearSystemString() { return "writeLinearSystem"; } + /// @return string for the usePhysicsScaling wrapper + static constexpr char const * usePhysicsScalingString() { return "usePhysicsScaling"; } + /// @return string for the allowNonConvergedLinearSolverSolution wrapper static constexpr char const * allowNonConvergedLinearSolverSolutionString() { return "allowNonConvergedLinearSolverSolution"; } }; @@ -1027,6 +1030,12 @@ class PhysicsSolverBase : public ExecutableGroup /// System solution vector ParallelVector m_solution; + /// Scaling vector (A^\tilde = D * A * D, b^\tilde = D * b, x = D * x^\tilde) + ParallelVector m_scaling; + + /// Flag to decide whether to apply physics-based scaling to the linear system + integer m_usePhysicsScaling; + /// Local system matrix and rhs CRSMatrix< real64, globalIndex > m_localMatrix; From af55bba2f359849050e885420aeca5e6e46f1a7b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 31 May 2025 11:39:21 -0700 Subject: [PATCH 31/67] Fix build with/without SuperLU and SuiteSparse --- src/coreComponents/common/GeosxConfig.hpp.in | 6 ++-- .../linearAlgebra/CMakeLists.txt | 2 +- .../interfaces/direct/SuiteSparse.cpp | 3 -- .../interfaces/direct/SuiteSparse.hpp | 2 -- .../interfaces/hypre/HypreInterface.cpp | 12 ++++---- .../interfaces/hypre/HypreMGR.hpp | 4 +-- .../interfaces/petsc/PetscInterface.cpp | 11 ++++++++ .../interfaces/trilinos/TrilinosInterface.cpp | 9 ++++++ src/coreComponents/mainInterface/version.cpp | 28 +++++++++---------- 9 files changed, 47 insertions(+), 30 deletions(-) diff --git a/src/coreComponents/common/GeosxConfig.hpp.in b/src/coreComponents/common/GeosxConfig.hpp.in index bc20648df6a..544429f2670 100644 --- a/src/coreComponents/common/GeosxConfig.hpp.in +++ b/src/coreComponents/common/GeosxConfig.hpp.in @@ -5,7 +5,7 @@ * Copyright (c) 2016-2024 Lawrence Livermore National Security LLC * Copyright (c) 2018-2024 TotalEnergies * Copyright (c) 2018-2024 The Board of Trustees of the Leland Stanford Junior University - * Copyright (c) 2023-2024 Chevron + * Copyright (c) 2023-2024 Chevron * Copyright (c) 2019- GEOS/GEOSX Contributors * All rights reserved * @@ -95,6 +95,9 @@ /// Enables use of SuperLU_dist library through HYPRE (CMake option ENABLE_SUPERLU_DIST) #cmakedefine GEOS_USE_SUPERLU_DIST +/// Enables use of SuiteSparse library (CMake option ENABLE_SUITESPARSE) +#cmakedefine GEOS_USE_SUITESPARSE + /// Enables use of PETSc library (CMake option ENABLE_PETSC) #cmakedefine GEOS_USE_PETSC @@ -200,4 +203,3 @@ #endif #endif /* GEOS_CONFIG_HPP */ - diff --git a/src/coreComponents/linearAlgebra/CMakeLists.txt b/src/coreComponents/linearAlgebra/CMakeLists.txt index c6458b84073..9650c973ac6 100644 --- a/src/coreComponents/linearAlgebra/CMakeLists.txt +++ b/src/coreComponents/linearAlgebra/CMakeLists.txt @@ -79,7 +79,7 @@ set( linearAlgebra_sources # Workaround for link issue with cce-18 if( NOT ${ENABLE_HIP} ) - list( APPEND linearAlgebra_sources solvers/GmresSolver.hpp ) + list( APPEND linearAlgebra_sources solvers/GmresSolver.cpp ) endif() set( dependencyList ${parallelDeps} mesh denseLinearAlgebra finiteVolume ) diff --git a/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.cpp b/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.cpp index 30cc3a9fc63..53a45e2ce7f 100644 --- a/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.cpp @@ -19,7 +19,6 @@ #include "SuiteSparse.hpp" -#if defined(suitesparse_VERSION) #include "codingUtilities/Utilities.hpp" #include "common/Stopwatch.hpp" #include "linearAlgebra/common/common.hpp" @@ -385,5 +384,3 @@ template class SuiteSparse< PetscInterface >; #endif } - -#endif /* if defined(suitesparse_VERSION) */ diff --git a/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.hpp b/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.hpp index 501b2befc40..77534242052 100644 --- a/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.hpp @@ -20,7 +20,6 @@ #ifndef GEOS_LINEARALGEBRA_INTERFACES_SUITESPARSE_HPP_ #define GEOS_LINEARALGEBRA_INTERFACES_SUITESPARSE_HPP_ -#if defined(suitesparse_VERSION) #include "common/DataTypes.hpp" #include "common/LinearSolverBase.hpp" #include "common/PreconditionerBase.hpp" @@ -143,5 +142,4 @@ class SuiteSparse final : public LinearSolverBase< LAI > } -#endif /* if defined(suitesparse_VERSION) */ #endif /*GEOS_LINEARALGEBRA_INTERFACES_SUITESPARSE_HPP_*/ diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp index 339262da651..7365861ba6d 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp @@ -19,16 +19,17 @@ #include "HypreInterface.hpp" +#if defined(GEOS_USE_SUITESPARSE) #include "linearAlgebra/interfaces/direct/SuiteSparse.hpp" +#endif +#if defined(GEOS_USE_SUPERLU_DIST) +#include "linearAlgebra/interfaces/direct/SuperLUDist.hpp" +#endif #include "linearAlgebra/interfaces/hypre/HypreMatrix.hpp" #include "linearAlgebra/interfaces/hypre/HyprePreconditioner.hpp" #include "linearAlgebra/interfaces/hypre/HypreSolver.hpp" #include "linearAlgebra/interfaces/hypre/HypreUtils.hpp" -#if defined(GEOS_USE_SUPERLU_DIST) -#include "linearAlgebra/interfaces/direct/SuperLUDist.hpp" -#endif - #include "HYPRE_utilities.h" #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP #include "_hypre_utilities.h" @@ -98,8 +99,7 @@ HypreInterface::createSolver( LinearSolverParameters params ) } else { - /* TODO: add GEOS_USE_SUITESPARSE and use it below */ -#if defined(suitesparse_VERSION) +#if defined(GEOS_USE_SUITESPARSE) return std::make_unique< SuiteSparse< HypreInterface > >( std::move( params ) ); #else GEOS_ERROR( "GEOS is configured without support for SuiteSparse." ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp index 2224ffbc41d..c33f8d211ce 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp @@ -180,7 +180,7 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxRowSum( solver.ptr, 1.0 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetStrongThreshold( solver.ptr, 0.6 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType( solver.ptr, hypre::getAMGInterpolationType( LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); + //GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType( solver.ptr, hypre::getAMGInterpolationType( LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, hypre::getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::chebyshev ) ) ); @@ -211,7 +211,7 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::multipass ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::modifiedExtendedE ) ) ); + //GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::modifiedExtendedE ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::l1jacobi ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumSweeps( solver.ptr, 2 ) ); diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscInterface.cpp index bb404c744eb..d5b7de097dc 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscInterface.cpp @@ -19,8 +19,12 @@ #include "PetscInterface.hpp" +#if defined(GEOS_USE_SUITESPARSE) #include "linearAlgebra/interfaces/direct/SuiteSparse.hpp" +#endif +#if defined(GEOS_USE_SUPERLU_DIST) #include "linearAlgebra/interfaces/direct/SuperLUDist.hpp" +#endif #include "linearAlgebra/interfaces/petsc/PetscPreconditioner.hpp" #include "linearAlgebra/interfaces/petsc/PetscSolver.hpp" @@ -47,13 +51,20 @@ PetscInterface::createSolver( LinearSolverParameters params ) { if( params.solverType == LinearSolverParameters::SolverType::direct ) { +#if defined(GEOS_USE_SUPERLU_DIST) if( params.direct.parallel ) { return std::make_unique< SuperLUDist< PetscInterface > >( std::move( params ) ); } else +#endif { +#if defined(GEOS_USE_SUITESPARSE) return std::make_unique< SuiteSparse< PetscInterface > >( std::move( params ) ); +#else + GEOS_ERROR( "GEOS is configured without support for SuiteSparse." ); + return std::unique_ptr< LinearSolverBase< TrilinosInterface > >( nullptr ); +#endif } } else diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp index eb1244bf687..4f7cf52afff 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp @@ -19,8 +19,12 @@ #include "TrilinosInterface.hpp" +#if defined(GEOS_USE_SUITESPARSE) #include "linearAlgebra/interfaces/direct/SuiteSparse.hpp" +#endif +#if defined(GEOS_USE_SUPERLU_DIST) #include "linearAlgebra/interfaces/direct/SuperLUDist.hpp" +#endif #include "linearAlgebra/interfaces/trilinos/TrilinosPreconditioner.hpp" #include "linearAlgebra/interfaces/trilinos/TrilinosSolver.hpp" @@ -46,7 +50,12 @@ TrilinosInterface::createSolver( LinearSolverParameters params ) else #endif { +#if defined(GEOS_USE_SUITESPARSE) return std::make_unique< SuiteSparse< TrilinosInterface > >( std::move( params ) ); +#else + GEOS_ERROR( "GEOS is configured without support for SuiteSparse." ); + return std::unique_ptr< LinearSolverBase< TrilinosInterface > >( nullptr ); +#endif } } else diff --git a/src/coreComponents/mainInterface/version.cpp b/src/coreComponents/mainInterface/version.cpp index 57470db31f3..7999ede8c56 100644 --- a/src/coreComponents/mainInterface/version.cpp +++ b/src/coreComponents/mainInterface/version.cpp @@ -106,55 +106,55 @@ void outputVersionInfo() GEOS_LOG_RANK_0( " - VTK version: " << STRINGIZE( VTK_VERSION ) ); #endif -#if defined(RAJA_VERSION) +#if defined(GEOS_USE_RAJA) && defined(RAJA_VERSION) GEOS_LOG_RANK_0( " - RAJA version: " << STRINGIZE( RAJA_VERSION ) ); #endif -#if defined(umpire_VERSION) +#if defined(GEOS_USE_UMPIRE) && defined(umpire_VERSION) GEOS_LOG_RANK_0( " - Umpire version: " << STRINGIZE( umpire_VERSION ) ); #endif -#if defined(chai_VERSION) +#if defined(GEOS_USE_CHAI) && defined(chai_VERSION) GEOS_LOG_RANK_0( " - CHAI version: " << STRINGIZE( chai_VERSION ) ); #endif -#if defined(adiak_VERSION) +#if defined(GEOS_USE_ADIAK) && defined(adiak_VERSION) GEOS_LOG_RANK_0( " - adiak version: " << STRINGIZE( adiak_VERSION ) ); #endif -#if defined(caliper_VERSION) +#if defined(GEOS_USE_CALIPER) && defined(caliper_VERSION) GEOS_LOG_RANK_0( " - Caliper version: " << STRINGIZE( caliper_VERSION ) ); #endif -#if defined(metis_VERSION) +#if defined(GEOS_USE_METIS) && defined(metis_VERSION) GEOS_LOG_RANK_0( " - METIS version: " << STRINGIZE( metis_VERSION ) ); #endif -#if defined(parmetis_VERSION) +#if defined(GEOS_USE_PARMETIS) && defined(parmetis_VERSION) GEOS_LOG_RANK_0( " - PARMETIS version: " << STRINGIZE( parmetis_VERSION ) ); #endif -#if defined(scotch_VERSION) +#if defined(GEOS_USE_SCOTCH) && defined(scotch_VERSION) GEOS_LOG_RANK_0( " - Scotch version: " << STRINGIZE( scotch_VERSION ) ); #endif -#if defined(superlu_dist_VERSION) +#if defined(GEOS_USE_SUPERLU_DIST) && defined(superlu_dist_VERSION) GEOS_LOG_RANK_0( " - SuperLU_Dist version: " << STRINGIZE( superlu_dist_VERSION ) ); #endif -#if defined(suitesparse_VERSION) +#if defined(GEOS_USE_SUITESPARSE) && defined(suitesparse_VERSION) GEOS_LOG_RANK_0( " - SuiteSparse version: " << STRINGIZE( suitesparse_VERSION ) ); #endif -#if defined(hypre_VERSION) +#if defined(GEOS_USE_HYPRE) && defined(hypre_VERSION) GEOS_LOG_RANK_0( " - HYPRE version: " << STRINGIZE( hypre_VERSION ) ); #endif -#if defined(trilinos_VERSION) +#if defined(GEOS_USE_TRILINOS) && defined(trilinos_VERSION) GEOS_LOG_RANK_0( " - trilinos version: " << STRINGIZE( trilinos_VERSION ) ); #endif -#if defined(petsc_VERSION) +#if defined(GEOS_USE_PETSC) && defined(petsc_VERSION) GEOS_LOG_RANK_0( " - PETSc version: " << STRINGIZE( petsc_VERSION ) ); #endif @@ -172,7 +172,7 @@ void outputVersionInfo() ( GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CPU ) GEOS_LOG_RANK_0( "" ); GEOS_LOG_RANK_0( "*******************************************************" ); - GEOS_LOG_RANK_0( "* WARNING!!! *" ); + GEOS_LOG_RANK_0( "* WARNING!! WARNING!! WARNING!! *" ); GEOS_LOG_RANK_0( "* *" ); GEOS_LOG_RANK_0( "* GEOS has GPU support enabled, but HYPRE does not! *" ); GEOS_LOG_RANK_0( "*******************************************************" ); From a4cf2eb8a3e36f2cc4c89e4351cc132d80129c25 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 31 May 2025 11:41:44 -0700 Subject: [PATCH 32/67] Add matrix host configs --- host-configs/LLNL/matrix-base.cmake | 65 +++++++++++++++++++ host-configs/LLNL/matrix-gcc-12-cuda-12.cmake | 19 ++++++ 2 files changed, 84 insertions(+) create mode 100644 host-configs/LLNL/matrix-base.cmake create mode 100644 host-configs/LLNL/matrix-gcc-12-cuda-12.cmake diff --git a/host-configs/LLNL/matrix-base.cmake b/host-configs/LLNL/matrix-base.cmake new file mode 100644 index 00000000000..bcf374512ac --- /dev/null +++ b/host-configs/LLNL/matrix-base.cmake @@ -0,0 +1,65 @@ +############################################################################### +# +# Base configuration for LC Matrix builds +# Calling configuration file must define the following CMAKE variables: +# +# MPI_HOME +# +############################################################################### + +set(GEOS_BUILD_OBJ_LIBS OFF CACHE BOOL "") + +# Fortran +set(ENABLE_FORTRAN OFF CACHE BOOL "") + +# MPI +set(ENABLE_MPI ON CACHE BOOL "") +set(MPI_C_COMPILER ${MPI_HOME}/bin/mpicc CACHE PATH "") +set(MPI_CXX_COMPILER ${MPI_HOME}/bin/mpicxx CACHE PATH "") +set(MPIEXEC srun CACHE STRING "") +set(MPIEXEC_NUMPROC_FLAG -n CACHE STRING "") +set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "") + +# OpenMP +set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE) + +# CUDA +# LvArray sets this to the CMAKE_CXX_COMPILER. +set(CMAKE_CUDA_HOST_COMPILER ${MPI_CXX_COMPILER} CACHE STRING "") +set(ENABLE_CUDA_NVTOOLSEXT OFF CACHE BOOL "") + +# ESSL +set(ENABLE_ESSL OFF CACHE BOOL "" FORCE ) + +# TPL +set(ENABLE_PAPI OFF CACHE BOOL "") + +# GEOS specific options +set(ENABLE_PVTPackage ON CACHE BOOL "") +set(ENABLE_PETSC OFF CACHE BOOL "" FORCE ) +set(ENABLE_TRILINOS OFF CACHE BOOL "" FORCE ) +set(ENABLE_HYPRE_DEVICE "CUDA" CACHE STRING "" FORCE) + +# Documentation +set(ENABLE_UNCRUSTIFY OFF CACHE BOOL "" FORCE) +set(ENABLE_DOXYGEN OFF CACHE BOOL "" FORCE) + +# Other +set(ENABLE_MATHPRESSO OFF CACHE BOOL "") + +# YAPF python formatting +#set(YAPF_EXECUTABLE /usr/gapps/GEOSX/thirdPartyLibs/python/lassen-gcc-python/python/bin/yapf CACHE PATH "" FORCE) + +# PYGEOSX +set(ENABLE_PYGEOSX ON CACHE BOOL "") +set(PYTHON_EXECUTABLE /usr/tce/packages/python/python-3.9.12/bin/python CACHE PATH "") +set(Python3_ROOT_DIR /usr/tce/packages/python/python-3.9.12 CACHE PATH "") +set(Python3_EXECUTABLE /usr/tce/packages/python/python-3.9.12/bin/python3 CACHE PATH "") + +# ATS +set(ATS_ARGUMENTS "--ats jsrun_omp --ats jsrun_bind=packed" CACHE STRING "") +# set(USER $ENV{USER} CACHE STRING "") +# set(ATS_WORKING_DIR "/p/gpfs1/${USER}/integratedTestsGEOS/${CONFIG_NAME}" CACHE PATH "") +# set(ATS_BASELINE_DIR "/p/gpfs1/${USER}/integratedTestsGEOS/baselines" CACHE PATH "") + +include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake) diff --git a/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake b/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake new file mode 100644 index 00000000000..83c837c13af --- /dev/null +++ b/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake @@ -0,0 +1,19 @@ +include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake) + +# C++ +# The "-march=native -mtune=native" which LvArray adds breaks the PVT package. +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" CACHE STRING "" FORCE) + +# Fortran +set(CMAKE_Fortran_COMPILER /usr/tce/packages/gcc/gcc-12.1.1/bin/gfortran CACHE PATH "") +set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "") +set(FORTRAN_MANGLE_NO_UNDERSCORE OFF CACHE BOOL "") + +# MPI +set(MPI_HOME /usr/tce/packages/openmpi/openmpi-4.1.2-gcc-12.1.1 CACHE PATH "") +set(MPI_Fortran_COMPILER ${MPI_HOME}/bin/mpifort CACHE PATH "") + +set(ENABLE_CUDA_NVTOOLSEXT OFF CACHE BOOL "") + +include(${CMAKE_CURRENT_LIST_DIR}/matrix-base.cmake) From 2bbb7cb9d1c1cd1c7d255895efddc8e5352fbeaa Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 31 May 2025 11:42:01 -0700 Subject: [PATCH 33/67] Update LvArray hash --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index d3106866b2b..404f1d79a01 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit d3106866b2b7785bb0b5d6d1cf1a88ebe5364b96 +Subproject commit 404f1d79a01f29d42a153223b9b23c06236c743f From dd00e7fe4fafb9fdb57772ef4fe6f8f37462525d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 31 May 2025 14:47:15 -0400 Subject: [PATCH 34/67] Testing MGR parameters --- .../linearAlgebra/interfaces/hypre/HypreMGR.hpp | 2 +- .../interfaces/hypre/mgrStrategies/Hydrofracture.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp index 2224ffbc41d..71ac3e62490 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp @@ -180,7 +180,7 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxRowSum( solver.ptr, 1.0 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetStrongThreshold( solver.ptr, 0.6 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType( solver.ptr, hypre::getAMGInterpolationType( LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); + //GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType( solver.ptr, hypre::getAMGInterpolationType( LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, hypre::getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::chebyshev ) ) ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp index 3555d7d824e..023ef212abc 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp @@ -64,7 +64,7 @@ class Hydrofracture : public MGRStrategyBase< 1 > m_levelFRelaxIters[0] = 1; m_levelInterpType[0] = MGRInterpolationType::jacobi; m_levelRestrictType[0] = MGRRestrictionType::injection; - m_levelCoarseGridMethod[0] = MGRCoarseGridMethod::galerkin; + m_levelCoarseGridMethod[0] = MGRCoarseGridMethod::nonGalerkin; m_levelGlobalSmootherType[0] = MGRGlobalSmootherType::none; } @@ -85,7 +85,7 @@ class Hydrofracture : public MGRStrategyBase< 1 > // Configure the BoomerAMG solver used as mgr coarse solver for the pressure reduced system setPressureAMG( mgrData.coarseSolver ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMinCoarseSize( mgrData.coarseSolver.ptr, 1000 ) ); + //GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMinCoarseSize( mgrData.coarseSolver.ptr, 1000 ) ); } }; From d1f3aa818330be1dcaf1a20a353f5d328d9a4f96 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sun, 1 Jun 2025 23:42:09 -0400 Subject: [PATCH 35/67] Add kernel build options --- src/CMakeLists.txt | 8 +- src/cmake/GeosxOptions.cmake | 12 ++ src/coreComponents/CMakeLists.txt | 5 +- .../linearAlgebra/CMakeLists.txt | 12 +- .../physicsSolvers/fluidFlow/CMakeLists.txt | 132 ++++++++++++------ ...lSpecs.json => HybridFVM_KernelSpecs.json} | 103 -------------- .../fluidFlow/IsothermalFVM_KernelSpecs.json | 57 ++++++++ .../fluidFlow/Reactive_KernelSpecs.json | 67 +++++++++ .../fluidFlow/ThermalFVM_KernelSpecs.json | 57 ++++++++ .../multiphysics/CMakeLists.txt | 93 +++++++++--- .../PoromechanicsDamage_KernelSpecs.json | 55 ++++++++ .../PoromechanicsEFEM_KernelSpecs.json | 53 +++++++ ...cs.json => Poromechanics_KernelSpecs.json} | 88 ------------ .../ThermoPoromechanics_KernelSpecs.json | 58 ++++++++ .../solidMechanics/CMakeLists.txt | 108 +++++++++++--- .../SolidMechanicsElastic_KernelSpecs.json | 57 ++++++++ ...n => SolidMechanicsExtra_KernelSpecs.json} | 62 +------- ...edStressThermoPoroElastic_KernelSpecs.json | 52 +++++++ ...StressThermoPoromechanics_KernelSpecs.json | 56 ++++++++ 19 files changed, 791 insertions(+), 344 deletions(-) rename src/coreComponents/physicsSolvers/fluidFlow/{kernelSpecs.json => HybridFVM_KernelSpecs.json} (58%) create mode 100644 src/coreComponents/physicsSolvers/fluidFlow/IsothermalFVM_KernelSpecs.json create mode 100644 src/coreComponents/physicsSolvers/fluidFlow/Reactive_KernelSpecs.json create mode 100644 src/coreComponents/physicsSolvers/fluidFlow/ThermalFVM_KernelSpecs.json create mode 100644 src/coreComponents/physicsSolvers/multiphysics/PoromechanicsDamage_KernelSpecs.json create mode 100644 src/coreComponents/physicsSolvers/multiphysics/PoromechanicsEFEM_KernelSpecs.json rename src/coreComponents/physicsSolvers/multiphysics/{kernelSpecs.json => Poromechanics_KernelSpecs.json} (50%) create mode 100644 src/coreComponents/physicsSolvers/multiphysics/ThermoPoromechanics_KernelSpecs.json create mode 100644 src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsElastic_KernelSpecs.json rename src/coreComponents/physicsSolvers/solidMechanics/{kernelSpecs.json => SolidMechanicsExtra_KernelSpecs.json} (57%) create mode 100644 src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoroElastic_KernelSpecs.json create mode 100644 src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoromechanics_KernelSpecs.json diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d9c175fec20..7bab940e7ab 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,6 +6,7 @@ cmake_policy(SET CMP0104 OLD) # when using nvcc populate CMAKE_CUDA_ARCHITECTURE cmake_policy(SET CMP0074 NEW) # dont ignore _ROOT env vars when searching for packages via find_package() cmake_policy(SET CMP0066 NEW) # use CMAKE__FLAGS_ for try_compile() instead of only CMAKE__FLAGS cmake_policy(SET CMP0056 NEW) # use CMAKE_EXE_LINKER_FLAGS in try_compile() in addition to CMAKE__FLAGS +cmake_policy(SET CMP0144 NEW) # find_package uses upper-case _ROOT variables. ################################ # GEOSX @@ -85,7 +86,6 @@ add_subdirectory( coreComponents ) add_subdirectory( externalComponents ) add_subdirectory( main ) - ################################ # Add docs ################################ @@ -104,7 +104,7 @@ install( DIRECTORY ${PROJECT_BINARY_DIR}/include DESTINATION . ) # Generate version information ################################ include( cmake/GeosxVersion.cmake ) -message( STATUS "Configuring GEOSX version ${GEOS_VERSION_FULL}" ) +message( STATUS "Configuring GEOS version ${GEOS_VERSION_FULL}" ) ################################ @@ -135,7 +135,7 @@ endif() blt_add_executable( NAME geosx SOURCES main/main.cpp - DEPENDS_ON geosx_core + DEPENDS_ON geosx_core ${extraComponentsLinkList} ${externalComponentsLinkList} ) @@ -309,7 +309,7 @@ if ( ENABLE_ATS ) if (NOT Python3_EXECUTABLE) message( FATAL_ERROR "An appropriate version of python was not found (required for integrated testing system). Try setting Python3_ROOT_DIR and/or Python3_EXECUTABLE in your host config." ) - endif() + endif() # Setup testing set( ATS_SCRIPT diff --git a/src/cmake/GeosxOptions.cmake b/src/cmake/GeosxOptions.cmake index c9072b081a4..21c7bf38767 100644 --- a/src/cmake/GeosxOptions.cmake +++ b/src/cmake/GeosxOptions.cmake @@ -120,10 +120,22 @@ endif() # Physics packages option( GEOS_ENABLE_CONTACT "Enables contact physics package" ON ) option( GEOS_ENABLE_FLUIDFLOW "Enables fluid flow physics package" ON ) +option( GEOS_ENABLE_FLUIDFLOW_ISOTHERMAL_FVM "Enables isothermal FVM kernels in fluid flow physics package" ON ) +option( GEOS_ENABLE_FLUIDFLOW_THERMAL_FVM "Enables thermal FVM kernels in fluid flow physics package" ON ) +option( GEOS_ENABLE_FLUIDFLOW_REACTIVE "Enables reactive kernels in fluid flow physics package" ON ) +option( GEOS_ENABLE_FLUIDFLOW_HYBRIDFVM "Enables Hybrid FVM kernels in fluid flow physics package" ON ) option( GEOS_ENABLE_INDUCEDSEISMICITY "Enables induced seismicity physics package" ON ) option( GEOS_ENABLE_MULTIPHYSICS "Enables multiphysics physics package" ON ) +option( GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS "Enables poromechanics kernels in multiphysics package" ON ) +option( GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS_EFEM "Enables poromechanics EFEM kernels in multiphysics package" ON ) +option( GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS_DAMAGE "Enables poromechanics damage kernels in multiphysics package" ON ) +option( GEOS_ENABLE_MULTIPHYSICS_THERMOPOROMECHANICS "Enables thermoporomechanics kernels in multiphysics package" ON ) option( GEOS_ENABLE_SIMPLEPDE "Enables simple PDE physics package" ON ) option( GEOS_ENABLE_SOLIDMECHANICS "Enables solid mechanics physics package" ON ) +option( GEOS_ENABLE_SOLIDMECHANICS_ELASTIC "Enables elastic kernels in solid mechanics physics package" ON ) +option( GEOS_ENABLE_SOLIDMECHANICS_EXTRA "Enables extra kernels in solid mechanics physics package" ON ) +option( GEOS_ENABLE_SOLIDMECHANICS_THERMOPOROELASTIC "Enables thermoporoelastic kernels in solid mechanics physics package" OFF ) +option( GEOS_ENABLE_SOLIDMECHANICS_THERMOPOROMECHANICS "Enables thermoporomechanics kernels in solid mechanics physics package" ON ) option( GEOS_ENABLE_SURFACEGENERATION "Enables surface generation physics package" ON ) option( GEOS_ENABLE_WAVEPROPAGATION "Enables wave propagation physics package" ON ) diff --git a/src/coreComponents/CMakeLists.txt b/src/coreComponents/CMakeLists.txt index 7cf2532422d..4e3c2242478 100644 --- a/src/coreComponents/CMakeLists.txt +++ b/src/coreComponents/CMakeLists.txt @@ -1,3 +1,5 @@ +set(TOTAL_GENERATED_KERNELS 0 CACHE INTERNAL "Total number of generated kernels") + set( subdirs math common @@ -83,5 +85,6 @@ endif() if( GEOS_ENABLE_TESTS ) add_subdirectory( unitTests ) -endif( ) +endif() +message( STATUS "Total number of generated kernels: ${TOTAL_GENERATED_KERNELS}" ) \ No newline at end of file diff --git a/src/coreComponents/linearAlgebra/CMakeLists.txt b/src/coreComponents/linearAlgebra/CMakeLists.txt index 9650c973ac6..49ac36ab0be 100644 --- a/src/coreComponents/linearAlgebra/CMakeLists.txt +++ b/src/coreComponents/linearAlgebra/CMakeLists.txt @@ -60,7 +60,11 @@ set( linearAlgebra_headers utilities/ReverseCutHillMcKeeOrdering.hpp utilities/TransposeOperator.hpp ) -# Workaround for link issue with cce-18 +# TODO: workaround for issue with undefined hidden symbols +# LvArray::internal::toChaiExecutionSpace(camp::resources::v1::Platform) +# LvArray::internal::getArrayManager() +# chai::ArrayManager::move(void*, chai::PointerRecord*, chai::ExecutionSpace) +# geos::HypreVector::~HypreVector() if( NOT ${ENABLE_HIP} ) list( APPEND linearAlgebra_headers solvers/GmresSolver.hpp ) endif() @@ -77,7 +81,11 @@ set( linearAlgebra_sources solvers/SeparateComponentPreconditioner.cpp utilities/ReverseCutHillMcKeeOrdering.cpp ) -# Workaround for link issue with cce-18 +# TODO: workaround for issue with undefined hidden symbols +# LvArray::internal::toChaiExecutionSpace(camp::resources::v1::Platform) +# LvArray::internal::getArrayManager() +# chai::ArrayManager::move(void*, chai::PointerRecord*, chai::ExecutionSpace) +# geos::HypreVector::~HypreVector() if( NOT ${ENABLE_HIP} ) list( APPEND linearAlgebra_sources solvers/GmresSolver.cpp ) endif() diff --git a/src/coreComponents/physicsSolvers/fluidFlow/CMakeLists.txt b/src/coreComponents/physicsSolvers/fluidFlow/CMakeLists.txt index f0ce8147004..db3d9845636 100644 --- a/src/coreComponents/physicsSolvers/fluidFlow/CMakeLists.txt +++ b/src/coreComponents/physicsSolvers/fluidFlow/CMakeLists.txt @@ -20,6 +20,7 @@ Contains: set( fluidFlowSolvers_headers FlowSolverBase.hpp FlowSolverBaseFields.hpp + SinglePhaseHybridFVM.hpp CompositionalMultiphaseBase.hpp CompositionalMultiphaseBaseFields.hpp CompositionalMultiphaseStatistics.hpp @@ -36,14 +37,12 @@ set( fluidFlowSolvers_headers SinglePhaseBaseFields.hpp SinglePhaseStatistics.hpp SinglePhaseFVM.hpp - SinglePhaseHybridFVM.hpp SinglePhaseProppantBase.hpp StencilAccessors.hpp StencilDataCollection.hpp LogLevelsInfo.hpp kernels/MinPoreVolumeMaxPorosityKernel.hpp kernels/StencilWeightsUpdateKernel.hpp - kernels/HybridFVMHelperKernels.hpp kernels/singlePhase/AccumulationKernels.hpp kernels/singlePhase/AquiferBCKernel.hpp kernels/singlePhase/DirichletFluxComputeKernel.hpp @@ -54,7 +53,6 @@ set( fluidFlowSolvers_headers kernels/singlePhase/HydrostaticPressureKernel.hpp kernels/singlePhase/MobilityKernel.hpp kernels/singlePhase/ResidualNormKernel.hpp - kernels/singlePhase/SinglePhaseHybridFVMKernels.hpp kernels/singlePhase/SolidInternalEnergyUpdateKernel.hpp kernels/singlePhase/SolutionCheckKernel.hpp kernels/singlePhase/SolutionScalingKernel.hpp @@ -71,7 +69,6 @@ set( fluidFlowSolvers_headers kernels/compositional/C1PPUPhaseFlux.hpp kernels/compositional/CapillaryPressureUpdateKernel.hpp kernels/compositional/CFLKernel.hpp - kernels/compositional/CompositionalMultiphaseHybridFVMKernels.hpp kernels/compositional/DiffusionDispersionFluxComputeKernel.hpp kernels/compositional/DirichletFluxComputeKernel.hpp kernels/compositional/DissipationFluxComputeKernel.hpp @@ -126,31 +123,30 @@ set( fluidFlowSolvers_headers # Specify solver sources set( fluidFlowSolvers_sources - CompositionalMultiphaseBase.cpp - CompositionalMultiphaseFVM.cpp - CompositionalMultiphaseStatistics.cpp - CompositionalMultiphaseHybridFVM.cpp - ImmiscibleMultiphaseFlow.cpp - ReactiveCompositionalMultiphaseOBL.cpp FlowSolverBase.cpp SinglePhaseBase.cpp SinglePhaseStatistics.cpp SinglePhaseFVM.cpp SinglePhaseHybridFVM.cpp SinglePhaseProppantBase.cpp + CompositionalMultiphaseBase.cpp + CompositionalMultiphaseFVM.cpp + CompositionalMultiphaseHybridFVM.cpp + CompositionalMultiphaseStatistics.cpp + ImmiscibleMultiphaseFlow.cpp + ReactiveCompositionalMultiphaseOBL.cpp SourceFluxStatistics.cpp StencilDataCollection.cpp kernels/singlePhase/proppant/ProppantFluxKernels.cpp kernels/compositional/AquiferBCKernel.cpp kernels/compositional/CFLKernel.cpp kernels/compositional/FluxComputeKernelBase.cpp - kernels/compositional/CompositionalMultiphaseHybridFVMKernels.cpp wells/CompositionalMultiphaseWell.cpp wells/kernels/CompositionalMultiphaseWellKernels.cpp wells/SinglePhaseWell.cpp wells/kernels/SinglePhaseWellKernels.cpp wells/WellControls.cpp - wells/WellSolverBase.cpp + wells/WellSolverBase.cpp proppantTransport/ProppantTransport.cpp proppantTransport/ProppantTransportKernels.cpp ) @@ -158,54 +154,100 @@ set( dependencyList ${parallelDeps} physicsSolversBase ) geos_decorate_link_dependencies( LIST decoratedDependencies DEPENDENCIES ${dependencyList} ) +# Add isothermal FVM fluid flow kernels +if( GEOS_ENABLE_FLUIDFLOW_ISOTHERMAL_FVM ) + file( READ "${CMAKE_CURRENT_LIST_DIR}/IsothermalFVM_KernelSpecs.json" kernelSpecs ) + # TODO: The kernel below has non-matching file names and JSON keys. + # Either fix them to follow pattern, or come up with another mechanism. + generateKernels( TEMPLATE "CompositionalMultiphaseFVMKernel_isothermal.cpp.template" + JSON kernelSpecs + KEY "IsothermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels" + HEADERS headerFiles + SOURCES sourceFiles ) + list(APPEND fluidFlowSolvers_headers ${headerFiles}) + list(APPEND fluidFlowSolvers_sources ${sourceFiles}) + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") +endif() -file( READ "${CMAKE_CURRENT_LIST_DIR}/kernelSpecs.json" kernelSpecs ) -set( kernelTemplateFileList "" ) - -list( APPEND kernelTemplateFileList - SinglePhaseHybridFVMKernels.cpp.template - CompositionalMultiphaseHybridFVMKernels_upwinding.cpp.template - CompositionalMultiphaseHybridFVMKernels_assembly.cpp.template - CompositionalMultiphaseHybridFVMKernels_flux.cpp.template - ReactiveCompositionalMultiphaseOBLKernels.cpp.template ) - +# Add thermal FVM fluid flow kernels +if( GEOS_ENABLE_FLUIDFLOW_THERMAL_FVM ) + file( READ "${CMAKE_CURRENT_LIST_DIR}/ThermalFVM_KernelSpecs.json" kernelSpecs ) -foreach( kernelTemplateFile ${kernelTemplateFileList} ) - get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) - generateKernels( TEMPLATE ${kernelTemplateFile} + # TODO: The kernel below has non-matching file names and JSON keys. + # Either fix them to follow pattern, or come up with another mechanism. + generateKernels( TEMPLATE "CompositionalMultiphaseFVMKernel_thermal.cpp.template" JSON kernelSpecs - KEY ${jsonKey} + KEY "ThermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels" HEADERS headerFiles SOURCES sourceFiles ) + list(APPEND fluidFlowSolvers_headers ${headerFiles}) + list(APPEND fluidFlowSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") +endif() + +# Add reactive fluid flow kernels +if( GEOS_ENABLE_FLUIDFLOW_REACTIVE ) + file( READ "${CMAKE_CURRENT_LIST_DIR}/Reactive_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList + ReactiveCompositionalMultiphaseOBLKernels.cpp.template ) -list(APPEND fluidFlowSolvers_headers ${headerFiles}) -list(APPEND fluidFlowSolvers_sources ${sourceFiles}) -endforeach() + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) -# TODO: The two kernels below have non-matching file names and JSON keys. -# Either fix them to follow pattern, or come up with another mechanism. + list(APPEND fluidFlowSolvers_headers ${headerFiles}) + list(APPEND fluidFlowSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() -generateKernels( TEMPLATE "CompositionalMultiphaseFVMKernel_isothermal.cpp.template" - JSON kernelSpecs - KEY "IsothermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels" - HEADERS headerFiles - SOURCES sourceFiles ) -list(APPEND fluidFlowSolvers_headers ${headerFiles}) -list(APPEND fluidFlowSolvers_sources ${sourceFiles}) +# Add Hybrid FVM fluid flow code +if( GEOS_ENABLE_FLUIDFLOW_HYBRIDFVM ) + list( APPEND fluidFlowSolvers_headers + CompositionalMultiphaseHybridFVM.hpp + kernels/HybridFVMHelperKernels.hpp + kernels/singlePhase/SinglePhaseHybridFVMKernels.hpp + kernels/compositional/CompositionalMultiphaseHybridFVMKernels.hpp ) + list( APPEND fluidFlowSolvers_sources + SinglePhaseHybridFVM.cpp + CompositionalMultiphaseHybridFVM.cpp + kernels/compositional/CompositionalMultiphaseHybridFVMKernels.cpp ) -generateKernels( TEMPLATE "CompositionalMultiphaseFVMKernel_thermal.cpp.template" - JSON kernelSpecs - KEY "ThermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels" - HEADERS headerFiles - SOURCES sourceFiles ) -list(APPEND fluidFlowSolvers_headers ${headerFiles}) -list(APPEND fluidFlowSolvers_sources ${sourceFiles}) + file( READ "${CMAKE_CURRENT_LIST_DIR}/HybridFVM_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList + SinglePhaseHybridFVMKernels.cpp.template + CompositionalMultiphaseHybridFVMKernels_upwinding.cpp.template + CompositionalMultiphaseHybridFVMKernels_assembly.cpp.template + CompositionalMultiphaseHybridFVMKernels_flux.cpp.template ) + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + list(APPEND fluidFlowSolvers_headers ${headerFiles}) + list(APPEND fluidFlowSolvers_sources ${sourceFiles}) + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() +set(TOTAL_GENERATED_KERNELS ${TOTAL_GENERATED_KERNELS} CACHE INTERNAL "Total number of generated kernels") blt_add_library( NAME fluidFlowSolvers SOURCES ${fluidFlowSolvers_sources} diff --git a/src/coreComponents/physicsSolvers/fluidFlow/kernelSpecs.json b/src/coreComponents/physicsSolvers/fluidFlow/HybridFVM_KernelSpecs.json similarity index 58% rename from src/coreComponents/physicsSolvers/fluidFlow/kernelSpecs.json rename to src/coreComponents/physicsSolvers/fluidFlow/HybridFVM_KernelSpecs.json index e2c3ed6173c..a0cf8192228 100644 --- a/src/coreComponents/physicsSolvers/fluidFlow/kernelSpecs.json +++ b/src/coreComponents/physicsSolvers/fluidFlow/HybridFVM_KernelSpecs.json @@ -141,108 +141,5 @@ ] }, "explicit": [] - }, - - "ReactiveCompositionalMultiphaseOBLKernels": { - "vars": [ - "NAME", - "NDIM", - "NCOMP", - "ENABLE_ENERGY", - "STENCIL" - ], - "constants": [ - [ "FluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "NAME": [ - "FluxComputeKernel" - ], - "NDIM": [ - 1, - 2, - 3 - ], - "NCOMP": [ - 1, - 2, - 3, - 4, - 5 - ], - "ENABLE_ENERGY": [ - "true", - "false" - ], - "STENCIL": [ - "CellElementStencilTPFA", - "SurfaceElementStencil", - "EmbeddedSurfaceToCellStencil", - "FaceElementToCellStencil" - ] - }, - "explicit": [] - }, - - "IsothermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels": { - "vars": [ - "NAME", - "NCOMP_NDOF", - "KERNEL_TYPE" - ], - "constants": [ - [ "DirichletFluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "NAME": [ - "DirichletFluxComputeKernel" - ], - "NCOMP_NDOF": [ - "1, 2", - "2, 3", - "3, 4", - "4, 5", - "5, 6" - ], - "KERNEL_TYPE": [ - "constitutive::DeadOilFluid", - "constitutive::BlackOilFluid", - "constitutive::CO2BrinePhillipsFluid", - "constitutive::CO2BrineEzrokhiFluid", - "constitutive::CO2BrinePhillipsThermalFluid" - ] - }, - "explicit": [] - }, - - "ThermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels": { - "vars": [ - "NAME", - "NCOMP_NDOF", - "KERNEL_TYPE" - ], - "constants": [ - [ "DirichletFluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "NAME": [ - "DirichletFluxComputeKernel" - ], - "NCOMP_NDOF": [ - "1, 2", - "2, 3", - "3, 4", - "4, 5", - "5, 6" - ], - "KERNEL_TYPE": [ - "constitutive::DeadOilFluid", - "constitutive::BlackOilFluid", - "constitutive::CO2BrinePhillipsFluid", - "constitutive::CO2BrineEzrokhiFluid", - "constitutive::CO2BrinePhillipsThermalFluid" - ] - }, - "explicit": [] } } diff --git a/src/coreComponents/physicsSolvers/fluidFlow/IsothermalFVM_KernelSpecs.json b/src/coreComponents/physicsSolvers/fluidFlow/IsothermalFVM_KernelSpecs.json new file mode 100644 index 00000000000..0ba359c5b45 --- /dev/null +++ b/src/coreComponents/physicsSolvers/fluidFlow/IsothermalFVM_KernelSpecs.json @@ -0,0 +1,57 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "IsothermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels": { + "vars": [ + "NAME", + "NCOMP_NDOF", + "KERNEL_TYPE" + ], + "constants": [ + [ "DirichletFluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "NAME": [ + "DirichletFluxComputeKernel" + ], + "NCOMP_NDOF": [ + "1, 2", + "2, 3", + "3, 4", + "4, 5", + "5, 6" + ], + "KERNEL_TYPE": [ + "constitutive::DeadOilFluid", + "constitutive::BlackOilFluid", + "constitutive::CO2BrinePhillipsFluid", + "constitutive::CO2BrineEzrokhiFluid", + "constitutive::CO2BrinePhillipsThermalFluid" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/fluidFlow/Reactive_KernelSpecs.json b/src/coreComponents/physicsSolvers/fluidFlow/Reactive_KernelSpecs.json new file mode 100644 index 00000000000..1cfcc94784f --- /dev/null +++ b/src/coreComponents/physicsSolvers/fluidFlow/Reactive_KernelSpecs.json @@ -0,0 +1,67 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "ReactiveCompositionalMultiphaseOBLKernels": { + "vars": [ + "NAME", + "NDIM", + "NCOMP", + "ENABLE_ENERGY", + "STENCIL" + ], + "constants": [ + [ "FluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "NAME": [ + "FluxComputeKernel" + ], + "NDIM": [ + 1, + 2, + 3 + ], + "NCOMP": [ + 1, + 2, + 3, + 4, + 5 + ], + "ENABLE_ENERGY": [ + "true", + "false" + ], + "STENCIL": [ + "CellElementStencilTPFA", + "SurfaceElementStencil", + "EmbeddedSurfaceToCellStencil", + "FaceElementToCellStencil" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/fluidFlow/ThermalFVM_KernelSpecs.json b/src/coreComponents/physicsSolvers/fluidFlow/ThermalFVM_KernelSpecs.json new file mode 100644 index 00000000000..c311085c9ab --- /dev/null +++ b/src/coreComponents/physicsSolvers/fluidFlow/ThermalFVM_KernelSpecs.json @@ -0,0 +1,57 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "ThermalCompositionalMultiphaseFVM_DirichletFluxComputeKernels": { + "vars": [ + "NAME", + "NCOMP_NDOF", + "KERNEL_TYPE" + ], + "constants": [ + [ "DirichletFluxComputeKernelPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "NAME": [ + "DirichletFluxComputeKernel" + ], + "NCOMP_NDOF": [ + "1, 2", + "2, 3", + "3, 4", + "4, 5", + "5, 6" + ], + "KERNEL_TYPE": [ + "constitutive::DeadOilFluid", + "constitutive::BlackOilFluid", + "constitutive::CO2BrinePhillipsFluid", + "constitutive::CO2BrineEzrokhiFluid", + "constitutive::CO2BrinePhillipsThermalFluid" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt b/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt index be6090a67ed..2f7878ee036 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt +++ b/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt @@ -83,28 +83,87 @@ set( dependencyList ${parallelDeps} fluidFlowSolvers simplePDESolvers solidMecha geos_decorate_link_dependencies( LIST decoratedDependencies DEPENDENCIES ${dependencyList} ) +if (GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS) + file( READ "${CMAKE_CURRENT_LIST_DIR}/Poromechanics_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList poromechanicsKernels/PoromechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND multiPhysicsSolvers_headers ${headerFiles}) + list(APPEND multiPhysicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() + +if (GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS_EFEM) + file( READ "${CMAKE_CURRENT_LIST_DIR}/PoromechanicsEFEM_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList poromechanicsKernels/PoromechanicsEFEMKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND multiPhysicsSolvers_headers ${headerFiles}) + list(APPEND multiPhysicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() -file( READ "${CMAKE_CURRENT_LIST_DIR}/kernelSpecs.json" kernelSpecs ) -set( kernelTemplateFileList "" ) +if (GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS_DAMAGE) + file( READ "${CMAKE_CURRENT_LIST_DIR}/PoromechanicsDamage_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList poromechanicsKernels/PoromechanicsDamageKernels.cpp.template ) - list( APPEND kernelTemplateFileList - poromechanicsKernels/PoromechanicsKernels.cpp.template - poromechanicsKernels/PoromechanicsEFEMKernels.cpp.template - poromechanicsKernels/ThermoPoromechanicsKernels.cpp.template - poromechanicsKernels/PoromechanicsDamageKernels.cpp.template ) + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + list(APPEND multiPhysicsSolvers_headers ${headerFiles}) + list(APPEND multiPhysicsSolvers_sources ${sourceFiles}) -foreach( kernelTemplateFile ${kernelTemplateFileList} ) - get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) - generateKernels( TEMPLATE ${kernelTemplateFile} - JSON kernelSpecs - KEY ${jsonKey} - HEADERS headerFiles - SOURCES sourceFiles ) + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() + +if (GEOS_ENABLE_MULTIPHYSICS_THERMOPOROMECHANICS) + file( READ "${CMAKE_CURRENT_LIST_DIR}/ThermoPoromechanics_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList poromechanicsKernels/ThermoPoromechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND multiPhysicsSolvers_headers ${headerFiles}) + list(APPEND multiPhysicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() -list(APPEND multiPhysicsSolvers_headers ${headerFiles}) -list(APPEND multiPhysicsSolvers_sources ${sourceFiles}) -endforeach() +set(TOTAL_GENERATED_KERNELS ${TOTAL_GENERATED_KERNELS} CACHE INTERNAL "Total number of generated kernels") blt_add_library( NAME multiPhysicsSolvers SOURCES ${multiPhysicsSolvers_sources} diff --git a/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsDamage_KernelSpecs.json b/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsDamage_KernelSpecs.json new file mode 100644 index 00000000000..4201abb6b07 --- /dev/null +++ b/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsDamage_KernelSpecs.json @@ -0,0 +1,55 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "PoromechanicsDamageKernels": { + "vars": [ + "SUBREGION_TYPE", + "CONSTITUTIVE_TYPE", + "FE_TYPE" + ], + "constants": [ + [ "SinglePhasePoromechanicsDamagePolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "PorousDamageSolid>", + "PorousDamageSolid>", + "PorousDamageSolid>" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Tetrahedron_Lagrange1_Gauss14", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsEFEM_KernelSpecs.json b/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsEFEM_KernelSpecs.json new file mode 100644 index 00000000000..8fe3b139afd --- /dev/null +++ b/src/coreComponents/physicsSolvers/multiphysics/PoromechanicsEFEM_KernelSpecs.json @@ -0,0 +1,53 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "PoromechanicsEFEMKernels": { + "vars": [ + "SUBREGION_TYPE", + "CONSTITUTIVE_TYPE", + "FE_TYPE" + ], + "constants": [ + [ "SinglePhasePoromechanicsEFEMPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "PorousSolid" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Tetrahedron_Lagrange1_Gauss14", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/multiphysics/kernelSpecs.json b/src/coreComponents/physicsSolvers/multiphysics/Poromechanics_KernelSpecs.json similarity index 50% rename from src/coreComponents/physicsSolvers/multiphysics/kernelSpecs.json rename to src/coreComponents/physicsSolvers/multiphysics/Poromechanics_KernelSpecs.json index 875f6ff442f..a28953231ca 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/kernelSpecs.json +++ b/src/coreComponents/physicsSolvers/multiphysics/Poromechanics_KernelSpecs.json @@ -63,93 +63,5 @@ ] }, "explicit": [] - }, - - "PoromechanicsEFEMKernels": { - "vars": [ - "SUBREGION_TYPE", - "CONSTITUTIVE_TYPE", - "FE_TYPE" - ], - "constants": [ - [ "SinglePhasePoromechanicsEFEMPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "PorousSolid" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Tetrahedron_Lagrange1_Gauss14", - "H1_Pyramid_Lagrange1_Gauss5" - ] - }, - "explicit": [] - }, - - "ThermoPoromechanicsKernels": { - "vars": [ - "SUBREGION_TYPE", - "CONSTITUTIVE_TYPE", - "FE_TYPE" - ], - "constants": [ - [ "ThermalMultiphasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ThermalSinglePhasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ThermalSinglePhasePoromechanicsEFEMPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "PorousSolid", - "PorousSolid", - "PorousSolid", - "PorousSolid" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Pyramid_Lagrange1_Gauss5", - "H1_Tetrahedron_Lagrange1_Gauss14" - ] - }, - "explicit": [] - }, - - "PoromechanicsDamageKernels": { - "vars": [ - "SUBREGION_TYPE", - "CONSTITUTIVE_TYPE", - "FE_TYPE" - ], - "constants": [ - [ "SinglePhasePoromechanicsDamagePolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "PorousDamageSolid>", - "PorousDamageSolid>", - "PorousDamageSolid>" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Tetrahedron_Lagrange1_Gauss14", - "H1_Pyramid_Lagrange1_Gauss5" - ] - }, - "explicit": [] } } diff --git a/src/coreComponents/physicsSolvers/multiphysics/ThermoPoromechanics_KernelSpecs.json b/src/coreComponents/physicsSolvers/multiphysics/ThermoPoromechanics_KernelSpecs.json new file mode 100644 index 00000000000..db759c295d6 --- /dev/null +++ b/src/coreComponents/physicsSolvers/multiphysics/ThermoPoromechanics_KernelSpecs.json @@ -0,0 +1,58 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "ThermoPoromechanicsKernels": { + "vars": [ + "SUBREGION_TYPE", + "CONSTITUTIVE_TYPE", + "FE_TYPE" + ], + "constants": [ + [ "ThermalMultiphasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ThermalSinglePhasePoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ThermalSinglePhasePoromechanicsEFEMPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "PorousSolid", + "PorousSolid", + "PorousSolid", + "PorousSolid" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Pyramid_Lagrange1_Gauss5", + "H1_Tetrahedron_Lagrange1_Gauss14" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/solidMechanics/CMakeLists.txt b/src/coreComponents/physicsSolvers/solidMechanics/CMakeLists.txt index ced15c47ba5..da37d961c21 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/CMakeLists.txt +++ b/src/coreComponents/physicsSolvers/solidMechanics/CMakeLists.txt @@ -55,7 +55,7 @@ set( solidMechanicsSolvers_headers contact/kernels/SolidMechanicsALMKernels.hpp contact/kernels/SolidMechanicsConformingContactKernelsHelper.hpp contact/kernels/SolidMechanicsContactFaceBubbleKernels.hpp - contact/kernels/SolidMechanicsLagrangeContactKernels.hpp + contact/kernels/SolidMechanicsLagrangeContactKernels.hpp contact/LogLevelsInfo.hpp ) # Specify solver sources @@ -76,27 +76,89 @@ set( dependencyList ${parallelDeps} physicsSolversBase ) geos_decorate_link_dependencies( LIST decoratedDependencies DEPENDENCIES ${dependencyList} ) - -file( READ "${CMAKE_CURRENT_LIST_DIR}/kernelSpecs.json" kernelSpecs ) -set( kernelTemplateFileList "" ) - - list( APPEND kernelTemplateFileList - kernels/SolidMechanicsKernels.cpp.template - kernels/SolidMechanicsFixedStressThermoPoromechanicsKernels.cpp.template ) - - -foreach( kernelTemplateFile ${kernelTemplateFileList} ) - get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) - generateKernels( TEMPLATE ${kernelTemplateFile} - JSON kernelSpecs - KEY ${jsonKey} - HEADERS headerFiles - SOURCES sourceFiles ) - -list(APPEND solidMechanicsSolvers_headers ${headerFiles}) -list(APPEND solidMechanicsSolvers_sources ${sourceFiles}) -endforeach() - +if (GEOS_ENABLE_SOLIDMECHANICS_ELASTIC) + file( READ "${CMAKE_CURRENT_LIST_DIR}/SolidMechanicsElastic_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList kernels/SolidMechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND solidMechanicsSolvers_headers ${headerFiles}) + list(APPEND solidMechanicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() + +if (GEOS_ENABLE_SOLIDMECHANICS_EXTRA) + file( READ "${CMAKE_CURRENT_LIST_DIR}/SolidMechanicsExtra_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList kernels/SolidMechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND solidMechanicsSolvers_headers ${headerFiles}) + list(APPEND solidMechanicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() + +# TODO: There is an error if we turn this on. Are we missing SolidMechanicsFixedStressThermoPoroElasticKernels.cpp.template? +if (GEOS_ENABLE_SOLIDMECHANICS_THERMOPOROELASTIC) + file( READ "${CMAKE_CURRENT_LIST_DIR}/SolidMechanicsFixedStressThermoPoroElastic_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList + kernels/SolidMechanicsFixedStressThermoPoromechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND solidMechanicsSolvers_headers ${headerFiles}) + list(APPEND solidMechanicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() + +if (GEOS_ENABLE_SOLIDMECHANICS_THERMOPOROMECHANICS) + file( READ "${CMAKE_CURRENT_LIST_DIR}/SolidMechanicsFixedStressThermoPoromechanics_KernelSpecs.json" kernelSpecs ) + set( kernelTemplateFileList kernels/SolidMechanicsFixedStressThermoPoromechanicsKernels.cpp.template ) + + foreach( kernelTemplateFile ${kernelTemplateFileList} ) + get_filename_component( jsonKey ${kernelTemplateFile} NAME_WE ) + generateKernels( TEMPLATE ${kernelTemplateFile} + JSON kernelSpecs + KEY ${jsonKey} + HEADERS headerFiles + SOURCES sourceFiles ) + + list(APPEND solidMechanicsSolvers_headers ${headerFiles}) + list(APPEND solidMechanicsSolvers_sources ${sourceFiles}) + + list(LENGTH sourceFiles NUM_KERNELS) + math(EXPR TOTAL_GENERATED_KERNELS "${TOTAL_GENERATED_KERNELS} + ${NUM_KERNELS}") + endforeach() +endif() + +set(TOTAL_GENERATED_KERNELS ${TOTAL_GENERATED_KERNELS} CACHE INTERNAL "Total number of generated kernels") blt_add_library( NAME solidMechanicsSolvers SOURCES ${solidMechanicsSolvers_sources} @@ -112,4 +174,4 @@ install( TARGETS solidMechanicsSolvers LIBRARY DESTINATION ${CMAKE_INSTALL_PREFI if( externalComponentDeps ) target_include_directories( solidMechanicsSolvers PUBLIC ${CMAKE_SOURCE_DIR}/externalComponents ) -endif() \ No newline at end of file +endif() diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsElastic_KernelSpecs.json b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsElastic_KernelSpecs.json new file mode 100644 index 00000000000..970544949ce --- /dev/null +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsElastic_KernelSpecs.json @@ -0,0 +1,57 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "SolidMechanicsKernels": { + "vars": [ "SUBREGION_TYPE", "CONSTITUTIVE_TYPE", "FE_TYPE" ], + "constants": [ + [ "ExplicitSmallStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ExplicitFiniteStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ImplicitSmallStrainNewmarkPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ImplicitSmallStrainQuasiStaticPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "ElasticIsotropic", + "ElasticTransverseIsotropic", + "ElasticIsotropicPressureDependent", + "ElasticOrthotropic" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Tetrahedron_Lagrange1_Gauss5", + "H1_Tetrahedron_Lagrange1_Gauss14", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/solidMechanics/kernelSpecs.json b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsExtra_KernelSpecs.json similarity index 57% rename from src/coreComponents/physicsSolvers/solidMechanics/kernelSpecs.json rename to src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsExtra_KernelSpecs.json index 9781aa9b7c2..53ecc7653f5 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/kernelSpecs.json +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsExtra_KernelSpecs.json @@ -47,11 +47,7 @@ "DruckerPragerExtended", "ModifiedCamClay", "DelftEgg", - "DruckerPrager", - "ElasticIsotropic", - "ElasticTransverseIsotropic", - "ElasticIsotropicPressureDependent", - "ElasticOrthotropic" + "DruckerPrager" ], "FE_TYPE": [ "H1_Hexahedron_Lagrange1_GaussLegendre2", @@ -63,61 +59,5 @@ ] }, "explicit": [] - }, - - "SolidMechanicsFixedStressThermoPoromechanicsKernels": { - "vars": [ - "SUBREGION_TYPE", - "CONSTITUTIVE_TYPE", - "FE_TYPE" - ], - "constants": [ - [ "ExplicitSmallStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ExplicitFiniteStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ImplicitSmallStrainNewmarkPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], - [ "ImplicitSmallStrainQuasiStaticPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "PorousSolid" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Pyramid_Lagrange1_Gauss5" - ] - }, - "explicit": [] - }, - - "SolidMechanicsFixedStressThermoPoroElasticKernels": { - "vars": [ - "SUBREGION_TYPE", - "CONSTITUTIVE_TYPE", - "FE_TYPE" - ], - "constants": [ - [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] - ], - "combinations": { - "SUBREGION_TYPE": [ - "CellElementSubRegion" - ], - "CONSTITUTIVE_TYPE": [ - "PorousSolid" - ], - "FE_TYPE": [ - "H1_Hexahedron_Lagrange1_GaussLegendre2", - "H1_Wedge_Lagrange1_Gauss6", - "H1_Tetrahedron_Lagrange1_Gauss1", - "H1_Pyramid_Lagrange1_Gauss5" - ] - }, - "explicit": [] } } diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoroElastic_KernelSpecs.json b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoroElastic_KernelSpecs.json new file mode 100644 index 00000000000..d26e8604f90 --- /dev/null +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoroElastic_KernelSpecs.json @@ -0,0 +1,52 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "SolidMechanicsFixedStressThermoPoroElasticKernels": { + "vars": [ + "SUBREGION_TYPE", + "CONSTITUTIVE_TYPE", + "FE_TYPE" + ], + "constants": [ + [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "PorousSolid" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoromechanics_KernelSpecs.json b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoromechanics_KernelSpecs.json new file mode 100644 index 00000000000..009c4784647 --- /dev/null +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsFixedStressThermoPoromechanics_KernelSpecs.json @@ -0,0 +1,56 @@ +{ + "doc" : + { + "comment" : "All kernel template instantiation specs need all dict keys except 'comment' and 'explicit'", + "vars" : + [ + "This is a list of the keys in the combinations dict (keys aren't queryable in cmake)", + "they are also the names of the CMAKE variables which will be set to one of the values specified", + "in the combinations dict just prior to cmake calling configure_file() on the template file" + ], + "constants" : + [ + [ "key", "value" ], + [ "note", "All constants will be set for cmake to configure_file() for every instantiation." ] + ], + "combinations" : + { + "vars" : [ "each value in this list will be used to generate kernel template files", + "combinatorally with all other value lists" ] + }, + "explicit" : [ "specify a specific combination of the variables to generate", + "specify by choosing one value from each variable you want generated", + "and concating them with '#' (this can by modified internally at the", + "generation call). Duplicate entries are unified internally" ] + }, + + "SolidMechanicsFixedStressThermoPoromechanicsKernels": { + "vars": [ + "SUBREGION_TYPE", + "CONSTITUTIVE_TYPE", + "FE_TYPE" + ], + "constants": [ + [ "ExplicitSmallStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ExplicitFiniteStrainPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "FixedStressThermoPoromechanicsPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ImplicitSmallStrainNewmarkPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ], + [ "ImplicitSmallStrainQuasiStaticPolicy", "geos::parallelDevicePolicy< GEOS_BLOCK_SIZE >" ] + ], + "combinations": { + "SUBREGION_TYPE": [ + "CellElementSubRegion" + ], + "CONSTITUTIVE_TYPE": [ + "PorousSolid" + ], + "FE_TYPE": [ + "H1_Hexahedron_Lagrange1_GaussLegendre2", + "H1_Wedge_Lagrange1_Gauss6", + "H1_Tetrahedron_Lagrange1_Gauss1", + "H1_Pyramid_Lagrange1_Gauss5" + ] + }, + "explicit": [] + } +} From 19f78e30f6a95da81c18be1c8c2f1af2dbd0bfae Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 2 Jun 2025 00:43:32 -0400 Subject: [PATCH 36/67] Change Hypre APIs: Elm -> Pointwise --- .../linearAlgebra/interfaces/hypre/HypreVector.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp index 38813c0c021..26ffb9ade2c 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp @@ -162,7 +162,7 @@ void HypreVector::reciprocal() { GEOS_LAI_ASSERT( ready() ); - GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorElmInverse( m_vec, &m_vec ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorPointwiseInverse( m_vec, &m_vec ) ); touch(); } @@ -229,7 +229,7 @@ void HypreVector::pointwiseScale( HypreVector const & x ) GEOS_LAI_ASSERT( x.ready() ); GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); - GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorElmProduct( x.m_vec, m_vec, &m_vec ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorPointwiseProduct( x.m_vec, m_vec, &m_vec ) ); touch(); } @@ -239,7 +239,7 @@ void HypreVector::pointwiseDivide( HypreVector const & x ) GEOS_LAI_ASSERT( x.ready() ); GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); - GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorElmDivision( x.m_vec, m_vec, &m_vec ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorPointwiseDivision( x.m_vec, m_vec, &m_vec ) ); touch(); } From 900d5e94ba2bec2b381407bc40c850d4a12edf88 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 2 Jun 2025 00:58:54 -0400 Subject: [PATCH 37/67] Disable physics scaling for now --- src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index bbee986d0ac..1682c403045 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -43,7 +43,7 @@ PhysicsSolverBase::PhysicsSolverBase( string const & name, m_nextDt( 1e99 ), m_numTimestepsSinceLastDtCut( -1 ), m_dofManager( name ), - m_usePhysicsScaling( 1 ), + m_usePhysicsScaling( 0 ), m_linearSolverParameters( groupKeyStruct::linearSolverParametersString(), this ), m_nonlinearSolverParameters( groupKeyStruct::nonlinearSolverParametersString(), this ), m_solverStatistics( groupKeyStruct::solverStatisticsString(), this ), @@ -95,7 +95,7 @@ PhysicsSolverBase::PhysicsSolverBase( string const & name, setDescription( "Cut time step if linear solution fail without going until max nonlinear iterations." ); registerWrapper( viewKeyStruct::usePhysicsScalingString(), &m_usePhysicsScaling ) - .setApplyDefaultValue( 1 ) + .setApplyDefaultValue( 0 ) .setInputFlag( InputFlags::OPTIONAL ) .setDescription( "Enable physics-based scaling of the linear system. Default: true." ); From e9c7ee050f4e4e2524e3e725a4ad0ff45d562215 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 2 Jun 2025 00:59:10 -0400 Subject: [PATCH 38/67] Update LvArray hash --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index 404f1d79a01..5ef6481f204 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit 404f1d79a01f29d42a153223b9b23c06236c743f +Subproject commit 5ef6481f204b735a97bc788c57bddd39d2b25eb8 From eb672721955599daa5a6cd57663f3bbec29bd1ad Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 2 Jun 2025 13:05:32 -0400 Subject: [PATCH 39/67] Fixes --- .../multiphysics/CMakeLists.txt | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt b/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt index 2f7878ee036..53fc33e08d2 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt +++ b/src/coreComponents/physicsSolvers/multiphysics/CMakeLists.txt @@ -24,16 +24,9 @@ set( multiPhysicsSolvers_headers CoupledSolver.hpp PoromechanicsSolver.hpp FlowProppantTransportSolver.hpp - HydrofractureSolver.hpp - HydrofractureSolverKernels.hpp - MultiphasePoromechanics.hpp OneWayCoupledFractureFlowContactMechanics.hpp - MultiphasePoromechanicsConformingFractures.hpp PhaseFieldFractureSolver.hpp - PhaseFieldPoromechanicsSolver.hpp - PoromechanicsInitialization.hpp PoromechanicsFields.hpp - PoromechanicsInitialization.hpp poromechanicsKernels/MultiphasePoromechanics.hpp poromechanicsKernels/MultiphasePoromechanics_impl.hpp poromechanicsKernels/MultiphasePoromechanicsConformingFractures.hpp @@ -55,9 +48,6 @@ set( multiPhysicsSolvers_headers poromechanicsKernels/ThermalSinglePhasePoromechanicsEFEM_impl.hpp poromechanicsKernels/ThermalSinglePhasePoromechanicsConformingFractures.hpp poromechanicsKernels/ThermalSinglePhasePoromechanicsEmbeddedFractures.hpp - SinglePhasePoromechanics.hpp - SinglePhasePoromechanicsEmbeddedFractures.hpp - SinglePhasePoromechanicsConformingFractures.hpp SinglePhaseReservoirAndWells.hpp ) # Specify solver sources @@ -65,16 +55,8 @@ set( multiPhysicsSolvers_sources CompositionalMultiphaseReservoirAndWells.cpp CoupledReservoirAndWellsBase.cpp FlowProppantTransportSolver.cpp - HydrofractureSolver.cpp - MultiphasePoromechanics.cpp OneWayCoupledFractureFlowContactMechanics.cpp - MultiphasePoromechanicsConformingFractures.cpp PhaseFieldFractureSolver.cpp - PhaseFieldPoromechanicsSolver.cpp - PoromechanicsInitialization.cpp - SinglePhasePoromechanics.cpp - SinglePhasePoromechanicsEmbeddedFractures.cpp - SinglePhasePoromechanicsConformingFractures.cpp SinglePhaseReservoirAndWells.cpp ) #include(poromechanicsKernels/PoromechanicsKernels.cmake) @@ -84,6 +66,24 @@ set( dependencyList ${parallelDeps} fluidFlowSolvers simplePDESolvers solidMecha geos_decorate_link_dependencies( LIST decoratedDependencies DEPENDENCIES ${dependencyList} ) if (GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS) + list( APPEND multiPhysicsSolvers_headers + PoromechanicsInitialization.hpp + SinglePhasePoromechanics.hpp + SinglePhasePoromechanicsConformingFractures.hpp + MultiphasePoromechanics.hpp + MultiphasePoromechanicsConformingFractures.hpp + PhaseFieldPoromechanicsSolver.hpp + HydrofractureSolver.hpp + HydrofractureSolverKernels.hpp ) + list( APPEND multiPhysicsSolvers_sources + PoromechanicsInitialization.cpp + SinglePhasePoromechanics.cpp + SinglePhasePoromechanicsConformingFractures.cpp + MultiphasePoromechanics.cpp + MultiphasePoromechanicsConformingFractures.cpp + PhaseFieldPoromechanicsSolver.cpp + HydrofractureSolver.cpp ) + file( READ "${CMAKE_CURRENT_LIST_DIR}/Poromechanics_KernelSpecs.json" kernelSpecs ) set( kernelTemplateFileList poromechanicsKernels/PoromechanicsKernels.cpp.template ) @@ -104,6 +104,11 @@ if (GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS) endif() if (GEOS_ENABLE_MULTIPHYSICS_POROMECHANICS_EFEM) + list( APPEND multiPhysicsSolvers_headers + SinglePhasePoromechanicsEmbeddedFractures.hpp ) + list( APPEND multiPhysicsSolvers_sources + SinglePhasePoromechanicsEmbeddedFractures.cpp ) + file( READ "${CMAKE_CURRENT_LIST_DIR}/PoromechanicsEFEM_KernelSpecs.json" kernelSpecs ) set( kernelTemplateFileList poromechanicsKernels/PoromechanicsEFEMKernels.cpp.template ) From a855a246fc7614f84f6db9586a7f0854a844030c Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 2 Jun 2025 12:30:23 -0700 Subject: [PATCH 40/67] Fix base HCs --- host-configs/LLNL/amdgpu-base.cmake | 3 ++- host-configs/LLNL/tuo-base.cmake | 41 ----------------------------- 2 files changed, 2 insertions(+), 42 deletions(-) delete mode 100644 host-configs/LLNL/tuo-base.cmake diff --git a/host-configs/LLNL/amdgpu-base.cmake b/host-configs/LLNL/amdgpu-base.cmake index ab92058b034..de498a6ae7a 100644 --- a/host-configs/LLNL/amdgpu-base.cmake +++ b/host-configs/LLNL/amdgpu-base.cmake @@ -28,10 +28,11 @@ set( ENABLE_DOCS OFF CACHE BOOL "" FORCE ) set( ENABLE_SCOTCH OFF CACHE BOOL "" FORCE ) set( ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE ) +set( ENABLE_SUITESPARSE OFF CACHE BOOL "" FORCE ) # HYPRE options set( ENABLE_HYPRE_DEVICE "HIP" CACHE STRING "" ) -set( ENABLE_HYPRE_MIXINT ON CACHE STRING "" ) +set( ENABLE_HYPRE_MIXINT ON CACHE BOOL "" ) # TPLs include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake) diff --git a/host-configs/LLNL/tuo-base.cmake b/host-configs/LLNL/tuo-base.cmake deleted file mode 100644 index 207236dc9eb..00000000000 --- a/host-configs/LLNL/tuo-base.cmake +++ /dev/null @@ -1,41 +0,0 @@ -set( GEOS_ENABLE_VEM OFF CACHE BOOL "" FORCE ) - -set( ENABLE_MATHPRESSO OFF CACHE BOOL "" ) -set( ENABLE_PAMELA OFF CACHE BOOL "" ) -set( ENABLE_PVTPackage ON CACHE BOOL "" ) -set( ENABLE_PETSC OFF CACHE BOOL "" FORCE ) -set( ENABLE_CALIPER ON CACHE BOOL "" ) -set( ENABLE_PAPI OFF CACHE BOOL "" ) -set( ENABLE_ESSL OFF CACHE BOOL "" ) -set( ENABLE_TRILINOS OFF CACHE BOOL "" ) -set( ENABLE_VTK ON CACHE BOOL "" ) -set( ENABLE_OPENMP OFF CACHE BOOL "" FORCE ) - -set( CAMP_STANDALONE TRUE CACHE BOOL "" ) - -# ROCM options -set( ENABLE_ROCM ON CACHE BOOL "" FORCE ) -set( ROCM_ROOT "${HIP_ROOT}" CACHE PATH "" ) - -set( GEOS_BUILD_OBJ_LIBS OFF CACHE BOOL "" FORCE ) -set( ENABLE_GTEST_DEATH_TESTS OFF CACHE BOOL "" ) -set( gtest_disable_pthreads ON CACHE BOOL "" ) - -set( ENABLE_TESTS ON CACHE BOOL "" FORCE ) -set( ENABLE_EXAMPLES ON CACHE BOOL "" FORCE ) -set( ENABLE_BENCHMARKS ON CACHE BOOL "" FORCE ) -set( ENABLE_DOCS OFF CACHE BOOL "" FORCE ) - -set( ENABLE_SCOTCH OFF CACHE BOOL "" FORCE ) -set( ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE ) - -# HYPRE options -set( ENABLE_HYPRE_MIXINT ON CACHE STRING "" ) - -# TPLs -include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake) - -# ATS -set(USER $ENV{USER} CACHE STRING "") -set(ATS_WORKING_DIR "/p/lustre2/${USER}/integratedTestsGEOS/${CONFIG_NAME}" CACHE PATH "") -set(ATS_BASELINE_DIR "/p/lustre2/${USER}/integratedTestsGEOS/baselines" CACHE PATH "") From 336b5b2ab8303e56612eef9dc056ef8f3b1b987b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 2 Jun 2025 12:34:33 -0700 Subject: [PATCH 41/67] Update LvArray hash --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index 5ef6481f204..71ce0db5160 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit 5ef6481f204b735a97bc788c57bddd39d2b25eb8 +Subproject commit 71ce0db51605a37681a3f3b76efcac067014f487 From ba4295f05fc76cc1f13d916f7cff571330519861 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 2 Jun 2025 16:07:13 -0400 Subject: [PATCH 42/67] Fix cmake policy --- src/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7bab940e7ab..c515eb25ae7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,7 +6,9 @@ cmake_policy(SET CMP0104 OLD) # when using nvcc populate CMAKE_CUDA_ARCHITECTURE cmake_policy(SET CMP0074 NEW) # dont ignore _ROOT env vars when searching for packages via find_package() cmake_policy(SET CMP0066 NEW) # use CMAKE__FLAGS_ for try_compile() instead of only CMAKE__FLAGS cmake_policy(SET CMP0056 NEW) # use CMAKE_EXE_LINKER_FLAGS in try_compile() in addition to CMAKE__FLAGS -cmake_policy(SET CMP0144 NEW) # find_package uses upper-case _ROOT variables. +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.27") + cmake_policy(SET CMP0144 NEW) # find_package uses upper-case _ROOT variables. +endif() ################################ # GEOSX From 97296b4161dcb3c8726fb373c4c869459d1aebb3 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 2 Jun 2025 13:58:54 -0700 Subject: [PATCH 43/67] Add tioga HC --- host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake diff --git a/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake b/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake new file mode 100644 index 00000000000..b60c0e5247e --- /dev/null +++ b/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake @@ -0,0 +1,5 @@ +include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake) + +# MPI +set(MPI_HOME /opt/cray/pe/mpich/8.1.33.1/ofi/crayclang/18.0 CACHE PATH "") From 22a9e9e7a61829d0201003d3ba75425d54947802 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 2 Jun 2025 14:56:02 -0700 Subject: [PATCH 44/67] Update matrix HCs --- host-configs/LLNL/matrix-base.cmake | 10 +++++++--- host-configs/LLNL/matrix-gcc-12-cuda-12.cmake | 4 +--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/host-configs/LLNL/matrix-base.cmake b/host-configs/LLNL/matrix-base.cmake index bcf374512ac..6193ea8ebde 100644 --- a/host-configs/LLNL/matrix-base.cmake +++ b/host-configs/LLNL/matrix-base.cmake @@ -14,6 +14,7 @@ set(ENABLE_FORTRAN OFF CACHE BOOL "") # MPI set(ENABLE_MPI ON CACHE BOOL "") +set(MPI_HOME "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1" CACHE PATH "") set(MPI_C_COMPILER ${MPI_HOME}/bin/mpicc CACHE PATH "") set(MPI_CXX_COMPILER ${MPI_HOME}/bin/mpicxx CACHE PATH "") set(MPIEXEC srun CACHE STRING "") @@ -38,7 +39,10 @@ set(ENABLE_PAPI OFF CACHE BOOL "") set(ENABLE_PVTPackage ON CACHE BOOL "") set(ENABLE_PETSC OFF CACHE BOOL "" FORCE ) set(ENABLE_TRILINOS OFF CACHE BOOL "" FORCE ) +set(ENABLE_SUITESPARSE OFF CACHE BOOL "" FORCE ) set(ENABLE_HYPRE_DEVICE "CUDA" CACHE STRING "" FORCE) +set(ENABLE_HYPRE_MIXINT ON CACHE BOOL "" FORCE) +set(ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE ) # Documentation set(ENABLE_UNCRUSTIFY OFF CACHE BOOL "" FORCE) @@ -58,8 +62,8 @@ set(Python3_EXECUTABLE /usr/tce/packages/python/python-3.9.12/bin/python3 CACHE # ATS set(ATS_ARGUMENTS "--ats jsrun_omp --ats jsrun_bind=packed" CACHE STRING "") -# set(USER $ENV{USER} CACHE STRING "") -# set(ATS_WORKING_DIR "/p/gpfs1/${USER}/integratedTestsGEOS/${CONFIG_NAME}" CACHE PATH "") -# set(ATS_BASELINE_DIR "/p/gpfs1/${USER}/integratedTestsGEOS/baselines" CACHE PATH "") +set(USER $ENV{USER} CACHE STRING "") +set(ATS_WORKING_DIR "/p/lustre2/${USER}/integratedTestsGEOS/${CONFIG_NAME}" CACHE PATH "") +set(ATS_BASELINE_DIR "/p/lustre2/${USER}/integratedTestsGEOS/baselines" CACHE PATH "") include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake) diff --git a/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake b/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake index 83c837c13af..df82f1531f4 100644 --- a/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake +++ b/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake @@ -1,4 +1,5 @@ include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/matrix-gcc-12-cuda-12.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/matrix-base.cmake) # C++ # The "-march=native -mtune=native" which LvArray adds breaks the PVT package. @@ -11,9 +12,6 @@ set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "") set(FORTRAN_MANGLE_NO_UNDERSCORE OFF CACHE BOOL "") # MPI -set(MPI_HOME /usr/tce/packages/openmpi/openmpi-4.1.2-gcc-12.1.1 CACHE PATH "") set(MPI_Fortran_COMPILER ${MPI_HOME}/bin/mpifort CACHE PATH "") set(ENABLE_CUDA_NVTOOLSEXT OFF CACHE BOOL "") - -include(${CMAKE_CURRENT_LIST_DIR}/matrix-base.cmake) From fb9bb2bc437dded8c3736002018f1a26839be385 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 3 Jun 2025 12:27:54 -0700 Subject: [PATCH 45/67] Add runtime control on GPU-aware MPI for hypre --- .../linearAlgebra/interfaces/hypre/HypreInterface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp index 7365861ba6d..c7b5630e937 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp @@ -76,6 +76,7 @@ void HypreInterface::initialize() #endif HYPRE_SetLogLevel( getenv( "HYPRE_LOG_LEVEL" ) ? atoi( getenv( "HYPRE_LOG_LEVEL" ) ) : 0 ); + HYPRE_SetGpuAwareMPI( getenv( "MPICH_GPU_SUPPORT_ENABLED" ) ? atoi( getenv( "MPICH_GPU_SUPPORT_ENABLED" ) ) : 0 ); } void HypreInterface::finalize() From f758790cae03d525f9bf63f190b8e91be5083426 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 3 Jun 2025 12:29:36 -0700 Subject: [PATCH 46/67] Build info about GPU-aware MPI --- host-configs/LLNL/amdgpu-base.cmake | 5 +++-- host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake | 5 +++++ src/cmake/thirdparty/SetupGeosxThirdParty.cmake | 10 +++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/host-configs/LLNL/amdgpu-base.cmake b/host-configs/LLNL/amdgpu-base.cmake index de498a6ae7a..59bb2a27572 100644 --- a/host-configs/LLNL/amdgpu-base.cmake +++ b/host-configs/LLNL/amdgpu-base.cmake @@ -31,8 +31,9 @@ set( ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE ) set( ENABLE_SUITESPARSE OFF CACHE BOOL "" FORCE ) # HYPRE options -set( ENABLE_HYPRE_DEVICE "HIP" CACHE STRING "" ) -set( ENABLE_HYPRE_MIXINT ON CACHE BOOL "" ) +set( ENABLE_HYPRE_DEVICE "HIP" CACHE STRING "Enable HIP backend for hypre" ) +set( ENABLE_HYPRE_MIXINT ON CACHE BOOL "Enable 64-bit integers for hypre" ) +set( ENABLE_HYPRE_GPU_AWARE_MPI OFF CACHE BOOL "Enable GPU-aware MPI for hypre" ) # TPLs include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake) diff --git a/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake b/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake index b60c0e5247e..204d3d6821f 100644 --- a/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake +++ b/host-configs/LLNL/tioga-cce-19-rocm-6.4.0.cmake @@ -3,3 +3,8 @@ include(${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake) # MPI set(MPI_HOME /opt/cray/pe/mpich/8.1.33.1/ofi/crayclang/18.0 CACHE PATH "") + +# GPU-aware MPI option +if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( CMAKE_HIP_LINK_FLAGS "${CMAKE_HIP_LINK_FLAGS} $ENV{PE_MPICH_GTL_DIR_amd_gfx90a} $ENV{PE_MPICH_GTL_LIBS_amd_gfx90a}" CACHE STRING "" FORCE ) +endif() diff --git a/src/cmake/thirdparty/SetupGeosxThirdParty.cmake b/src/cmake/thirdparty/SetupGeosxThirdParty.cmake index 6736e46062c..88765d34157 100644 --- a/src/cmake/thirdparty/SetupGeosxThirdParty.cmake +++ b/src/cmake/thirdparty/SetupGeosxThirdParty.cmake @@ -211,7 +211,7 @@ else() endif() ################################ -# ZLIB +# ZLIB ################################ if(DEFINED ZLIB_DIR) list(PREPEND CMAKE_PREFIX_PATH "${ZLIB_DIR}") @@ -719,6 +719,14 @@ if(DEFINED HYPRE_DIR AND ENABLE_HYPRE) message( " ----> hypre_VERSION = ${hypre_VERSION}" ) endif() + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + if( "${header_file}" MATCHES "HYPRE_USING_GPU_AWARE_MPI 1" ) + message( " ----> GPU-aware MPI support activated" ) + else () + message(FATAL_ERROR "GEOS configured with ENABLE_HYPRE_GPU_AWARE_MPI=ON, however hypre was not built with GPU-aware MPI support") + endif () + endif () + # Prepend Hypre to link flags, fix for Umpire appearing before Hypre on the link line # if (NOT CMAKE_HOST_APPLE) # blt_add_target_link_flags (TO hypre FLAGS "-Wl,--whole-archive ${HYPRE_DIR}/lib/libHYPRE.a -Wl,--no-whole-archive") From 6274614f6e1e2b72e6191254f4cf9e4961191f11 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 6 Jun 2025 13:04:20 -0700 Subject: [PATCH 47/67] Add cmake parameters --- scripts/config-build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/config-build.py b/scripts/config-build.py index 505d213b4e3..2aefa4d0f69 100755 --- a/scripts/config-build.py +++ b/scripts/config-build.py @@ -214,6 +214,12 @@ def main(calling_script, args, unknown_args): # Append cache file at the end of the command line to make previous argument visible to the cache. cmake_line.append("-C%s" % cache_file) + # Add them back + for unknown_arg in unknown_args: + if not unknown_arg.startswith('-D'): + logging.warning("Additional argument '%s' does not start with '-D'. Keeping it nevertheless." % unknown_arg) + cmake_line.append(unknown_arg) + cmake_line.append(os.path.normpath(os.path.join(scripts_dir, "..", "src"))) # Dump the cmake command to file for convenience From 60a346189c047c9279ab846a6f087aca0a393d2b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 6 Jun 2025 13:04:37 -0700 Subject: [PATCH 48/67] Fix warning --- src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index 1682c403045..74956b2e948 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -795,8 +795,8 @@ real64 PhysicsSolverBase::eisenstatWalker( real64 const newNewtonNorm, void MemoryUsageOutput( std::string const & message ) { -#if 0 GEOS_UNUSED_VAR(message); +#if 0 #if defined( GEOS_USE_HIP ) size_t free, total; From 5f6613d1ebca346730e0707d195bc156fceaca04 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 7 Jun 2025 21:19:37 -0700 Subject: [PATCH 49/67] cuda 12.8 fixes --- .../compositional/functions/CubicEOSPhaseModel.hpp | 4 ++-- .../functions/NegativeTwoPhaseFlash.hpp | 12 ++++++++---- .../models/ImmiscibleWaterFlashModel.hpp | 2 +- .../compositional/models/PhillipsBrineDensity.cpp | 4 +++- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/CubicEOSPhaseModel.hpp b/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/CubicEOSPhaseModel.hpp index 1b9be2bf38a..0961134ae17 100644 --- a/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/CubicEOSPhaseModel.hpp +++ b/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/CubicEOSPhaseModel.hpp @@ -874,8 +874,8 @@ computeCompressibilityFactor( integer const numComps, } } - stackArray1d< real64, MultiFluidConstants::MAX_NUM_COMPONENTS > logFugacityCoefficientsMax( numComps ); - stackArray1d< real64, MultiFluidConstants::MAX_NUM_COMPONENTS > logFugacityCoefficientsMin( numComps ); + StackArray< real64, 1, MultiFluidConstants::MAX_NUM_COMPONENTS > logFugacityCoefficientsMax( numComps ); + StackArray< real64, 1, MultiFluidConstants::MAX_NUM_COMPONENTS > logFugacityCoefficientsMin( numComps ); computeLogFugacityCoefficients( numComps, composition, binaryInteractionCoefficients, zMin, aPureCoefficient, bPureCoefficient, aMixtureCoefficient, bMixtureCoefficient, logFugacityCoefficientsMin.toSlice() ); diff --git a/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/NegativeTwoPhaseFlash.hpp b/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/NegativeTwoPhaseFlash.hpp index 3027f0a2ec1..942f1475fe7 100644 --- a/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/NegativeTwoPhaseFlash.hpp +++ b/src/coreComponents/constitutive/fluid/multifluid/compositional/functions/NegativeTwoPhaseFlash.hpp @@ -376,10 +376,14 @@ void NegativeTwoPhaseFlash::computeDerivatives( else { // Calculate the liquid and vapour fugacities and derivatives - stackArray1d< real64, maxNumComps > logLiquidFugacity( numComps ); - stackArray1d< real64, maxNumComps > logVapourFugacity( numComps ); - stackArray2d< real64, maxNumComps * maxNumDofs > logLiquidFugacityDerivs( numComps, numDofs ); - stackArray2d< real64, maxNumComps * maxNumDofs > logVapourFugacityDerivs( numComps, numDofs ); + //stackArray1d< real64, maxNumComps > logLiquidFugacity( numComps ); + //stackArray1d< real64, maxNumComps > logVapourFugacity( numComps ); + //stackArray2d< real64, maxNumComps * maxNumDofs > logLiquidFugacityDerivs( numComps, numDofs ); + //stackArray2d< real64, maxNumComps * maxNumDofs > logVapourFugacityDerivs( numComps, numDofs ); + StackArray< real64, 1, maxNumComps > logLiquidFugacity( numComps ); + StackArray< real64, 1, maxNumComps > logVapourFugacity( numComps ); + StackArray< real64, 2, maxNumComps * maxNumDofs, MatrixLayout::ROW_MAJOR_PERM > logLiquidFugacityDerivs( numComps, numDofs ); + StackArray< real64, 2, maxNumComps * maxNumDofs, MatrixLayout::ROW_MAJOR_PERM > logVapourFugacityDerivs( numComps, numDofs ); FugacityCalculator::computeLogFugacity( numComps, pressure, diff --git a/src/coreComponents/constitutive/fluid/multifluid/compositional/models/ImmiscibleWaterFlashModel.hpp b/src/coreComponents/constitutive/fluid/multifluid/compositional/models/ImmiscibleWaterFlashModel.hpp index 59a32c87a4a..cc338739c4e 100644 --- a/src/coreComponents/constitutive/fluid/multifluid/compositional/models/ImmiscibleWaterFlashModel.hpp +++ b/src/coreComponents/constitutive/fluid/multifluid/compositional/models/ImmiscibleWaterFlashModel.hpp @@ -172,7 +172,7 @@ void ImmiscibleWaterFlashModelUpdate::compute( ComponentProperties::KernelWrappe // Hydrocarbon phases // Calculate normalised hyrdocarbon composition - stackArray1d< real64, maxNumComps > composition( m_numComponents ); + StackArray< real64, 1, maxNumComps > composition( m_numComponents ); for( integer ic = 0; ic < m_numComponents; ++ic ) { composition[ic] = compFraction[ic] / z_hc; diff --git a/src/coreComponents/constitutive/fluid/multifluid/compositional/models/PhillipsBrineDensity.cpp b/src/coreComponents/constitutive/fluid/multifluid/compositional/models/PhillipsBrineDensity.cpp index 5094c7f9b81..a66bdade08d 100644 --- a/src/coreComponents/constitutive/fluid/multifluid/compositional/models/PhillipsBrineDensity.cpp +++ b/src/coreComponents/constitutive/fluid/multifluid/compositional/models/PhillipsBrineDensity.cpp @@ -233,7 +233,9 @@ void PhillipsBrineDensity::calculateEosWaterMolarVolume( arraySlice1d< real64 co // Create pure water composition stackArray1d< real64, MultiFluidConstants::MAX_NUM_COMPONENTS > waterComposition( numComps ); - stackArray1d< real64, 2 + MultiFluidConstants::MAX_NUM_COMPONENTS > tempDerivs( numDofs ); + //stackArray1d< real64, 2 + MultiFluidConstants::MAX_NUM_COMPONENTS > tempDerivs( numDofs ); + //stackArray1d< real64, 2 + MultiFluidConstants::MAX_NUM_COMPONENTS > tempDerivs; + StackArray< real64, 1, 2 + MultiFluidConstants::MAX_NUM_COMPONENTS > tempDerivs( numDofs ); for( integer ic = 0; ic < numComps; ++ic ) { waterComposition[ic] = 0.0; From 6f82249dffe0ce5afd6a6bef1cdfcb9fb13266d4 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 7 Jun 2025 21:19:50 -0700 Subject: [PATCH 50/67] Minor --- host-configs/LLNL/matrix-base.cmake | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/host-configs/LLNL/matrix-base.cmake b/host-configs/LLNL/matrix-base.cmake index 6193ea8ebde..a27ef79f9b8 100644 --- a/host-configs/LLNL/matrix-base.cmake +++ b/host-configs/LLNL/matrix-base.cmake @@ -37,20 +37,18 @@ set(ENABLE_PAPI OFF CACHE BOOL "") # GEOS specific options set(ENABLE_PVTPackage ON CACHE BOOL "") +set(ENABLE_MATHPRESSO OFF CACHE BOOL "") set(ENABLE_PETSC OFF CACHE BOOL "" FORCE ) set(ENABLE_TRILINOS OFF CACHE BOOL "" FORCE ) set(ENABLE_SUITESPARSE OFF CACHE BOOL "" FORCE ) +set(ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE ) set(ENABLE_HYPRE_DEVICE "CUDA" CACHE STRING "" FORCE) set(ENABLE_HYPRE_MIXINT ON CACHE BOOL "" FORCE) -set(ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE ) # Documentation set(ENABLE_UNCRUSTIFY OFF CACHE BOOL "" FORCE) set(ENABLE_DOXYGEN OFF CACHE BOOL "" FORCE) -# Other -set(ENABLE_MATHPRESSO OFF CACHE BOOL "") - # YAPF python formatting #set(YAPF_EXECUTABLE /usr/gapps/GEOSX/thirdPartyLibs/python/lassen-gcc-python/python/bin/yapf CACHE PATH "" FORCE) From 08d5b69dc52d1163ab5d4620138c7ea1632a2eab Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 7 Jun 2025 21:20:39 -0700 Subject: [PATCH 51/67] Add nvml option --- src/CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c515eb25ae7..10a11fc8a9a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -125,10 +125,16 @@ endif() if( ENABLE_CUDA ) list( APPEND extraComponentsLinkList cuda ) -endif() -if( ENABLE_CUDA AND ENABLE_CUDA_NVTOOLSEXT ) - list( APPEND extraComponentsLinkList CUDA::nvToolsExt ) + if (ENABLE_CUDA_NVTOOLSEXT) + list( APPEND extraComponentsLinkList CUDA::nvToolsExt ) + message( STATUS "Enabling CUDA::nvToolsExt" ) + endif() + + if (ENABLE_CUDA_NVML) + list( APPEND extraComponentsLinkList CUDA::nvml ) + message( STATUS "Enabling CUDA::nvml" ) + endif() endif() if( ENABLE_HIP ) From dda59d98f2753ee5cb07b38644ad8911d94b58c6 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 18 Jun 2025 12:35:35 -0400 Subject: [PATCH 52/67] Fix APIs --- .../linearAlgebra/interfaces/hypre/HypreVector.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp index 38813c0c021..26ffb9ade2c 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp @@ -162,7 +162,7 @@ void HypreVector::reciprocal() { GEOS_LAI_ASSERT( ready() ); - GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorElmInverse( m_vec, &m_vec ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorPointwiseInverse( m_vec, &m_vec ) ); touch(); } @@ -229,7 +229,7 @@ void HypreVector::pointwiseScale( HypreVector const & x ) GEOS_LAI_ASSERT( x.ready() ); GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); - GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorElmProduct( x.m_vec, m_vec, &m_vec ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorPointwiseProduct( x.m_vec, m_vec, &m_vec ) ); touch(); } @@ -239,7 +239,7 @@ void HypreVector::pointwiseDivide( HypreVector const & x ) GEOS_LAI_ASSERT( x.ready() ); GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); - GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorElmDivision( x.m_vec, m_vec, &m_vec ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_ParVectorPointwiseDivision( x.m_vec, m_vec, &m_vec ) ); touch(); } From f37410a7dc0596cfecf0b8ae8c73047c9a1bf20d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 19 Jun 2025 23:06:44 -0400 Subject: [PATCH 53/67] Fixes --- .../linearAlgebra/interfaces/hypre/HypreMatrix.cpp | 2 +- .../linearAlgebra/interfaces/hypre/HypreVector.cpp | 1 - src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp | 4 ++++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp index 76329e865c4..1b21388dc27 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp @@ -1483,7 +1483,7 @@ void HypreMatrix::computeScalingVector(HypreVector & scaling) const HYPRE_ParVector *hypre_vec_ptr = &hypre_vec; // Compute scaling vector - GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixCompScalingTagged( m_parcsr_mat, 0, num_tags, pointMarkers.data(), hypre_vec_ptr ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixCompScalingTagged( m_parcsr_mat, 1, num_tags, pointMarkers.data(), hypre_vec_ptr ) ); } }// end namespace geos diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp index 26ffb9ade2c..bf136aeb0f9 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp @@ -102,7 +102,6 @@ void HypreVector::create( localIndex const localSize, // Set up the parallel and local vector data structures m_vec = hypre_ParVectorCreate( comm, globalSize, partitioning ); - hypre_ParVectorOwnsData( m_vec ) = false; hypre_Vector * const localVector = hypre_ParVectorLocalVector( m_vec ); hypre_VectorOwnsData( localVector ) = false; diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index db9c6ee13f4..eedbd2363c8 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -1319,6 +1319,8 @@ void PhysicsSolverBase::solveLinearSystem( DofManager const & dofManager, // Apply physics-based scaling to the linear system if enabled if( m_usePhysicsScaling ) { + Timer timer_setup( m_timers["linear solver scaling"] ); + matrix.computeScalingVector( m_scaling ); matrix.leftRightScale( m_scaling, m_scaling ); rhs.pointwiseScale( m_scaling ); @@ -1369,6 +1371,8 @@ void PhysicsSolverBase::solveLinearSystem( DofManager const & dofManager, // Unscale the solution vector if physics-based scaling was applied if( m_usePhysicsScaling ) { + Timer timer_setup( m_timers["linear solver scaling"] ); + solution.pointwiseScale( m_scaling ); } } From 0688fa12e99453d83e0a47bdc4d141d21ef56ce6 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 19 Jun 2025 23:07:23 -0400 Subject: [PATCH 54/67] Fixes --- src/coreComponents/common/MpiWrapper.hpp | 1 + .../linearAlgebra/interfaces/hypre/HypreMatrix.cpp | 2 +- .../linearAlgebra/interfaces/hypre/HypreVector.cpp | 2 +- src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp | 4 ++++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/coreComponents/common/MpiWrapper.hpp b/src/coreComponents/common/MpiWrapper.hpp index b884e15a8bc..6efc6f0e6de 100644 --- a/src/coreComponents/common/MpiWrapper.hpp +++ b/src/coreComponents/common/MpiWrapper.hpp @@ -1280,6 +1280,7 @@ U MpiWrapper::prefixSum( T const value, MPI_Comm comm ) U const convertedValue = value; int const error = MPI_Exscan( &convertedValue, &localResult, 1, internal::getMpiType< U >(), MPI_SUM, comm ); MPI_CHECK_ERROR( error ); + MPI_Barrier( comm ); #endif if( commRank() == 0 ) { diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp index 76329e865c4..1b21388dc27 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp @@ -1483,7 +1483,7 @@ void HypreMatrix::computeScalingVector(HypreVector & scaling) const HYPRE_ParVector *hypre_vec_ptr = &hypre_vec; // Compute scaling vector - GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixCompScalingTagged( m_parcsr_mat, 0, num_tags, pointMarkers.data(), hypre_vec_ptr ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixCompScalingTagged( m_parcsr_mat, 1, num_tags, pointMarkers.data(), hypre_vec_ptr ) ); } }// end namespace geos diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp index 26ffb9ade2c..0d04bbee943 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp @@ -102,7 +102,7 @@ void HypreVector::create( localIndex const localSize, // Set up the parallel and local vector data structures m_vec = hypre_ParVectorCreate( comm, globalSize, partitioning ); - hypre_ParVectorOwnsData( m_vec ) = false; + //hypre_ParVectorOwnsData( m_vec ) = false; hypre_Vector * const localVector = hypre_ParVectorLocalVector( m_vec ); hypre_VectorOwnsData( localVector ) = false; diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index 74956b2e948..c899988d15a 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -1370,6 +1370,8 @@ void PhysicsSolverBase::solveLinearSystem( DofManager const & dofManager, // Apply physics-based scaling to the linear system if enabled if( m_usePhysicsScaling ) { + Timer timer_setup( m_timers["linear solver scaling"] ); + matrix.computeScalingVector( m_scaling ); matrix.leftRightScale( m_scaling, m_scaling ); rhs.pointwiseScale( m_scaling ); @@ -1420,6 +1422,8 @@ void PhysicsSolverBase::solveLinearSystem( DofManager const & dofManager, // Unscale the solution vector if physics-based scaling was applied if( m_usePhysicsScaling ) { + Timer timer_setup( m_timers["linear solver scaling"] ); + solution.pointwiseScale( m_scaling ); } } From ce597766ab4387b36bef42abf3d8216211bb4b6c Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 26 Jun 2025 16:14:46 -0400 Subject: [PATCH 55/67] Update API --- .../linearAlgebra/interfaces/hypre/HypreMatrix.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp index 1b21388dc27..f2808f51d0a 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp @@ -1483,7 +1483,8 @@ void HypreMatrix::computeScalingVector(HypreVector & scaling) const HYPRE_ParVector *hypre_vec_ptr = &hypre_vec; // Compute scaling vector - GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixCompScalingTagged( m_parcsr_mat, 1, num_tags, pointMarkers.data(), hypre_vec_ptr ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_ParCSRMatrixComputeScalingTagged( m_parcsr_mat, 1, HYPRE_MEMORY_HOST, num_tags, + pointMarkers.data(), hypre_vec_ptr ) ); } }// end namespace geos From c20a79701d341a47fb554baf541e888f5fa49e83 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 26 Jun 2025 16:39:16 -0400 Subject: [PATCH 56/67] Uncrustify --- src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp | 4 ++-- .../linearAlgebra/interfaces/hypre/HypreMGR.hpp | 3 ++- .../linearAlgebra/interfaces/hypre/HypreMatrix.cpp | 6 +++--- .../linearAlgebra/interfaces/hypre/HypreMatrix.hpp | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp b/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp index ebdc0d096a9..05f7012450f 100644 --- a/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp @@ -167,11 +167,11 @@ class MatrixBase : public virtual LinearOperator< VECTOR > return m_dofManager; } - /** + /** * @brief Compute left and right scaling vectors for diagonal scaling. * @param scaling Vector to be filled with scaling factors. */ - virtual void computeScalingVector(Vector & scaling) const = 0; + virtual void computeScalingVector( Vector & scaling ) const = 0; ///@} diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp index 71ac3e62490..81764c7210a 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp @@ -180,7 +180,8 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxRowSum( solver.ptr, 1.0 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetStrongThreshold( solver.ptr, 0.6 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) ); - //GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType( solver.ptr, hypre::getAMGInterpolationType( LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); + //GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType( solver.ptr, hypre::getAMGInterpolationType( + // LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, hypre::getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::chebyshev ) ) ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp index f2808f51d0a..bf74d3d6324 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp @@ -1464,14 +1464,14 @@ MPI_Comm HypreMatrix::comm() const return hypre_IJMatrixComm( m_ij_mat ); } -void HypreMatrix::computeScalingVector(HypreVector & scaling) const +void HypreMatrix::computeScalingVector( HypreVector & scaling ) const { GEOS_LAI_ASSERT( ready() ); // Get number of components HYPRE_Int num_tags = LvArray::integerConversion< HYPRE_Int >( - (m_dofManager->*static_cast(&geos::DofManager::numComponents))() - ); + (m_dofManager->*static_cast< integer (geos::DofManager::*)() const >(&geos::DofManager::numComponents))() + ); // Get local dof component labels array1d< HYPRE_Int > pointMarkers( numLocalRows() ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp index 672fbae498b..7a268dfda03 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp @@ -273,7 +273,7 @@ class HypreMatrix final : public virtual LinearOperator< HypreVector >, /** * @copydoc MatrixBase::computeScalingVector */ - virtual void computeScalingVector(HypreVector & scaling) const override; + virtual void computeScalingVector( HypreVector & scaling ) const override; /** * @copydoc MatrixBase::leftRightScale From d107af30725cb48469f06b8ec6736d0625994a56 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" <50467563+victorapm@users.noreply.github.com> Date: Thu, 26 Jun 2025 16:42:13 -0700 Subject: [PATCH 57/67] Update TPL tag --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 867b9a93abc..41c5047c09b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ "build": { "dockerfile": "Dockerfile", "args": { - "GEOS_TPL_TAG": "307-681" + "GEOS_TPL_TAG": "312-714" } }, "runArgs": [ From 9270a8ce9c8fef966a02a598ec531e1137181e20 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 26 Jun 2025 20:22:59 -0400 Subject: [PATCH 58/67] Add missing interfaces --- src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp | 2 +- .../linearAlgebra/interfaces/hypre/HypreVector.cpp | 2 +- .../linearAlgebra/interfaces/petsc/PetscMatrix.cpp | 6 ++++++ .../linearAlgebra/interfaces/petsc/PetscMatrix.hpp | 2 ++ .../linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp | 6 ++++++ .../linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp | 2 ++ src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp | 4 ++-- 7 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp b/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp index a821cecd4cc..cb56748fbc1 100644 --- a/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp @@ -246,7 +246,7 @@ class VectorBase * @param x Vector to multiply with. * @note This modifies the current vector in place: this[i] *= x[i] */ - virtual void pointwiseScale( Vector const & x ) = 0; + virtual void pointwiseProduct( Vector const & x ) = 0; /** * @brief Element-wise division of this vector by another vector. diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp index bf136aeb0f9..048a299c3b7 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.cpp @@ -222,7 +222,7 @@ void HypreVector::axpby( real64 const alpha, } } -void HypreVector::pointwiseScale( HypreVector const & x ) +void HypreVector::pointwiseProduct( HypreVector const & x ) { GEOS_LAI_ASSERT( ready() ); GEOS_LAI_ASSERT( x.ready() ); diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp index 1ddf73808e2..6eedec61d56 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp @@ -653,6 +653,12 @@ void PetscMatrix::leftRightScale( PetscVector const & vecLeft, GEOS_LAI_CHECK_ERROR( MatDiagonalScale( m_mat, vecLeft.unwrapped(), vecRight.unwrapped() ) ); } +void PetscMatrix::computeScalingVector( PetscVector & scaling ) +{ + GEOS_UNUSED_VAR( scaling ); + GEOS_ERROR( "Not implemented!!!" ); +} + void PetscMatrix::multiplyRAP( PetscMatrix const & R, PetscMatrix const & P, PetscMatrix & dst ) const diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp index 8c1666d4c42..cd680455679 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp @@ -255,6 +255,8 @@ class PetscMatrix final : public virtual LinearOperator< PetscVector >, virtual void leftRightScale( PetscVector const & vecLeft, PetscVector const & vecRight ) override; + virtual void computeScalingVector( PetscVector & scaling ) override; + virtual void rescaleRows( arrayView1d< globalIndex const > const & rowIndices, RowSumType const rowSumType ) override; diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp index bfd54eef9e8..38e5566c49c 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp @@ -531,6 +531,12 @@ void EpetraMatrix::leftRightScale( EpetraVector const & vecLeft, rightScale( vecRight ); } +void EpetraMatrix::computeScalingVector( EpetraVector & scaling ) +{ + GEOS_UNUSED_VAR( scaling ); + GEOS_ERROR( "Not implemented!!!" ); +} + void EpetraMatrix::transpose( EpetraMatrix & dst ) const { GEOS_LAI_ASSERT( ready() ); diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp index 501e5d03916..dfcafd98516 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp @@ -244,6 +244,8 @@ class EpetraMatrix final : public virtual LinearOperator< EpetraVector >, virtual void leftRightScale( EpetraVector const & vecLeft, EpetraVector const & vecRight ) override; + virtual void computeScalingVector( EpetraVector & scaling ) override; + virtual void rescaleRows( arrayView1d< globalIndex const > const & rowIndices, RowSumType const rowSumType ) override; diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index eedbd2363c8..fee91a2f938 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -1323,7 +1323,7 @@ void PhysicsSolverBase::solveLinearSystem( DofManager const & dofManager, matrix.computeScalingVector( m_scaling ); matrix.leftRightScale( m_scaling, m_scaling ); - rhs.pointwiseScale( m_scaling ); + rhs.pointwiseProduct( m_scaling ); // Assume the solution is zeroed out, thus no need to scale it } @@ -1373,7 +1373,7 @@ void PhysicsSolverBase::solveLinearSystem( DofManager const & dofManager, { Timer timer_setup( m_timers["linear solver scaling"] ); - solution.pointwiseScale( m_scaling ); + solution.pointwiseProduct( m_scaling ); } } From 782ff7d436cab579ae27adb9e0be8f644eec0433 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 26 Jun 2025 20:31:25 -0400 Subject: [PATCH 59/67] Scale -> Pointwise --- .../linearAlgebra/interfaces/hypre/HypreVector.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp index 66d6fe7d89a..36df6e3c40b 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreVector.hpp @@ -141,9 +141,9 @@ class HypreVector final : private VectorBase< HypreVector > real64 const beta ) override; /** - * @copydoc VectorBase::pointwiseScale + * @copydoc VectorBase::pointwiseProduct */ - virtual void pointwiseScale( HypreVector const & x ) override; + virtual void pointwiseProduct( HypreVector const & x ) override; /** * @copydoc VectorBase::pointwiseDivide From 192a4c11d6cca49c8d56c63861ab1204655e062a Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 26 Jun 2025 21:04:09 -0400 Subject: [PATCH 60/67] Compilation fixes --- .../linearAlgebra/interfaces/MatrixBase.hpp | 12 +++++------ .../interfaces/petsc/PetscMatrix.cpp | 2 +- .../interfaces/petsc/PetscMatrix.hpp | 2 +- .../interfaces/petsc/PetscVector.cpp | 21 +++++++++++++------ .../interfaces/petsc/PetscVector.hpp | 5 +++-- .../interfaces/trilinos/EpetraMatrix.cpp | 2 +- .../interfaces/trilinos/EpetraMatrix.hpp | 2 +- .../interfaces/trilinos/EpetraVector.cpp | 16 ++++++++------ .../interfaces/trilinos/EpetraVector.hpp | 5 +++-- 9 files changed, 41 insertions(+), 26 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp b/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp index 05f7012450f..42a1b8db354 100644 --- a/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp @@ -167,12 +167,6 @@ class MatrixBase : public virtual LinearOperator< VECTOR > return m_dofManager; } - /** - * @brief Compute left and right scaling vectors for diagonal scaling. - * @param scaling Vector to be filled with scaling factors. - */ - virtual void computeScalingVector( Vector & scaling ) const = 0; - ///@} /** @@ -710,6 +704,12 @@ class MatrixBase : public virtual LinearOperator< VECTOR > virtual void leftRightScale( Vector const & vecLeft, Vector const & vecRight ) = 0; + /** + * @brief Compute left and right scaling vectors for diagonal scaling. + * @param scaling Vector to be filled with scaling factors. + */ + virtual void computeScalingVector( Vector & scaling ) const = 0; + /** * @brief Rescales selected rows of matrix using row sum reciprocal as a factor. * @param rowIndices global indicies of rows to scale (all must be locally owned) diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp index 6eedec61d56..dac9a208a7c 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.cpp @@ -653,7 +653,7 @@ void PetscMatrix::leftRightScale( PetscVector const & vecLeft, GEOS_LAI_CHECK_ERROR( MatDiagonalScale( m_mat, vecLeft.unwrapped(), vecRight.unwrapped() ) ); } -void PetscMatrix::computeScalingVector( PetscVector & scaling ) +void PetscMatrix::computeScalingVector( PetscVector & scaling ) const { GEOS_UNUSED_VAR( scaling ); GEOS_ERROR( "Not implemented!!!" ); diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp index cd680455679..847ed89cc0a 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscMatrix.hpp @@ -255,7 +255,7 @@ class PetscMatrix final : public virtual LinearOperator< PetscVector >, virtual void leftRightScale( PetscVector const & vecLeft, PetscVector const & vecRight ) override; - virtual void computeScalingVector( PetscVector & scaling ) override; + virtual void computeScalingVector( PetscVector & scaling ) const override; virtual void rescaleRows( arrayView1d< globalIndex const > const & rowIndices, RowSumType const rowSumType ) override; diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.cpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.cpp index 5e2106127bf..52f9bb6ee11 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.cpp @@ -214,17 +214,26 @@ void PetscVector::axpby( real64 const alpha, } } -void PetscVector::pointwiseProduct( PetscVector const & x, - PetscVector & y ) const +void PetscVector::pointwiseProduct( PetscVector const & x ) { GEOS_LAI_ASSERT( ready() ); GEOS_LAI_ASSERT( x.ready() ); - GEOS_LAI_ASSERT( y.ready() ); + GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); GEOS_LAI_ASSERT_EQ( globalSize(), x.globalSize() ); - GEOS_LAI_ASSERT_EQ( globalSize(), y.globalSize() ); - GEOS_LAI_CHECK_ERROR( VecPointwiseMult( y.m_vec, m_vec, x.m_vec ) ); - y.touch(); + GEOS_LAI_CHECK_ERROR( VecPointwiseMult( m_vec, m_vec, x.m_vec ) ); + touch(); +} + +void PetscVector::pointwiseDivide( PetscVector const & x ) +{ + GEOS_LAI_ASSERT( ready() ); + GEOS_LAI_ASSERT( x.ready() ); + GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); + GEOS_LAI_ASSERT_EQ( globalSize(), x.globalSize() ); + + GEOS_LAI_CHECK_ERROR( VecPointwiseDivide( m_vec, m_vec, x.m_vec ) ); + touch(); } real64 PetscVector::norm1() const diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp index 16a325323ff..07c8bb87a67 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp @@ -134,8 +134,9 @@ class PetscVector final : private VectorBase< PetscVector > PetscVector const & x, real64 const beta ) override; - virtual void pointwiseProduct( PetscVector const & x, - PetscVector & y ) const override; + virtual void pointwiseProduct( PetscVector const & x ) override; + + virtual void pointwiseDivision( PetscVector const & x ) override; /** * @copydoc VectorBase::norm1 diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp index 38e5566c49c..295714afa91 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.cpp @@ -531,7 +531,7 @@ void EpetraMatrix::leftRightScale( EpetraVector const & vecLeft, rightScale( vecRight ); } -void EpetraMatrix::computeScalingVector( EpetraVector & scaling ) +void EpetraMatrix::computeScalingVector( EpetraVector & scaling ) const { GEOS_UNUSED_VAR( scaling ); GEOS_ERROR( "Not implemented!!!" ); diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp index dfcafd98516..49b30321dfe 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraMatrix.hpp @@ -244,7 +244,7 @@ class EpetraMatrix final : public virtual LinearOperator< EpetraVector >, virtual void leftRightScale( EpetraVector const & vecLeft, EpetraVector const & vecRight ) override; - virtual void computeScalingVector( EpetraVector & scaling ) override; + virtual void computeScalingVector( EpetraVector & scaling ) const override; virtual void rescaleRows( arrayView1d< globalIndex const > const & rowIndices, RowSumType const rowSumType ) override; diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp index a9285597103..8a53bdd27ea 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp @@ -187,17 +187,21 @@ void EpetraVector::axpby( real64 const alpha, touch(); } -void EpetraVector::pointwiseProduct( EpetraVector const & x, - EpetraVector & y ) const +void EpetraVector::pointwiseProduct( EpetraVector const & x ) { GEOS_LAI_ASSERT( ready() ); GEOS_LAI_ASSERT( x.ready() ); - GEOS_LAI_ASSERT( y.ready() ); + GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); GEOS_LAI_ASSERT_EQ( globalSize(), x.globalSize() ); - GEOS_LAI_ASSERT_EQ( globalSize(), y.globalSize() ); - GEOS_LAI_CHECK_ERROR( ( y.unwrapped() ).Multiply( 1.0, unwrapped(), x.unwrapped(), 0.0 ) ); - y.touch(); + GEOS_LAI_CHECK_ERROR( m_vec->Multiply( 1.0, m_vec, x.unwrapped(), 0.0 ) ); + touch(); +} + +void EpetraVector::pointwiseDivide( EpetraVector const & x ) +{ + GEOS_UNUSED_VAR( x ); + GEOS_ERROR( "Not implemented!!!" ); } real64 EpetraVector::norm1() const diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp index 0d1b394a005..5e4dd08963c 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp @@ -123,8 +123,9 @@ class EpetraVector final : private VectorBase< EpetraVector > EpetraVector const & x, real64 const beta ) override; - virtual void pointwiseProduct( EpetraVector const & x, - EpetraVector & y ) const override; + virtual void pointwiseProduct( EpetraVector const & x ) override; + + virtual void pointwiseDivision( EpetraVector const & x ) override; /** * @copydoc VectorBase::norm1 From 82a001a34cff1b4ad8e3b16685df68cf2d20b541 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 26 Jun 2025 21:08:15 -0400 Subject: [PATCH 61/67] Division -> Divide --- .../linearAlgebra/interfaces/petsc/PetscVector.hpp | 2 +- .../linearAlgebra/interfaces/trilinos/EpetraVector.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp index 07c8bb87a67..e75d3b30881 100644 --- a/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/petsc/PetscVector.hpp @@ -136,7 +136,7 @@ class PetscVector final : private VectorBase< PetscVector > virtual void pointwiseProduct( PetscVector const & x ) override; - virtual void pointwiseDivision( PetscVector const & x ) override; + virtual void pointwiseDivide( PetscVector const & x ) override; /** * @copydoc VectorBase::norm1 diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp index 5e4dd08963c..92df009dac0 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.hpp @@ -125,7 +125,7 @@ class EpetraVector final : private VectorBase< EpetraVector > virtual void pointwiseProduct( EpetraVector const & x ) override; - virtual void pointwiseDivision( EpetraVector const & x ) override; + virtual void pointwiseDivide( EpetraVector const & x ) override; /** * @copydoc VectorBase::norm1 From a2388efc3a0282b935029d3c5cbf258c37ef93e1 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 26 Jun 2025 21:15:29 -0400 Subject: [PATCH 62/67] Update doc --- src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp index bb0907b9941..0e981ce804f 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.hpp @@ -1030,7 +1030,7 @@ class PhysicsSolverBase : public ExecutableGroup /// System solution vector ParallelVector m_solution; - /// Scaling vector (A^\tilde = D * A * D, b^\tilde = D * b, x = D * x^\tilde) + /// Diagonal scaling vector D (Ahat = D * A * D, bhat = D * b, x = D * xhat) ParallelVector m_scaling; /// Flag to decide whether to apply physics-based scaling to the linear system From 0e76fa1f3fe9b4e2bceb4e2a1483163aec7830d9 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 26 Jun 2025 21:59:26 -0400 Subject: [PATCH 63/67] Compilation fix --- .../linearAlgebra/interfaces/trilinos/EpetraVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp index 8a53bdd27ea..f5e0b213d26 100644 --- a/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/trilinos/EpetraVector.cpp @@ -194,7 +194,7 @@ void EpetraVector::pointwiseProduct( EpetraVector const & x ) GEOS_LAI_ASSERT_EQ( localSize(), x.localSize() ); GEOS_LAI_ASSERT_EQ( globalSize(), x.globalSize() ); - GEOS_LAI_CHECK_ERROR( m_vec->Multiply( 1.0, m_vec, x.unwrapped(), 0.0 ) ); + GEOS_LAI_CHECK_ERROR( ( unwrapped() ).Multiply( 1.0, unwrapped(), x.unwrapped(), 0.0 ) ); touch(); } From 53ea0dc47d32fc66322ae3324dd8aa0c918fc174 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" <50467563+victorapm@users.noreply.github.com> Date: Thu, 26 Jun 2025 21:35:33 -0700 Subject: [PATCH 64/67] Update TPL tag --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 41c5047c09b..3942e66fa09 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ "build": { "dockerfile": "Dockerfile", "args": { - "GEOS_TPL_TAG": "312-714" + "GEOS_TPL_TAG": "312-715" } }, "runArgs": [ From 7a2a849f81a44293ba1bdbc64875ff09f1ed4695 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 27 Jun 2025 06:41:08 -0400 Subject: [PATCH 65/67] Enable physics scaling by default only with hypre --- .../physicsSolvers/PhysicsSolverBase.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index fee91a2f938..e720ff2ec8f 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -43,12 +43,19 @@ PhysicsSolverBase::PhysicsSolverBase( string const & name, m_nextDt( 1e99 ), m_numTimestepsSinceLastDtCut( -1 ), m_dofManager( name ), - m_usePhysicsScaling( 1 ), + m_usePhysicsScaling(), m_linearSolverParameters( groupKeyStruct::linearSolverParametersString(), this ), m_nonlinearSolverParameters( groupKeyStruct::nonlinearSolverParametersString(), this ), m_solverStatistics( groupKeyStruct::solverStatisticsString(), this ), m_systemSetupTimestamp( 0 ) { + // Physics-scaling is enabled by default only with hypre builds +#ifdef (GEOS_USE_HYPRE) + integer usePhysicsScaling = 1; +#else + integer usePhysicsScaling = 0; +#endif + setInputFlags( InputFlags::OPTIONAL_NONUNIQUE ); // This sets a flag to indicate that this object is going to select the time step size @@ -95,7 +102,7 @@ PhysicsSolverBase::PhysicsSolverBase( string const & name, setDescription( "Cut time step if linear solution fail without going until max nonlinear iterations." ); registerWrapper( viewKeyStruct::usePhysicsScalingString(), &m_usePhysicsScaling ) - .setApplyDefaultValue( 1 ) + .setApplyDefaultValue( usePhysicsScaling ) .setInputFlag( InputFlags::OPTIONAL ) .setDescription( "Enable physics-based scaling of the linear system. Default: true." ); From 12b9c931da73e60b9e90913f2fca7646760fe044 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 27 Jun 2025 06:50:23 -0400 Subject: [PATCH 66/67] Fix --- src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp index e720ff2ec8f..4dfc1210de5 100644 --- a/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/PhysicsSolverBase.cpp @@ -50,7 +50,7 @@ PhysicsSolverBase::PhysicsSolverBase( string const & name, m_systemSetupTimestamp( 0 ) { // Physics-scaling is enabled by default only with hypre builds -#ifdef (GEOS_USE_HYPRE) +#ifdef GEOS_USE_HYPRE integer usePhysicsScaling = 1; #else integer usePhysicsScaling = 0; From 1087d668702f2185f451f9504f073e17ca912d82 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" <50467563+victorapm@users.noreply.github.com> Date: Fri, 27 Jun 2025 14:39:12 -0700 Subject: [PATCH 67/67] Update TPL tag --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3942e66fa09..9853a691e0c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ "build": { "dockerfile": "Dockerfile", "args": { - "GEOS_TPL_TAG": "312-715" + "GEOS_TPL_TAG": "312-717" } }, "runArgs": [