Update PAPI SDE to support new versions

.gitlab-ci.yml

@@ -113,6 +113,7 @@ build/cuda101/nompi/clang/cuda_wo_omp/release/shared:
     CUDA_ARCH: 35
 
 # Job with example runs.
+# Also explicitly test PAPI SDE
 build/cuda101/openmpi/gcc/all/debug/shared:
   extends:
     - .build_template
@@ -126,6 +127,7 @@ build/cuda101/openmpi/gcc/all/debug/shared:
     MPI_AS_ROOT: "ON"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Debug"
+    BUILD_PAPI_SDE: "ON"
     RUN_EXAMPLES: "ON"
     CUDA_ARCH: 35
 

.gitlab/scripts.yml

@@ -43,6 +43,7 @@
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
         -DGINKGO_BUILD_MPI=${BUILD_MPI} ${MPI_STR}
         -DGINKGO_BUILD_HWLOC=${BUILD_HWLOC}
+        -DGINKGO_BUILD_PAPI_SDE=${BUILD_PAPI_SDE}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_FAST_TESTS=${FAST_TESTS}
         -DGINKGO_TEST_NONDEFAULT_STREAM=${NONDEFAULT_STREAM}
@@ -87,6 +88,7 @@
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
         -DGINKGO_BUILD_MPI=${BUILD_MPI} ${MPI_STR}
         -DGINKGO_BUILD_HWLOC=${BUILD_HWLOC}
+        -DGINKGO_BUILD_PAPI_SDE=${BUILD_PAPI_SDE}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_FAST_TESTS=${FAST_TESTS}
         -DGINKGO_MIXED_PRECISION=${MIXED_PRECISION}

.gitlab/variables.yml

@@ -12,6 +12,7 @@
     BUILD_CUDA: "OFF"
     BUILD_HIP: "OFF"
     BUILD_HWLOC: "ON"
+    BUILD_PAPI_SDE: "OFF"
     BUILD_MPI: "OFF"
     GKO_COMPILER_FLAGS: ""
     MPI_AS_ROOT: "OFF"

CMakeLists.txt

@@ -9,6 +9,7 @@ include(cmake/hip_path.cmake)
 include(cmake/autodetect_executors.cmake)
 
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules/")
+include(cmake/autodetect_system_libs.cmake)
 
 # Ginkgo configuration options
 option(GINKGO_DEVEL_TOOLS "Add development tools to the build system" OFF)
@@ -77,8 +78,9 @@ option(BUILD_SHARED_LIBS "Build shared (.so, .dylib, .dll) libraries" ON)
 if(MSVC OR WIN32 OR CYGWIN OR APPLE)
     option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is OFF. Ginkgo does not support HWLOC on Windows/MacOS" OFF)
 else()
-    option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is ON. If a system HWLOC is not found, then we try to build it ourselves. Switch this OFF to disable HWLOC." ON)
+    option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Enabled if a system installation is found." ${HWLOC_FOUND})
 endif()
+option(GINKGO_BUILD_PAPI_SDE "Build Ginkgo with PAPI SDE. Enabled if a system installation is found." ${PAPI_SDE_FOUND})
 option(GINKGO_DPCPP_SINGLE_MODE "Do not compile double kernels for the DPC++ backend." OFF)
 option(GINKGO_INSTALL_RPATH "Set the RPATH when installing its libraries." ON)
 option(GINKGO_INSTALL_RPATH_ORIGIN "Add $ORIGIN (Linux) or @loader_path (MacOS) to the installation RPATH." ON)
@@ -189,14 +191,6 @@ endif()
 include(CheckIncludeFileCXX)
 check_include_file_cxx(cxxabi.h GKO_HAVE_CXXABI_H)
 
-# Automatically find PAPI and search for the required 'sde' component
-set(GINKGO_HAVE_PAPI_SDE 0)
-# PAPI is temporarily disabled
-#find_package(PAPI OPTIONAL_COMPONENTS sde)
-#if(PAPI_sde_FOUND)
-#    set(GINKGO_HAVE_PAPI_SDE 1)
-#endif()
-
 # Automatically find TAU
 set(GINKGO_HAVE_TAU 0)
 find_package(PerfStubs QUIET)
@@ -226,12 +220,6 @@ if(GINKGO_BUILD_HWLOC AND (MSVC OR WIN32 OR CYGWIN OR APPLE))
     set(GINKGO_BUILD_HWLOC OFF CACHE BOOL "Build Ginkgo with HWLOC. Default is OFF. Ginkgo does not support HWLOC on Windows/MacOS" FORCE)
     message(WARNING "Ginkgo does not support HWLOC on Windows/MacOS, switch GINKGO_BUILD_HWLOC to OFF")
 endif()
-if(GINKGO_BUILD_HWLOC)
-    set(GINKGO_HAVE_HWLOC 1)
-else()
-    set(GINKGO_HAVE_HWLOC 0)
-    message(STATUS "HWLOC is being forcibly switched off")
-endif()
 
 set(GINKGO_HAVE_GPU_AWARE_MPI OFF)
 set(GINKGO_FORCE_SPMV_BLOCKING_COMM OFF)
@@ -261,17 +249,38 @@ if(GINKGO_BUILD_MPI)
 endif()
 
 # Try to find the third party packages before using our subdirectories
-include(cmake/package_helpers.cmake)
 if(GINKGO_BUILD_TESTS)
     find_package(GTest 1.10.0) # No need for QUIET as CMake ships FindGTest
 endif()
 if(GINKGO_BUILD_BENCHMARKS)
     find_package(gflags 2.2.2 QUIET)
     find_package(RapidJSON 1.1.0 QUIET)
 endif()
+
+# System provided, third party libraries (not bundled!)
+set(GINKGO_HAVE_HWLOC 0)
 if(GINKGO_BUILD_HWLOC)
-    find_package(HWLOC 2.1) # No need for QUIET as we ship FindHWLOC
+    find_package(HWLOC 2.1)
+    if (HWLOC_FOUND)
+        set(GINKGO_HAVE_HWLOC 1)
+    else()
+        message(WARNING "HWLOC could not be found. HWLOC support will be disabled.")
+        set(GINKGO_BUILD_HWLOC OFF CACHE BOOL "HWLOC support was disabled because a system package could not be found." FORCE)
+    endif()
 endif()
+
+set(GINKGO_HAVE_PAPI_SDE 0)
+if(GINKGO_BUILD_PAPI_SDE)
+    find_package(PAPI 7.0.1.0 COMPONENTS sde)
+    if (PAPI_SDE_FOUND)
+        set(GINKGO_HAVE_PAPI_SDE 1)
+    else()
+        message(WARNING "PAPI (SDE) could not be found. PAPI_SDE support will be disabled.")
+        set(GINKGO_BUILD_PAPI_SDE OFF CACHE BOOL "PAPI_SDE support was disabled because a system package could not be found." FORCE)
+    endif()
+endif()
+
+# Bundled third party libraries
 add_subdirectory(third_party)    # Third-party tools and libraries
 
 if(MSVC)

cmake/GinkgoConfig.cmake.in

@@ -143,7 +143,7 @@ set(VTune_PATH "@VTune_PATH@")
 #     so `third_party` libraries are currently unneeded.
 
 if(GINKGO_HAVE_PAPI_SDE)
-    find_package(PAPI REQUIRED OPTIONAL_COMPONENTS sde)
+    find_package(PAPI REQUIRED COMPONENTS sde)
 endif()
 
 if(GINKGO_HAVE_HWLOC)

cmake/Modules/FindPAPI.cmake

@@ -57,6 +57,7 @@ if(NOT PAPI_LIBRARY)
     select_library_configurations(PAPI)
 endif()
 
+set(WORK_DIR "${PROJECT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/FindPAPI")
 if(PAPI_INCLUDE_DIR)
     if(EXISTS "${PAPI_INCLUDE_DIR}/papi.h")
         file(STRINGS "${PAPI_INCLUDE_DIR}/papi.h" papi_version_str REGEX "^#define[\t ]+PAPI_VERSION[\t ]+.*")
@@ -70,25 +71,28 @@ if(PAPI_INCLUDE_DIR)
         # find the components
         enable_language(C)
         foreach(component IN LISTS PAPI_FIND_COMPONENTS)
-            file(WRITE "${PROJECT_BINARY_DIR}/papi_${component}_detect.c"
+            set(SRC_FILE "${WORK_DIR}/papi_${component}_detect.c")
+            set(BIN_FILE "${WORK_DIR}/papi_${component}_detect.bin")
+            file(WRITE "${SRC_FILE}"
                 "
                 #include <papi.h>
                 int main() {
                  int retval;
                  retval = PAPI_library_init(PAPI_VER_CURRENT);
                    if (retval != PAPI_VER_CURRENT && retval > 0)
                     return -1;
-                   if (PAPI_get_component_index(\"${component}\") < 0)
+                   if (PAPI_get_component_index(\"${component}\") == PAPI_ENOCMP)
                     return 0;
                    return 1;
                 }"
                 )
             try_run(PAPI_${component}_FOUND
                 gko_result_unused
-                "${PROJECT_BINARY_DIR}"
-                "${PROJECT_BINARY_DIR}/papi_${component}_detect.c"
+                "${WORK_DIR}"
+                "${SRC_FILE}"
                 CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${PAPI_INCLUDE_DIR}
                 LINK_LIBRARIES ${PAPI_LIBRARY}
+                COPY_FILE ${BIN_FILE}
                 )
 
             if (NOT PAPI_${component}_FOUND EQUAL 1)
@@ -105,6 +109,33 @@ find_package_handle_standard_args(PAPI
                                   VERSION_VAR PAPI_VERSION_STRING
                                   HANDLE_COMPONENTS)
 
+if(PAPI_sde_FOUND)
+    # PAPI SDE is another library and header, let's try to find them
+    find_path(PAPI_SDE_INCLUDE_DIR NAMES sde_lib.h)
+    mark_as_advanced(PAPI_SDE_INCLUDE_DIR)
+
+    if(NOT PAPI_SDE_LIBRARY)
+        find_library(PAPI_SDE_LIBRARY_RELEASE NAMES
+            sde
+        )
+        mark_as_advanced(PAPI_SDE_LIBRARY_RELEASE)
+
+        find_library(PAPI_SDE_LIBRARY_DEBUG NAMES
+            sded
+            sde-d
+        )
+        mark_as_advanced(PAPI_SDE_LIBRARY_DEBUG)
+
+        include(SelectLibraryConfigurations)
+        select_library_configurations(PAPI_SDE)
+    endif()
+
+    # FIXME: with CMake>=3.17, use NAME_MISMATCHED to get rid of the warning
+    find_package_handle_standard_args(PAPI_SDE
+        REQUIRED_VARS PAPI_SDE_LIBRARY PAPI_SDE_INCLUDE_DIR
+        VERSION_VAR PAPI_VERSION_STRING)
+endif()
+
 if(PAPI_FOUND)
     set(PAPI_LIBRARIES ${PAPI_LIBRARY})
     set(PAPI_INCLUDE_DIRS ${PAPI_INCLUDE_DIR})
@@ -142,3 +173,41 @@ if(PAPI_FOUND)
         endif()
     endif()
 endif()
+
+if (PAPI_SDE_FOUND AND NOT TARGET PAPI::PAPI_SDE)
+    set(PAPI_SDE_LIBRARIES ${PAPI_SDE_LIBRARY})
+    set(PAPI_SDE_INCLUDE_DIRS ${PAPI_SDE_INCLUDE_DIR})
+    unset(PAPI_SDE_LIBRARY)
+    unset(PAPI_SDE_INCLUDE_DIR)
+
+    if(NOT TARGET PAPI::PAPI_SDE)
+        add_library(PAPI::PAPI_SDE UNKNOWN IMPORTED)
+        set_target_properties(PAPI::PAPI_SDE PROPERTIES
+            INTERFACE_INCLUDE_DIRECTORIES "${PAPI_SDE_INCLUDE_DIRS}")
+
+        if(EXISTS "${PAPI_SDE_LIBRARIES}")
+            set_target_properties(PAPI::PAPI_SDE PROPERTIES
+                IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+                INTERFACE_LINK_LIBRARIES "${PAPI_SDE_LIBRARIES}"
+                IMPORTED_LOCATION "${PAPI_SDE_LIBRARIES}")
+        endif()
+        if(PAPI_SDE_LIBRARY_RELEASE)
+            set_property(TARGET PAPI::PAPI_SDE APPEND PROPERTY
+                IMPORTED_CONFIGURATIONS RELEASE)
+            set_target_properties(PAPI::PAPI_SDE PROPERTIES
+                IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+                INTERFACE_LINK_LIBRARIES_RELEASE "${PAPI_SDE_LIBRARY_RELEASE}"
+                IMPORTED_LOCATION_RELEASE "${PAPI_SDE_LIBRARY_RELEASE}")
+            unset(PAPI_SDE_LIBRARY_RELEASE)
+        endif()
+        if(PAPI_SDE_LIBRARY_DEBUG)
+            set_property(TARGET PAPI::PAPI_SDE APPEND PROPERTY
+                IMPORTED_CONFIGURATIONS DEBUG)
+            set_target_properties(PAPI::PAPI_SDE PROPERTIES
+                IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+                INTERFACE_LINK_LIBRARIES_DEBUG "${PAPI_SDE_LIBRARY_DEBUG}"
+                IMPORTED_LOCATION_DEBUG "${PAPI_SDE_LIBRARY_DEBUG}")
+            unset(PAPI_SDE_LIBRARY_DEBUG)
+        endif()
+    endif()
+endif()

cmake/autodetect_system_libs.cmake

@@ -0,0 +1,7 @@
+if (NOT DEFINED GINKGO_BUILD_HWLOC)
+    find_package(HWLOC 2.1)
+endif()
+
+if (NOT DEFINED GINKGO_BUILD_PAPI_SDE)
+    find_package(PAPI 7.0.1.0 COMPONENTS sde)
+endif()

cmake/get_info.cmake

@@ -190,16 +190,21 @@ ginkgo_print_module_footer(${detailed_log} "")
 
 ginkgo_print_generic_header(${minimal_log} "  Components:")
 ginkgo_print_generic_header(${detailed_log} "  Components:")
-if(PAPI_sde_FOUND)
+ginkgo_print_variable(${minimal_log} "GINKGO_BUILD_PAPI_SDE")
+ginkgo_print_variable(${detailed_log} "GINKGO_BUILD_PAPI_SDE")
+if(TARGET PAPI::PAPI)
     ginkgo_print_variable(${detailed_log} "PAPI_VERSION")
     ginkgo_print_variable(${detailed_log} "PAPI_INCLUDE_DIR")
     ginkgo_print_flags(${detailed_log} "PAPI_LIBRARY")
 endif()
+
 ginkgo_print_variable(${minimal_log} "GINKGO_BUILD_HWLOC")
 ginkgo_print_variable(${detailed_log} "GINKGO_BUILD_HWLOC")
-ginkgo_print_variable(${detailed_log} "HWLOC_VERSION")
-ginkgo_print_variable(${detailed_log} "HWLOC_LIBRARIES")
-ginkgo_print_variable(${detailed_log} "HWLOC_INCLUDE_DIRS")
+if(TARGET hwloc)
+    ginkgo_print_variable(${detailed_log} "HWLOC_VERSION")
+    ginkgo_print_variable(${detailed_log} "HWLOC_LIBRARIES")
+    ginkgo_print_variable(${detailed_log} "HWLOC_INCLUDE_DIRS")
+endif()
 
 _minimal(
     "

cmake/information_helpers.cmake

@@ -78,11 +78,7 @@ macro(ginkgo_interface_libraries_recursively INTERFACE_LIBS)
             list(TRANSFORM GINKGO_LIBS_INTERFACE_LIBS REPLACE "\\$<LINK_ONLY:(.*)>" "\\1")
             ginkgo_interface_libraries_recursively("${GINKGO_LIBS_INTERFACE_LIBS}")
         elseif(EXISTS "${_libs}")
-            if ("${_libs}" MATCHES "${PROJECT_BINARY_DIR}.*hwloc.so")
-                list(APPEND GINKGO_INTERFACE_LIBS_FOUND "${CMAKE_INSTALL_FULL_LIBDIR}/libhwloc.so")
-            else()
-                list(APPEND GINKGO_INTERFACE_LIBS_FOUND "${_libs}")
-            endif()
+            list(APPEND GINKGO_INTERFACE_LIBS_FOUND "${_libs}")
         elseif("${_libs}" STREQUAL "${CMAKE_DL_LIBS}")
             list(APPEND GINKGO_INTERFACE_LIBS_FOUND "-l${_libs}")
         endif()

cmake/install_helpers.cmake

@@ -30,10 +30,6 @@ function(ginkgo_add_install_rpath name)
     endif()
     if (GINKGO_INSTALL_RPATH_DEPENDENCIES)
         set(RPATH_DEPENDENCIES "${ARGN}")
-        if(GINKGO_HAVE_HWLOC AND HWLOC_FOUND)
-            get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)
-            list(APPEND RPATH_DEPENDENCIES "${HWLOC_LIBRARIES}")
-        endif()
     endif()
     if (GINKGO_INSTALL_RPATH)
         set_property(TARGET "${name}" PROPERTY INSTALL_RPATH
@@ -80,11 +76,6 @@ function(ginkgo_install)
     install(FILES "${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp"
         DESTINATION "${CMAKE_INSTALL_FULL_INCLUDEDIR}/ginkgo"
         )
-    if (GINKGO_HAVE_PAPI_SDE)
-        install(FILES "${Ginkgo_SOURCE_DIR}/third_party/papi_sde/papi_sde_interface.h"
-            DESTINATION "${CMAKE_INSTALL_FULL_INCLUDEDIR}/third_party/papi_sde"
-            )
-    endif()
 
     if  (GINKGO_HAVE_HWLOC AND NOT HWLOC_FOUND)
         get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)