Add optional singularity filesystem encryption

docs/developer/containers.rst

@@ -59,7 +59,7 @@ whether they break the image.
 Singularity usage
 -----------------
 
-Singularity images are like docker images (https://sylabs.io/guides/3.0/user-guide/quick_start.html) that work better 
+Singularity images are like docker images (https://sylabs.io/guides/3.0/user-guide/quick_start.html) that work better
 in HPC environments (https://singularity-tutorial.github.io/).
 The latest image can be downloaded from the GitHub Container registry. It should be downloaded to a `libexec` folder
 under :git_url:`singularity` with the name `kqclib`. This is performed as follows at the root of the repo::
@@ -70,6 +70,11 @@ The image can also be built manually in the :git_url:`singularity` folder by run
 
    ./singularity.sh
 
+.. note::
+    If a `singularity.pem` RSA public key is present in the `singularity` folder then the image will be encrypted. To
+    successfully use this image the user also needs the corresponding `$HOME/singularity_private.pem` private key. See
+    the `Singularity docs <https://docs.sylabs.io/guides/3.4/user-guide/encryption.html>`_ for further details.
+
 After pulling or building, you can now run (again, in the :git_url:`singularity` folder)::
 
    ./create_links.sh

docs/user_guide/simulation/elmer_remote_workflow.rst

@@ -7,7 +7,7 @@ Setup:
 * Build and push singularity image to the remote using
     * `kqc singularity --build`: Compile singularity image
        The desired package versions such as OpenMPI can be chosen by modifying
-       the definitions in the beginning of `KQCircuits/singularity/install_kqcircuits.sh` script
+       the definitions in the beginning of `KQCircuits/singularity/install_software.sh` script
        (**NOTE that MPI implementation and its version has to match those of the remote machine!**).
     * `kqc singularity --push user@host`: Send singularity image to remote and setup symbolic links. 
        You can directory choose the remote directory by specifying `--singularity-remote-path SINGULARITY_PATH`.

klayout_package/python/console_scripts/run.py

@@ -120,14 +120,14 @@ def run():
     elif args.command == "singularity":
         if args.build:
             chdir(Path(ROOT_PATH / "singularity"))
-            with open('install_kqcircuits.sh','r') as f:
+            with open('install_software.sh','r') as f:
                 for line in f:
                     if 'export MPI_VERSION=' in line:
                         mpi_v = line.partition('export MPI_VERSION=')[2].strip()
                         print((f"Singularity will use MPI version {mpi_v}. "
                                 "Make sure this corresponds to the MPI version on the target machine\n"
                                 "MPI and other package versions used by singularity can be changed "
-                                "in the beginning of the /singularity/install_kqcircuits.sh script"))
+                                "in the beginning of the /singularity/install_software.sh script"))
                         break
             subprocess.call("./singularity.sh", shell=True)
         elif args.push is not None:

klayout_package/python/kqcircuits/util/export_helper.py

@@ -189,15 +189,15 @@ def write_export_machine_versions_file(path: Path):
 
 def open_with_klayout_or_default_application(filepath):
     """
-    Tries to open file with Klayout. If Klayout is not found, opens file with operating system's default application.
+    Tries to open file with KLayout. If KLayout is not found, opens file with operating system's default application.
     Implementation supports Windows, macOS, and Linux.
     """
     if argv[-1] == "-q":  # quiet mode, do not run viewer
         return
 
     exe = klayout_executable_command()
     if not exe:
-        logging.warning("Klayout executable not found.")
+        logging.warning("KLayout executable not found.")
     else:
         subprocess.call((exe, filepath))
 

singularity/create_links.sh

@@ -1,12 +1,13 @@
 #!/bin/bash
 
 function create_link () {
-	ln -sfv /"$PWD"/libexec/kqclib.sh bin/"$1" 
+	ln -sfv "$PWD"/libexec/kqclib.sh bin/"$1"
 }
 
 EXECUTABLES=("ElmerSolver" "ElmerSolver_mpi" "ElmerGrid" "klayout" "kqclib" "paraview" "python")
 
 echo "Creating symbolic links to the singularity image software"
+mkdir -p bin
 for EXECUTABLE in "${EXECUTABLES[@]}"
 do
 	create_link "$EXECUTABLE"
@@ -15,16 +16,16 @@ done
 # move python away from bin such that it does not over-ride the system python if path is set to $PWD/bin
 mv -v bin/python .
 
-echo 
+echo
 echo "Among other executables, the image contains the following executables that are needed for the simulation workflow:"
 echo "EXECUTABLES=(\"ElmerSolver\" \"ElmerSolver_mpi\" \"ElmerGrid\" \"klayout\" \"kqclib\" \"paraview\" \"python\")"
-echo 
+echo
 echo "You could add your own executable in the list in ./create_links.sh (it is just a symbolic link named like the"
 echo "executable that then needs to be found in the image)."
 echo "Remember to add $PWD/bin to your PATH environment variable."
-echo 
+echo
 echo "You can now prepare KQC simulations using the image:"
-echo "For example go to ../klayout_package/python/scripts/simulations/" and run 
+echo "For example go to ../klayout_package/python/scripts/simulations/" and run
 echo
 echo "'kqclib waveguides_sim_compare.py'"
 echo
@@ -49,6 +50,8 @@ echo
 echo "The simulation scripts are then prepared in a subfolder (for example \$KQC_TMP_PATH/waveguides_sim_elmer in the affore mentioned example)."
 echo "\$KQC_TMP_PATH folder is normally in ../tmp/, remember to set it! If you do not, you might get a read-only error when the singularity image tries to write to the image tmp folder that is *read-only*"
 echo
+echo "You will also likely need 'export DISPLAY=:0.0' to run GUI applications like KLayout or paraview"
+echo
 echo "In order to run the actual simulations:"
 echo "Go to the folder and run ./simulation.sh"
 

singularity/install_kqcircuits.sh → singularity/install_software.sh

@@ -8,8 +8,6 @@ define_variables () {
     export PYTHON_VERSION="3.10.11"
     export OPENSSL_VERSION="1.1.1t"
     # KLayout version
-    export KL_FILE="klayout-0.28.8-0.x86_64.rpm"
-    export KL_HASH="8e5083035046f806d16072f3203ad782"
     # Target CPU architecture for compilation, see https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html
     # Examples include `znver3` for AMD Zen 3, and `alderlake` for Intel 12th gen.
     #export MARCH="znver2"
@@ -52,31 +50,28 @@ install_yum_packages () {
 }
 
 install_deb_packages () {
-    echo "deb http://fi.archive.ubuntu.com/ubuntu/ jammy main universe multiverse" > /etc/apt/sources.list
-    apt-get update && apt-get install -y apt-utils && apt-get upgrade -y && \
-    DEBIAN_FRONTEND=noninteractive apt install -y tzdata && \ # TODO: region data can be setup non-interactively
-    apt-get install -y wget xvfb python-is-python3 python3-pip git libcurl4 libglu1-mesa libxft-dev && \
-    apt-get install -y paraview && \
-    wget -q https://www.klayout.org/downloads/Ubuntu-20/klayout_0.27.3-1_amd64.deb && \
-    echo "5779feaecc03d85e31b174088b6232c7  klayout_0.27.3-1_amd64.deb" > klayout.md5 && \
-    md5sum --check klayout.md5  && \
-    apt-get install -y ./klayout_0.27.3-1_amd64.deb && \
-    apt-get clean -y && rm -rf /var/lib/apt/lists/* ./klayout* && apt-get clean && \
-    python -m pip install --upgrade pip && \
+    KL_FILE="klayout_0.28.11-1_amd64.deb"
+    KL_HASH="78ea7cb55a1f21353e1b8ed5bf30a8ee"
+
+    apt update ; apt install -y apt-utils; apt upgrade -y
+    DEBIAN_FRONTEND=noninteractive apt install -y tzdata
+    apt install -y wget python-is-python3 python3-pip git libcurl4 libglu1-mesa libxft-dev
+    apt install -y libopenblas-dev m4 libhdf5-dev gfortran paraview build-essential cmake
+
+    wget -q https://www.klayout.org/downloads/Ubuntu-22/$KL_FILE
+    echo "$KL_HASH  $KL_FILE" > klayout.md5
+    md5sum --check klayout.md5 || exit
+    apt install -y ./$KL_FILE
+    apt clean -y
+    rm -rf /var/lib/apt/lists/* ./klayout*
+
+    python -m pip install --upgrade pip
     rm -rf /usr/lib/python3/dist-packages/klayout /usr/lib/python3/dist-packages/klayout.egg-info
-    git clone https://github.com/iqm-finland/KQCircuits.git && cd KQCircuits && \
-    python -m pip install -e klayout_package/python && \
-    python -m pip install gmsh pandas && \
-    ln -sf /usr/bin/python /usr/bin/kqclib && \
-    apt-get install -y libopenblas-dev && \
-    apt-get install -y m4 && \
-    #apt-get install -y libcurl-dev && \
-    apt-get install -y libhdf5-dev && \
-    apt-get install -y gfortran && \
-    apt-get install -y build-essential && \
-    apt-get install -y cmake && \
-    apt-get install -y wget && \
-    apt-get install -y git
+    git clone https://github.com/iqm-finland/KQCircuits.git
+    cd KQCircuits || exit
+    python -m pip install -e klayout_package/python
+    python -m pip install gmsh pandas
+    ln -sf /usr/bin/python /usr/bin/kqclib
 }
 
 ######################
@@ -86,8 +81,8 @@ install_deb_packages () {
 
 # Install mpich
 compile_mpich () {
-    
-    wget -qO- http://www.mpich.org/static/downloads/$MPI_VERSION/mpich-$MPI_VERSION.tar.gz | tar xvz
+
+    wget -qO- "http://www.mpich.org/static/downloads/$MPI_VERSION/mpich-$MPI_VERSION.tar.gz" | tar xvz
     cd $MPI_VERSION || exit
 
     ./configure --enable-fast=all,O3 --prefix=/usr FFLAGS="-std=legacy" FCFLAGS="-std=legacy"
@@ -100,8 +95,8 @@ compile_mpich () {
 compile_openmpi () {
     MPI_VERSION_FOLDER="v${MPI_VERSION%.*}"
     MPI_TARBALL="openmpi-$MPI_VERSION.tar.gz"
-    wget -qO- https://download.open-mpi.org/release/open-mpi/$MPI_VERSION_FOLDER/$MPI_TARBALL | tar xvz
-    cd openmpi-$MPI_VERSION || exit
+    wget -qO- "https://download.open-mpi.org/release/open-mpi/$MPI_VERSION_FOLDER/$MPI_TARBALL" | tar xvz
+    cd "openmpi-$MPI_VERSION" || exit
 
     ./configure --enable-fast=all,O3 --prefix=/usr FFLAGS="-std=legacy" FCFLAGS="-std=legacy"
 
@@ -122,12 +117,12 @@ compile_mpi () {
         echo "TARGET_MPI environment variable is not found! Choosing 'openmpi'."
         TARGET_MPI='openmpi'
     fi
-    
+
     if [[ "${TARGET_MPI}" = "mpich" ]]; then
-        echo "TARGET_MPI environment variable is set to 'mpich'. If you need a specific version, you need to modify install_kqcircuits.sh:compile_mpich()"
+        echo "TARGET_MPI environment variable is set to 'mpich'. If you need a specific version, you need to modify install_software.sh:compile_mpich()"
         compile_mpich
     elif [[ "${TARGET_MPI}" = "openmpi" ]]; then
-        echo "TARGET_MPI environment variable is set to 'openmpi'. If you need a specific version, you need to modify install_kqcircuits.sh:compile_openmpi()"
+        echo "TARGET_MPI environment variable is set to 'openmpi'. If you need a specific version, you need to modify install_software.sh:compile_openmpi()"
         compile_openmpi
     else
         echo "TARGET_MPI environment variable is neither 'mpich' nor 'openmpi'! Choosing 'openmpi'."
@@ -149,7 +144,8 @@ compile_netcdf () {
     cd netcdf-fortran/ || exit
     mkdir build
     cd build/ || exit
-    cmake -DCMAKE_INSTALL_PREFIX=/opt/netcdf   -DENABLE_HDF5:BOOL=FALSE -DUSE_HDF5:BOOL=FALSE -DCMAKE_Fortran_FLAGS="-std=legacy -O3 -fopenmp -funroll-loops" -DCMAKE_C_FLAGS="-O3 -fopenmp -funroll-loops"  ..
+    cmake -DCMAKE_INSTALL_PREFIX=/opt/netcdf -DENABLE_HDF5:BOOL=FALSE -DUSE_HDF5:BOOL=FALSE \
+          -DCMAKE_Fortran_FLAGS="-std=legacy -O3 -fopenmp -funroll-loops" -DCMAKE_C_FLAGS="-O3 -fopenmp -funroll-loops"  ..
     make -j "$(nproc)"
     make install
     cd /opt/src || exit
@@ -173,7 +169,8 @@ compile_scalapack () {
     cd scalapack || exit
     mkdir build
     cd build || exit
-    cmake -DCMAKE_INSTALL_PREFIX=/opt/scalapack  -DBUILD_SHARED_LIBS=ON -DCMAKE_C_FLAGS="-O3 -fopenmp -funroll-loops" -DCMAKE_Fortran_FLAGS="-O3 -fPIC -funroll-loops" ..
+    cmake -DCMAKE_INSTALL_PREFIX=/opt/scalapack  -DBUILD_SHARED_LIBS=ON -DCMAKE_C_FLAGS="-O3 -fopenmp -funroll-loops" \
+          -DCMAKE_Fortran_FLAGS="-O3 -fPIC -funroll-loops" ..
     make -j "$(nproc)" install
     cd /opt/src || exit
     #rm -rf scalapack
@@ -186,7 +183,7 @@ compile_MUMPS () {
     cd MUMPS_5.6.0 || exit
     cp /opt/Makefile.inc ./
     # this hack is needed because include directory is again wrongly set up
-    ln -sf /usr/include /include 
+    ln -sf /usr/include /include
     make -j "$(nproc)"
     mkdir /opt/mumps
     mv lib /opt/mumps
@@ -204,7 +201,8 @@ compile_mmg_and_parmmg () {
     git checkout develop
     mkdir build
     cd build || exit
-    cmake -DCMAKE_INSTALL_PREFIX="/opt/mmg" -D CMAKE_BUILD_TYPE=RelWithDebInfo -D BUILD_SHARED_LIBS:BOOL=TRUE -D MMG_INSTALL_PRIVATE_HEADERS=ON -D CMAKE_C_FLAGS="-fPIC  -g" -D CMAKE_CXX_FLAGS="-fPIC -std=c++11 -g"  ..
+    cmake -DCMAKE_INSTALL_PREFIX="/opt/mmg" -D CMAKE_BUILD_TYPE=RelWithDebInfo -D BUILD_SHARED_LIBS:BOOL=TRUE \
+          -D MMG_INSTALL_PRIVATE_HEADERS=ON -D CMAKE_C_FLAGS="-fPIC  -g" -D CMAKE_CXX_FLAGS="-fPIC -std=c++11 -g"  ..
     make -j "$(nproc)" install
     cd /opt/src || exit
     git clone https://github.com/MmgTools/ParMmg.git
@@ -213,7 +211,9 @@ compile_mmg_and_parmmg () {
     git checkout develop
     mkdir build
     cd build || exit
-    cmake -D CMAKE_INSTALL_PREFIX="/opt/parmmg" -D CMAKE_BUILD_TYPE=RelWithDebInfo -D USE_VTK:BOOL=FALSE -D BUILD_SHARED_LIBS:BOOL=TRUE -D DOWNLOAD_MMG=OFF  -D MMG_DIR="/opt/mmg"  -D MMG_DIR_FOUND="/opt/mmg" -D MMG_libmmgtypes.h_DIRS="/opt/mmg/include/mmg/common" -D MMG_mmg_LIBRARY="/opt/mmg/lib/libmmg.so"  ..
+    cmake -D CMAKE_INSTALL_PREFIX="/opt/parmmg" -D CMAKE_BUILD_TYPE=RelWithDebInfo -D USE_VTK:BOOL=FALSE \
+          -D BUILD_SHARED_LIBS:BOOL=TRUE -D DOWNLOAD_MMG=OFF  -D MMG_DIR="/opt/mmg"  -D MMG_DIR_FOUND="/opt/mmg" \
+          -D MMG_libmmgtypes.h_DIRS="/opt/mmg/include/mmg/common" -D MMG_mmg_LIBRARY="/opt/mmg/lib/libmmg.so"  ..
     make -j "$(nproc)" install
     export LD_LIBRARY_PATH="/opt/mmg/lib:/opt/parmmg/lib:$LD_LIBRARY_PATH"
     cd /opt/src || exit
@@ -257,8 +257,8 @@ compile_elmer () {
             -DWITH_MPI:BOOL=TRUE \
             -DWITH_LUA:BOOL=TRUE \
             -DWITH_OpenMP:BOOL=TRUE \
-        -DWITH_ElmerIce:BOOL=TRUE \
-        -DWITH_NETCDF:BOOL=TRUE \
+            -DWITH_ElmerIce:BOOL=TRUE \
+            -DWITH_NETCDF:BOOL=TRUE \
             -DWITH_GridDataReader:BOOL=TRUE \
             -DNETCDF_INCLUDE_DIR="/opt/netcdf/include" \
             -DNETCDF_LIBRARY="/opt/netcdf/lib/libnetcdf.so" \
@@ -274,12 +274,12 @@ compile_elmer () {
             -DCSA_INCLUDE_DIR="/opt/csa/include" \
             -DNN_INCLUDE_DIR="/opt/nn/include" \
             -DNN_LIBRARY="/opt/nn/lib/libnn.a" \
-	    -DWITH_MMG:BOOL=TRUE \
-	    -DMMG_ROOT="/opt/mmg" \
-	    -DMMG_LIBRARY="/opt/mmg/lib/libmmg.so" \
-	    -DMMG_INCLUDE_DIR="/opt/mmg/include" \
-	    -DWITH_PARMMG:BOOL=TRUE \
-	    -DPARMMGROOT="/opt/parmmg" \
+            -DWITH_MMG:BOOL=TRUE \
+            -DMMG_ROOT="/opt/mmg" \
+            -DMMG_LIBRARY="/opt/mmg/lib/libmmg.so" \
+            -DMMG_INCLUDE_DIR="/opt/mmg/include" \
+            -DWITH_PARMMG:BOOL=TRUE \
+            -DPARMMGROOT="/opt/parmmg" \
             -DCMAKE_C_FLAGS="-O3 -fopenmp -funroll-loops -march=$MARCH" \
             -DCMAKE_Fortran_FLAGS="-O3 -fPIC -funroll-loops -march=$MARCH"
 
@@ -296,6 +296,9 @@ install_kqcircuits_compile () {
     # KQCircuits installation #
     ###########################
 
+    KL_FILE="klayout-0.28.8-0.x86_64.rpm"
+    KL_HASH="8e5083035046f806d16072f3203ad782"
+
     yum -y install -y xorg-x11-server-Xvfb mesa-libGL libXft-devel
     yum -y install -y paraview
 

singularity/libexec/kqclib.sh

@@ -7,6 +7,13 @@ else
 	cmd=$(basename "$0")
 fi
 
+
+if [ -e "$HOME/singularity_private.pem" ]; then
+    run_cmd="singularity exec --pem-path=$HOME/singularity_private.pem"
+else
+    run_cmd="singularity exec"
+fi
+
 # Check if running on WSL and use higher compatibility then
 if grep -qi "microsoft" /proc/version; then
 
@@ -16,13 +23,13 @@ if grep -qi "microsoft" /proc/version; then
     single_cmd="mpirun $2 $3 $cmd $1"
   else
     # If in wsl but using other tool than ElmerSolver
-    single_cmd="$cmd $*" 
+    single_cmd="$cmd $*"
   fi
-  run_cmd="singularity exec --containall --home ${PWD} ${dir}/${img} $single_cmd"
+  run_cmd+=" --containall --home ${PWD} ${dir}/${img} $single_cmd"
   echo running: "$run_cmd"
   $run_cmd
 else
-  run_cmd="singularity exec --home $HOME ${dir}/${img} $cmd $*"
+  run_cmd+=" --home $HOME ${dir}/${img} $cmd $*"
   echo running: "$run_cmd"
   $run_cmd
 fi

singularity/singularity.def

@@ -1,10 +1,9 @@
 Bootstrap: docker
-From: centos:7.9.2009
+From: ubuntu:latest
+MirrorURL: http://us.archive.ubuntu.com/ubuntu/
 
 %help
-    This container can be used to run Elmer simulations for KQCircuits.
-
-    The main `runscript` corresponds to using the `kqc` command provided by KQCircuits. Alternatively, to get into a shell in the container use `singularity shell image` 
+    This container can be used to run Elmer simulations for KQCircuits simulations
 
 %labels
     org.opencontainers.image.title "KQCircuits simulations image"
@@ -16,23 +15,21 @@ From: centos:7.9.2009
 
 %files
     MUMPS_Makefile.inc /opt/Makefile.inc
-    install_kqcircuits.sh /opt/install_kqcircuits.sh
+    install_software.sh /opt/install_software.sh
 
 %runscript
     exec kqc "$@"
 
 %environment
     export SLURM_MPI_TYPE=pmi2
     export PMIX_MCA_gds=hash
-    source /opt/rh/devtoolset-11/enable
-    source /opt/rh/rh-git218/enable
     export ELMER_HOME="/opt/elmer"
     export PATH=$ELMER_HOME/bin:$PATH
-    export LD_LIBRARY_PATH=$ELMER_HOME/include:$ELMER_HOME/lib:/opt/elmer/share/elmersolver/lib:/opt/hypre/lib:/opt/scalapack/lib:/opt/mumps/lib:/opt/netcdf/lib64:/opt/mmg/lib64:/opt/parmmg/lib64:/opt/nn/lib:/opt/csa/lib:$LD_LIBRARY_PATH
+    export LD_LIBRARY_PATH=/usr/lib/:$ELMER_HOME/include:$ELMER_HOME/lib:/opt/elmer/share/elmersolver/lib:/opt/hypre/lib:/opt/scalapack/lib:/opt/mumps/lib:/opt/netcdf/lib:/opt/mmg/lib:/opt/parmmg/lib:/opt/nn/lib:/opt/csa/lib:$LD_LIBRARY_PATH
 
 %post -c /bin/bash
     #!/bin/bash
-    chmod u+x /opt/install_kqcircuits.sh
-    /opt/install_kqcircuits.sh
+    chmod u+x /opt/install_software.sh
+    /opt/install_software.sh
 
 

singularity/singularity.sh

@@ -1,16 +1,14 @@
 #!/bin/bash
 
-SINGULARITY_DEF_FILE="singularity_ubuntu.def"
-
-USE_SUDO="true"
-{ echo "Building singularity image with fakeroot..." && singularity build --fakeroot kqclib $SINGULARITY_DEF_FILE && USE_SUDO="false"; } || \
-	{ echo "Fakeroot build failed, let's try using sudo" && sudo singularity build kqclib $SINGULARITY_DEF_FILE; } && \
-	mv kqclib libexec && mkdir -p bin && ln -s ../libexec/kqclib.sh bin/kqclib || exit
-
-if [[ "${USE_SUDO}" == "true" ]]; then
-	sudo chown -R "$USER":"$USER" bin
+if [ -e singularity.pem ]; then
+    echo "Building singularity image with encryption, needs sudo."
+    sudo singularity build --pem-path singularity.pem kqclib singularity.def
+else
+    echo "Building singularity image."
+    singularity build --fakeroot kqclib singularity.def
 fi
-chmod 755 libexec/kqclib.sh
+
+mv kqclib libexec
 
 echo "Singularity image is now built!"
 echo "You can now run ./create_links.sh in order to get the executables for using the software in the image (instead of in your own system)"