WIP: Do not install optional requirements when building docker images…

… for PyPI (#850) * Do not install optional requirements when building docker images for pypi. * Added openssl-dev to musllinux image * Skip installing dependencies for Python 12 on i686 * Do not build Windows wheel for Python 3.12 * Fixed typo * Try to allow python executable being named python3 * Trying to build Windows wheels * Force standard behaviour when looking for virtual env on Windows * Fixed typo * Force CI to run * Added explicit python version * Cleanup. Removed deprecated cmake options++
SINTEF · Jun 13, 2024 · 509732b · 509732b
1 parent 15b9ad3
commit 509732b
Show file tree

Hide file tree

Showing 8 changed files with 69 additions and 136 deletions.
diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template
@@ -25,16 +25,14 @@
 # Reference: https://github.com/pypa/manylinux#manylinux2014-centos-7-based
 FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest
 
-ARG PY_MINORS="7 8 9 10"
-
-#COPY requirements.txt /tmp/requirements.txt
-COPY requirements_full.txt requirements_dev.txt /tmp/
+ARG PY_MINORS="7 8 9 10 11 12"
 
 {{ EXTRA_PRE }}
 
 # Enable rpmfusion for additional packages
-RUN yum update -y
-RUN yum localinstall -y --skip-broken \
+RUN \
+  yum update -y && \
+  yum localinstall -y --skip-broken \
     https://mirrors.rpmfusion.org/free/el/rpmfusion-free-release-$(rpm --eval %{centos_ver}).noarch.rpm \
     https://mirrors.rpmfusion.org/nonfree/el/rpmfusion-nonfree-release-$(rpm --eval %{centos_ver}).noarch.rpm && \
   yum install -y \
@@ -43,39 +41,23 @@ RUN yum localinstall -y --skip-broken \
     swig \
     libcurl-devel \
     libxslt-devel \
-    libxml2-devel
-
-# Rust and cargo are needed by pydantic2, but may not be available
-RUN yum install -y rust cargo || true
-
-# hdf5-devel is needed for building the hdf5 plugin
-RUN if [ "{{ TYPE }}{{ ARCH }}" != "2014i686" ]; then yum install -y hdf5-devel; fi
-
-# Unpack static libraries
-# It's necessary to be in /opt/_internal because the internal libraries
-# exist here.
-RUN  cd /opt/_internal && \
+    libxml2-devel && \
+  # hdf5-devel is needed for building the hdf5 plugin
+  yum install -y hdf5-devel || true && \
+  # Unpack static libraries
+  # It's necessary to be in /opt/_internal because the internal libraries
+  # exist here.
+  cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
-# Change required version of pydantic to be <2 and remove challenging packages
-  sed -e 's/^\(pydantic>.*<\).*$/\12/' \
-      -e '/^psycopg/d' \
-      -e '/^matplotlib/d' \
-      -e '/^scikit-image/d' \
-      -e '/^ncempy/d' \
-      -e '/^h5py/d' \
-      -e '/oteapi/d' \
-      -e '/^otelib/d' \
-      -e '/^pyarrow/d' \
-      -i /tmp/requirements_full.txt
-
-# Install required Python packages
-RUN mkdir -p /ci/pip_cache && \
-  if [ -f "/etc/yum.repos.d/pgdg-91.repo" ]; then export PATH="$PATH:/usr/pgsql-9.1/bin"; fi && \
+  # Install required Python packages
+  mkdir -p /ci/pip_cache && \
   for minor in ${PY_MINORS}; do \
+    which python3.${minor} || continue && \
     python3.${minor} -m pip install -U pip && \
     python3.${minor} -m pip install -U setuptools wheel && \
-    python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \
-    python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt -r /tmp/requirements_dev.txt; \
-  done
+    python3.${minor} -m pip install -U --cache-dir /ci/pip_cache \
+      cmake oldest-supported-numpy; \
+  done && \
+  rm -rf /ci/pip_cache
 
 {{ EXTRA_POST }}
diff --git a/.github/docker/Dockerfile-manylinux_x_y.template b/.github/docker/Dockerfile-manylinux_x_y.template
@@ -27,15 +27,12 @@ FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}
 
 ARG PY_MINORS="7 8 9 10 11 12"
 
-#COPY requirements.txt /tmp/requirements.txt
-COPY requirements_full.txt requirements_dev.txt /tmp/
-
 {{ EXTRA_PRE }}
 
 # Enable rpmfusion for additional packages
-# Rust and cargo are needed by pydantic2
-RUN dnf update -y
-RUN dnf localinstall -y --skip-broken \
+RUN \
+  dnf update -y && \
+  dnf localinstall -y --skip-broken \
     https://mirrors.rpmfusion.org/free/el/rpmfusion-free-release-$(rpm --eval %{centos_ver}).noarch.rpm \
     https://mirrors.rpmfusion.org/nonfree/el/rpmfusion-nonfree-release-$(rpm --eval %{centos_ver}).noarch.rpm && \
   dnf install -y \
@@ -45,31 +42,21 @@ RUN dnf localinstall -y --skip-broken \
     libcurl-devel \
     libxslt-devel \
     libxml2-devel \
-    hdf5-devel \
-    rust \
-    cargo
-
-# Unpack static libraries
-# It's necessary to be in /opt/_internal because the internal libraries
-# exist here.
-RUN  cd /opt/_internal && \
-    tar -Jxvf static-libs-for-embedding-only.tar.xz
-
-# Change required version of pydantic to be <2
-#RUN  sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt
-
-# Remove psycopg from requirements
-RUN  sed '/^psycopg/d' -i /tmp/requirements_full.txt
-
-# Install required Python packages
-RUN mkdir -p /ci/pip_cache && \
-    if [ -f "/etc/yum.repos.d/pgdg-91.repo" ]; then export PATH="$PATH:/usr/pgsql-9.1/bin"; fi
-
-RUN for minor in ${PY_MINORS}; do \
+    hdf5-devel && \
+  # Unpack static libraries
+  # It's necessary to be in /opt/_internal because the internal libraries
+  # exist here.
+  cd /opt/_internal && \
+    tar -Jxvf static-libs-for-embedding-only.tar.xz && \
+  # Install required Python packages
+  mkdir -p /ci/pip_cache && \
+  for minor in ${PY_MINORS}; do \
+    which python3.${minor} || continue && \
+    echo "==================== python3.${minor} ====================" && \
     python3.${minor} -m pip install -U pip && \
     python3.${minor} -m pip install -U setuptools wheel && \
-    python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \
-    python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt -r /tmp/requirements_dev.txt; \
-  done
+    python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy; \
+  done && \
+  rm -rf /ci/pip_cache
 
 {{ EXTRA_POST }}
diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template
@@ -4,9 +4,8 @@
 # Usage:
 #
 # Copy this template file and replace:
-# - `{{ TYPE }}` with a valid musllinux type.
-#   For now only '_1_1' is allowed.
-# - `{{ ARCH }}` with a valid arch, e.g., x86_64 or i686.
+# - `{{ TYPE }}` with a valid musllinux type, e.g. '_1_1' or '_1_2'
+# - `{{ ARCH }}` with a valid arch, e.g., x86_64 or i686
 # Remove the `.template` suffix from the copy.
 #
 # Build:
@@ -25,9 +24,6 @@ FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest
 
 ARG PY_MINORS="7 8 9 10 11 12"
 
-#COPY requirements.txt /tmp/requirements.txt
-COPY requirements_full.txt requirements_dev.txt /tmp/
-
 # Do not use distutils distributed with setuptools
 # This is due to base changes in the distutils API, removing msvccompiler,
 # which is necessary for building the numpy wheel.
@@ -36,36 +32,27 @@ ENV SETUPTOOLS_USE_DISTUTILS="stdlib"
 RUN apk add -u \
   redland \
   rasqal \
-  hdf5 \
-  swig \
-  rust \
-  cargo && \
+  hdf5-dev \
+  openssl-dev \
+  swig && \
   # Unpack static libraries
   # It's necessary to be in /opt/_internal because the internal libraries
   # exist here.
   cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
-  # Change required version of pydantic to be <2 and remove challenging packages
-  sed -e 's/^\(pydantic>.*<\).*$/\12/' \
-      -e '/^psycopg/d' \
-      -e '/^matplotlib/d' \
-      -e '/^scikit-image/d' \
-      -e '/^ncempy/d' \
-      -e '/^h5py/d' \
-      -e '/oteapi/d' \
-      -e '/^otelib/d' \
-      -e '/^pyarrow/d' \
-      -i /tmp/requirements_full.txt && \
+  # Filter minor versions
+  minors=$( for n in $PY_MINORS; do \
+    if [ $(uname -m) == "i686" -o $n -lt 12 ]; then echo $n; fi; \
+  done ) && \
+  echo "*** Minors: $minors" && \
   # Install required Python packages
   mkdir -p /ci/pip_cache && \
-  for minor in ${PY_MINORS}; do \
-    # musllinux_1_1 only support Python up to 3.11
-    [ "{{ TYPE }}" = "_1_1" ] && [ $minor -gt 11 ] && continue || \
+  for minor in ${minors}; do \
+    which python3.${minor} 2>/dev/null || continue && \
+    echo "==================== python3.${minor} ====================" && \
     python3.${minor} -m pip install -U pip && \
-    python3.${minor} -m pip install -U setuptools wheel && \
+    python3.${minor} -m pip install -U setuptools wheel; \
     python3.${minor} -m pip install -U --cache-dir /ci/pip_cache \
-      cmake oldest-supported-numpy && \
-    python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary \
-      -r /tmp/requirements_full.txt -r /tmp/requirements_dev.txt; \
+      oldest-supported-numpy; \
   done && \
-  rm -rf /ci/pip_cache /tmp/requirements*.txt
+  rm -rf /ci/pip_cache
diff --git a/.github/workflows/cd_release.yml b/.github/workflows/cd_release.yml
@@ -177,9 +177,9 @@ jobs:
       env:
         CIBW_BUILD: cp3${{ contains(matrix.py_minors, ',') && format('{{{0}}}', matrix.py_minors) || matrix.py_minors }}-${{ matrix.system_type[0] }}${{ matrix.arch != '' && '_' || '' }}${{ matrix.arch }}
         CIBW_MANYLINUX_X86_64_IMAGE: ${{ env.CONTAINER_REGISTRY }}/sintef/dlite-python-manylinux${{ matrix.system_type[1] }}_x86_64:latest
-        CIBW_MANYLINUX_I686_IMAGE: ${{ env.CONTAINER_REGISTRY }}/sintef/dlite-python-manylinux${{ matrix.system_type[1] }}_i686:latest
-        CIBW_MUSLLINUX_X86_64_IMAGE: ${{ env.CONTAINER_REGISTRY }}/sintef/dlite-python-musllinux_1_1_x86_64:latest
-        CIBW_MUSLLINUX_I686_IMAGE: ${{ env.CONTAINER_REGISTRY }}/sintef/dlite-python-musllinux_1_1_i686:latest
+        CIBW_MANYLINUX_I686_IMAGE:   ${{ env.CONTAINER_REGISTRY }}/sintef/dlite-python-manylinux${{ matrix.system_type[1] }}_i686:latest
+        CIBW_MUSLLINUX_X86_64_IMAGE: ${{ env.CONTAINER_REGISTRY }}/sintef/dlite-python-musllinux${{ matrix.system_type[1] }}_x86_64:latest
+        CIBW_MUSLLINUX_I686_IMAGE:   ${{ env.CONTAINER_REGISTRY }}/sintef/dlite-python-musllinux${{ matrix.system_type[1] }}_i686:latest
 
     - name: Store wheels for publishing
       uses: actions/upload-artifact@v4

diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml
@@ -34,7 +34,7 @@ jobs:
           #- os: ubuntu-20.04
           #  system_type: ["musllinux", "_1_2"]
           #  arch: i686
-          #  py_minors: 10,12
+          #  py_minors: 10,11
 
           # 64-bit linux
           - os: ubuntu-20.04
@@ -130,7 +130,7 @@ jobs:
           python -m pip install -U cibuildwheel
 
       # See pyproject.toml (under python/) for cibuildwheel configuration
-      - name: Build wheels
+      - name: Build wheels with cibuildwheel
         run: |
           python -m cibuildwheel --print-build-identifiers python
           python -m cibuildwheel --output-dir wheelhouse python

diff --git a/.github/workflows/container_builds_weekly.yml b/.github/workflows/container_builds_weekly.yml
@@ -43,8 +43,6 @@ jobs:
           type: "_1_1"
           arch: "x86_64"
           py_minors: "7 8 9 10 11"
-
-          # tests...
         - system: "musllinux"
           template: "musllinux"
           type: "_1_2"
@@ -72,8 +70,6 @@ jobs:
           type: "_1_1"
           arch: "i686"
           py_minors: "7 8 9"
-
-          # tests
         - system: "musllinux"
           template: "musllinux"
           type: "_1_2"
@@ -83,7 +79,9 @@ jobs:
           template: "musllinux"
           type: "_1_2"
           arch: "i686"
-          py_minors: "10 11 12"
+          # Python 3.12 fails while building numpy: missing distutils
+          #py_minors: "10 11 12"
+          py_minors: "10 11"
 
     steps:
       - name: Checkout repository

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -253,7 +253,7 @@ if(WITH_PYTHON)
     if(NOT PYTHON_VERSION)
       find_program(
         python_exe
-        python
+        NAMES python python3
         PATHS $ENV{VIRTUAL_ENV}/bin
         REQUIRED
         NO_DEFAULT_PATH

diff --git a/python/setup.py b/python/setup.py
@@ -14,6 +14,7 @@
 if TYPE_CHECKING:
     from typing import Union
 
+
 # Based on
 # https://github.com/galois-advertising/cmake_setup/blob/master/cmake_setup/cmake/__init__.py
 
@@ -26,7 +27,6 @@
 
     CMAKE_ARGS = [
         "-DWITH_DOC=OFF",
-        "-DWITH_JSON=ON",
         "-DWITH_HDF5=OFF",
         "-DALLOW_WARNINGS=ON",
         "-Ddlite_PYTHON_BUILD_REDISTRIBUTABLE_PACKAGE=YES",
@@ -49,15 +49,16 @@
     dlite_compiled_ext = "_dlite.pyd"
     dlite_compiled_dll_suffix = "*.dll"
     is_64bits = sys.maxsize > 2**32
-
+    v = sys.version_info
     CMAKE_ARGS = [
         #"-G", "Visual Studio 15 2017",
         "-A", "x64",
         "-DWITH_DOC=OFF",
-        "-DWITH_JSON=ON",
         "-DWITH_HDF5=OFF",
+        f"-DPYTHON_VERSION={v.major}.{v.minor}",
         "-Ddlite_PYTHON_BUILD_REDISTRIBUTABLE_PACKAGE=YES",
-        f"-DCMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE={'x64' if is_64bits else 'x86'}"
+        f"-DCMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE={'x64' if is_64bits else 'x86'}",
+        "-DPython3_FIND_VIRTUALENV=STANDARD",
     ]
 
 else:
@@ -156,30 +157,6 @@ def build_extension(self, ext: CMakeExtension) -> None:
             str(cmake_bdist_dir / ext.name), str(Path(output_dir) / ext.name)
         )
 
-# Read requirements from requireemtns.txt
-with open(SOURCE_DIR / "requirements.txt", "r") as f:
-    requirements = [
-        line.strip() for line in f.readlines() if not line.startswith("#")
-    ]
-
-# Populate extra_requirements from requirements_*.txt
-extra_requirements = {}
-for name in "mappings", "full", "dev", "doc":
-    with open(SOURCE_DIR / f"requirements_{name}.txt", "r") as f:
-        extra_requirements[name] = [
-            line.strip() for line in f.readlines() if not line.startswith("#")
-        ]
-
-# Temporary workaround!
-# Require pydantic <2. Needed before we have managed to install rust in
-# the docker image for wheels
-extras = extra_requirements["full"]
-for i, pkg in enumerate(extras):
-    match = re.match("^(pydantic>.*<)", pkg)
-    if match:
-        extras[i] = f"{match.groups()[0]}2"
-
-
 version = re.search(
     r"project\([^)]*VERSION\s+([0-9.]+)",
     (SOURCE_DIR / "CMakeLists.txt").read_text(),
@@ -213,10 +190,12 @@ def build_extension(self, ext: CMakeExtension) -> None:
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
-    install_requires=requirements,
-    extras_require=extra_requirements,
+    install_requires="numpy",
+    #install_requires=requirements,
+    #extras_require=extra_requirements,
     packages=["dlite"],
     scripts=[
         str(SOURCE_DIR / "bindings" / "python" / "scripts" / "dlite-validate"),