Deprecate eol python versions 3.6 and 3.7, enable support for 3.11

.github/workflows/deployment-tests.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        os_container: ['centos7', 'centosstream8', 'rockylinux8', 'rockylinux9', 'ubuntu20_04', 'ubuntu22_04']
+        os_container: ['centos7', 'rockylinux8', 'rockylinux9', 'ubuntu20_04', 'ubuntu22_04']
     steps:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
@@ -36,7 +36,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        os_container: ['centos7', 'centosstream8', 'rockylinux8', 'rockylinux9', 'ubuntu20_04', 'ubuntu22_04']
+        os_container: ['centos7', 'rockylinux8', 'rockylinux9', 'ubuntu20_04', 'ubuntu22_04']
     container: matterminers/cobald-tardis-deployment-test-env:${{ matrix.os_container }}
     steps:
       - uses: actions/checkout@v3
@@ -58,9 +58,9 @@ jobs:
       - uses: actions/checkout@v3
       - name: Install dependencies on ${{ matrix.platform }}
         run: |
-          python3.10 -m pip install --upgrade pip
-          python3.10 -m pip install .[contrib]
-          python3.10 -m pip install coverage codecov
+          python3 -m pip install --upgrade pip
+          python3 -m pip install .[contrib]
+          python3 -m pip install coverage codecov
       - name: Test with unittest on ${{ matrix.platform }}
         run: |
           coverage run -m unittest -v

.github/workflows/unittests.yml

@@ -4,10 +4,10 @@ on: [push, pull_request]
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
 
     steps:
     - uses: actions/checkout@v3

.pre-commit-config.yaml

@@ -1,27 +1,33 @@
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.1.0
     hooks:
       - id: check-merge-conflict
       - id: end-of-file-fixer
--   repo: https://github.com/psf/black
-    rev: 23.1.0
+  - repo: https://github.com/psf/black
+    rev: 23.7.0
     hooks:
-    -   id: black
+      - id: black
         args:
-        -   --target-version=py36
--   repo: https://github.com/PyCQA/flake8
+          - --target-version=py38
+  - repo: https://github.com/PyCQA/flake8
     rev: 6.0.0
     hooks:
       - id: flake8
--   repo: https://github.com/Matterminers/dev-tools
+  - repo: https://github.com/Matterminers/dev-tools
     rev: "87c2435b377bd932acbcc1b619320d98e67b630a"
     hooks:
-      -  id: contributors
-      -  id: changelog
-         args:
+      - id: contributors
+      - id: changelog
+        args:
           - docs/source/changes
           - compile
+          - --categories
+          - Added
+          - Changed
+          - Fixed
+          - Security
+          - Deprecated
           - --output=docs/source/changelog.rst
-         additional_dependencies:
-          - PyYAML  
+        additional_dependencies:
+          - PyYAML

containers/cobald-tardis-deployment-test-env/Dockerfile.centos7

@@ -1,16 +1,22 @@
 FROM centos:7
 LABEL maintainer="Manuel Giffels <giffels@gmail.com>"
 
-RUN yum -y install epel-release curl && yum clean all
+RUN yum -y install epel-release centos-release-scl-rh curl && yum clean all
+RUN yum --enablerepo=centos-sclo-rh -y install rh-python38 && yum clean all
 
 RUN curl -sL https://rpm.nodesource.com/setup_16.x | bash -
 
 RUN yum -y update \
     && yum -y install git \
-                      python3 \
                       gcc \
-                      python3-devel \
                       nodejs \
     && yum clean all
 
+# Enable rh-python38 from SCL
+ENV PATH=/opt/rh/rh-python38/root/usr/local/bin:/opt/rh/rh-python38/root/usr/bin${PATH:+:${PATH}}
+ENV LD_LIBRARY_PATH=/opt/rh/rh-python38/root/usr/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+ENV MANPATH=/opt/rh/rh-python38/root/usr/share/man:$MANPATH
+ENV PKG_CONFIG_PATH=/opt/rh/rh-python38/root/usr/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}
+ENV XDG_DATA_DIRS="/opt/rh/rh-python38/root/usr/share:${XDG_DATA_DIRS:-/usr/local/share:/usr/share}"
+
 SHELL [ "/bin/bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c" ]

containers/cobald-tardis-deployment-test-env/Dockerfile.rockylinux8

@@ -7,11 +7,14 @@ RUN curl -sL https://rpm.nodesource.com/setup_18.x | bash -
 
 RUN yum -y update \
     && yum -y install git \
-                      python38 \
+                      python39 \
                       gcc \
-                      python38-devel \
+                      python39-devel \
                       nodejs \
                       glibc-langpack-en \
     && yum clean all
 
+# Make Python3.9 default
+RUN alternatives --set python3 /usr/bin/python3.9
+
 SHELL [ "/bin/bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c" ]

containers/cobald-tardis-htcondor/Dockerfile

@@ -1,28 +1,29 @@
-FROM centos:7
+FROM rockylinux:8
 LABEL maintainer="Manuel Giffels <giffels@gmail.com>"
 
 ARG SOURCE_BRANCH=master
 ARG SOURCE_REPO_URL=https://github.com/MatterMiners/tardis
 
 RUN rpm --import http://research.cs.wisc.edu/htcondor/yum/RPM-GPG-KEY-HTCondor
-RUN yum install -y https://research.cs.wisc.edu/htcondor/repo/9.0/htcondor-release-current.el7.noarch.rpm
+RUN dnf install -y https://research.cs.wisc.edu/htcondor/repo/10/10.0/htcondor-release-current.el8.noarch.rpm
 
-RUN yum -y install epel-release && yum clean all
+RUN dnf -y install epel-release && dnf clean all
+RUN dnf config-manager --set-enabled powertools
 
-RUN yum -y update \
-    && yum -y install condor \
+RUN dnf -y update \
+    && dnf -y install condor \
                       git \
-                      python3 \
+                      python39 \
                       gcc \
-                      python3-devel \
-    && yum clean all
+                      python39-devel \
+    && dnf clean all
 
-RUN python3 -m pip install --no-cache-dir --upgrade pip \
-    && python3 -m pip install --no-cache-dir git+$SOURCE_REPO_URL@$SOURCE_BRANCH
+RUN python3.9 -m pip install --no-cache-dir --upgrade pip \
+    && python3.9 -m pip install --no-cache-dir git+$SOURCE_REPO_URL@$SOURCE_BRANCH
 
 WORKDIR /srv
 
 COPY cobald.yml /srv/cobald.yml
 
-ENTRYPOINT ["python3", "-m", "cobald.daemon"]
+ENTRYPOINT ["python3.9", "-m", "cobald.daemon"]
 CMD ["/srv/cobald.yml"]

containers/cobald-tardis/Dockerfile

@@ -1,5 +1,5 @@
-FROM python:3.8-alpine
-MAINTAINER Manuel Giffels <giffels@gmail.com>
+FROM python:3.10-alpine
+LABEL maintainer="Manuel Giffels <giffels@gmail.com>"
 ARG SOURCE_BRANCH=master
 ARG SOURCE_REPO_URL=https://github.com/MatterMiners/tardis
 

docs/source/changelog.rst

@@ -1,11 +1,19 @@
 .. Created by changelog.py at 2023-08-10, command
-   '/Users/giffler/.cache/pre-commit/repor6pnmwlm/py_env-python3.10/bin/changelog docs/source/changes compile --output=docs/source/changelog.rst'
+   '/Users/giffler/.cache/pre-commit/repor6pnmwlm/py_env-python3.10/bin/changelog docs/source/changes compile --categories Added Changed Fixed Security Deprecated --output=docs/source/changelog.rst'
    based on the format of 'https://keepachangelog.com/'
 
 #########
 CHANGELOG
 #########
 
+[Unreleased] - 2023-07-31
+=========================
+
+Deprecated
+----------
+
+* Minimal Python version is 3.8
+
 [0.7.1] - 2023-05-16
 ====================
 

docs/source/changes/304.deprecate_python_36_37.yaml

@@ -0,0 +1,10 @@
+category: deprecated
+summary: "Minimal Python version is 3.8"
+description: |
+  Support for Python versions 3.6 and 3.7 has been depracted. This has no further
+  functional changes for users but reflects the end of life of these versions
+  and allows TARDIS and plugins to drop outdated dependencies and compatibility layers.
+pull requests:
+  - 304
+issues:
+  - 303

setup.py

@@ -15,10 +15,9 @@
 TESTS_REQUIRE = ["flake8", "httpx"]
 REST_REQUIRES = [
     "fastapi-jwt-auth",
-    "fastapi<0.84.0; python_version<'3.7'",  # to support python3.6 (Centos 7)
-    "fastapi; python_version>='3.7'",
+    "fastapi",
     "python-jose",
-    "uvicorn[standard]<=0.14.0",  # to support python3.6 (Centos 7)
+    "uvicorn[standard]",
     "typer",
     "bcrypt",
     "python-multipart",
@@ -53,11 +52,10 @@ def get_cryptography_version():
         "Topic :: Utilities",
         "Framework :: AsyncIO",
         "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
     entry_points={
         "console_scripts": [
@@ -78,14 +76,14 @@ def get_cryptography_version():
     },
     keywords=package_about["__keywords__"],
     packages=find_packages(exclude=["tests"]),
+    python_requires=">=3.8",
     install_requires=[
-        "aiohttp<4.0; python_version<'3.7'",  # to support python3.6 (Centos 7)
-        "aiohttp; python_version>='3.7'",
+        "aiohttp",
         get_cryptography_version(),
         "CloudStackAIO>=0.0.8",
         "PyYAML",
         "AsyncOpenStackClient>=0.9.0",
-        "cobald>=0.12.3",
+        "cobald>=0.14.0 ",
         "asyncssh",
         "aiotelegraf",
         "elasticsearch>=7.17,<8.0.0",
@@ -94,7 +92,6 @@ def get_cryptography_version():
         "pydantic<2.0.0",
         "asyncstdlib",
         "typing_extensions",
-        "backports.cached_property",
         "python-auditor==0.1.0",
         "pytz",
         "tzlocal",

tardis/resources/drone.py

@@ -12,8 +12,8 @@
 from cobald.daemon import service
 from cobald.interfaces import Pool
 
-from backports.cached_property import cached_property
 from datetime import datetime
+from functools import cached_property
 
 import asyncio
 import logging

tardis/utilities/asyncbulkcall.py

@@ -1,11 +1,10 @@
+from functools import cached_property
 from typing import TypeVar, Generic, Iterable, List, Tuple, Optional, Set
 from typing_extensions import Protocol
 import asyncio
 import time
 import sys
 
-from backports.cached_property import cached_property
-
 
 T = TypeVar("T")
 R = TypeVar("R")

tests/adapters_t/sites_t/test_fakesite.py

@@ -3,7 +3,7 @@
 from tardis.interfaces.siteadapter import ResourceStatus
 from tardis.utilities.attributedict import AttributeDict
 
-from ...utilities.utilities import run_async
+from tests.utilities.utilities import run_async
 
 from datetime import datetime
 from datetime import timedelta

tests/adapters_t/sites_t/test_htcondorsiteadapter.py

@@ -4,8 +4,8 @@
 from tardis.exceptions.tardisexceptions import TardisResourceStatusUpdateFailed
 from tardis.interfaces.siteadapter import ResourceStatus
 from tardis.utilities.attributedict import AttributeDict
-from ...utilities.utilities import mock_executor_run_command
-from ...utilities.utilities import run_async
+from tests.utilities.utilities import mock_executor_run_command
+from tests.utilities.utilities import run_async
 
 from datetime import datetime
 from datetime import timedelta

tests/adapters_t/sites_t/test_slurm.py

@@ -5,8 +5,8 @@
 from tardis.exceptions.executorexceptions import CommandExecutionFailure
 from tardis.interfaces.siteadapter import ResourceStatus
 from tardis.utilities.attributedict import AttributeDict
-from ...utilities.utilities import mock_executor_run_command
-from ...utilities.utilities import run_async
+from tests.utilities.utilities import mock_executor_run_command
+from tests.utilities.utilities import run_async
 
 from unittest import TestCase
 from unittest.mock import MagicMock, patch

tests/agents_t/test_batchsystemagent.py

@@ -1,5 +1,5 @@
-from ..utilities.utilities import run_async
-from ..utilities.utilities import async_return
+from tests.utilities.utilities import run_async
+from tests.utilities.utilities import async_return
 from tardis.agents.batchsystemagent import BatchSystemAgent
 from tardis.interfaces.batchsystemadapter import BatchSystemAdapter
 from tardis.utilities.attributedict import AttributeDict

tests/agents_t/test_siteagent.py

@@ -1,5 +1,5 @@
-from ..utilities.utilities import run_async
-from ..utilities.utilities import async_return
+from tests.utilities.utilities import run_async
+from tests.utilities.utilities import async_return
 from tardis.agents.siteagent import SiteAgent
 from tardis.interfaces.siteadapter import SiteAdapter
 

tests/interfaces_t/test_batchsystemadapter.py

@@ -1,6 +1,6 @@
 from tardis.interfaces.batchsystemadapter import BatchSystemAdapter
 
-from ..utilities.utilities import run_async
+from tests.utilities.utilities import run_async
 
 from unittest import TestCase
 from unittest.mock import patch

tests/interfaces_t/test_siteadapter.py

@@ -1,7 +1,7 @@
 from tardis.interfaces.siteadapter import SiteAdapter
 from tardis.utilities.attributedict import AttributeDict
 
-from ..utilities.utilities import run_async
+from tests.utilities.utilities import run_async
 
 from cobald.utility.primitives import infinity as inf
 from unittest import TestCase

tests/plugins_t/test_auditor.py

@@ -18,8 +18,8 @@
 from unittest import TestCase
 from unittest.mock import patch
 
-from ..utilities.utilities import async_return
-from ..utilities.utilities import run_async
+from tests.utilities.utilities import async_return
+from tests.utilities.utilities import run_async
 
 
 class TestAuditor(TestCase):

tests/plugins_t/test_elasticsearchmonitoring.py

@@ -7,7 +7,7 @@
 from unittest import TestCase
 from unittest.mock import patch
 
-from ..utilities.utilities import run_async
+from tests.utilities.utilities import run_async
 
 
 class TestElasticsearchMonitoring(TestCase):

tests/plugins_t/test_prometheusmonitoring.py

@@ -8,7 +8,7 @@
 from unittest.mock import Mock
 from unittest.mock import patch
 
-from ..utilities.utilities import get_free_port, run_async
+from tests.utilities.utilities import get_free_port, run_async
 
 
 class TestPrometheusMonitoring(TestCase):