diff --git a/.github/workflows/deployment-tests.yml b/.github/workflows/deployment-tests.yml index 95c65215..967c9171 100644 --- a/.github/workflows/deployment-tests.yml +++ b/.github/workflows/deployment-tests.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - os_container: ['centos7', 'centosstream8', 'rockylinux8', 'rockylinux9', 'ubuntu20_04', 'ubuntu22_04'] + os_container: ['centos7', 'rockylinux8', 'rockylinux9', 'ubuntu20_04', 'ubuntu22_04'] steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - os_container: ['centos7', 'centosstream8', 'rockylinux8', 'rockylinux9', 'ubuntu20_04', 'ubuntu22_04'] + os_container: ['centos7', 'rockylinux8', 'rockylinux9', 'ubuntu20_04', 'ubuntu22_04'] container: matterminers/cobald-tardis-deployment-test-env:${{ matrix.os_container }} steps: - uses: actions/checkout@v3 @@ -58,9 +58,9 @@ jobs: - uses: actions/checkout@v3 - name: Install dependencies on ${{ matrix.platform }} run: | - python3.10 -m pip install --upgrade pip - python3.10 -m pip install .[contrib] - python3.10 -m pip install coverage codecov + python3 -m pip install --upgrade pip + python3 -m pip install .[contrib] + python3 -m pip install coverage codecov - name: Test with unittest on ${{ matrix.platform }} run: | coverage run -m unittest -v diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 2533bb38..709e6ab8 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -4,10 +4,10 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57c0925a..673d2d30 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,27 +1,33 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.1.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer -- repo: https://github.com/psf/black - rev: 23.1.0 + - repo: https://github.com/psf/black + rev: 23.7.0 hooks: - - id: black + - id: black args: - - --target-version=py36 -- repo: https://github.com/PyCQA/flake8 + - --target-version=py38 + - repo: https://github.com/PyCQA/flake8 rev: 6.0.0 hooks: - id: flake8 -- repo: https://github.com/Matterminers/dev-tools + - repo: https://github.com/Matterminers/dev-tools rev: "87c2435b377bd932acbcc1b619320d98e67b630a" hooks: - - id: contributors - - id: changelog - args: + - id: contributors + - id: changelog + args: - docs/source/changes - compile + - --categories + - Added + - Changed + - Fixed + - Security + - Deprecated - --output=docs/source/changelog.rst - additional_dependencies: - - PyYAML + additional_dependencies: + - PyYAML diff --git a/containers/cobald-tardis-deployment-test-env/Dockerfile.centos7 b/containers/cobald-tardis-deployment-test-env/Dockerfile.centos7 index 6c5ef110..e7bc528b 100644 --- a/containers/cobald-tardis-deployment-test-env/Dockerfile.centos7 +++ b/containers/cobald-tardis-deployment-test-env/Dockerfile.centos7 @@ -1,16 +1,22 @@ FROM centos:7 LABEL maintainer="Manuel Giffels " -RUN yum -y install epel-release curl && yum clean all +RUN yum -y install epel-release centos-release-scl-rh curl && yum clean all +RUN yum --enablerepo=centos-sclo-rh -y install rh-python38 && yum clean all RUN curl -sL https://rpm.nodesource.com/setup_16.x | bash - RUN yum -y update \ && yum -y install git \ - python3 \ gcc \ - python3-devel \ nodejs \ && yum clean all +# Enable rh-python38 from SCL +ENV PATH=/opt/rh/rh-python38/root/usr/local/bin:/opt/rh/rh-python38/root/usr/bin${PATH:+:${PATH}} +ENV LD_LIBRARY_PATH=/opt/rh/rh-python38/root/usr/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} +ENV MANPATH=/opt/rh/rh-python38/root/usr/share/man:$MANPATH +ENV PKG_CONFIG_PATH=/opt/rh/rh-python38/root/usr/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} +ENV XDG_DATA_DIRS="/opt/rh/rh-python38/root/usr/share:${XDG_DATA_DIRS:-/usr/local/share:/usr/share}" + SHELL [ "/bin/bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c" ] diff --git a/containers/cobald-tardis-deployment-test-env/Dockerfile.centosstream8 b/containers/cobald-tardis-deployment-test-env/Dockerfile.centosstream8 deleted file mode 100644 index f652c287..00000000 --- a/containers/cobald-tardis-deployment-test-env/Dockerfile.centosstream8 +++ /dev/null @@ -1,15 +0,0 @@ -FROM quay.io/centos/centos:stream8 -LABEL maintainer="Manuel Giffels " - -RUN curl -sL https://rpm.nodesource.com/setup_18.x | bash - - -RUN dnf -y update \ - && dnf -y install git \ - python39 \ - gcc-8.5.0-7.el8 \ - python39-devel \ - nodejs \ - glibc-langpack-en \ - && dnf clean all - -SHELL [ "/bin/bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c" ] diff --git a/containers/cobald-tardis-deployment-test-env/Dockerfile.rockylinux8 b/containers/cobald-tardis-deployment-test-env/Dockerfile.rockylinux8 index 3da3a174..bf7382fb 100644 --- a/containers/cobald-tardis-deployment-test-env/Dockerfile.rockylinux8 +++ b/containers/cobald-tardis-deployment-test-env/Dockerfile.rockylinux8 @@ -7,11 +7,14 @@ RUN curl -sL https://rpm.nodesource.com/setup_18.x | bash - RUN yum -y update \ && yum -y install git \ - python38 \ + python39 \ gcc \ - python38-devel \ + python39-devel \ nodejs \ glibc-langpack-en \ && yum clean all +# Make Python3.9 default +RUN alternatives --set python3 /usr/bin/python3.9 + SHELL [ "/bin/bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c" ] diff --git a/containers/cobald-tardis-htcondor/Dockerfile b/containers/cobald-tardis-htcondor/Dockerfile index f8808d11..a067406d 100644 --- a/containers/cobald-tardis-htcondor/Dockerfile +++ b/containers/cobald-tardis-htcondor/Dockerfile @@ -1,28 +1,29 @@ -FROM centos:7 +FROM rockylinux:8 LABEL maintainer="Manuel Giffels " ARG SOURCE_BRANCH=master ARG SOURCE_REPO_URL=https://github.com/MatterMiners/tardis RUN rpm --import http://research.cs.wisc.edu/htcondor/yum/RPM-GPG-KEY-HTCondor -RUN yum install -y https://research.cs.wisc.edu/htcondor/repo/9.0/htcondor-release-current.el7.noarch.rpm +RUN dnf install -y https://research.cs.wisc.edu/htcondor/repo/10/10.0/htcondor-release-current.el8.noarch.rpm -RUN yum -y install epel-release && yum clean all +RUN dnf -y install epel-release && dnf clean all +RUN dnf config-manager --set-enabled powertools -RUN yum -y update \ - && yum -y install condor \ +RUN dnf -y update \ + && dnf -y install condor \ git \ - python3 \ + python39 \ gcc \ - python3-devel \ - && yum clean all + python39-devel \ + && dnf clean all -RUN python3 -m pip install --no-cache-dir --upgrade pip \ - && python3 -m pip install --no-cache-dir git+$SOURCE_REPO_URL@$SOURCE_BRANCH +RUN python3.9 -m pip install --no-cache-dir --upgrade pip \ + && python3.9 -m pip install --no-cache-dir git+$SOURCE_REPO_URL@$SOURCE_BRANCH WORKDIR /srv COPY cobald.yml /srv/cobald.yml -ENTRYPOINT ["python3", "-m", "cobald.daemon"] +ENTRYPOINT ["python3.9", "-m", "cobald.daemon"] CMD ["/srv/cobald.yml"] diff --git a/containers/cobald-tardis/Dockerfile b/containers/cobald-tardis/Dockerfile index 37153520..b8035e70 100644 --- a/containers/cobald-tardis/Dockerfile +++ b/containers/cobald-tardis/Dockerfile @@ -1,5 +1,5 @@ -FROM python:3.8-alpine -MAINTAINER Manuel Giffels +FROM python:3.10-alpine +LABEL maintainer="Manuel Giffels " ARG SOURCE_BRANCH=master ARG SOURCE_REPO_URL=https://github.com/MatterMiners/tardis diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index ce4fcb54..b74122a0 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -1,11 +1,19 @@ .. Created by changelog.py at 2023-08-10, command - '/Users/giffler/.cache/pre-commit/repor6pnmwlm/py_env-python3.10/bin/changelog docs/source/changes compile --output=docs/source/changelog.rst' + '/Users/giffler/.cache/pre-commit/repor6pnmwlm/py_env-python3.10/bin/changelog docs/source/changes compile --categories Added Changed Fixed Security Deprecated --output=docs/source/changelog.rst' based on the format of 'https://keepachangelog.com/' ######### CHANGELOG ######### +[Unreleased] - 2023-07-31 +========================= + +Deprecated +---------- + +* Minimal Python version is 3.8 + [0.7.1] - 2023-05-16 ==================== diff --git a/docs/source/changes/304.deprecate_python_36_37.yaml b/docs/source/changes/304.deprecate_python_36_37.yaml new file mode 100644 index 00000000..77ac6671 --- /dev/null +++ b/docs/source/changes/304.deprecate_python_36_37.yaml @@ -0,0 +1,10 @@ +category: deprecated +summary: "Minimal Python version is 3.8" +description: | + Support for Python versions 3.6 and 3.7 has been depracted. This has no further + functional changes for users but reflects the end of life of these versions + and allows TARDIS and plugins to drop outdated dependencies and compatibility layers. +pull requests: + - 304 +issues: + - 303 diff --git a/setup.py b/setup.py index 24c1fafe..1fd2b4e3 100644 --- a/setup.py +++ b/setup.py @@ -15,10 +15,9 @@ TESTS_REQUIRE = ["flake8", "httpx"] REST_REQUIRES = [ "fastapi-jwt-auth", - "fastapi<0.84.0; python_version<'3.7'", # to support python3.6 (Centos 7) - "fastapi; python_version>='3.7'", + "fastapi", "python-jose", - "uvicorn[standard]<=0.14.0", # to support python3.6 (Centos 7) + "uvicorn[standard]", "typer", "bcrypt", "python-multipart", @@ -53,11 +52,10 @@ def get_cryptography_version(): "Topic :: Utilities", "Framework :: AsyncIO", "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], entry_points={ "console_scripts": [ @@ -78,14 +76,14 @@ def get_cryptography_version(): }, keywords=package_about["__keywords__"], packages=find_packages(exclude=["tests"]), + python_requires=">=3.8", install_requires=[ - "aiohttp<4.0; python_version<'3.7'", # to support python3.6 (Centos 7) - "aiohttp; python_version>='3.7'", + "aiohttp", get_cryptography_version(), "CloudStackAIO>=0.0.8", "PyYAML", "AsyncOpenStackClient>=0.9.0", - "cobald>=0.12.3", + "cobald>=0.14.0 ", "asyncssh", "aiotelegraf", "elasticsearch>=7.17,<8.0.0", @@ -94,7 +92,6 @@ def get_cryptography_version(): "pydantic<2.0.0", "asyncstdlib", "typing_extensions", - "backports.cached_property", "python-auditor==0.1.0", "pytz", "tzlocal", diff --git a/tardis/resources/drone.py b/tardis/resources/drone.py index a5b3c13c..43ba3ec6 100644 --- a/tardis/resources/drone.py +++ b/tardis/resources/drone.py @@ -12,8 +12,8 @@ from cobald.daemon import service from cobald.interfaces import Pool -from backports.cached_property import cached_property from datetime import datetime +from functools import cached_property import asyncio import logging diff --git a/tardis/utilities/asyncbulkcall.py b/tardis/utilities/asyncbulkcall.py index dff49c60..97e3bd58 100644 --- a/tardis/utilities/asyncbulkcall.py +++ b/tardis/utilities/asyncbulkcall.py @@ -1,11 +1,10 @@ +from functools import cached_property from typing import TypeVar, Generic, Iterable, List, Tuple, Optional, Set from typing_extensions import Protocol import asyncio import time import sys -from backports.cached_property import cached_property - T = TypeVar("T") R = TypeVar("R") diff --git a/tests/adapters_t/sites_t/test_fakesite.py b/tests/adapters_t/sites_t/test_fakesite.py index a77f5ff3..55bb489d 100644 --- a/tests/adapters_t/sites_t/test_fakesite.py +++ b/tests/adapters_t/sites_t/test_fakesite.py @@ -3,7 +3,7 @@ from tardis.interfaces.siteadapter import ResourceStatus from tardis.utilities.attributedict import AttributeDict -from ...utilities.utilities import run_async +from tests.utilities.utilities import run_async from datetime import datetime from datetime import timedelta diff --git a/tests/adapters_t/sites_t/test_htcondorsiteadapter.py b/tests/adapters_t/sites_t/test_htcondorsiteadapter.py index 2c80c242..ea908d5d 100644 --- a/tests/adapters_t/sites_t/test_htcondorsiteadapter.py +++ b/tests/adapters_t/sites_t/test_htcondorsiteadapter.py @@ -4,8 +4,8 @@ from tardis.exceptions.tardisexceptions import TardisResourceStatusUpdateFailed from tardis.interfaces.siteadapter import ResourceStatus from tardis.utilities.attributedict import AttributeDict -from ...utilities.utilities import mock_executor_run_command -from ...utilities.utilities import run_async +from tests.utilities.utilities import mock_executor_run_command +from tests.utilities.utilities import run_async from datetime import datetime from datetime import timedelta diff --git a/tests/adapters_t/sites_t/test_slurm.py b/tests/adapters_t/sites_t/test_slurm.py index a372abb0..13a5c29b 100644 --- a/tests/adapters_t/sites_t/test_slurm.py +++ b/tests/adapters_t/sites_t/test_slurm.py @@ -5,8 +5,8 @@ from tardis.exceptions.executorexceptions import CommandExecutionFailure from tardis.interfaces.siteadapter import ResourceStatus from tardis.utilities.attributedict import AttributeDict -from ...utilities.utilities import mock_executor_run_command -from ...utilities.utilities import run_async +from tests.utilities.utilities import mock_executor_run_command +from tests.utilities.utilities import run_async from unittest import TestCase from unittest.mock import MagicMock, patch diff --git a/tests/agents_t/test_batchsystemagent.py b/tests/agents_t/test_batchsystemagent.py index 74fe6804..f825c18a 100644 --- a/tests/agents_t/test_batchsystemagent.py +++ b/tests/agents_t/test_batchsystemagent.py @@ -1,5 +1,5 @@ -from ..utilities.utilities import run_async -from ..utilities.utilities import async_return +from tests.utilities.utilities import run_async +from tests.utilities.utilities import async_return from tardis.agents.batchsystemagent import BatchSystemAgent from tardis.interfaces.batchsystemadapter import BatchSystemAdapter from tardis.utilities.attributedict import AttributeDict diff --git a/tests/agents_t/test_siteagent.py b/tests/agents_t/test_siteagent.py index 54dd80b9..3cc45def 100644 --- a/tests/agents_t/test_siteagent.py +++ b/tests/agents_t/test_siteagent.py @@ -1,5 +1,5 @@ -from ..utilities.utilities import run_async -from ..utilities.utilities import async_return +from tests.utilities.utilities import run_async +from tests.utilities.utilities import async_return from tardis.agents.siteagent import SiteAgent from tardis.interfaces.siteadapter import SiteAdapter diff --git a/tests/interfaces_t/test_batchsystemadapter.py b/tests/interfaces_t/test_batchsystemadapter.py index e2ec3add..fa4b273c 100644 --- a/tests/interfaces_t/test_batchsystemadapter.py +++ b/tests/interfaces_t/test_batchsystemadapter.py @@ -1,6 +1,6 @@ from tardis.interfaces.batchsystemadapter import BatchSystemAdapter -from ..utilities.utilities import run_async +from tests.utilities.utilities import run_async from unittest import TestCase from unittest.mock import patch diff --git a/tests/interfaces_t/test_siteadapter.py b/tests/interfaces_t/test_siteadapter.py index 614b3ac2..10a7c505 100644 --- a/tests/interfaces_t/test_siteadapter.py +++ b/tests/interfaces_t/test_siteadapter.py @@ -1,7 +1,7 @@ from tardis.interfaces.siteadapter import SiteAdapter from tardis.utilities.attributedict import AttributeDict -from ..utilities.utilities import run_async +from tests.utilities.utilities import run_async from cobald.utility.primitives import infinity as inf from unittest import TestCase diff --git a/tests/plugins_t/test_auditor.py b/tests/plugins_t/test_auditor.py index faf4b812..18ed435e 100644 --- a/tests/plugins_t/test_auditor.py +++ b/tests/plugins_t/test_auditor.py @@ -18,8 +18,8 @@ from unittest import TestCase from unittest.mock import patch -from ..utilities.utilities import async_return -from ..utilities.utilities import run_async +from tests.utilities.utilities import async_return +from tests.utilities.utilities import run_async class TestAuditor(TestCase): diff --git a/tests/plugins_t/test_elasticsearchmonitoring.py b/tests/plugins_t/test_elasticsearchmonitoring.py index 9867cb1d..2a87f33a 100644 --- a/tests/plugins_t/test_elasticsearchmonitoring.py +++ b/tests/plugins_t/test_elasticsearchmonitoring.py @@ -7,7 +7,7 @@ from unittest import TestCase from unittest.mock import patch -from ..utilities.utilities import run_async +from tests.utilities.utilities import run_async class TestElasticsearchMonitoring(TestCase): diff --git a/tests/plugins_t/test_prometheusmonitoring.py b/tests/plugins_t/test_prometheusmonitoring.py index ee721487..28aef634 100644 --- a/tests/plugins_t/test_prometheusmonitoring.py +++ b/tests/plugins_t/test_prometheusmonitoring.py @@ -8,7 +8,7 @@ from unittest.mock import Mock from unittest.mock import patch -from ..utilities.utilities import get_free_port, run_async +from tests.utilities.utilities import get_free_port, run_async class TestPrometheusMonitoring(TestCase): diff --git a/tests/plugins_t/test_sqliteregistry.py b/tests/plugins_t/test_sqliteregistry.py index 0cd20797..5e738a70 100644 --- a/tests/plugins_t/test_sqliteregistry.py +++ b/tests/plugins_t/test_sqliteregistry.py @@ -5,7 +5,7 @@ from tardis.interfaces.state import State from tardis.plugins.sqliteregistry import SqliteRegistry from tardis.utilities.attributedict import AttributeDict -from ..utilities.utilities import run_async +from tests.utilities.utilities import run_async from unittest import TestCase from unittest.mock import patch diff --git a/tests/plugins_t/test_telegrafmonitoring.py b/tests/plugins_t/test_telegrafmonitoring.py index 4ddafffd..a1a1f139 100644 --- a/tests/plugins_t/test_telegrafmonitoring.py +++ b/tests/plugins_t/test_telegrafmonitoring.py @@ -7,8 +7,8 @@ from unittest.mock import Mock from unittest.mock import patch -from ..utilities.utilities import async_return -from ..utilities.utilities import run_async +from tests.utilities.utilities import async_return +from tests.utilities.utilities import run_async import platform diff --git a/tests/resources_t/test_drone.py b/tests/resources_t/test_drone.py index 1d38efe0..f122a2ae 100644 --- a/tests/resources_t/test_drone.py +++ b/tests/resources_t/test_drone.py @@ -1,4 +1,8 @@ -from ..utilities.utilities import async_return, run_async, set_awaitable_return_value +from tests.utilities.utilities import ( + async_return, + run_async, + set_awaitable_return_value, +) from tardis.interfaces.plugin import Plugin from tardis.interfaces.siteadapter import ResourceStatus diff --git a/tests/resources_t/test_dronestates.py b/tests/resources_t/test_dronestates.py index 4aa835ba..dbe03ab5 100644 --- a/tests/resources_t/test_dronestates.py +++ b/tests/resources_t/test_dronestates.py @@ -18,8 +18,8 @@ from tardis.resources.dronestates import CleanupState from tardis.resources.dronestates import DownState from tardis.utilities.attributedict import AttributeDict -from ..utilities.utilities import async_return -from ..utilities.utilities import run_async +from tests.utilities.utilities import async_return +from tests.utilities.utilities import run_async from functools import partial from datetime import datetime, timedelta diff --git a/tests/rest_t/test_service.py b/tests/rest_t/test_service.py index a5d3dd78..6f468b04 100644 --- a/tests/rest_t/test_service.py +++ b/tests/rest_t/test_service.py @@ -1,5 +1,5 @@ from tardis.rest.service import RestService -from ..utilities.utilities import async_return, run_async +from tests.utilities.utilities import async_return, run_async from unittest import TestCase from unittest.mock import patch diff --git a/tests/utilities_t/test_asyncbulkcall.py b/tests/utilities_t/test_asyncbulkcall.py index 6ffd6a04..1ebd69a0 100644 --- a/tests/utilities_t/test_asyncbulkcall.py +++ b/tests/utilities_t/test_asyncbulkcall.py @@ -6,7 +6,7 @@ from tardis.utilities.asyncbulkcall import AsyncBulkCall -from ..utilities.utilities import run_async +from tests.utilities.utilities import run_async class CallCounter: diff --git a/tests/utilities_t/test_asynccachemap.py b/tests/utilities_t/test_asynccachemap.py index 53939f17..e67825bd 100644 --- a/tests/utilities_t/test_asynccachemap.py +++ b/tests/utilities_t/test_asynccachemap.py @@ -1,7 +1,7 @@ from tardis.exceptions.executorexceptions import CommandExecutionFailure from tardis.utilities.asynccachemap import AsyncCacheMap -from ..utilities.utilities import run_async +from tests.utilities.utilities import run_async from json.decoder import JSONDecodeError from datetime import datetime diff --git a/tests/utilities_t/test_pipeline.py b/tests/utilities_t/test_pipeline.py index 9cb46a0d..098efa73 100644 --- a/tests/utilities_t/test_pipeline.py +++ b/tests/utilities_t/test_pipeline.py @@ -1,6 +1,6 @@ from tardis.utilities.pipeline import PipelineProcessor from tardis.utilities.pipeline import StopProcessing -from ..utilities.utilities import run_async +from tests.utilities.utilities import run_async from unittest import TestCase