From 423bde759c0a4bf6ea311f26b86a4418deaf2493 Mon Sep 17 00:00:00 2001 From: Stephane Odul Date: Mon, 8 May 2023 19:03:28 -0700 Subject: [PATCH] Add ability to run CI checks manually. Add 15m timeout to CI checks. Update CI checks to pip install green in developer mode. --- .github/workflows/ci.yml | 20 +++-- green/process.py | 129 +++++++++++++++++---------------- green/test/test_integration.py | 32 ++++---- pyproject.toml | 4 +- release.md | 3 - requirements-optional.txt | 2 + 6 files changed, 99 insertions(+), 91 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e93ff5..f9ee32f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,10 +5,12 @@ on: branches: [main] pull_request: branches: [main] + workflow_dispatch: jobs: tests: runs-on: ${{ matrix.os }} + timeout-minutes: 15 strategy: matrix: os: [macos-latest, ubuntu-latest, windows-latest] @@ -26,27 +28,29 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install run: | python -m pip install --upgrade pip pip install --upgrade -r requirements-optional.txt + pip install --upgrade -e . - - name: Lint - run: | - pip install --upgrade black mypy - black --check --diff green example - mypy . + - name: Format + run: black --check --diff green example + if: matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest' + + - name: Mypy + run: mypy . if: matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest' - name: Test run: | - python -m green.cmdline -tvvvv green + green -tvvvv green cd example && python -m green.cmdline -tvvvv proj - name: Generate coverage run: | pip install --upgrade coveralls - ./g -tvvvvr green + green -tvvvvr green if: matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest' - name: Coveralls diff --git a/green/process.py b/green/process.py index 60357d9..4a26ad9 100644 --- a/green/process.py +++ b/green/process.py @@ -6,7 +6,6 @@ import sys import tempfile import traceback -from typing import Union import coverage @@ -60,14 +59,73 @@ def __call__(self, *args, **kwargs): return result -class LoggingDaemonlessPool(Pool): +# ------------------------------------------------------------------------- +# I started with code from cpython/Lib/multiprocessing/pool.py from version +# 3.5.0a4+ of the main python mercurial repository. Then altered it to run +# on 2.7+ and added the finalizer/finalargs parameter handling. This approach +# worked until we hit Python 3.8, when it broke. +class LoggingDaemonlessPool37(Pool): # pragma: no cover """ I make a pool of workers which can get crash output to the logger, run processes not as daemons, - and which run finalizers. + and which run finalizers...in a way which works on Python 2.7 to 3.7, inclusive. """ + @staticmethod + def Process(*args, **kwargs): + kwargs["daemon"] = False + return multiprocessing.Process(*args, **kwargs) + + def apply_async(self, func, args=(), kwds={}, callback=None): + return Pool.apply_async(self, ProcessLogger(func), args, kwds, callback) + _wrap_exception = True + def __init__( + self, + processes=None, + initializer=None, + initargs=(), + maxtasksperchild=None, + context=None, + finalizer=None, + finalargs=(), + ): + self._finalizer = finalizer + self._finalargs = finalargs + super().__init__(processes, initializer, initargs, maxtasksperchild) + + def _repopulate_pool(self): + """ + Bring the number of pool processes up to the specified number, for use + after reaping workers which have exited. + """ + for i in range(self._processes - len(self._pool)): + w = self.Process( + target=worker, + args=( + self._inqueue, + self._outqueue, + self._initializer, + self._initargs, + self._maxtasksperchild, + self._wrap_exception, + self._finalizer, + self._finalargs, + ), + ) + self._pool.append(w) + w.name = w.name.replace("Process", "PoolWorker") + w.start() + util.debug("added worker") + + +class LoggingDaemonlessPool38(Pool): + """ + I make a pool of workers which can get crash output to the logger, run processes not as daemons, + and which run finalizers...in a way which works on Python 3.8+. + + """ + @staticmethod def Process(ctx, *args, **kwds): process = ctx.Process(daemon=False, *args, **kwds) @@ -78,6 +136,8 @@ def apply_async(self, func, args=(), kwds={}, callback=None, error_callback=None self, ProcessLogger(func), args, kwds, callback, error_callback ) + _wrap_exception = True + def __init__( self, processes=None, @@ -148,70 +208,13 @@ def _repopulate_pool_static( util.debug("added worker") -# ------------------------------------------------------------------------- -# I started with code from cpython/Lib/multiprocessing/pool.py from version -# 3.5.0a4+ of the main python mercurial repository. Then altered it to run -# on 2.7+ and added the finalizer/finalargs parameter handling. This approach -# worked until we hit Python 3.8, when it broke. -class LoggingDaemonlessPool37(LoggingDaemonlessPool): # pragma: no cover - """ - I make a pool of workers which can get crash output to the logger, run processes not as daemons, - and which run finalizers...in a way which works on Python 3.6 and 3.7. - """ - - @staticmethod - def Process(*args, **kwargs): - kwargs["daemon"] = False - return multiprocessing.Process(*args, **kwargs) - - def apply_async(self, func, args=(), kwds={}, callback=None): - return Pool.apply_async(self, ProcessLogger(func), args, kwds, callback) - - def __init__( - self, - processes=None, - initializer=None, - initargs=(), - maxtasksperchild=None, - context=None, - finalizer=None, - finalargs=(), - ): - self._finalizer = finalizer - self._finalargs = finalargs - super().__init__(processes, initializer, initargs, maxtasksperchild) - - def _repopulate_pool(self): - """ - Bring the number of pool processes up to the specified number, for use - after reaping workers which have exited. - """ - for i in range(self._processes - len(self._pool)): - w = self.Process( - target=worker, - args=( - self._inqueue, - self._outqueue, - self._initializer, - self._initargs, - self._maxtasksperchild, - self._wrap_exception, - self._finalizer, - self._finalargs, - ), - ) - self._pool.append(w) - w.name = w.name.replace("Process", "PoolWorker") - w.start() - util.debug("added worker") - - +LoggingDaemonlessPool = LoggingDaemonlessPool38 if tuple(map(int, platform.python_version_tuple()[:2])) < (3, 8): # pragma: no cover LoggingDaemonlessPool = LoggingDaemonlessPool37 # type: ignore import multiprocessing.pool -from multiprocessing import util +from multiprocessing import util # type: ignore from multiprocessing.pool import MaybeEncodingError # type: ignore @@ -225,7 +228,7 @@ def worker( finalizer=None, finalargs=(), ): # pragma: no cover - assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0) + assert maxtasks is None or (isinstance(maxtasks, int) and maxtasks > 0) put = outqueue.put get = inqueue.get if hasattr(inqueue, "_writer"): diff --git a/green/test/test_integration.py b/green/test/test_integration.py index a643865..9dc3d25 100644 --- a/green/test/test_integration.py +++ b/green/test/test_integration.py @@ -1,29 +1,30 @@ import copy import multiprocessing import os -from pathlib import PurePath +import pathlib +import shutil import subprocess import sys import tempfile from textwrap import dedent import unittest -from unittest.mock import MagicMock - -from green import cmdline class TestFinalizer(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() + def setUp(self) -> None: + self.tmpdir = pathlib.Path(tempfile.mkdtemp()) + + def tearDown(self) -> None: + shutil.rmtree(self.tmpdir, ignore_errors=True) - def test_finalizer(self): + def test_finalizer(self) -> None: """ - Test that the finalizer works on Python 3.8+ + Test that the finalizer works on all supported versions of Python. """ - sub_tmpdir = tempfile.mkdtemp(dir=self.tmpdir) + sub_tmpdir = pathlib.Path(tempfile.mkdtemp(dir=self.tmpdir)) for i in range(multiprocessing.cpu_count() * 2): - fh = open(os.path.join(sub_tmpdir, f"test_finalizer{i}.py"), "w") - fh.write( + finalizer_path = sub_tmpdir / f"test_finalizer{i}.py" + finalizer_path.write_text( dedent( f""" import unittest @@ -35,7 +36,6 @@ def msg(): """ ) ) - fh.close() args = [ sys.executable, "-m", @@ -43,17 +43,19 @@ def msg(): "--finalizer=test_finalizer0.msg", "--maxtasksperchild=1", ] - pythonpath = str(PurePath(__file__).parent.parent.parent) + pythonpath = str(pathlib.Path(__file__).parent.parent.parent) env = copy.deepcopy(os.environ) env["PYTHONPATH"] = pythonpath output = subprocess.run( args, - cwd=sub_tmpdir, + cwd=str(sub_tmpdir), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, timeout=10, - ).stdout.decode("utf-8") + encoding="utf-8", + check=True, + ).stdout self.assertIn("finalizer worked", output) diff --git a/pyproject.toml b/pyproject.toml index 68f79fe..9a9032a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,8 +11,8 @@ build-backend = "setuptools.build_meta" [tool.mypy] -# Targetting supported versions of python since third party libraries will have -# support for newer python syntax. +# Targetting versions of python officially supported by python.org since third +# party libraries will have support for newer python syntax, throwing errors. python_version = "3.8" sqlite_cache = true namespace_packages = true diff --git a/release.md b/release.md index e384dea..cba324a 100644 --- a/release.md +++ b/release.md @@ -10,7 +10,6 @@ Very First Time =============== 1. Set up `~/.pypirc` - ``` [distutils] index-servers = @@ -28,5 +27,3 @@ Very First Time ``` 2. `python setup.py register -r pypi` - - diff --git a/requirements-optional.txt b/requirements-optional.txt index 57df946..97c2e38 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -1,3 +1,5 @@ +black django +mypy testtools -r requirements.txt