Add ability to run CI checks manually.

Add 15m timeout to CI checks.
Update CI checks to pip install green in developer mode.

.github/workflows/ci.yml

@@ -5,10 +5,12 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+  workflow_dispatch:
 
 jobs:
   tests:
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
@@ -26,27 +28,29 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies
+      - name: Install
         run: |
           python -m pip install --upgrade pip
           pip install --upgrade -r requirements-optional.txt
+          pip install --upgrade -e .
 
-      - name: Lint
-        run: |
-          pip install --upgrade black mypy
-          black --check --diff green example
-          mypy .
+      - name: Format
+        run: black --check --diff green example
+        if: matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest'
+
+      - name: Mypy
+        run: mypy .
         if: matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest'
 
       - name: Test
         run: |
-          python -m green.cmdline -tvvvv green
+          green -tvvvv green
           cd example && python -m green.cmdline -tvvvv proj
 
       - name: Generate coverage
         run: |
           pip install --upgrade coveralls
-          ./g -tvvvvr green
+          green -tvvvvr green
         if: matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest'
 
       - name: Coveralls

green/process.py

@@ -6,7 +6,6 @@
 import sys
 import tempfile
 import traceback
-from typing import Union
 
 import coverage
 
@@ -60,14 +59,73 @@ def __call__(self, *args, **kwargs):
         return result
 
 
-class LoggingDaemonlessPool(Pool):
+# -------------------------------------------------------------------------
+# I started with code from cpython/Lib/multiprocessing/pool.py from version
+# 3.5.0a4+ of the main python mercurial repository.  Then altered it to run
+# on 2.7+ and added the finalizer/finalargs parameter handling. This approach
+# worked until we hit Python 3.8, when it broke.
+class LoggingDaemonlessPool37(Pool):  # pragma: no cover
     """
     I make a pool of workers which can get crash output to the logger, run processes not as daemons,
-    and which run finalizers.
+    and which run finalizers...in a way which works on Python 2.7 to 3.7, inclusive.
     """
 
+    @staticmethod
+    def Process(*args, **kwargs):
+        kwargs["daemon"] = False
+        return multiprocessing.Process(*args, **kwargs)
+
+    def apply_async(self, func, args=(), kwds={}, callback=None):
+        return Pool.apply_async(self, ProcessLogger(func), args, kwds, callback)
+
     _wrap_exception = True
 
+    def __init__(
+        self,
+        processes=None,
+        initializer=None,
+        initargs=(),
+        maxtasksperchild=None,
+        context=None,
+        finalizer=None,
+        finalargs=(),
+    ):
+        self._finalizer = finalizer
+        self._finalargs = finalargs
+        super().__init__(processes, initializer, initargs, maxtasksperchild)
+
+    def _repopulate_pool(self):
+        """
+        Bring the number of pool processes up to the specified number, for use
+        after reaping workers which have exited.
+        """
+        for i in range(self._processes - len(self._pool)):
+            w = self.Process(
+                target=worker,
+                args=(
+                    self._inqueue,
+                    self._outqueue,
+                    self._initializer,
+                    self._initargs,
+                    self._maxtasksperchild,
+                    self._wrap_exception,
+                    self._finalizer,
+                    self._finalargs,
+                ),
+            )
+            self._pool.append(w)
+            w.name = w.name.replace("Process", "PoolWorker")
+            w.start()
+            util.debug("added worker")
+
+
+class LoggingDaemonlessPool38(Pool):
+    """
+    I make a pool of workers which can get crash output to the logger, run processes not as daemons,
+    and which run finalizers...in a way which works on Python 3.8+.
+
+    """
+
     @staticmethod
     def Process(ctx, *args, **kwds):
         process = ctx.Process(daemon=False, *args, **kwds)
@@ -78,6 +136,8 @@ def apply_async(self, func, args=(), kwds={}, callback=None, error_callback=None
             self, ProcessLogger(func), args, kwds, callback, error_callback
         )
 
+    _wrap_exception = True
+
     def __init__(
         self,
         processes=None,
@@ -148,70 +208,13 @@ def _repopulate_pool_static(
             util.debug("added worker")
 
 
-# -------------------------------------------------------------------------
-# I started with code from cpython/Lib/multiprocessing/pool.py from version
-# 3.5.0a4+ of the main python mercurial repository.  Then altered it to run
-# on 2.7+ and added the finalizer/finalargs parameter handling. This approach
-# worked until we hit Python 3.8, when it broke.
-class LoggingDaemonlessPool37(LoggingDaemonlessPool):  # pragma: no cover
-    """
-    I make a pool of workers which can get crash output to the logger, run processes not as daemons,
-    and which run finalizers...in a way which works on Python 3.6 and 3.7.
-    """
-
-    @staticmethod
-    def Process(*args, **kwargs):
-        kwargs["daemon"] = False
-        return multiprocessing.Process(*args, **kwargs)
-
-    def apply_async(self, func, args=(), kwds={}, callback=None):
-        return Pool.apply_async(self, ProcessLogger(func), args, kwds, callback)
-
-    def __init__(
-        self,
-        processes=None,
-        initializer=None,
-        initargs=(),
-        maxtasksperchild=None,
-        context=None,
-        finalizer=None,
-        finalargs=(),
-    ):
-        self._finalizer = finalizer
-        self._finalargs = finalargs
-        super().__init__(processes, initializer, initargs, maxtasksperchild)
-
-    def _repopulate_pool(self):
-        """
-        Bring the number of pool processes up to the specified number, for use
-        after reaping workers which have exited.
-        """
-        for i in range(self._processes - len(self._pool)):
-            w = self.Process(
-                target=worker,
-                args=(
-                    self._inqueue,
-                    self._outqueue,
-                    self._initializer,
-                    self._initargs,
-                    self._maxtasksperchild,
-                    self._wrap_exception,
-                    self._finalizer,
-                    self._finalargs,
-                ),
-            )
-            self._pool.append(w)
-            w.name = w.name.replace("Process", "PoolWorker")
-            w.start()
-            util.debug("added worker")
-
-
+LoggingDaemonlessPool = LoggingDaemonlessPool38
 if tuple(map(int, platform.python_version_tuple()[:2])) < (3, 8):  # pragma: no cover
     LoggingDaemonlessPool = LoggingDaemonlessPool37  # type: ignore
 
 
 import multiprocessing.pool
-from multiprocessing import util
+from multiprocessing import util  # type: ignore
 from multiprocessing.pool import MaybeEncodingError  # type: ignore
 
 
@@ -225,7 +228,7 @@ def worker(
     finalizer=None,
     finalargs=(),
 ):  # pragma: no cover
-    assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0)
+    assert maxtasks is None or (isinstance(maxtasks, int) and maxtasks > 0)
     put = outqueue.put
     get = inqueue.get
     if hasattr(inqueue, "_writer"):

green/test/test_integration.py

@@ -1,29 +1,30 @@
 import copy
 import multiprocessing
 import os
-from pathlib import PurePath
+import pathlib
+import shutil
 import subprocess
 import sys
 import tempfile
 from textwrap import dedent
 import unittest
-from unittest.mock import MagicMock
-
-from green import cmdline
 
 
 class TestFinalizer(unittest.TestCase):
-    def setUp(self):
-        self.tmpdir = tempfile.mkdtemp()
+    def setUp(self) -> None:
+        self.tmpdir = pathlib.Path(tempfile.mkdtemp())
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
 
-    def test_finalizer(self):
+    def test_finalizer(self) -> None:
         """
-        Test that the finalizer works on Python 3.8+
+        Test that the finalizer works on all supported versions of Python.
         """
-        sub_tmpdir = tempfile.mkdtemp(dir=self.tmpdir)
+        sub_tmpdir = pathlib.Path(tempfile.mkdtemp(dir=self.tmpdir))
         for i in range(multiprocessing.cpu_count() * 2):
-            fh = open(os.path.join(sub_tmpdir, f"test_finalizer{i}.py"), "w")
-            fh.write(
+            finalizer_path = sub_tmpdir / f"test_finalizer{i}.py"
+            finalizer_path.write_text(
                 dedent(
                     f"""
                 import unittest
@@ -35,25 +36,26 @@ def msg():
                 """
                 )
             )
-            fh.close()
         args = [
             sys.executable,
             "-m",
             "green.cmdline",
             "--finalizer=test_finalizer0.msg",
             "--maxtasksperchild=1",
         ]
-        pythonpath = str(PurePath(__file__).parent.parent.parent)
+        pythonpath = str(pathlib.Path(__file__).parent.parent.parent)
 
         env = copy.deepcopy(os.environ)
         env["PYTHONPATH"] = pythonpath
 
         output = subprocess.run(
             args,
-            cwd=sub_tmpdir,
+            cwd=str(sub_tmpdir),
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             env=env,
             timeout=10,
-        ).stdout.decode("utf-8")
+            encoding="utf-8",
+            check=True,
+        ).stdout
         self.assertIn("finalizer worked", output)

pyproject.toml

@@ -11,8 +11,8 @@ build-backend = "setuptools.build_meta"
 
 
 [tool.mypy]
-# Targetting supported versions of python since third party libraries will have
-# support for newer python syntax.
+# Targetting versions of python officially supported by python.org since third
+# party libraries will have support for newer python syntax, throwing errors.
 python_version = "3.8"
 sqlite_cache = true
 namespace_packages = true

release.md

@@ -10,7 +10,6 @@ Very First Time
 ===============
 
 1. Set up `~/.pypirc`
-
 ```
     [distutils]
     index-servers =
@@ -28,5 +27,3 @@ Very First Time
 ```
 
 2. `python setup.py register -r pypi`
-
-

requirements-optional.txt

@@ -1,3 +1,5 @@
+black
 django
+mypy
 testtools
 -r requirements.txt