[WIP] RHOAIENG-9707 ci: dynamic testing of container images with pytest

README.md

@@ -64,6 +64,20 @@ poetry install --sync
 poetry run pytest
 ```
 
+##### Container selftests
+
+We're using [Dagger.io](https://dagger.io) to run containers from Python tests.
+It has very nice verbose logging by default for every action that is running.
+
+**TODO** need to decide which is more suitable;
+ideally the tests should allow switching between Kubernetes and Docker/Podman.
+That means tying it to this fancy Dagger thing may not be the smartest thing to do.
+
+We also considered [Testcontainers.com](https://testcontainers.com/).
+The main difference between the two is that Dagger creates more abstractions over the container engine.
+Especially Dagger [does not allow bind-mounting local directories](https://docs.dagger.io/cookbook/#modify-a-copied-directory-or-remote-repository-in-a-container)
+directly to the container but always copies files in and out.
+
 #### Notebooks
 
 Deploy the notebook images in your Kubernetes environment using:

pyproject.toml

@@ -9,10 +9,12 @@ package-mode = false
 [tool.poetry.dependencies]
 python = "~3.12"
 
-
 [tool.poetry.group.dev.dependencies]
-pytest = "^8.2.2"
-pytest-subtests = "^0.12.1"
+pytest = "^8.3.2"
+pytest-subtests = "^0.13.1"
+pytest-logger = "^1.1.1"
+testcontainers = "^4.7.2"
+pyfakefs = "^5.6.0"
 
 [build-system]
 requires = ["poetry-core"]

tests/__init__.py

@@ -0,0 +1,10 @@
+import os
+
+import pathlib
+
+# Absolute path to the top level directory
+ROOT_PATH = pathlib.Path(__file__).parent.parent
+
+# Disable Dagger telemetry and PaaS offering
+os.environ["DO_NOT_TRACK"]= "1"
+os.environ["NOTHANKS"]= "1"

tests/conftest.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import testcontainers.core.container
+import testcontainers.core.config
+
+if TYPE_CHECKING:
+    from pytest import ExitCode, Session
+
+# We'd get selinux violations with podman otherwise, so either ryuk must be privileged, or we need to disable selinux.
+# https://github.com/testcontainers/testcontainers-java/issues/2088#issuecomment-1169830358
+testcontainers.core.config.testcontainers_config.ryuk_privileged = True
+
+
+# https://docs.pytest.org/en/latest/reference/reference.html#pytest.hookspec.pytest_sessionfinish
+def pytest_sessionfinish(session: Session, exitstatus: int | ExitCode) -> None:
+    testcontainers.core.container.Reaper.delete_instance()

tests/docker_utils.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import io
+import logging
+import os.path
+import sys
+import tarfile
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from docker.models.containers import Container
+
+
+def container_cp(container: Container, src: str, dst: str,
+                 user: int | None = None, group: int | None = None) -> None:
+    """
+    Copies a directory into a container
+    From https://stackoverflow.com/questions/46390309/how-to-copy-a-file-from-host-to-container-using-docker-py-docker-sdk
+    """
+    fh = io.BytesIO()
+    tar = tarfile.open(fileobj=fh, mode="w:gz")
+
+    tar_filter = None
+    if user or group:
+        def tar_filter(f: tarfile.TarInfo) -> tarfile.TarInfo:
+            if user:
+                f.uid = user
+            if group:
+                f.gid = group
+            return f
+
+    logging.debug(f"Adding {src=} to archive {dst=}")
+    try:
+        tar.add(src, arcname=os.path.basename(src), filter=tar_filter)
+    finally:
+        tar.close()
+
+    fh.seek(0)
+    container.put_archive(dst, fh)
+
+
+def container_exec(
+        container: Container,
+        cmd: str | list[str],
+        stdout: bool = True,
+        stderr: bool = True,
+        stdin: bool = False,
+        tty: bool = False,
+        privileged: bool = False,
+        user: str = "",
+        detach: bool = False,
+        stream: bool = False,
+        socket: bool = False,
+        environment: dict[str, str] | None = None,
+        workdir: str | None = None,
+) -> ContainerExec:
+    """
+    An enhanced version of #docker.Container.exec_run() which returns an object
+    that can be properly inspected for the status of the executed commands.
+    Usage example:
+    result = tools.container_exec(container, cmd, stream=True, **kwargs)
+    res = result.communicate(line_prefix=b'--> ')
+    if res != 0:
+        error('exit code {!r}'.format(res))
+    From https://github.com/docker/docker-py/issues/1989
+    """
+
+    exec_id = container.client.api.exec_create(
+        container.id,
+        cmd,
+        stdout=stdout,
+        stderr=stderr,
+        stdin=stdin,
+        tty=tty,
+        privileged=privileged,
+        user=user,
+        environment=environment,
+        workdir=workdir,
+    )["Id"]
+
+    output = container.client.api.exec_start(exec_id, detach=detach, tty=tty, stream=stream, socket=socket)
+
+    return ContainerExec(container.client, exec_id, output)
+
+
+class ContainerExec:
+    def __init__(self, client, id, output):
+        self.client = client
+        self.id = id
+        self.output = output
+
+    def inspect(self):
+        return self.client.api.exec_inspect(self.id)
+
+    def poll(self):
+        return self.inspect()["ExitCode"]
+
+    def communicate(self, line_prefix=b""):
+        for data in self.output:
+            if not data:
+                continue
+            offset = 0
+            while offset < len(data):
+                sys.stdout.buffer.write(line_prefix)
+                nl = data.find(b"\n", offset)
+                if nl >= 0:
+                    slice = data[offset: nl + 1]
+                    offset = nl + 1
+                else:
+                    slice = data[offset:]
+                    offset += len(slice)
+                sys.stdout.buffer.write(slice)
+            sys.stdout.flush()
+        while self.poll() is None:
+            raise RuntimeError("Hm could that really happen?")
+        return self.poll()

tests/logs/.gitignore

@@ -0,0 +1 @@
+pytest-logs.txt