diff --git a/README.md b/README.md index d2498cc..f53bb45 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,18 @@ Get the list of docker_compose commands to be executed for test clean-up actions Override this fixture in your tests if you need to change clean-up actions. Returning anything that would evaluate to False will skip this command. +## Docker Live Output + +```python +@pytest.fixture(scope="session") +def http_service(docker_ip, docker_services): + docker_services.display_live_logs("service_name") +``` + +```bash +pytest --capture=tee-sys +``` + # Development Use of a virtual environment is recommended. See the diff --git a/src/pytest_docker/plugin.py b/src/pytest_docker/plugin.py index bcb7a31..e023c25 100644 --- a/src/pytest_docker/plugin.py +++ b/src/pytest_docker/plugin.py @@ -1,16 +1,20 @@ +from concurrent.futures import Future, ThreadPoolExecutor import contextlib import os import re import subprocess import time import timeit -from typing import Any, Dict, Iterable, Iterator, List, Tuple, Union +from types import TracebackType +from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Type, Union import attr import pytest from _pytest.config import Config from _pytest.fixtures import FixtureRequest +_MAX_LOG_WORKERS = 100 + @pytest.fixture def container_scope_fixture(request: FixtureRequest) -> Any: @@ -21,7 +25,7 @@ def containers_scope(fixture_name: str, config: Config) -> Any: # pylint: disab return config.getoption("--container-scope", "session") -def execute(command: str, success_codes: Iterable[int] = (0,)) -> Union[bytes, Any]: +def execute_and_get_output(command: str, success_codes: Iterable[int] = (0,)) -> Union[bytes, Any]: """Run a shell command.""" try: output = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) @@ -38,6 +42,20 @@ def execute(command: str, success_codes: Iterable[int] = (0,)) -> Union[bytes, A return output +def execute(command: str, success_codes: Iterable[int] = (0,)) -> None: + try: + process = subprocess.run(command, stderr=subprocess.STDOUT, shell=True) + returncode = process.returncode + except subprocess.CalledProcessError as error: + returncode = error.returncode + command = error.cmd + + if returncode not in success_codes: + raise Exception( + 'Command {} returned {}'.format(command, returncode) + ) + + def get_docker_ip() -> Union[str, Any]: # When talking to the Docker daemon via a UNIX socket, route all TCP # traffic to docker containers via the TCP loopback interface. @@ -59,9 +77,18 @@ def docker_ip() -> Union[str, Any]: @attr.s(frozen=True) -class Services: - _docker_compose: Any = attr.ib() - _services: Dict[Any, Dict[Any, Any]] = attr.ib(init=False, default=attr.Factory(dict)) +class Services(contextlib.AbstractContextManager): # type: ignore + _docker_compose: "DockerComposeExecutor" = attr.ib() + _services: Dict[Any, Dict[Any, Any]] = attr.ib( + init=False, default=attr.Factory(dict) + ) + _live_logs: Dict[str, Future[Any]] = attr.ib(init=False, default=attr.Factory(dict)) + _thread_pool_executor: ThreadPoolExecutor = attr.ib( + init=False, + default=attr.Factory( + lambda: ThreadPoolExecutor(max_workers=_MAX_LOG_WORKERS, thread_name_prefix="docker_") + ), + ) def port_for(self, service: str, container_port: int) -> int: """Return the "host" port for `service` and `container_port`. @@ -83,7 +110,7 @@ def port_for(self, service: str, container_port: int) -> int: if cache is not None: return cache - output = self._docker_compose.execute("port %s %d" % (service, container_port)) + output = self._docker_compose.execute_and_get_output("port %s %d" % (service, container_port)) endpoint = output.strip().decode("utf-8") if not endpoint: raise ValueError('Could not detect port for "%s:%d".' % (service, container_port)) @@ -119,6 +146,36 @@ def wait_until_responsive( raise Exception("Timeout reached while waiting on service!") + def display_live_logs(self, service: str) -> None: + """Run `logs` command with the follow flag to show live logs of a service.""" + if service in self._live_logs: + return + + if len(self._live_logs) == _MAX_LOG_WORKERS: + raise NotImplementedError( + f"""\ +{_MAX_LOG_WORKERS} worker threads are supported to display live logs. \ +Please submit a PR if you want to change that.""" + ) + + self._live_logs[service] = self._thread_pool_executor.submit( + self._docker_compose.execute, f"logs {service} -f" + ) + + def close(self) -> None: + for _, fut in self._live_logs.items(): + _ = fut.cancel() + self._thread_pool_executor.shutdown(wait=False) + + def __exit__( + self, + _exc_type: Optional[Type[BaseException]], + __exc_value: Optional[BaseException], + __traceback: Optional[TracebackType], + ) -> None: + self.close() + return None + def str_to_list(arg: Union[str, List[Any], Tuple[Any]]) -> Union[List[Any], Tuple[Any]]: if isinstance(arg, (list, tuple)): @@ -132,12 +189,18 @@ class DockerComposeExecutor: _compose_files: Any = attr.ib(converter=str_to_list) _compose_project_name: str = attr.ib() - def execute(self, subcommand: str) -> Union[bytes, Any]: + def execute_and_get_output(self, subcommand: str) -> Union[bytes, Any]: + return execute_and_get_output(self._format_cmd(subcommand)) + + def execute(self, subcommand: str) -> None: + execute(self._format_cmd(subcommand)) + + def _format_cmd(self, subcommand: str) -> str: command = self._compose_command for compose_file in self._compose_files: command += ' -f "{}"'.format(compose_file) command += ' -p "{}" {}'.format(self._compose_project_name, subcommand) - return execute(command) + return command @pytest.fixture(scope=containers_scope) @@ -213,7 +276,8 @@ def get_docker_services( try: # Let test(s) run. - yield Services(docker_compose) + with Services(docker_compose) as services: + yield services finally: # Clean up. if docker_cleanup: diff --git a/tests/test_docker_services.py b/tests/test_docker_services.py index 27e385d..fea633f 100644 --- a/tests/test_docker_services.py +++ b/tests/test_docker_services.py @@ -15,51 +15,56 @@ def test_docker_services() -> None: """Automatic teardown of all services.""" with mock.patch("subprocess.check_output") as check_output: - check_output.side_effect = [b"", b"0.0.0.0:32770", b""] - check_output.returncode = 0 + with mock.patch("subprocess.run") as run: + check_output.side_effect = [b"0.0.0.0:32770"] + check_output.returncode = 0 + run.return_value = subprocess.CompletedProcess([], returncode=0) - assert check_output.call_count == 0 + assert check_output.call_count == 0 - # The fixture is a context-manager. - with get_docker_services( - "docker compose", - "docker-compose.yml", - docker_compose_project_name="pytest123", - docker_setup=get_setup_command(), - docker_cleanup=get_cleanup_command(), - ) as services: - assert isinstance(services, Services) + # The fixture is a context-manager. + with get_docker_services( + "docker compose", + "docker-compose.yml", + docker_compose_project_name="pytest123", + docker_setup=get_setup_command(), + docker_cleanup=get_cleanup_command(), + ) as services: + assert isinstance(services, Services) - assert check_output.call_count == 1 + assert run.call_count == 1 + assert check_output.call_count == 0 - # Can request port for services. - port = services.port_for("abc", 123) - assert port == 32770 + # Can request port for services. + port = services.port_for("abc", 123) + assert port == 32770 - assert check_output.call_count == 2 + assert check_output.call_count == 1 - # 2nd request for same service should hit the cache. - port = services.port_for("abc", 123) - assert port == 32770 + # 2nd request for same service should hit the cache. + port = services.port_for("abc", 123) + assert port == 32770 - assert check_output.call_count == 2 + assert check_output.call_count == 1 - assert check_output.call_count == 3 + assert run.call_count == 2 # Both should have been called. - assert check_output.call_args_list == [ + assert run.call_args_list == [ mock.call( 'docker compose -f "docker-compose.yml" -p "pytest123" up --build -d', stderr=subprocess.STDOUT, shell=True, ), mock.call( - 'docker compose -f "docker-compose.yml" -p "pytest123" port abc 123', + 'docker compose -f "docker-compose.yml" -p "pytest123" down -v', stderr=subprocess.STDOUT, shell=True, ), + ] + assert check_output.call_args_list == [ mock.call( - 'docker compose -f "docker-compose.yml" -p "pytest123" down -v', + 'docker compose -f "docker-compose.yml" -p "pytest123" port abc 123', stderr=subprocess.STDOUT, shell=True, ), @@ -70,34 +75,38 @@ def test_docker_services_unused_port() -> None: """Complain loudly when the requested port is not used by the service.""" with mock.patch("subprocess.check_output") as check_output: - check_output.side_effect = [b"", b"", b""] - check_output.returncode = 0 + with mock.patch("subprocess.run") as run: + check_output.side_effect = [b"", b"", b""] + check_output.returncode = 0 + run.return_value = subprocess.CompletedProcess([], returncode=0) - assert check_output.call_count == 0 + assert check_output.call_count == 0 - # The fixture is a context-manager. - with get_docker_services( - "docker compose", - "docker-compose.yml", - docker_compose_project_name="pytest123", - docker_setup=get_setup_command(), - docker_cleanup=get_cleanup_command(), - ) as services: - assert isinstance(services, Services) + # The fixture is a context-manager. + with get_docker_services( + "docker compose", + "docker-compose.yml", + docker_compose_project_name="pytest123", + docker_setup=get_setup_command(), + docker_cleanup=get_cleanup_command(), + ) as services: + assert isinstance(services, Services) - assert check_output.call_count == 1 + assert run.call_count == 1 + assert check_output.call_count == 0 - # Can request port for services. - with pytest.raises(ValueError) as exc: - print(services.port_for("abc", 123)) - assert str(exc.value) == ('Could not detect port for "%s:%d".' % ("abc", 123)) + # Can request port for services. + with pytest.raises(ValueError) as exc: + print(services.port_for("abc", 123)) + assert str(exc.value) == ('Could not detect port for "%s:%d".' % ("abc", 123)) - assert check_output.call_count == 2 + assert check_output.call_count == 1 - assert check_output.call_count == 3 + assert run.call_count == 2 + assert check_output.call_count == 1 # Both should have been called. - assert check_output.call_args_list == [ + assert run.call_args_list == [ mock.call( 'docker compose -f "docker-compose.yml" -p "pytest123" ' "up --build -d", # pylint: disable:=implicit-str-concat @@ -105,13 +114,15 @@ def test_docker_services_unused_port() -> None: stderr=subprocess.STDOUT, ), mock.call( - 'docker compose -f "docker-compose.yml" -p "pytest123" ' - "port abc 123", # pylint: disable:=implicit-str-concat + 'docker compose -f "docker-compose.yml" -p "pytest123" down -v', # pylint: disable:=implicit-str-concat shell=True, stderr=subprocess.STDOUT, ), + ] + assert check_output.call_args_list == [ mock.call( - 'docker compose -f "docker-compose.yml" -p "pytest123" down -v', # pylint: disable:=implicit-str-concat + 'docker compose -f "docker-compose.yml" -p "pytest123" ' + "port abc 123", # pylint: disable:=implicit-str-concat shell=True, stderr=subprocess.STDOUT, ), @@ -121,9 +132,9 @@ def test_docker_services_unused_port() -> None: def test_docker_services_failure() -> None: """Propagate failure to start service.""" - with mock.patch("subprocess.check_output") as check_output: - check_output.side_effect = [subprocess.CalledProcessError(1, "the command", b"the output")] - check_output.returncode = 1 + with mock.patch("subprocess.run") as run: + run.side_effect = [subprocess.CalledProcessError(1, "the command")] + run.returncode = 1 # The fixture is a context-manager. with pytest.raises(Exception) as exc: @@ -138,13 +149,13 @@ def test_docker_services_failure() -> None: # Failure propagates with improved diagnoatics. assert str(exc.value) == ( - 'Command {} returned {}: """{}""".'.format("the command", 1, "the output") + 'Command {} returned {}'.format("the command", 1) ) - assert check_output.call_count == 1 + assert run.call_count == 1 # Tear down code should not be called. - assert check_output.call_args_list == [ + assert run.call_args_list == [ mock.call( 'docker compose -f "docker-compose.yml" -p "pytest123" ' "up --build -d", # pylint: disable:=implicit-str-concat @@ -179,50 +190,56 @@ def test_single_commands() -> None: """Ensures backwards compatibility with single command strings for setup and cleanup.""" with mock.patch("subprocess.check_output") as check_output: - check_output.returncode = 0 + with mock.patch("subprocess.run") as run: + run.return_value = subprocess.CompletedProcess([], returncode=0) + check_output.returncode = 0 - assert check_output.call_count == 0 + assert check_output.call_count == 0 - # The fixture is a context-manager. - with get_docker_services( - "docker compose", - "docker-compose.yml", - docker_compose_project_name="pytest123", - docker_setup="up --build -d", - docker_cleanup="down -v", - ) as services: - assert isinstance(services, Services) + # The fixture is a context-manager. + with get_docker_services( + "docker compose", + "docker-compose.yml", + docker_compose_project_name="pytest123", + docker_setup="up --build -d", + docker_cleanup="down -v", + ) as services: + assert isinstance(services, Services) - assert check_output.call_count == 1 + assert run.call_count == 1 + assert check_output.call_count == 0 - # Can request port for services. - port = services.port_for("hello", 80) - assert port == 1 + # Can request port for services. + port = services.port_for("hello", 80) + assert port == 1 - assert check_output.call_count == 2 + assert check_output.call_count == 1 - # 2nd request for same service should hit the cache. - port = services.port_for("hello", 80) - assert port == 1 + # 2nd request for same service should hit the cache. + port = services.port_for("hello", 80) + assert port == 1 - assert check_output.call_count == 2 + assert check_output.call_count == 1 - assert check_output.call_count == 3 + assert run.call_count == 2 + assert check_output.call_count == 1 # Both should have been called. - assert check_output.call_args_list == [ + assert run.call_args_list == [ mock.call( 'docker compose -f "docker-compose.yml" -p "pytest123" up --build -d', stderr=subprocess.STDOUT, shell=True, ), mock.call( - 'docker compose -f "docker-compose.yml" -p "pytest123" port hello 80', + 'docker compose -f "docker-compose.yml" -p "pytest123" down -v', stderr=subprocess.STDOUT, shell=True, ), + ] + assert check_output.call_args_list == [ mock.call( - 'docker compose -f "docker-compose.yml" -p "pytest123" down -v', + 'docker compose -f "docker-compose.yml" -p "pytest123" port hello 80', stderr=subprocess.STDOUT, shell=True, ), @@ -233,38 +250,42 @@ def test_multiple_commands() -> None: """Multiple startup and cleanup commands should be executed.""" with mock.patch("subprocess.check_output") as check_output: - check_output.returncode = 0 + with mock.patch("subprocess.run") as run: + run.return_value = subprocess.CompletedProcess([], returncode=0) + check_output.returncode = 0 - assert check_output.call_count == 0 + assert check_output.call_count == 0 - # The fixture is a context-manager. - with get_docker_services( - "docker compose", - "docker-compose.yml", - docker_compose_project_name="pytest123", - docker_setup=["ps", "up --build -d"], - docker_cleanup=["down -v", "ps"], - ) as services: - assert isinstance(services, Services) + # The fixture is a context-manager. + with get_docker_services( + "docker compose", + "docker-compose.yml", + docker_compose_project_name="pytest123", + docker_setup=["ps", "up --build -d"], + docker_cleanup=["down -v", "ps"], + ) as services: + assert isinstance(services, Services) - assert check_output.call_count == 2 + assert run.call_count == 2 + assert check_output.call_count == 0 - # Can request port for services. - port = services.port_for("hello", 80) - assert port == 1 + # Can request port for services. + port = services.port_for("hello", 80) + assert port == 1 - assert check_output.call_count == 3 + assert check_output.call_count == 1 - # 2nd request for same service should hit the cache. - port = services.port_for("hello", 80) - assert port == 1 + # 2nd request for same service should hit the cache. + port = services.port_for("hello", 80) + assert port == 1 - assert check_output.call_count == 3 + assert check_output.call_count == 1 - assert check_output.call_count == 5 + assert run.call_count == 4 + assert check_output.call_count == 1 # Both should have been called. - assert check_output.call_args_list == [ + assert run.call_args_list == [ mock.call( 'docker compose -f "docker-compose.yml" -p "pytest123" ps', stderr=subprocess.STDOUT, @@ -276,17 +297,19 @@ def test_multiple_commands() -> None: shell=True, ), mock.call( - 'docker compose -f "docker-compose.yml" -p "pytest123" port hello 80', + 'docker compose -f "docker-compose.yml" -p "pytest123" down -v', stderr=subprocess.STDOUT, shell=True, ), mock.call( - 'docker compose -f "docker-compose.yml" -p "pytest123" down -v', + 'docker compose -f "docker-compose.yml" -p "pytest123" ps', stderr=subprocess.STDOUT, shell=True, ), + ] + assert check_output.call_args_list == [ mock.call( - 'docker compose -f "docker-compose.yml" -p "pytest123" ps', + 'docker compose -f "docker-compose.yml" -p "pytest123" port hello 80', stderr=subprocess.STDOUT, shell=True, ), diff --git a/tests/test_dockercomposeexecutor.py b/tests/test_dockercomposeexecutor.py index 04b8094..aae9ba1 100644 --- a/tests/test_dockercomposeexecutor.py +++ b/tests/test_dockercomposeexecutor.py @@ -7,9 +7,10 @@ def test_execute() -> None: docker_compose = DockerComposeExecutor("docker compose", "docker-compose.yml", "pytest123") - with mock.patch("subprocess.check_output") as check_output: + with mock.patch("subprocess.run") as run: + run.return_value = subprocess.CompletedProcess([], returncode=0) docker_compose.execute("up") - assert check_output.call_args_list == [ + assert run.call_args_list == [ mock.call( 'docker compose -f "docker-compose.yml" -p "pytest123" up', shell=True, @@ -20,9 +21,10 @@ def test_execute() -> None: def test_execute_docker_compose_v2() -> None: docker_compose = DockerComposeExecutor("docker compose", "docker-compose.yml", "pytest123") - with mock.patch("subprocess.check_output") as check_output: + with mock.patch("subprocess.run") as run: + run.return_value = subprocess.CompletedProcess([], returncode=0) docker_compose.execute("up") - assert check_output.call_args_list == [ + assert run.call_args_list == [ mock.call( 'docker compose -f "docker-compose.yml" -p "pytest123" up', shell=True, @@ -34,16 +36,17 @@ def test_execute_docker_compose_v2() -> None: def test_pypath_compose_files() -> None: compose_file: py.path.local = py.path.local("/tmp/docker-compose.yml") docker_compose = DockerComposeExecutor("docker compose", compose_file, "pytest123") # type: ignore - with mock.patch("subprocess.check_output") as check_output: + with mock.patch("subprocess.run") as run: + run.return_value = subprocess.CompletedProcess([], returncode=0) docker_compose.execute("up") - assert check_output.call_args_list == [ + assert run.call_args_list == [ mock.call( 'docker compose -f "/tmp/docker-compose.yml"' ' -p "pytest123" up', # pylint: disable:=implicit-str-concat shell=True, stderr=subprocess.STDOUT, ) - ] or check_output.call_args_list == [ + ] or run.call_args_list == [ mock.call( 'docker compose -f "C:\\tmp\\docker-compose.yml"' ' -p "pytest123" up', # pylint: disable:=implicit-str-concat @@ -57,9 +60,10 @@ def test_multiple_compose_files() -> None: docker_compose = DockerComposeExecutor( "docker compose", ["docker-compose.yml", "other-compose.yml"], "pytest123" ) - with mock.patch("subprocess.check_output") as check_output: + with mock.patch("subprocess.run") as run: + run.return_value = subprocess.CompletedProcess([], returncode=0) docker_compose.execute("up") - assert check_output.call_args_list == [ + assert run.call_args_list == [ mock.call( 'docker compose -f "docker-compose.yml" -f "other-compose.yml"' ' -p "pytest123" up',