From 33e0da75af8e666f41f65c05a5f5edf6d90f339c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Wieczorek?= Date: Tue, 17 Jun 2025 10:57:58 +0200 Subject: [PATCH 1/4] Revert "src/trigger.py: add --build-configs optional argument" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 81387ab4786d5c7a00afd22eae98c39dda323093. Build configurations list will not be known up front when external configuration files are enabled in Maestro. Dropping this filter will allow simplifying Trigger Service logic taking into account only parameters that are still relevant. Signed-off-by: Paweł Wieczorek --- src/trigger.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/trigger.py b/src/trigger.py index 794601556..cc53bfaf0 100755 --- a/src/trigger.py +++ b/src/trigger.py @@ -134,27 +134,23 @@ def _run_trigger(self, build_config, force, timeout, trees): except Exception as ex: self.traceback(ex) - def _iterate_build_configs(self, force, build_configs_list, - timeout, trees): + def _iterate_build_configs(self, force, timeout, trees): for name, config in self._build_configs.items(): - if not build_configs_list or name in build_configs_list: - self._run_trigger(config, force, timeout, trees) + self._run_trigger(config, force, timeout, trees) def _setup(self, args): return { 'poll_period': int(args.poll_period), 'force': args.force, - 'build_configs_list': (args.build_configs or '').split(), 'startup_delay': int(args.startup_delay or 0), 'timeout': args.timeout, 'trees': args.trees, } def _run(self, ctx): - poll_period, force, build_configs_list, startup_delay, timeout, trees = ( + poll_period, force, startup_delay, timeout, trees = ( ctx[key] for key in ( - 'poll_period', 'force', 'build_configs_list', 'startup_delay', - 'timeout', 'trees' + 'poll_period', 'force', 'startup_delay', 'timeout', 'trees' ) ) @@ -163,8 +159,7 @@ def _run(self, ctx): time.sleep(startup_delay) while True: - self._iterate_build_configs(force, build_configs_list, - timeout, trees) + self._iterate_build_configs(force, timeout, trees) if poll_period: self.log.info(f"Sleeping for {poll_period}s") time.sleep(poll_period) @@ -191,10 +186,6 @@ class cmd_run(Command): 'action': 'store_true', 'help': "Always create a new checkout node", }, - { - 'name': '--build-configs', - 'help': "List of build configurations to monitor", - }, { 'name': '--name', 'help': "Name of pipeline instance", From a2f30ddace30c561782de895188ac5461a912286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Wieczorek?= Date: Thu, 19 Jun 2025 20:50:43 +0200 Subject: [PATCH 2/4] src/trigger.py: Switch key data structure for main loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build configurations list will not be known up front when external configuration files are enabled in Maestro. Switching main loop to iterate over monitored trees will allow obtaining this information right before its use. Signed-off-by: Paweł Wieczorek --- src/trigger.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/trigger.py b/src/trigger.py index cc53bfaf0..a3b6d400b 100755 --- a/src/trigger.py +++ b/src/trigger.py @@ -26,6 +26,7 @@ class Trigger(Service): def __init__(self, configs, args): super().__init__(configs, args, 'trigger') self._build_configs = configs['build_configs'] + self._trees = configs['trees'] self._current_user = self._api.user.whoami() def _log_revision(self, message, build_config, head_commit): @@ -134,9 +135,11 @@ def _run_trigger(self, build_config, force, timeout, trees): except Exception as ex: self.traceback(ex) - def _iterate_build_configs(self, force, timeout, trees): - for name, config in self._build_configs.items(): - self._run_trigger(config, force, timeout, trees) + def _iterate_trees(self, force, timeout, trees): + for tree in self._trees.keys(): + build_configs = {name: config for name, config in self._build_configs.items() if config.tree.name == tree} + for name, config in build_configs.items(): + self._run_trigger(config, force, timeout, trees) def _setup(self, args): return { @@ -159,7 +162,7 @@ def _run(self, ctx): time.sleep(startup_delay) while True: - self._iterate_build_configs(force, timeout, trees) + self._iterate_trees(force, timeout, trees) if poll_period: self.log.info(f"Sleeping for {poll_period}s") time.sleep(poll_period) From 833dda8539d5ca65ad723da7facb5b903f0309db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Wieczorek?= Date: Wed, 2 Jul 2025 13:34:36 +0200 Subject: [PATCH 3/4] src/trigger.py: Retrieve externally hosted build configurations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch enables external config retrieval. It is currently limited to trees hosted on Kernel.org and GitHub and will be soon extended to cover a generic use-case. Signed-off-by: Paweł Wieczorek --- src/trigger.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/trigger.py b/src/trigger.py index a3b6d400b..0d790f744 100755 --- a/src/trigger.py +++ b/src/trigger.py @@ -9,6 +9,7 @@ import copy from datetime import datetime, timedelta import sys +import tempfile import time import kernelci.build @@ -136,8 +137,28 @@ def _run_trigger(self, build_config, force, timeout, trees): self.traceback(ex) def _iterate_trees(self, force, timeout, trees): - for tree in self._trees.keys(): - build_configs = {name: config for name, config in self._build_configs.items() if config.tree.name == tree} + for tree in self._trees.values(): + build_configs = {name: config for name, config in self._build_configs.items() if config.tree.name == tree.name} + if not build_configs: + try: + # Remove hardcoded remote build config paths + if tree.url.startswith("https://git.kernel.org"): + config_url = f"{tree.url}/plain/.kernelci.yaml?h=kernelci" + elif tree.url.startswith("https://github.com"): + config_url = f"https://raw.githubusercontent.com/{tree}/linux/refs/heads/kernelci/.kernelci.yaml" + else: + raise Exception("Hosting service not supported") + remote_config = requests.get(config_url) + except Exception as ex: + self.log.error(f"Failed to get remote build config for {tree.name}, ignoring") + self.traceback(ex) + continue + with tempfile.NamedTemporaryFile(suffix='.yaml', delete_on_close=False) as tmp: + tmp.write(remote_config.text.encode()) + tmp.close() + # Remove hardcoded trees config path + tmp_configs = kernelci.config.load(['config/trees.yaml', tmp.name]) + build_configs = tmp_configs['build_configs'] for name, config in build_configs.items(): self._run_trigger(config, force, timeout, trees) From 03865ccf70be292801a26e885e7a86e4daf4b72d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Wieczorek?= Date: Thu, 3 Jul 2025 09:19:11 +0200 Subject: [PATCH 4/4] src/tarball.py: Trust BuildConfig data from the Nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch drops the double-check with static Pipeline configuration since it may be dynamically loaded from an external source in the Trigger Service. Signed-off-by: Paweł Wieczorek --- src/tarball.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/tarball.py b/src/tarball.py index 1e2aec43d..bebe47380 100755 --- a/src/tarball.py +++ b/src/tarball.py @@ -18,6 +18,7 @@ import kernelci import kernelci.build import kernelci.config +from kernelci.legacy.config.build import BuildConfig, Tree from kernelci.legacy.cli import Args, Command, parse_opts import kernelci.storage @@ -41,7 +42,6 @@ class Tarball(Service): def __init__(self, global_configs, service_config, name): super().__init__(global_configs, service_config, name) self._service_config = service_config - self._build_configs = global_configs['build_configs'] if not os.path.exists(self._service_config.output): os.makedirs(self._service_config.output) storage_config = global_configs['storage_configs'][ @@ -51,17 +51,12 @@ def __init__(self, global_configs, service_config, name): storage_config, service_config.storage_cred ) - def _find_build_config(self, node): + def _create_build_config(self, node): revision = node['data']['kernel_revision'] tree = revision['tree'] + url = revision['url'] branch = revision['branch'] - for config in self._build_configs.values(): - if config.tree.name == tree and config.branch == branch: - return config - if config.tree.name == tree and config.branch.startswith('http'): - current = copy.copy(config) - current._branch = branch - return current + return BuildConfig(tree, Tree(tree, url), branch, {}) def _find_build_commit(self, node): revision = node['data'].get('kernel_revision') @@ -229,10 +224,7 @@ def _run(self, sub_id): continue subscribe_retries = 0 - build_config = self._find_build_config(checkout_node) - if build_config is None: - continue - + build_config = self._create_build_config(checkout_node) if self._update_repo(build_config): self.log.error("Failed to update repo, retrying") if self._update_repo(build_config):