From 1d4423416e793d165886ab44606162346820e81d Mon Sep 17 00:00:00 2001 From: Arnaud Ferraris Date: Thu, 19 Jun 2025 17:11:43 +0200 Subject: [PATCH] config: runtime: base: python: make more generic The base `python` template was designed with post-checkout jobs (`kbuild`, `kunit`...) in mind and doesn't work too well with test or post-processing jobs due to the following issues: * it assumes we need the kernel source from the parent `checkout` node * it doesn't provide a way to retrieve arbitrary artifacts from a node or its parent * the `_get_source()` function doesn't allow specifying an arbitrary target folder to extract tarballs to Ensure we only download/extract/cd to the kernel source if the current node is a `kbuild` one, otherwise let the job definition prepare its workspace as needed. In order to make things easier to jobs needing to work on the original kernel source, add a new `_get_kernel_source()` so such jobs (e.g. kunit) need a single function call. Also add a generic `_get_artifact_url()` for downloading arbitrary artifacts and make the existing `_get_tarball_url()` use it to avoid duplicating code. Finally, add an optional `path` parameter to `_get_source()` so the job can specify the destination folder if needed. Signed-off-by: Arnaud Ferraris --- config/runtime/base/python.jinja2 | 42 ++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/config/runtime/base/python.jinja2 b/config/runtime/base/python.jinja2 index 7858d422c0..a1bbd71dca 100644 --- a/config/runtime/base/python.jinja2 +++ b/config/runtime/base/python.jinja2 @@ -86,22 +86,36 @@ class BaseJob: storage_cred = os.getenv('KCI_STORAGE_CREDENTIALS') return kernelci.storage.get_storage(storage_config, storage_cred) - def _get_tarball_url(self, node): - if node.get('artifacts') and 'tarball' in node['artifacts']: - return node['artifacts']['tarball'] + def _get_artifact_url(self, node, artifact): + if node.get('artifacts') and artifact in node['artifacts']: + return node['artifacts'][artifact] if node.get('parent'): parent = self._api.node.get(node['parent']) - return self._get_tarball_url(parent) - raise ValueError(f"'tarball' artifact not found in node {self._node['id']} ancestors") + return self._get_artifact_url(parent, artifact) + raise ValueError(f"'{artifact}' artifact not found in node {node['id']} ancestors") + + def _get_tarball_url(self, node): + return self._get_artifact_url(node, 'tarball') - def _get_source(self, url): + def _get_source(self, url, path=None): + if path is None: + path = self._workspace + # Ensure our destination folder exists + os.makedirs(path, exist_ok=True) resp = requests.get(url, stream=True) resp.raise_for_status() tarball_name = os.path.basename(urllib.parse.urlparse(url).path) base, ext = tarball_name.split('.tar.') with tarfile.open(fileobj=resp.raw, mode=f'r|{ext}') as tarball: - tarball.extractall(path=self._workspace) - return os.path.join(self._workspace, base) + tarball.extractall(path=path) + return os.path.join(path, base) + + def _get_kernel_source(self): + print("Getting kernel source tree...") + tarball_url = self._get_tarball_url(self._node) + src_path = self._get_source(tarball_url) + print(f"Source directory: {src_path}") + return src_path def _run(self, src_path): raise NotImplementedError("_run() method required to run job") @@ -119,10 +133,14 @@ class BaseJob: if self._node.get('debug') and 'dry_run' in self._node['debug']: return self.dry_run(self._node['debug']['result']) - print("Getting kernel source tree...") - tarball_url = self._get_tarball_url(self._node) - src_path = self._get_source(tarball_url) - print(f"Source directory: {src_path}") + # Ensure our workspace folder exists + os.makedirs(self._workspace, exist_ok=True) + + if self._node.get('kind') == 'kbuild': + src_path = self._get_kernel_source() + else: + src_path = self._workspace + print("Running job...") return self._run(src_path)