config: runtime: base: python: make more generic

The base `python` template was designed with post-checkout jobs
(`kbuild`, `kunit`...) in mind and doesn't work too well with test or
post-processing jobs due to the following issues:
* it assumes we need the kernel source from the parent `checkout` node
* it doesn't provide a way to retrieve arbitrary artifacts from a node
  or its parent
* the `_get_source()` function doesn't allow specifying an arbitrary
  target folder to extract tarballs to

Ensure we only download/extract/cd to the kernel source if the current
node is a `kbuild` one, otherwise let the job definition prepare its
workspace as needed. In order to make things easier to jobs needing to
work on the original kernel source, add a new `_get_kernel_source()` so
such jobs (e.g. kunit) need a single function call.

Also add a generic `_get_artifact_url()` for downloading arbitrary
artifacts and make the existing `_get_tarball_url()` use it to avoid
duplicating code.

Finally, add an optional `path` parameter to `_get_source()` so the job
can specify the destination folder if needed.

Signed-off-by: Arnaud Ferraris <arnaud.ferraris@collabora.com>

Author: a-wai

config/runtime/base/python.jinja2

@@ -86,22 +86,36 @@ class BaseJob:
         storage_cred = os.getenv('KCI_STORAGE_CREDENTIALS')
         return kernelci.storage.get_storage(storage_config, storage_cred)
 
-    def _get_tarball_url(self, node):
-        if node.get('artifacts') and 'tarball' in node['artifacts']:
-            return node['artifacts']['tarball']
+    def _get_artifact_url(self, node, artifact):
+        if node.get('artifacts') and artifact in node['artifacts']:
+            return node['artifacts'][artifact]
         if node.get('parent'):
             parent = self._api.node.get(node['parent'])
-            return self._get_tarball_url(parent)
-        raise ValueError(f"'tarball' artifact not found in node {self._node['id']} ancestors")
+            return self._get_artifact_url(parent, artifact)
+        raise ValueError(f"'{artifact}' artifact not found in node {node['id']} ancestors")
+
+    def _get_tarball_url(self, node):
+        return self._get_artifact_url(node, 'tarball')
 
-    def _get_source(self, url):
+    def _get_source(self, url, path=None):
+        if path is None:
+            path = self._workspace
+        # Ensure our destination folder exists
+        os.makedirs(path, exist_ok=True)
         resp = requests.get(url, stream=True)
         resp.raise_for_status()
         tarball_name = os.path.basename(urllib.parse.urlparse(url).path)
         base, ext = tarball_name.split('.tar.')
         with tarfile.open(fileobj=resp.raw, mode=f'r|{ext}') as tarball:
-            tarball.extractall(path=self._workspace)
-        return os.path.join(self._workspace, base)
+            tarball.extractall(path=path)
+        return os.path.join(path, base)
+
+    def _get_kernel_source(self):
+        print("Getting kernel source tree...")
+        tarball_url = self._get_tarball_url(self._node)
+        src_path = self._get_source(tarball_url)
+        print(f"Source directory: {src_path}")
+        return src_path
 
     def _run(self, src_path):
         raise NotImplementedError("_run() method required to run job")
@@ -119,10 +133,14 @@ class BaseJob:
         if self._node.get('debug') and 'dry_run' in self._node['debug']:
             return self.dry_run(self._node['debug']['result'])
 
-        print("Getting kernel source tree...")
-        tarball_url = self._get_tarball_url(self._node)
-        src_path = self._get_source(tarball_url)
-        print(f"Source directory: {src_path}")
+        # Ensure our workspace folder exists
+        os.makedirs(self._workspace, exist_ok=True)
+
+        if self._node.get('kind') == 'kbuild':
+            src_path = self._get_kernel_source()
+        else:
+            src_path = self._workspace
+
         print("Running job...")
         return self._run(src_path)