From c155ae37a571122ce3acb0f3cc3ff5468301d910 Mon Sep 17 00:00:00 2001 From: Arnaud Ferraris Date: Thu, 19 Jun 2025 17:19:25 +0200 Subject: [PATCH 1/2] config: runtime: add new coverage report template This new template is meant to be executed as a post-processing job, once every test job for a given `kbuild` node are complete. It gathers the raw coverage data from each of those test jobs and processes it as follows: * create a single JSON tracefile using `gcovr` * extract lines/functions coverage percentages for each job * create `test` child nodes for each job, reporting those percentages * merge all tracefiles in a single results file and generate both an HTML report and `lcov`-compatible tracefile; the HTML report gives a quick overview of the code coverage, while the tracefile can then be downloaded by developers for more targeted processing * create `test` child nodes for the `kbuild` job, reporting global lines/functions coverage percentages for this run Signed-off-by: Arnaud Ferraris --- config/runtime/coverage-report.jinja2 | 228 ++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 config/runtime/coverage-report.jinja2 diff --git a/config/runtime/coverage-report.jinja2 b/config/runtime/coverage-report.jinja2 new file mode 100644 index 000000000..1d44f0cbe --- /dev/null +++ b/config/runtime/coverage-report.jinja2 @@ -0,0 +1,228 @@ +{# -*- mode: Python -*- -#} +{# SPDX-License-Identifier: LGPL-2.1-or-later -#} + +{%- extends 'base/python.jinja2' %} + +{%- block python_imports %} +{{ super() }} +import gzip +import json +import shutil +import subprocess +{%- endblock %} + +{%- block python_local_imports %} +{{ super() }} +import kernelci.api.helper +{%- endblock %} + +{%- block python_globals %} +{{ super() }} +{% endblock %} + +{% block python_job -%} +class Job(BaseJob): + def _upload_artifacts(self): + artifacts = {} + storage = self._get_storage() + if storage and self._node: + root_path = '-'.join([JOB_NAME, self._node['id']]) + print(f"Uploading artifacts to {root_path}") + for name, file_path in self._artifacts.items(): + if os.path.exists(file_path): + file_url = storage.upload_single( + (file_path, os.path.basename(file_path)), root_path + ) + print(file_url) + artifacts[name] = file_url + return artifacts + + def _extract_coverage(self, summary_file, node=None): + if node is None: + node = self._node + + child_nodes = [] + + with open(summary_file, encoding='utf-8') as summary_json: + summary = json.load(summary_json) + node_data = node['data'] + + func_data = node_data.copy() + func_percent = summary.get('function_percent') + if func_percent is not None: + func_data['misc'] = {} + func_data['misc']['measurement'] = func_percent + child_nodes += [ + { + 'node': { + 'kind': 'test', + 'name': 'coverage.functions', + 'result': 'pass', + 'state': 'done', + 'data': func_data, + }, + 'child_nodes': [], + }, + ] + + line_data = node_data.copy() + line_percent = summary.get('line_percent') + if line_percent is not None: + line_data['misc'] = {} + line_data['misc']['measurement'] = line_percent + child_nodes += [ + { + 'node': { + 'kind': 'test', + 'name': 'coverage.lines', + 'result': 'pass', + 'state': 'done', + 'data': line_data, + }, + 'child_nodes': [], + }, + ] + + return { + 'node': { + 'result': 'pass' if node['id'] == self._node['id'] else node['result'], + 'artifacts': {}, + }, + 'child_nodes': child_nodes, + } + + def _run(self, src_path): + self._artifacts = {} + api_helper = kernelci.api.helper.APIHelper(self._api) + child_nodes = self._api.node.findfast({'parent': self._node['parent']}) + + log_path = os.path.join(self._workspace, f"log.txt") + log_file = open(log_path, mode='w') + + log_file.write("Getting coverage source...\n") + tarball_url = self._get_artifact_url(self._node, 'coverage_source_tar_xz') + self._get_source(tarball_url) + # Not getting src_path from _get_source() as it doesn't work in our case + # We do know that the top-level folder is named 'linux' however, so let's + # just use that + src_path = os.path.join(self._workspace, 'linux') + log_file.write(f"Coverage source downloaded from {tarball_url}\n") + + base_cmd = ['gcovr', '--root', src_path] + tracefiles = [] + + # Download and process coverage data for all child nodes + for cnode in child_nodes: + if cnode['id'] == self._node['id']: + log_file.write(f"Skipping self ({cnode['id']})\n") + continue + + coverage_dir = os.path.join(self._workspace, f"coverage-{cnode['id']}") + json_summary = coverage_dir + '.summary.json' + try: + data_url = self._get_artifact_url(cnode, 'coverage_data') + tracefile = coverage_dir + '.json' + self._get_source(data_url, path=coverage_dir) + log_file.write(f"Downloaded coverage data from {data_url}\n") + except: + log_file.write(f"WARNING: Unable to download coverage data for {cnode['id']}\n") + continue + + # We now have raw coverage data available, process it + log_file.write(f"--- Processing coverage data for {cnode['id']} ---\n") + cmd = subprocess.run(base_cmd + [ + '--gcov-ignore-parse-errors', + '--object-directory', coverage_dir, + '--json', tracefile, + '--json-summary', json_summary, + ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + log_file.write(cmd.stdout) + + try: + cmd.check_returncode() + except: + log_file.write(f"WARNING: Unable to process coverage data for {cnode['id']}") + continue + + tracefiles += [tracefile] + results = self._extract_coverage(json_summary, node=cnode) + # We only want to create child nodes reporting coverage percentages, not actually + # update the test node + if len(results['child_nodes']) > 0: + api_helper.submit_results(results, cnode) + + # Coverage data has been processed for all child nodes, we can now merge the tracefiles + args = base_cmd + for trace in tracefiles: + args += ['--add-tracefile', trace] + + output_base = os.path.join(self._workspace, f"coverage-{self._node['parent']}") + json_summary = output_base + '.summary.json' + html_report = output_base + '.html' + lcov_tracefile = output_base + '.info' + args += [ + '--json-summary', json_summary, + '--html', html_report, + '--lcov', lcov_tracefile, + ] + + log_file.write("--- Merging tracefiles ---\n") + cmd = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) + log_file.write(cmd.stdout) + + # Ensure job completed successfully or report failure + try: + cmd.check_returncode() + except: + log_file.write(f"ERROR: Unable to generate coverage report\n") + log_file.close() + + self._artifacts = {'log': log_path} + return { + 'node': { + 'result': 'fail', + 'artifacts': {}, + }, + 'child_nodes': [], + } + + log_file.write("--- Compressing artifacts ---\n") + compressed_lcov = lcov_tracefile + '.gz' + with open(lcov_tracefile, 'rb') as f_in: + with gzip.open(compressed_lcov, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + # Finish writing the job log and upload it along with other artifacts + log_file.write("--- Job successful ---\n") + log_file.close() + + self._artifacts = { + 'coverage_report': html_report, + 'tracefile': compressed_lcov, + 'log': log_path, + } + + return self._extract_coverage(json_summary) + + return results + + def _submit(self, result): + # Ensure top-level name is kept the same + result = result.copy() + # Update node from API, as we might have new fields + # such as k8s_context + node_id = self._node['id'] + self._node = self._api.node.get(node_id) + # Upload artifacts and update node accordingly + artifacts = self._upload_artifacts() + result['node']['name'] = self._node['name'] + result['node']['state'] = 'done' + result['node']['artifacts'] = artifacts + # Actually submit the results + api_helper = kernelci.api.helper.APIHelper(self._api) + api_helper.submit_results(result, self._node) + +{% endblock %} From e79ee7133b12757394481a4f9ad94ff4e19cf74f Mon Sep 17 00:00:00 2001 From: Arnaud Ferraris Date: Thu, 19 Jun 2025 17:22:17 +0200 Subject: [PATCH 2/2] config: create and enable coverage data post-processing job Create a generic job definition meant for post-processing coverage data for kernel builds where GCOV support is enabled. This job is enabled for all kernels built with the `coverage` config fragment. Signed-off-by: Arnaud Ferraris --- config/jobs.yaml | 8 ++++++++ config/scheduler.yaml | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/config/jobs.yaml b/config/jobs.yaml index 3377769c4..db0a383c0 100644 --- a/config/jobs.yaml +++ b/config/jobs.yaml @@ -155,6 +155,14 @@ jobs: baseline-x86-mfd: *baseline-job baseline-x86-qualcomm: *baseline-job + coverage-report: + template: coverage-report.jinja2 + kind: job + image: ghcr.io/kernelci/{image_prefix}gcc-12:x86-kselftest-kernelci + rules: + fragments: + - coverage + kbuild-clang-17-arm: &kbuild-clang-17-arm-job <<: *kbuild-job image: ghcr.io/kernelci/{image_prefix}clang-17:arm-kselftest-kernelci diff --git a/config/scheduler.yaml b/config/scheduler.yaml index 008836628..810b50262 100644 --- a/config/scheduler.yaml +++ b/config/scheduler.yaml @@ -332,6 +332,13 @@ scheduler: platforms: - supermicro-as-2015hr-tnr + - job: coverage-report + <<: *build-k8s-all + event: + <<: *node-event-kbuild + state: done + result: pass + - job: kbuild-clang-17-arm <<: *build-k8s-all