Skip to content

Create and enable post-processing job for code coverage reporting #1211

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions config/jobs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,14 @@ jobs:
baseline-x86-mfd: *baseline-job
baseline-x86-qualcomm: *baseline-job

coverage-report:
template: coverage-report.jinja2
kind: job
image: ghcr.io/kernelci/{image_prefix}gcc-12:x86-kselftest-kernelci
rules:
fragments:
- coverage

kbuild-clang-17-arm: &kbuild-clang-17-arm-job
<<: *kbuild-job
image: ghcr.io/kernelci/{image_prefix}clang-17:arm-kselftest-kernelci
Expand Down
228 changes: 228 additions & 0 deletions config/runtime/coverage-report.jinja2
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
{# -*- mode: Python -*- -#}
{# SPDX-License-Identifier: LGPL-2.1-or-later -#}

{%- extends 'base/python.jinja2' %}

{%- block python_imports %}
{{ super() }}
import gzip
import json
import shutil
import subprocess
{%- endblock %}

{%- block python_local_imports %}
{{ super() }}
import kernelci.api.helper
{%- endblock %}

{%- block python_globals %}
{{ super() }}
{% endblock %}

{% block python_job -%}
class Job(BaseJob):
def _upload_artifacts(self):
artifacts = {}
storage = self._get_storage()
if storage and self._node:
root_path = '-'.join([JOB_NAME, self._node['id']])
print(f"Uploading artifacts to {root_path}")
for name, file_path in self._artifacts.items():
if os.path.exists(file_path):
file_url = storage.upload_single(
(file_path, os.path.basename(file_path)), root_path
)
print(file_url)
artifacts[name] = file_url
return artifacts

def _extract_coverage(self, summary_file, node=None):
if node is None:
node = self._node

child_nodes = []

with open(summary_file, encoding='utf-8') as summary_json:
summary = json.load(summary_json)
node_data = node['data']

func_data = node_data.copy()
func_percent = summary.get('function_percent')
if func_percent is not None:
func_data['misc'] = {}
func_data['misc']['measurement'] = func_percent
child_nodes += [
{
'node': {
'kind': 'test',
'name': 'coverage.functions',
'result': 'pass',
'state': 'done',
'data': func_data,
},
'child_nodes': [],
},
]

line_data = node_data.copy()
line_percent = summary.get('line_percent')
if line_percent is not None:
line_data['misc'] = {}
line_data['misc']['measurement'] = line_percent
child_nodes += [
{
'node': {
'kind': 'test',
'name': 'coverage.lines',
'result': 'pass',
'state': 'done',
'data': line_data,
},
'child_nodes': [],
},
]

return {
'node': {
'result': 'pass' if node['id'] == self._node['id'] else node['result'],
'artifacts': {},
},
'child_nodes': child_nodes,
}

def _run(self, src_path):
self._artifacts = {}
api_helper = kernelci.api.helper.APIHelper(self._api)
child_nodes = self._api.node.findfast({'parent': self._node['parent']})

log_path = os.path.join(self._workspace, f"log.txt")
log_file = open(log_path, mode='w')

log_file.write("Getting coverage source...\n")
tarball_url = self._get_artifact_url(self._node, 'coverage_source_tar_xz')
self._get_source(tarball_url)
# Not getting src_path from _get_source() as it doesn't work in our case
# We do know that the top-level folder is named 'linux' however, so let's
# just use that
src_path = os.path.join(self._workspace, 'linux')
log_file.write(f"Coverage source downloaded from {tarball_url}\n")

base_cmd = ['gcovr', '--root', src_path]
tracefiles = []

# Download and process coverage data for all child nodes
for cnode in child_nodes:
if cnode['id'] == self._node['id']:
log_file.write(f"Skipping self ({cnode['id']})\n")
continue

coverage_dir = os.path.join(self._workspace, f"coverage-{cnode['id']}")
json_summary = coverage_dir + '.summary.json'
try:
data_url = self._get_artifact_url(cnode, 'coverage_data')
tracefile = coverage_dir + '.json'
self._get_source(data_url, path=coverage_dir)
log_file.write(f"Downloaded coverage data from {data_url}\n")
except:
log_file.write(f"WARNING: Unable to download coverage data for {cnode['id']}\n")
continue

# We now have raw coverage data available, process it
log_file.write(f"--- Processing coverage data for {cnode['id']} ---\n")
cmd = subprocess.run(base_cmd + [
'--gcov-ignore-parse-errors',
'--object-directory', coverage_dir,
'--json', tracefile,
'--json-summary', json_summary,
], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
log_file.write(cmd.stdout)

try:
cmd.check_returncode()
except:
log_file.write(f"WARNING: Unable to process coverage data for {cnode['id']}")
continue

tracefiles += [tracefile]
results = self._extract_coverage(json_summary, node=cnode)
# We only want to create child nodes reporting coverage percentages, not actually
# update the test node
if len(results['child_nodes']) > 0:
api_helper.submit_results(results, cnode)

# Coverage data has been processed for all child nodes, we can now merge the tracefiles
args = base_cmd
for trace in tracefiles:
args += ['--add-tracefile', trace]

output_base = os.path.join(self._workspace, f"coverage-{self._node['parent']}")
json_summary = output_base + '.summary.json'
html_report = output_base + '.html'
lcov_tracefile = output_base + '.info'
args += [
'--json-summary', json_summary,
'--html', html_report,
'--lcov', lcov_tracefile,
]

log_file.write("--- Merging tracefiles ---\n")
cmd = subprocess.run(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True)
log_file.write(cmd.stdout)

# Ensure job completed successfully or report failure
try:
cmd.check_returncode()
except:
log_file.write(f"ERROR: Unable to generate coverage report\n")
log_file.close()

self._artifacts = {'log': log_path}
return {
'node': {
'result': 'fail',
'artifacts': {},
},
'child_nodes': [],
}

log_file.write("--- Compressing artifacts ---\n")
compressed_lcov = lcov_tracefile + '.gz'
with open(lcov_tracefile, 'rb') as f_in:
with gzip.open(compressed_lcov, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)

# Finish writing the job log and upload it along with other artifacts
log_file.write("--- Job successful ---\n")
log_file.close()

self._artifacts = {
'coverage_report': html_report,
'tracefile': compressed_lcov,
'log': log_path,
}

return self._extract_coverage(json_summary)

return results

def _submit(self, result):
# Ensure top-level name is kept the same
result = result.copy()
# Update node from API, as we might have new fields
# such as k8s_context
node_id = self._node['id']
self._node = self._api.node.get(node_id)
# Upload artifacts and update node accordingly
artifacts = self._upload_artifacts()
result['node']['name'] = self._node['name']
result['node']['state'] = 'done'
result['node']['artifacts'] = artifacts
# Actually submit the results
api_helper = kernelci.api.helper.APIHelper(self._api)
api_helper.submit_results(result, self._node)

{% endblock %}
7 changes: 7 additions & 0 deletions config/scheduler.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,13 @@ scheduler:
platforms:
- supermicro-as-2015hr-tnr

- job: coverage-report
<<: *build-k8s-all
event:
<<: *node-event-kbuild
state: done
result: pass

- job: kbuild-clang-17-arm
<<: *build-k8s-all

Expand Down