Generation of a RPM status dashboard to compare with xs8

scripts/repo_status/.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+*.dot
+*.html

scripts/repo_status/.python-version

@@ -0,0 +1 @@
+3.11.11

scripts/repo_status/create_rebase_cards.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+
+# from icecream import ic
+from tabulate import tabulate
+
+import repoquery
+from lib import collect_data_xcpng, collect_data_xs8, get_xs8_rpm_updates, read_package_status_metadata
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-v', '--verbose', action='count', default=0)
+args = parser.parse_args()
+
+loglevel = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}.get(args.verbose, logging.DEBUG)
+logging.basicConfig(format='[%(levelname)s] %(message)s', level=loglevel)
+
+PACKAGE_STATUS = read_package_status_metadata()
+
+xcp_set = collect_data_xcpng()
+(xs8_srpms_set, xs8_rpms_sources_set) = collect_data_xs8()
+srpm_updates = get_xs8_rpm_updates()
+
+res = []
+for n in sorted(set(xs8_srpms_set.keys()) | xs8_rpms_sources_set.keys()):
+    if n in PACKAGE_STATUS and PACKAGE_STATUS[n].status == 'ignored':
+        logging.debug(f"ignoring {n}")
+        continue
+    xs8_srpms_evr = xs8_srpms_set.get(n)
+    xs8_rpms_sources_evr = xs8_rpms_sources_set.get(n)
+    if xs8_srpms_evr is not None and xs8_rpms_sources_evr is not None:
+        xs8_evr = max(xs8_srpms_evr, xs8_rpms_sources_evr)
+    else:
+        xs8_evr = xs8_srpms_evr or xs8_rpms_sources_evr
+    xcp_evr = xcp_set.get(n)
+    xs8_update = srpm_updates.get(f'{n}-{xs8_evr}.xs8', '?')
+    # if xcp_evr is not None and xcp_evr < xs8_evr:
+    if xcp_evr is None:
+        if not repoquery.is_pristine_upstream(str(xs8_evr)):
+            res.append((xs8_update, n, xcp_evr, xs8_evr))
+    elif xcp_evr < xs8_evr:
+        res.append((xs8_update, n, xcp_evr, xs8_evr))
+res.sort()
+print(tabulate(res, headers=['xs8 update', 'SRPM', 'XCP-ng version', 'XS8 version']))

scripts/repo_status/lib.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+import csv
+import gzip
+import logging
+import tempfile
+import xml.etree.ElementTree as ET
+from collections import namedtuple
+from typing import Iterable, Iterator
+from urllib.request import urlopen
+
+import rpm  # type: ignore
+
+import repoquery
+
+ARCH = "x86_64"
+XCP_VERSION = "8.3"
+
+class EVR:
+    def __init__(self, e: str, v: str, r: str):
+        self._evr = ('0' if e in [None, 'None'] else e, v, r)
+
+    def __eq__(self, other):
+        if isinstance(other, EVR):
+            return self._evr == other._evr
+        else:
+            return self._evr == other
+
+    def __gt__(self, other):
+        if isinstance(other, EVR):
+            return rpm.labelCompare(self._evr, other._evr) > 0  # type: ignore
+        else:
+            return self._evr > other
+
+    def __lt__(self, other):
+        return other > self
+
+    def __str__(self):
+        if self._evr[0] != '0':
+            return f'{self._evr[0]}:{self._evr[1]}-{self._evr[2]}'
+        else:
+            return f'{self._evr[1]}-{self._evr[2]}'
+
+# Filters an iterator of (n, e, v, r) for newest evr of each `n`.
+# Older versions are allowed to appear before the newer ones.
+def filter_best_evr(nevrs: Iterable[tuple[str, str, str, str]]) -> Iterator[tuple[str, str, str, str]]:
+    best: dict[str, tuple[str, str, str]] = {}
+    for (n, e, v, r) in nevrs:
+        if n not in best or rpm.labelCompare(best[n], (e, v, r)) < 0:  # type: ignore
+            best[n] = (e, v, r)
+            yield (n, e, v, r)
+        # else (e, v, r) is older than a previously-seen version, drop
+
+def collect_data_xcpng() -> dict[str, EVR]:
+    with (tempfile.NamedTemporaryFile() as dnfconf,
+          tempfile.TemporaryDirectory() as yumrepod):
+        repoquery.setup_xcpng_yum_repos(yum_repo_d=yumrepod,
+                                        sections=['base', 'updates'],
+                                        bin_arch=None,
+                                        version=XCP_VERSION)
+        repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod)
+
+        xcp_nevr = {
+            n: EVR(e, v, r)
+            for (n, e, v, r)
+            in filter_best_evr(repoquery.rpm_parse_nevr(nevr, f".xcpng{XCP_VERSION}")
+                               for nevr in repoquery.all_srpms())}
+
+    return xcp_nevr
+
+def collect_data_xs8():
+    with (tempfile.NamedTemporaryFile() as dnfconf,
+          tempfile.TemporaryDirectory() as yumrepod):
+
+        repoquery.setup_xs8_yum_repos(yum_repo_d=yumrepod,
+                                      sections=['base', 'normal'],
+                                      )
+        repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod)
+        logging.debug("fill cache with XS info")
+        repoquery.fill_srpm_binrpms_cache()
+
+        logging.debug("get all XS SRPMs")
+        xs8_srpms = {nevr for nevr in repoquery.all_srpms()}
+        xs8_rpms_sources = {nevr for nevr in repoquery.SRPM_BINRPMS_CACHE}
+
+        xs8_srpms_set = {n: EVR(e, v, r)
+                         for (n, e, v, r)
+                         in filter_best_evr(repoquery.rpm_parse_nevr(nevr, ".xs8")
+                                            for nevr in xs8_srpms)}
+        xs8_rpms_sources_set = {n: EVR(e, v, r)
+                                for (n, e, v, r)
+                                in filter_best_evr(repoquery.rpm_parse_nevr(nevr, ".xs8")
+                                                   for nevr in xs8_rpms_sources)}
+
+    return (xs8_srpms_set, xs8_rpms_sources_set)
+
+def read_package_status_metadata():
+    with open('package_status.csv', newline='') as csvfile:
+        csvreader = csv.reader(csvfile, delimiter=';', quotechar='|')
+        headers = next(csvreader)
+        assert headers == ["SRPM_name", "status", "comment"], f"unexpected headers {headers!r}"
+        PackageStatus = namedtuple("PackageStatus", headers[1:]) # type: ignore[misc]
+        return {row[0]: PackageStatus(*row[1:])
+                for row in csvreader}
+
+def get_xs8_rpm_updates():
+    NS = {'repo': 'http://linux.duke.edu/metadata/repo'}
+    BASE_URL = 'http://repos/repos/XS8/normal/xs8p-normal'
+
+    # read the update info path from repomd.xml
+    with urlopen(f'{BASE_URL}/repodata/repomd.xml') as f:
+        repomd = f.read()
+    data = ET.fromstring(repomd).find("repo:data[@type='updateinfo']", NS)
+    assert data is not None
+    location = data.find('repo:location', NS)
+    assert location is not None
+    path = location.attrib['href']
+
+    # read the update info file
+    res = {}
+    with urlopen(f'{BASE_URL}/{path}') as cf, gzip.open(cf, 'rb') as f:
+        updateinfo = f.read()
+    updates = ET.fromstring(updateinfo).findall('update')
+    for update in updates:
+        update_id = update.find('id')
+        assert update_id is not None
+        update_id = update_id.text
+        pkglist = update.find('pkglist')
+        assert pkglist is not None
+        collection = pkglist.find('collection')
+        assert collection is not None
+        packages = collection.findall('package')
+        for package in packages:
+            evr = EVR(package.attrib['epoch'], package.attrib['version'], package.attrib['release'])
+            rpm = f'{package.attrib["name"]}-{evr}'
+            srpm = repoquery.rpm_source_package(rpm, default=rpm)
+            res[srpm] = update_id
+    return res

scripts/repo_status/package_status.csv

@@ -0,0 +1,60 @@
+SRPM_name;status;comment
+auto-cert-kit;ignored;unused, why?
+automake16;ignored;unused, why?
+bpftool;ignored;unused, why?
+capstone;ignored;unused, why?
+citrix-crypto-module;ignored;proprietary, patched FIPS openssl
+compiler-rt18;ignored;unused, why?
+dlm;ignored;previous dependency for corosync
+emu-manager;ignored;proprietary, replaced by xcp-emu-manager
+epel-release;ignored;unused, why?
+forkexecd;ignored;now in xapi
+fuse;;breq for e2fsprogs 1.47
+gfs2-utils;ignored;unsupported fs
+glib2;ignored;same version as el7, why?
+golang;ignored;for newer xe-guest-utilities
+hcp_nss;ignored;unused, “enforce any permitted user login as root”
+hwloc;ignored;unused, why?
+libbpf;ignored;unused, why?
+libcgroup;ignored;unused, same version as el7, why?
+libhbalinux;;unused? el7 fork, “Fix crash in fcoeadm/elxhbamgr on certain machines”
+libnbd;ignored;unused, dep for xapi-storage-plugins
+linuxconsoletools;ignored;unused, same version as el7, why?
+mbootpack;;for secureboot?
+message-switch;ignored;now in xapi
+mpdecimal;ignored;unused, why?
+ninja-build;ignored;unused
+pbis-open;ignored;unused, likely linked to upgrade-pbis-to-winbind
+pbis-open-upgrade;ignored;unused, likely linked to upgrade-pbis-to-winbind
+pvsproxy;ignored;proprietary
+python-monotonic;ignored;previous dependency for sm
+python-tqdm;ignored;unused, dependency for pvsproxy
+rrdd-plugins;ignored;now in xapi
+ruby;ignored;unused, why?
+sbd;;unused? "storage-based death functionality"
+sm-cli;ignored;now in xapi
+secureboot-certificates;ignored;proprietary, needs alternative?
+security-tools;ignored;proprietary, pool_secret tool
+sm-transport-lib;ignored;proprietary
+squeezed;ignored;now in xapi
+tix;ignored;unused, why?
+upgrade-pbis-to-winbind;ignored;proprietary
+v6d;ignored;proprietary, replaced by xcp-featured
+varstored-guard;ignored;now in xapi
+vendor-update-keys;ignored;proprietary
+vgpu;ignored;proprietary
+vhd-tool;ignored;now in xapi
+wsproxy;ignored;now in xapi
+xapi-clusterd;ignored;proprietary
+xapi-nbd;ignored;now in xapi
+xapi-storage;ignored;now in xapi
+xapi-storage-plugins;ignored;proprietarized, forked as xcp-ng-xapi-storage
+xapi-storage-script;ignored;now in xapi
+xcp-networkd;ignored;now in xapi
+xcp-rrdd;ignored;now in xapi
+xencert;;"automated testkit for certifying storage hardware with XenServer"
+xenopsd;ignored;now in xapi
+xenserver-release;forked;xcp-ng-release
+xenserver-snmp-agent;ignored;proprietary, SNMP MIB
+xenserver-telemetry;ignored;proprietary, xapi plugin
+xs-clipboardd;ignored;proprietary, replaced by xcp-clipboardd

scripts/repo_status/pyproject.toml

@@ -0,0 +1,50 @@
+[project]
+name = "repo_status"
+version = "0.1.0"
+description = "Repository status and comparison"
+readme = "README.md"
+requires-python = "~=3.11"
+dependencies = [
+  "requests",
+  "rpm",
+  "tabulate",
+]
+
+[dependency-groups]
+dev = [
+  "icecream",
+  "mypy",
+  "flake8",
+  "pydocstyle",
+  "pyright",
+  "ruff",
+  "types-requests",
+  "typing-extensions",
+]
+
+[tool.pyright]
+typeCheckingMode = "standard"
+
+[tool.ruff]
+preview = true
+line-length = 120
+exclude = [".git"]
+
+[tool.ruff.format]
+quote-style = "preserve"
+
+[tool.ruff.lint]
+select = [
+  "F",  # Pyflakes
+  "I",  # isort
+  "SLF",  # flake8-self
+  "SIM",  # flake8-simplify
+]
+# don't use some of the SIM rules
+ignore = [
+  "SIM105",  # suppressible-exception
+  "SIM108",  # if-else-block-instead-of-if-exp
+]
+
+[tool.ruff.lint.isort]
+lines-after-imports = 1