From 87bba9d31a9202dcc3783b0c8ad4af742e6da8bd Mon Sep 17 00:00:00 2001 From: Florian Greinacher Date: Wed, 27 Mar 2024 10:49:05 +0100 Subject: [PATCH] refactor(datasource/nuget): move v2/v3 API logic to classes (#28117) --- .../nuget/{v3.spec.ts => common.spec.ts} | 4 +- lib/modules/datasource/nuget/common.ts | 21 + lib/modules/datasource/nuget/index.ts | 19 +- lib/modules/datasource/nuget/v2.ts | 108 ++--- lib/modules/datasource/nuget/v3.ts | 386 +++++++++--------- 5 files changed, 282 insertions(+), 256 deletions(-) rename lib/modules/datasource/nuget/{v3.spec.ts => common.spec.ts} (85%) diff --git a/lib/modules/datasource/nuget/v3.spec.ts b/lib/modules/datasource/nuget/common.spec.ts similarity index 85% rename from lib/modules/datasource/nuget/v3.spec.ts rename to lib/modules/datasource/nuget/common.spec.ts index 0a2bd8f315ec53..8296ca85e35345 100644 --- a/lib/modules/datasource/nuget/v3.spec.ts +++ b/lib/modules/datasource/nuget/common.spec.ts @@ -1,6 +1,6 @@ -import { sortNugetVersions } from './v3'; +import { sortNugetVersions } from './common'; -describe('modules/datasource/nuget/v3', () => { +describe('modules/datasource/nuget/common', () => { it.each<{ version: string; other: string; result: number }>` version | other | result ${'invalid1'} | ${'invalid2'} | ${0} diff --git a/lib/modules/datasource/nuget/common.ts b/lib/modules/datasource/nuget/common.ts index 55ce19a78f768b..e3c9afb28fd2d6 100644 --- a/lib/modules/datasource/nuget/common.ts +++ b/lib/modules/datasource/nuget/common.ts @@ -1,6 +1,7 @@ import { logger } from '../../../logger'; import { regEx } from '../../../util/regex'; import { parseUrl } from '../../../util/url'; +import { api as versioning } from '../../versioning/nuget'; import type { ParsedRegistryUrl } from './types'; const buildMetaRe = regEx(/\+.+$/g); @@ -47,3 +48,23 @@ export function parseRegistryUrl(registryUrl: string): ParsedRegistryUrl { const feedUrl = parsedUrl.href; return { feedUrl, protocolVersion }; } + +/** + * Compare two versions. Return: + * - `1` if `a > b` or `b` is invalid + * - `-1` if `a < b` or `a` is invalid + * - `0` if `a == b` or both `a` and `b` are invalid + */ +export function sortNugetVersions(a: string, b: string): number { + if (versioning.isValid(a)) { + if (versioning.isValid(b)) { + return versioning.sortVersions(a, b); + } else { + return 1; + } + } else if (versioning.isValid(b)) { + return -1; + } else { + return 0; + } +} diff --git a/lib/modules/datasource/nuget/index.ts b/lib/modules/datasource/nuget/index.ts index a34bcb00881adc..7b18f7010095c9 100644 --- a/lib/modules/datasource/nuget/index.ts +++ b/lib/modules/datasource/nuget/index.ts @@ -3,8 +3,8 @@ import * as nugetVersioning from '../../versioning/nuget'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, ReleaseResult } from '../types'; import { parseRegistryUrl } from './common'; -import * as v2 from './v2'; -import * as v3 from './v3'; +import { NugetV2Api } from './v2'; +import { NugetV3Api } from './v3'; // https://api.nuget.org/v3/index.json is a default official nuget feed export const nugetOrg = 'https://api.nuget.org/v3/index.json'; @@ -18,6 +18,10 @@ export class NugetDatasource extends Datasource { override readonly registryStrategy = 'merge'; + readonly v2Api = new NugetV2Api(); + + readonly v3Api = new NugetV3Api(); + constructor() { super(NugetDatasource.id); } @@ -33,12 +37,17 @@ export class NugetDatasource extends Datasource { } const { feedUrl, protocolVersion } = parseRegistryUrl(registryUrl); if (protocolVersion === 2) { - return v2.getReleases(this.http, feedUrl, packageName); + return this.v2Api.getReleases(this.http, feedUrl, packageName); } if (protocolVersion === 3) { - const queryUrl = await v3.getResourceUrl(this.http, feedUrl); + const queryUrl = await this.v3Api.getResourceUrl(this.http, feedUrl); if (queryUrl) { - return v3.getReleases(this.http, feedUrl, queryUrl, packageName); + return this.v3Api.getReleases( + this.http, + feedUrl, + queryUrl, + packageName, + ); } } return null; diff --git a/lib/modules/datasource/nuget/v2.ts b/lib/modules/datasource/nuget/v2.ts index 71629f90f8c436..618cd30c1283e3 100644 --- a/lib/modules/datasource/nuget/v2.ts +++ b/lib/modules/datasource/nuget/v2.ts @@ -1,70 +1,74 @@ import { XmlDocument, XmlElement } from 'xmldoc'; import { logger } from '../../../logger'; import type { Http } from '../../../util/http'; -import type { HttpResponse } from '../../../util/http/types'; import { regEx } from '../../../util/regex'; import type { ReleaseResult } from '../types'; import { massageUrl, removeBuildMeta } from './common'; -function getPkgProp(pkgInfo: XmlElement, propName: string): string | undefined { - return pkgInfo.childNamed('m:properties')?.childNamed(`d:${propName}`)?.val; -} +export class NugetV2Api { + getPkgProp(pkgInfo: XmlElement, propName: string): string | undefined { + return pkgInfo.childNamed('m:properties')?.childNamed(`d:${propName}`)?.val; + } -export async function getReleases( - http: Http, - feedUrl: string, - pkgName: string, -): Promise { - const dep: ReleaseResult = { - releases: [], - }; - let pkgUrlList: string | null = `${feedUrl.replace( - regEx(/\/+$/), - '', - )}/FindPackagesById()?id=%27${pkgName}%27&$select=Version,IsLatestVersion,ProjectUrl,Published`; - while (pkgUrlList !== null) { - // typescript issue - const pkgVersionsListRaw: HttpResponse = await http.get(pkgUrlList); - const pkgVersionsListDoc = new XmlDocument(pkgVersionsListRaw.body); + async getReleases( + http: Http, + feedUrl: string, + pkgName: string, + ): Promise { + const dep: ReleaseResult = { + releases: [], + }; + let pkgUrlList: string | null = `${feedUrl.replace( + regEx(/\/+$/), + '', + )}/FindPackagesById()?id=%27${pkgName}%27&$select=Version,IsLatestVersion,ProjectUrl,Published`; + while (pkgUrlList !== null) { + // typescript issue + const pkgVersionsListRaw = await http.get(pkgUrlList); + const pkgVersionsListDoc = new XmlDocument(pkgVersionsListRaw.body); - const pkgInfoList = pkgVersionsListDoc.childrenNamed('entry'); + const pkgInfoList = pkgVersionsListDoc.childrenNamed('entry'); - for (const pkgInfo of pkgInfoList) { - const version = getPkgProp(pkgInfo, 'Version'); - const releaseTimestamp = getPkgProp(pkgInfo, 'Published'); - dep.releases.push({ - // TODO: types (#22198) - version: removeBuildMeta(`${version}`), - releaseTimestamp, - }); - try { - const pkgIsLatestVersion = getPkgProp(pkgInfo, 'IsLatestVersion'); - if (pkgIsLatestVersion === 'true') { - dep['tags'] = { latest: removeBuildMeta(`${version}`) }; - const projectUrl = getPkgProp(pkgInfo, 'ProjectUrl'); - if (projectUrl) { - dep.sourceUrl = massageUrl(projectUrl); + for (const pkgInfo of pkgInfoList) { + const version = this.getPkgProp(pkgInfo, 'Version'); + const releaseTimestamp = this.getPkgProp(pkgInfo, 'Published'); + dep.releases.push({ + // TODO: types (#22198) + version: removeBuildMeta(`${version}`), + releaseTimestamp, + }); + try { + const pkgIsLatestVersion = this.getPkgProp( + pkgInfo, + 'IsLatestVersion', + ); + if (pkgIsLatestVersion === 'true') { + dep['tags'] = { latest: removeBuildMeta(`${version}`) }; + const projectUrl = this.getPkgProp(pkgInfo, 'ProjectUrl'); + if (projectUrl) { + dep.sourceUrl = massageUrl(projectUrl); + } } + } catch (err) /* istanbul ignore next */ { + logger.debug( + { err, pkgName, feedUrl }, + `nuget registry failure: can't parse pkg info for project url`, + ); } - } catch (err) /* istanbul ignore next */ { - logger.debug( - { err, pkgName, feedUrl }, - `nuget registry failure: can't parse pkg info for project url`, - ); } - } - const nextPkgUrlListLink = pkgVersionsListDoc - .childrenNamed('link') - .find((node) => node.attr.rel === 'next'); + const nextPkgUrlListLink = pkgVersionsListDoc + .childrenNamed('link') + .find((node) => node.attr.rel === 'next'); - pkgUrlList = nextPkgUrlListLink ? nextPkgUrlListLink.attr.href : null; - } + pkgUrlList = nextPkgUrlListLink ? nextPkgUrlListLink.attr.href : null; + } - // dep not found if no release, so we can try next registry - if (dep.releases.length === 0) { - return null; - } + // dep not found if no release, so we can try next registry + if (dep.releases.length === 0) { + return null; + } - return dep; + return dep; + } } diff --git a/lib/modules/datasource/nuget/v3.ts b/lib/modules/datasource/nuget/v3.ts index 6f53906b3afc6a..2024c175cbee6a 100644 --- a/lib/modules/datasource/nuget/v3.ts +++ b/lib/modules/datasource/nuget/v3.ts @@ -10,7 +10,7 @@ import { regEx } from '../../../util/regex'; import { ensureTrailingSlash } from '../../../util/url'; import { api as versioning } from '../../versioning/nuget'; import type { Release, ReleaseResult } from '../types'; -import { massageUrl, removeBuildMeta } from './common'; +import { massageUrl, removeBuildMeta, sortNugetVersions } from './common'; import type { CatalogEntry, CatalogPage, @@ -18,227 +18,219 @@ import type { ServicesIndexRaw, } from './types'; -const cacheNamespace = 'datasource-nuget'; - -export async function getResourceUrl( - http: Http, - url: string, - resourceType = 'RegistrationsBaseUrl', -): Promise { - // https://docs.microsoft.com/en-us/nuget/api/service-index - const resultCacheKey = `${url}:${resourceType}`; - const cachedResult = await packageCache.get( - cacheNamespace, - resultCacheKey, - ); - - // istanbul ignore if - if (cachedResult) { - return cachedResult; - } - let servicesIndexRaw: ServicesIndexRaw | undefined; - try { - const responseCacheKey = url; - servicesIndexRaw = await packageCache.get( - cacheNamespace, - responseCacheKey, +export class NugetV3Api { + static readonly cacheNamespace = 'datasource-nuget'; + + async getResourceUrl( + http: Http, + url: string, + resourceType = 'RegistrationsBaseUrl', + ): Promise { + // https://docs.microsoft.com/en-us/nuget/api/service-index + const resultCacheKey = `${url}:${resourceType}`; + const cachedResult = await packageCache.get( + NugetV3Api.cacheNamespace, + resultCacheKey, ); - // istanbul ignore else: currently not testable - if (!servicesIndexRaw) { - servicesIndexRaw = (await http.getJson(url)).body; - await packageCache.set( - cacheNamespace, + + // istanbul ignore if + if (cachedResult) { + return cachedResult; + } + let servicesIndexRaw: ServicesIndexRaw | undefined; + try { + const responseCacheKey = url; + servicesIndexRaw = await packageCache.get( + NugetV3Api.cacheNamespace, responseCacheKey, - servicesIndexRaw, - 3 * 24 * 60, ); - } + // istanbul ignore else: currently not testable + if (!servicesIndexRaw) { + servicesIndexRaw = (await http.getJson(url)).body; + await packageCache.set( + NugetV3Api.cacheNamespace, + responseCacheKey, + servicesIndexRaw, + 3 * 24 * 60, + ); + } + + const services = servicesIndexRaw.resources + .map(({ '@id': serviceId, '@type': t }) => ({ + serviceId, + type: t?.split('/')?.shift(), + version: t?.split('/')?.pop(), + })) + .filter( + ({ type, version }) => type === resourceType && semver.valid(version), + ) + .sort((x, y) => + x.version && y.version + ? semver.compare(x.version, y.version) + : /* istanbul ignore next: hard to test */ 0, + ); + + if (services.length === 0) { + await packageCache.set( + NugetV3Api.cacheNamespace, + resultCacheKey, + null, + 60, + ); + logger.debug( + { url, servicesIndexRaw }, + `no ${resourceType} services found`, + ); + return null; + } - const services = servicesIndexRaw.resources - .map(({ '@id': serviceId, '@type': t }) => ({ + const { serviceId, version } = services.pop()!; + + // istanbul ignore if + if ( + resourceType === 'RegistrationsBaseUrl' && + version && + !version.startsWith('3.0.0-') && + !semver.satisfies(version, '^3.0.0') + ) { + logger.warn( + { url, version }, + `Nuget: Unknown version returned. Only v3 is supported`, + ); + } + + await packageCache.set( + NugetV3Api.cacheNamespace, + resultCacheKey, serviceId, - type: t?.split('/')?.shift(), - version: t?.split('/')?.pop(), - })) - .filter( - ({ type, version }) => type === resourceType && semver.valid(version), - ) - .sort((x, y) => - x.version && y.version - ? semver.compare(x.version, y.version) - : /* istanbul ignore next: hard to test */ 0, + 60, ); - - if (services.length === 0) { - await packageCache.set(cacheNamespace, resultCacheKey, null, 60); + return serviceId; + } catch (err) { + // istanbul ignore if: not easy testable with nock + if (err instanceof ExternalHostError) { + throw err; + } logger.debug( - { url, servicesIndexRaw }, - `no ${resourceType} services found`, + { err, url, servicesIndexRaw }, + `nuget registry failure: can't get ${resourceType}`, ); return null; } + } - const { serviceId, version } = services.pop()!; - - // istanbul ignore if - if ( - resourceType === 'RegistrationsBaseUrl' && - version && - !version.startsWith('3.0.0-') && - !semver.satisfies(version, '^3.0.0') - ) { - logger.warn( - { url, version }, - `Nuget: Unknown version returned. Only v3 is supported`, - ); + async getCatalogEntry( + http: Http, + catalogPage: CatalogPage, + ): Promise { + let items = catalogPage.items; + if (!items) { + const url = catalogPage['@id']; + const catalogPageFull = await http.getJson(url); + items = catalogPageFull.body.items; } + return items.map(({ catalogEntry }) => catalogEntry); + } - await packageCache.set(cacheNamespace, resultCacheKey, serviceId, 60); - return serviceId; - } catch (err) { - // istanbul ignore if: not easy testable with nock - if (err instanceof ExternalHostError) { - throw err; - } - logger.debug( - { err, url, servicesIndexRaw }, - `nuget registry failure: can't get ${resourceType}`, + async getReleases( + http: Http, + registryUrl: string, + feedUrl: string, + pkgName: string, + ): Promise { + const baseUrl = feedUrl.replace(regEx(/\/*$/), ''); + const url = `${baseUrl}/${pkgName.toLowerCase()}/index.json`; + const packageRegistration = await http.getJson(url); + const catalogPages = packageRegistration.body.items || []; + const catalogPagesQueue = catalogPages.map( + (page) => (): Promise => this.getCatalogEntry(http, page), + ); + const catalogEntries = (await p.all(catalogPagesQueue)) + .flat() + .sort((a, b) => sortNugetVersions(a.version, b.version)); + + let homepage: string | null = null; + let latestStable: string | null = null; + const releases = catalogEntries.map( + ({ version, published: releaseTimestamp, projectUrl, listed }) => { + const release: Release = { version: removeBuildMeta(version) }; + if (releaseTimestamp) { + release.releaseTimestamp = releaseTimestamp; + } + if (versioning.isValid(version) && versioning.isStable(version)) { + latestStable = removeBuildMeta(version); + homepage = projectUrl ? massageUrl(projectUrl) : homepage; + } + if (listed === false) { + release.isDeprecated = true; + } + return release; + }, ); - return null; - } -} -async function getCatalogEntry( - http: Http, - catalogPage: CatalogPage, -): Promise { - let items = catalogPage.items; - if (!items) { - const url = catalogPage['@id']; - const catalogPageFull = await http.getJson(url); - items = catalogPageFull.body.items; - } - return items.map(({ catalogEntry }) => catalogEntry); -} + if (!releases.length) { + return null; + } -/** - * Compare two versions. Return: - * - `1` if `a > b` or `b` is invalid - * - `-1` if `a < b` or `a` is invalid - * - `0` if `a == b` or both `a` and `b` are invalid - */ -export function sortNugetVersions(a: string, b: string): number { - if (versioning.isValid(a)) { - if (versioning.isValid(b)) { - return versioning.sortVersions(a, b); - } else { - return 1; + // istanbul ignore next: only happens when no stable version exists + if (latestStable === null && catalogPages.length) { + const last = catalogEntries.pop()!; + latestStable = removeBuildMeta(last.version); + homepage ??= last.projectUrl ?? null; } - } else if (versioning.isValid(b)) { - return -1; - } else { - return 0; - } -} -export async function getReleases( - http: Http, - registryUrl: string, - feedUrl: string, - pkgName: string, -): Promise { - const baseUrl = feedUrl.replace(regEx(/\/*$/), ''); - const url = `${baseUrl}/${pkgName.toLowerCase()}/index.json`; - const packageRegistration = await http.getJson(url); - const catalogPages = packageRegistration.body.items || []; - const catalogPagesQueue = catalogPages.map( - (page) => (): Promise => getCatalogEntry(http, page), - ); - const catalogEntries = (await p.all(catalogPagesQueue)) - .flat() - .sort((a, b) => sortNugetVersions(a.version, b.version)); - - let homepage: string | null = null; - let latestStable: string | null = null; - const releases = catalogEntries.map( - ({ version, published: releaseTimestamp, projectUrl, listed }) => { - const release: Release = { version: removeBuildMeta(version) }; - if (releaseTimestamp) { - release.releaseTimestamp = releaseTimestamp; - } - if (versioning.isValid(version) && versioning.isStable(version)) { - latestStable = removeBuildMeta(version); - homepage = projectUrl ? massageUrl(projectUrl) : homepage; + const dep: ReleaseResult = { + releases, + }; + + try { + const packageBaseAddress = await this.getResourceUrl( + http, + registryUrl, + 'PackageBaseAddress', + ); + // istanbul ignore else: this is a required v3 api + if (is.nonEmptyString(packageBaseAddress)) { + const nuspecUrl = `${ensureTrailingSlash( + packageBaseAddress, + )}${pkgName.toLowerCase()}/${ + // TODO: types (#22198) + latestStable + }/${pkgName.toLowerCase()}.nuspec`; + const metaresult = await http.get(nuspecUrl); + const nuspec = new XmlDocument(metaresult.body); + const sourceUrl = nuspec.valueWithPath('metadata.repository@url'); + if (sourceUrl) { + dep.sourceUrl = massageUrl(sourceUrl); + } } - if (listed === false) { - release.isDeprecated = true; + } catch (err) { + // istanbul ignore if: not easy testable with nock + if (err instanceof ExternalHostError) { + throw err; } - return release; - }, - ); - - if (!releases.length) { - return null; - } - - // istanbul ignore next: only happens when no stable version exists - if (latestStable === null && catalogPages.length) { - const last = catalogEntries.pop()!; - latestStable = removeBuildMeta(last.version); - homepage ??= last.projectUrl ?? null; - } - - const dep: ReleaseResult = { - releases, - }; - - try { - const packageBaseAddress = await getResourceUrl( - http, - registryUrl, - 'PackageBaseAddress', - ); - // istanbul ignore else: this is a required v3 api - if (is.nonEmptyString(packageBaseAddress)) { - const nuspecUrl = `${ensureTrailingSlash( - packageBaseAddress, - )}${pkgName.toLowerCase()}/${ - // TODO: types (#22198) - latestStable - }/${pkgName.toLowerCase()}.nuspec`; - const metaresult = await http.get(nuspecUrl); - const nuspec = new XmlDocument(metaresult.body); - const sourceUrl = nuspec.valueWithPath('metadata.repository@url'); - if (sourceUrl) { - dep.sourceUrl = massageUrl(sourceUrl); + // ignore / silence 404. Seen on proget, if remote connector is used and package is not yet cached + if (err instanceof HttpError && err.response?.statusCode === 404) { + logger.debug( + { registryUrl, pkgName, pkgVersion: latestStable }, + `package manifest (.nuspec) not found`, + ); + return dep; } - } - } catch (err) { - // istanbul ignore if: not easy testable with nock - if (err instanceof ExternalHostError) { - throw err; - } - // ignore / silence 404. Seen on proget, if remote connector is used and package is not yet cached - if (err instanceof HttpError && err.response?.statusCode === 404) { logger.debug( - { registryUrl, pkgName, pkgVersion: latestStable }, - `package manifest (.nuspec) not found`, + { err, registryUrl, pkgName, pkgVersion: latestStable }, + `Cannot obtain sourceUrl`, ); return dep; } - logger.debug( - { err, registryUrl, pkgName, pkgVersion: latestStable }, - `Cannot obtain sourceUrl`, - ); - return dep; - } - // istanbul ignore else: not easy testable - if (homepage) { - // only assign if not assigned - dep.sourceUrl ??= homepage; - dep.homepage ??= homepage; - } + // istanbul ignore else: not easy testable + if (homepage) { + // only assign if not assigned + dep.sourceUrl ??= homepage; + dep.homepage ??= homepage; + } - return dep; + return dep; + } }