diff --git a/.projen/deps.json b/.projen/deps.json index ceb0dac2c..5145bfc80 100644 --- a/.projen/deps.json +++ b/.projen/deps.json @@ -225,6 +225,10 @@ "name": "markdown-it-emoji", "type": "build" }, + { + "name": "nock", + "type": "build" + }, { "name": "normalize-registry-metadata", "type": "build" diff --git a/.projen/tasks.json b/.projen/tasks.json index f73cea4c6..1faf99f2d 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -1160,13 +1160,13 @@ "exec": "yarn upgrade npm-check-updates" }, { - "exec": "npm-check-updates --upgrade --target=minor --peer --dep=dev,peer,prod,optional --filter=@aws-sdk/client-sfn,@jsii/spec,@octokit/rest,@types/aws-lambda,@types/changelog-filename-regex,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/node,@types/semver,@types/streamx,@types/tar-stream,@types/tough-cookie,@types/uuid,@typescript-eslint/eslint-plugin,@typescript-eslint/parser,async-sema,aws-cdk,aws-embedded-metrics,aws-sdk,aws-sdk-mock,aws-xray-sdk-core,case,cdk-dia,cdklabs-projen-project-types,changelog-filename-regex,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-node,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,eslint,feed,fs-extra,glob,got,jest,jest-junit,jsii-diff,jsii-docgen,jsii-pacmak,jsii-rosetta,JSONStream,markdown-it,markdown-it-emoji,normalize-registry-metadata,npm-check-updates,prettier,projen,semver,spdx-license-list,standard-version,streamx,tar-stream,ts-node,typescript,uuid,yaml" + "exec": "npm-check-updates --upgrade --target=minor --peer --dep=dev,peer,prod,optional --filter=@aws-sdk/client-sfn,@jsii/spec,@octokit/rest,@types/aws-lambda,@types/changelog-filename-regex,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/node,@types/semver,@types/streamx,@types/tar-stream,@types/tough-cookie,@types/uuid,@typescript-eslint/eslint-plugin,@typescript-eslint/parser,async-sema,aws-cdk,aws-embedded-metrics,aws-sdk,aws-sdk-mock,aws-xray-sdk-core,case,cdk-dia,cdklabs-projen-project-types,changelog-filename-regex,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-node,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,eslint,feed,fs-extra,glob,got,jest,jest-junit,jsii-diff,jsii-docgen,jsii-pacmak,jsii-rosetta,JSONStream,markdown-it,markdown-it-emoji,nock,normalize-registry-metadata,npm-check-updates,prettier,projen,semver,spdx-license-list,standard-version,streamx,tar-stream,ts-node,typescript,uuid,yaml" }, { "exec": "yarn install --check-files" }, { - "exec": "yarn upgrade @aws-sdk/client-sfn @jsii/spec @octokit/rest @types/aws-lambda @types/changelog-filename-regex @types/fs-extra @types/jest @types/markdown-it @types/markdown-it-emoji @types/node @types/semver @types/streamx @types/tar-stream @types/tough-cookie @types/uuid @typescript-eslint/eslint-plugin @typescript-eslint/parser async-sema aws-cdk aws-embedded-metrics aws-sdk aws-sdk-mock aws-xray-sdk-core case cdk-dia cdklabs-projen-project-types changelog-filename-regex construct-hub-webapp dotenv esbuild eslint-config-prettier eslint-import-resolver-node eslint-import-resolver-typescript eslint-plugin-import eslint-plugin-prettier eslint feed fs-extra glob got jest jest-junit jsii-diff jsii-docgen jsii-pacmak jsii-rosetta JSONStream markdown-it markdown-it-emoji normalize-registry-metadata npm-check-updates prettier projen semver spdx-license-list standard-version streamx tar-stream ts-node typescript uuid yaml" + "exec": "yarn upgrade @aws-sdk/client-sfn @jsii/spec @octokit/rest @types/aws-lambda @types/changelog-filename-regex @types/fs-extra @types/jest @types/markdown-it @types/markdown-it-emoji @types/node @types/semver @types/streamx @types/tar-stream @types/tough-cookie @types/uuid @typescript-eslint/eslint-plugin @typescript-eslint/parser async-sema aws-cdk aws-embedded-metrics aws-sdk aws-sdk-mock aws-xray-sdk-core case cdk-dia cdklabs-projen-project-types changelog-filename-regex construct-hub-webapp dotenv esbuild eslint-config-prettier eslint-import-resolver-node eslint-import-resolver-typescript eslint-plugin-import eslint-plugin-prettier eslint feed fs-extra glob got jest jest-junit jsii-diff jsii-docgen jsii-pacmak jsii-rosetta JSONStream markdown-it markdown-it-emoji nock normalize-registry-metadata npm-check-updates prettier projen semver spdx-license-list standard-version streamx tar-stream ts-node typescript uuid yaml" }, { "exec": "npx projen" diff --git a/.projenrc.ts b/.projenrc.ts index c59078ca8..1e9ef5878 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -65,6 +65,7 @@ const project = new CdklabsConstructLibrary({ 'tar-stream', 'uuid', 'yaml', + 'nock', 'normalize-registry-metadata', '@octokit/rest', 'markdown-it', diff --git a/package.json b/package.json index c2adef4ba..6a6525266 100644 --- a/package.json +++ b/package.json @@ -167,6 +167,7 @@ "JSONStream": "^1.3.5", "markdown-it": "^13.0.1", "markdown-it-emoji": "^2.0.2", + "nock": "^13.3.3", "normalize-registry-metadata": "^1.1.2", "npm-check-updates": "^16", "prettier": "^2.8.8", diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index 59979380f..a24e604f5 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -10208,7 +10208,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "32aa9eacdc7aaee2ff1f1305f61efbbaaf00d95a064cbc1403433a1aff4ce947.zip", + "S3Key": "75a854616ef4b16cf16661572c2ecebd0122d11416836782afaa3d527d0b2897.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -23273,7 +23273,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "32aa9eacdc7aaee2ff1f1305f61efbbaaf00d95a064cbc1403433a1aff4ce947.zip", + "S3Key": "75a854616ef4b16cf16661572c2ecebd0122d11416836782afaa3d527d0b2897.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -35902,7 +35902,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "32aa9eacdc7aaee2ff1f1305f61efbbaaf00d95a064cbc1403433a1aff4ce947.zip", + "S3Key": "75a854616ef4b16cf16661572c2ecebd0122d11416836782afaa3d527d0b2897.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -48675,7 +48675,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "32aa9eacdc7aaee2ff1f1305f61efbbaaf00d95a064cbc1403433a1aff4ce947.zip", + "S3Key": "75a854616ef4b16cf16661572c2ecebd0122d11416836782afaa3d527d0b2897.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -61455,7 +61455,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "32aa9eacdc7aaee2ff1f1305f61efbbaaf00d95a064cbc1403433a1aff4ce947.zip", + "S3Key": "75a854616ef4b16cf16661572c2ecebd0122d11416836782afaa3d527d0b2897.zip", }, "DeadLetterConfig": { "TargetArn": { diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index d03623b4c..ee384007c 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -11495,7 +11495,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "32aa9eacdc7aaee2ff1f1305f61efbbaaf00d95a064cbc1403433a1aff4ce947.zip", + "S3Key": "75a854616ef4b16cf16661572c2ecebd0122d11416836782afaa3d527d0b2897.zip", }, "DeadLetterConfig": { "TargetArn": { diff --git a/src/__tests__/package-sources/npmjs/stage-and-notify.lambda.test.ts b/src/__tests__/package-sources/npmjs/stage-and-notify.lambda.test.ts new file mode 100644 index 000000000..296ecd407 --- /dev/null +++ b/src/__tests__/package-sources/npmjs/stage-and-notify.lambda.test.ts @@ -0,0 +1,59 @@ +import { Context } from 'aws-lambda'; +import * as AWS from 'aws-sdk'; +import * as AWSMock from 'aws-sdk-mock'; +import * as nock from 'nock'; +import { + ENV_DENY_LIST_BUCKET_NAME, + ENV_DENY_LIST_OBJECT_KEY, +} from '../../../backend/deny-list/constants'; +import { + handler, + PackageVersion, +} from '../../../package-sources/npmjs/stage-and-notify.lambda'; + +const MOCK_DENY_LIST_BUCKET = 'deny-list-bucket-name'; +const MOCK_DENY_LIST_OBJECT = 'my-deny-list.json'; + +type Response = (err: AWS.AWSError | null, data?: T) => void; + +beforeEach(() => { + process.env.BUCKET_NAME = 'foo'; + process.env.QUEUE_URL = 'bar'; + process.env[ENV_DENY_LIST_BUCKET_NAME] = MOCK_DENY_LIST_BUCKET; + process.env[ENV_DENY_LIST_OBJECT_KEY] = MOCK_DENY_LIST_OBJECT; +}); + +afterEach(() => { + process.env.BUCKET_NAME = undefined; + process.env.QUEUE_URL = undefined; + delete process.env[ENV_DENY_LIST_BUCKET_NAME]; + delete process.env[ENV_DENY_LIST_OBJECT_KEY]; +}); + +test('ignores 404', async () => { + const basePath = 'https://registry.npmjs.org'; + const uri = '/@pepperize/cdk-vpc/-/cdk-vpc-0.0.785.tgz'; + + AWSMock.mock( + 'S3', + 'getObject', + (_req: AWS.S3.GetObjectRequest, cb: Response) => { + cb(null, { Body: JSON.stringify({}) }); + } + ); + + nock(basePath).get(uri).reply(404); + + const event: PackageVersion = { + tarballUrl: `${basePath}${uri}`, + integrity: '09d37ec93c5518bf4842ac8e381a5c06452500e5', + modified: '2023-09-22T15:48:10.381Z', + name: '@pepper/cdk-vpc', + seq: '26437963', + version: '0.0.785', + }; + + const context: Context = {} as any; + + await expect(handler(event, context)).resolves.toBe(undefined); +}); diff --git a/src/package-sources/npmjs/stage-and-notify.lambda.ts b/src/package-sources/npmjs/stage-and-notify.lambda.ts index dc75ac3ec..b6908dd3f 100644 --- a/src/package-sources/npmjs/stage-and-notify.lambda.ts +++ b/src/package-sources/npmjs/stage-and-notify.lambda.ts @@ -7,6 +7,8 @@ import { s3, sqs } from '../../backend/shared/aws.lambda-shared'; import { requireEnv } from '../../backend/shared/env.lambda-shared'; import { integrity } from '../../backend/shared/integrity.lambda-shared'; +class HttpNotFoundError extends Error {} + /** * This function is invoked by the `npm-js-follower.lambda` with a `PackageVersion` object, or by * an SQS trigger feeding from this function's Dead-Letter Queue (for re-trying purposes). @@ -38,9 +40,23 @@ export async function handler( return; } - // Download the tarball - console.log(`Downloading tarball from URL: ${event.tarballUrl}`); - const tarball = await httpGet(event.tarballUrl); + let tarball: Buffer = Buffer.from(''); + try { + // Download the tarball + console.log(`Downloading tarball from URL: ${event.tarballUrl}`); + tarball = await httpGet(event.tarballUrl); + } catch (e) { + if (e instanceof HttpNotFoundError) { + // We received a message to download a file that should exist but doesn't. + // If we throw an error, the message will be sent to the DLQ, to be processed + // again in a re-drive. But given that this file will probably never be + // available, the re-drives will also fail, and we'll never be able to get rid + // of the message in the DLQ. Instead, we ignore this version by returning silently. + return; + } else { + throw e; + } + } // Store the tarball into the staging bucket // - infos.dist.tarball => https://registry.npmjs.org/<@scope>//-/-.tgz @@ -135,7 +151,10 @@ export interface PackageVersion { function httpGet(url: string) { return new Promise((ok, ko) => { https.get(url, (response) => { - if (response.statusCode !== 200) { + console.log(response.statusCode); + if (response.statusCode === 404) { + ko(new HttpNotFoundError()); + } else if (response.statusCode !== 200) { ko( new Error( `Unsuccessful GET: ${response.statusCode} - ${response.statusMessage}` diff --git a/yarn.lock b/yarn.lock index a4ba19ab9..b053cff21 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11816,6 +11816,16 @@ no-case@^3.0.4: lower-case "^2.0.2" tslib "^2.0.3" +nock@^13.3.3: + version "13.3.3" + resolved "https://registry.yarnpkg.com/nock/-/nock-13.3.3.tgz#179759c07d3f88ad3e794ace885629c1adfd3fe7" + integrity sha512-z+KUlILy9SK/RjpeXDiDUEAq4T94ADPHE3qaRkf66mpEhzc/ytOMm3Bwdrbq6k1tMWkbdujiKim3G2tfQARuJw== + dependencies: + debug "^4.1.0" + json-stringify-safe "^5.0.1" + lodash "^4.17.21" + propagate "^2.0.0" + node-emoji@^1.11.0: version "1.11.0" resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c" @@ -13229,6 +13239,11 @@ prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1: object-assign "^4.1.1" react-is "^16.13.1" +propagate@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45" + integrity sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag== + property-information@^5.0.0, property-information@^5.3.0: version "5.6.0" resolved "https://registry.yarnpkg.com/property-information/-/property-information-5.6.0.tgz#61675545fb23002f245c6540ec46077d4da3ed69"