From 0144652dca047240572ef2e4c69948f74d3c2674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=91=A8=F0=9F=8F=BB=E2=80=8D=F0=9F=92=BB=20Romain=20M?= =?UTF-8?q?arcadier?= Date: Wed, 16 Jun 2021 13:14:05 +0200 Subject: [PATCH] chore: self mutation # Conflicts: # src/__tests__/__snapshots__/construct-hub.test.ts.snap # src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap --- .eslintrc.json | 3 +- .gitattributes | 3 + .projen/deps.json | 1 - .projenrc.js | 2 + package.json | 2 +- .../__snapshots__/construct-hub.test.ts.snap | 386 +++++++-------- .../__snapshots__/snapshot.test.ts.snap | 108 ++--- src/backend/discovery/discovery.lambda.ts | 454 ++++++++++-------- src/backend/discovery/index.ts | 32 +- src/backend/shared/client.lambda-shared.ts | 26 + src/backend/shared/env.lambda-shared.ts | 9 + src/backend/shared/index.ts | 6 +- ...ut.ts => ingestion-input.lambda-shared.ts} | 0 ...ntegrity.ts => integrity.lambda-shared.ts} | 2 +- src/construct-hub.ts | 14 +- yarn.lock | 96 +--- 16 files changed, 558 insertions(+), 586 deletions(-) create mode 100644 src/backend/shared/client.lambda-shared.ts create mode 100644 src/backend/shared/env.lambda-shared.ts rename src/backend/shared/{ingestion-input.ts => ingestion-input.lambda-shared.ts} (100%) rename src/backend/shared/{integrity.ts => integrity.lambda-shared.ts} (93%) diff --git a/.eslintrc.json b/.eslintrc.json index b9d91ba8d..efac97727 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -145,7 +145,8 @@ "devDependencies": [ "**/src/__tests__/**", "**/build-tools/**", - "src/**/*.lambda.ts" + "src/**/*.lambda.ts", + "src/**/*.lambda-shared.ts" ], "optionalDependencies": false, "peerDependencies": true diff --git a/.gitattributes b/.gitattributes index 145cdad3f..acda92fb1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,9 +8,12 @@ /.github/workflows/build.yml linguist-generated /.github/workflows/release.yml linguist-generated /.github/workflows/upgrade-dependencies.yml linguist-generated +/.gitignore linguist-generated /.npmignore linguist-generated /.projen/** linguist-generated +/.projen/deps.json linguist-generated /.projen/jest-snapshot-resolver.js linguist-generated +/.projen/tasks.json linguist-generated /lib/__tests__/devapp/cdk.json linguist-generated /LICENSE linguist-generated /package.json linguist-generated diff --git a/.projen/deps.json b/.projen/deps.json index e6f9565f7..195538ac2 100644 --- a/.projen/deps.json +++ b/.projen/deps.json @@ -214,7 +214,6 @@ }, { "name": "projen", - "version": "^0.22.15", "type": "build" }, { diff --git a/.projenrc.js b/.projenrc.js index 8c650c842..4167b81e4 100644 --- a/.projenrc.js +++ b/.projenrc.js @@ -246,6 +246,8 @@ function newLambdaHandler(entrypoint) { function discoverLambdas() { // allow .lambda code to import dev-deps (since they are only needed during bundling) project.eslint.allowDevDeps('src/**/*.lambda.ts'); + // Allow .lambda-shared code to import dev-deps (these are not entry points, but are shared by several lambdas) + project.eslint.allowDevDeps('src/**/*.lambda-shared.ts'); project.addDevDeps('glob'); for (const entry of glob.sync('src/**/*.lambda.ts')) { newLambdaHandler(entry); diff --git a/package.json b/package.json index 717cd448d..ed98a1d13 100644 --- a/package.json +++ b/package.json @@ -91,7 +91,7 @@ "nano": "^9.0.3", "npm-check-updates": "^11", "pascal-case": "^3.1.2", - "projen": "^0.22.15", + "projen": "^0.24.7", "semver": "^7.3.5", "standard-version": "^9", "tar-stream": "^2.2.0", diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index c1f668c80..a298299cf 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -45,18 +45,6 @@ Object { "Description": "S3 key for asset version \\"67b7823b74bc135986aa72f889d6a8da058d0c4a20cbc2dfc6f78995fdd2fc24\\"", "Type": "String", }, - "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132ArtifactHashA47619D7": Object { - "Description": "Artifact hash for asset \\"7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132\\"", - "Type": "String", - }, - "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3BucketE20F2CD6": Object { - "Description": "S3 bucket for asset \\"7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132\\"", - "Type": "String", - }, - "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3VersionKeyA793E728": Object { - "Description": "S3 key for asset version \\"7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132\\"", - "Type": "String", - }, "AssetParameters866c466f9e9aaf86d825bb22132965b3f5d3418371a1ca6027f40a2863d892d4ArtifactHashDB7C5CF3": Object { "Description": "Artifact hash for asset \\"866c466f9e9aaf86d825bb22132965b3f5d3418371a1ca6027f40a2863d892d4\\"", "Type": "String", @@ -69,28 +57,28 @@ Object { "Description": "S3 key for asset version \\"866c466f9e9aaf86d825bb22132965b3f5d3418371a1ca6027f40a2863d892d4\\"", "Type": "String", }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": Object { - "Description": "Artifact hash for asset \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", + "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7ArtifactHashC15CEDBA": Object { + "Description": "Artifact hash for asset \\"ac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7\\"", "Type": "String", }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": Object { - "Description": "S3 bucket for asset \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", + "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787": Object { + "Description": "S3 bucket for asset \\"ac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7\\"", "Type": "String", }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": Object { - "Description": "S3 key for asset version \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", + "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3VersionKey4270971F": Object { + "Description": "S3 key for asset version \\"ac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7\\"", "Type": "String", }, - "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524ArtifactHashABFB44E7": Object { - "Description": "Artifact hash for asset \\"c9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524\\"", + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": Object { + "Description": "Artifact hash for asset \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", "Type": "String", }, - "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217": Object { - "Description": "S3 bucket for asset \\"c9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524\\"", + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": Object { + "Description": "S3 bucket for asset \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", "Type": "String", }, - "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3VersionKeyA48E92EB": Object { - "Description": "S3 key for asset version \\"c9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524\\"", + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": Object { + "Description": "S3 key for asset version \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", "Type": "String", }, "AssetParametersda254f731d96f448a42b847d5e631a6a426b230b48eab5b0862307875334e305ArtifactHashE87E66EE": Object { @@ -129,6 +117,18 @@ Object { "Description": "S3 key for asset version \\"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\\"", "Type": "String", }, + "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52ArtifactHash1E5261D1": Object { + "Description": "Artifact hash for asset \\"f9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52\\"", + "Type": "String", + }, + "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3Bucket91D6685B": Object { + "Description": "S3 bucket for asset \\"f9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52\\"", + "Type": "String", + }, + "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3VersionKeyD7CA3182": Object { + "Description": "S3 key for asset version \\"f9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52\\"", + "Type": "String", + }, }, "Resources": Object { "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691": Object { @@ -448,23 +448,15 @@ Object { }, "Type": "AWS::IAM::Policy", }, - "ConstructHubDiscoveryFunctionDeadLetterQueueFE498555": Object { - "DeletionPolicy": "Delete", - "Properties": Object { - "MessageRetentionPeriod": 1209600, - }, - "Type": "AWS::SQS::Queue", - "UpdateReplacePolicy": "Delete", - }, - "ConstructHubDiscoveryFunctionF5E139FD": Object { + "ConstructHubDiscoveryD6EEC2B8": Object { "DependsOn": Array [ - "ConstructHubDiscoveryFunctionServiceRoleDefaultPolicyA09546EC", - "ConstructHubDiscoveryFunctionServiceRoleD371794F", + "ConstructHubDiscoveryServiceRoleDefaultPolicy9D5F91B3", + "ConstructHubDiscoveryServiceRole1B3CFF96", ], "Properties": Object { "Code": Object { "S3Bucket": Object { - "Ref": "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3BucketE20F2CD6", + "Ref": "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3Bucket91D6685B", }, "S3Key": Object { "Fn::Join": Array [ @@ -477,7 +469,7 @@ Object { "Fn::Split": Array [ "||", Object { - "Ref": "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3VersionKeyA793E728", + "Ref": "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3VersionKeyD7CA3182", }, ], }, @@ -490,7 +482,7 @@ Object { "Fn::Split": Array [ "||", Object { - "Ref": "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3VersionKeyA793E728", + "Ref": "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3VersionKeyD7CA3182", }, ], }, @@ -500,30 +492,23 @@ Object { ], }, }, - "DeadLetterConfig": Object { - "TargetArn": Object { - "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionDeadLetterQueueFE498555", - "Arn", - ], - }, - }, - "Description": "Periodically query npm.js index for new Constructs", + "Description": "Periodically query npm.js index for new construct libraries", "Environment": Object { "Variables": Object { + "BUCKET_NAME": Object { + "Ref": "ConstructHubDiscoveryStagingBucket1F2F7AE8", + }, "QUEUE_URL": Object { "Ref": "ConstructHubIngestionQueue637E4740", }, - "STAGING_BUCKET_NAME": Object { - "Ref": "ConstructHubStagingBucket29942A98", - }, }, }, "Handler": "index.handler", "MemorySize": 10240, + "ReservedConcurrentExecutions": 1, "Role": Object { "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionServiceRoleD371794F", + "ConstructHubDiscoveryServiceRole1B3CFF96", "Arn", ], }, @@ -532,44 +517,44 @@ Object { }, "Type": "AWS::Lambda::Function", }, - "ConstructHubDiscoveryFunctionScheduleRuleAllowEventRuleTestConstructHubDiscoveryFunctionC36D63A5E62E305C": Object { + "ConstructHubDiscoveryScheduleRule90EE2E2A": Object { + "Properties": Object { + "ScheduleExpression": "rate(15 minutes)", + "State": "ENABLED", + "Targets": Array [ + Object { + "Arn": Object { + "Fn::GetAtt": Array [ + "ConstructHubDiscoveryD6EEC2B8", + "Arn", + ], + }, + "Id": "Target0", + }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "ConstructHubDiscoveryScheduleRuleAllowEventRuleTestConstructHubDiscovery5714D5BB9D860B10": Object { "Properties": Object { "Action": "lambda:InvokeFunction", "FunctionName": Object { "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionF5E139FD", + "ConstructHubDiscoveryD6EEC2B8", "Arn", ], }, "Principal": "events.amazonaws.com", "SourceArn": Object { "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionScheduleRuleE32FE899", + "ConstructHubDiscoveryScheduleRule90EE2E2A", "Arn", ], }, }, "Type": "AWS::Lambda::Permission", }, - "ConstructHubDiscoveryFunctionScheduleRuleE32FE899": Object { - "Properties": Object { - "ScheduleExpression": "rate(5 minutes)", - "State": "ENABLED", - "Targets": Array [ - Object { - "Arn": Object { - "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionF5E139FD", - "Arn", - ], - }, - "Id": "Target0", - }, - ], - }, - "Type": "AWS::Events::Rule", - }, - "ConstructHubDiscoveryFunctionServiceRoleD371794F": Object { + "ConstructHubDiscoveryServiceRole1B3CFF96": Object { "Properties": Object { "AssumeRolePolicyDocument": Object { "Statement": Array [ @@ -600,20 +585,10 @@ Object { }, "Type": "AWS::IAM::Role", }, - "ConstructHubDiscoveryFunctionServiceRoleDefaultPolicyA09546EC": Object { + "ConstructHubDiscoveryServiceRoleDefaultPolicy9D5F91B3": Object { "Properties": Object { "PolicyDocument": Object { "Statement": Array [ - Object { - "Action": "sqs:SendMessage", - "Effect": "Allow", - "Resource": Object { - "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionDeadLetterQueueFE498555", - "Arn", - ], - }, - }, Object { "Action": Array [ "s3:GetObject*", @@ -627,7 +602,7 @@ Object { "Resource": Array [ Object { "Fn::GetAtt": Array [ - "ConstructHubStagingBucket29942A98", + "ConstructHubDiscoveryStagingBucket1F2F7AE8", "Arn", ], }, @@ -637,7 +612,7 @@ Object { Array [ Object { "Fn::GetAtt": Array [ - "ConstructHubStagingBucket29942A98", + "ConstructHubDiscoveryStagingBucket1F2F7AE8", "Arn", ], }, @@ -664,15 +639,31 @@ Object { ], "Version": "2012-10-17", }, - "PolicyName": "ConstructHubDiscoveryFunctionServiceRoleDefaultPolicyA09546EC", + "PolicyName": "ConstructHubDiscoveryServiceRoleDefaultPolicy9D5F91B3", "Roles": Array [ Object { - "Ref": "ConstructHubDiscoveryFunctionServiceRoleD371794F", + "Ref": "ConstructHubDiscoveryServiceRole1B3CFF96", }, ], }, "Type": "AWS::IAM::Policy", }, + "ConstructHubDiscoveryStagingBucket1F2F7AE8": Object { + "DeletionPolicy": "Retain", + "Properties": Object { + "LifecycleConfiguration": Object { + "Rules": Array [ + Object { + "ExpirationInDays": 30, + "Prefix": "staged/", + "Status": "Enabled", + }, + ], + }, + }, + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubIngestionQueue637E4740": Object { "DeletionPolicy": "Delete", "Type": "AWS::SQS::Queue", @@ -1039,22 +1030,6 @@ Object { }, "Type": "Custom::S3BucketNotifications", }, - "ConstructHubStagingBucket29942A98": Object { - "DeletionPolicy": "Retain", - "Properties": Object { - "LifecycleConfiguration": Object { - "Rules": Array [ - Object { - "ExpirationInDays": 30, - "Prefix": "packages", - "Status": "Enabled", - }, - ], - }, - }, - "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Retain", - }, "ConstructHubTransliterator9C48708A": Object { "DependsOn": Array [ "ConstructHubTransliteratorServiceRoleDefaultPolicyB9C4BE06", @@ -1314,7 +1289,7 @@ Object { }, "SourceBucketNames": Array [ Object { - "Ref": "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217", + "Ref": "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787", }, ], "SourceObjectKeys": Array [ @@ -1329,7 +1304,7 @@ Object { "Fn::Split": Array [ "||", Object { - "Ref": "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3VersionKeyA48E92EB", + "Ref": "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3VersionKey4270971F", }, ], }, @@ -1342,7 +1317,7 @@ Object { "Fn::Split": Array [ "||", Object { - "Ref": "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3VersionKeyA48E92EB", + "Ref": "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3VersionKey4270971F", }, ], }, @@ -1601,7 +1576,7 @@ Object { }, ":s3:::", Object { - "Ref": "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217", + "Ref": "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787", }, ], ], @@ -1616,7 +1591,7 @@ Object { }, ":s3:::", Object { - "Ref": "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217", + "Ref": "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787", }, "/*", ], @@ -1858,18 +1833,6 @@ Object { "Description": "S3 key for asset version \\"7af6295e521fd55af94332393ceffb3e866aac4dc4956321f7918f21e72199e4\\"", "Type": "String", }, - "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132ArtifactHashA47619D7": Object { - "Description": "Artifact hash for asset \\"7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132\\"", - "Type": "String", - }, - "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3BucketE20F2CD6": Object { - "Description": "S3 bucket for asset \\"7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132\\"", - "Type": "String", - }, - "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3VersionKeyA793E728": Object { - "Description": "S3 key for asset version \\"7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132\\"", - "Type": "String", - }, "AssetParameters866c466f9e9aaf86d825bb22132965b3f5d3418371a1ca6027f40a2863d892d4ArtifactHashDB7C5CF3": Object { "Description": "Artifact hash for asset \\"866c466f9e9aaf86d825bb22132965b3f5d3418371a1ca6027f40a2863d892d4\\"", "Type": "String", @@ -1882,28 +1845,28 @@ Object { "Description": "S3 key for asset version \\"866c466f9e9aaf86d825bb22132965b3f5d3418371a1ca6027f40a2863d892d4\\"", "Type": "String", }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": Object { - "Description": "Artifact hash for asset \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", + "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7ArtifactHashC15CEDBA": Object { + "Description": "Artifact hash for asset \\"ac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7\\"", "Type": "String", }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": Object { - "Description": "S3 bucket for asset \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", + "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787": Object { + "Description": "S3 bucket for asset \\"ac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7\\"", "Type": "String", }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": Object { - "Description": "S3 key for asset version \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", + "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3VersionKey4270971F": Object { + "Description": "S3 key for asset version \\"ac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7\\"", "Type": "String", }, - "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524ArtifactHashABFB44E7": Object { - "Description": "Artifact hash for asset \\"c9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524\\"", + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": Object { + "Description": "Artifact hash for asset \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", "Type": "String", }, - "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217": Object { - "Description": "S3 bucket for asset \\"c9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524\\"", + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": Object { + "Description": "S3 bucket for asset \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", "Type": "String", }, - "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3VersionKeyA48E92EB": Object { - "Description": "S3 key for asset version \\"c9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524\\"", + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": Object { + "Description": "S3 key for asset version \\"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\\"", "Type": "String", }, "AssetParametersda254f731d96f448a42b847d5e631a6a426b230b48eab5b0862307875334e305ArtifactHashE87E66EE": Object { @@ -1942,6 +1905,18 @@ Object { "Description": "S3 key for asset version \\"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\\"", "Type": "String", }, + "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52ArtifactHash1E5261D1": Object { + "Description": "Artifact hash for asset \\"f9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52\\"", + "Type": "String", + }, + "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3Bucket91D6685B": Object { + "Description": "S3 bucket for asset \\"f9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52\\"", + "Type": "String", + }, + "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3VersionKeyD7CA3182": Object { + "Description": "S3 key for asset version \\"f9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52\\"", + "Type": "String", + }, }, "Resources": Object { "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691": Object { @@ -2410,23 +2385,15 @@ Object { }, "Type": "AWS::IAM::Policy", }, - "ConstructHubDiscoveryFunctionDeadLetterQueueFE498555": Object { - "DeletionPolicy": "Delete", - "Properties": Object { - "MessageRetentionPeriod": 1209600, - }, - "Type": "AWS::SQS::Queue", - "UpdateReplacePolicy": "Delete", - }, - "ConstructHubDiscoveryFunctionF5E139FD": Object { + "ConstructHubDiscoveryD6EEC2B8": Object { "DependsOn": Array [ - "ConstructHubDiscoveryFunctionServiceRoleDefaultPolicyA09546EC", - "ConstructHubDiscoveryFunctionServiceRoleD371794F", + "ConstructHubDiscoveryServiceRoleDefaultPolicy9D5F91B3", + "ConstructHubDiscoveryServiceRole1B3CFF96", ], "Properties": Object { "Code": Object { "S3Bucket": Object { - "Ref": "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3BucketE20F2CD6", + "Ref": "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3Bucket91D6685B", }, "S3Key": Object { "Fn::Join": Array [ @@ -2439,7 +2406,7 @@ Object { "Fn::Split": Array [ "||", Object { - "Ref": "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3VersionKeyA793E728", + "Ref": "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3VersionKeyD7CA3182", }, ], }, @@ -2452,7 +2419,7 @@ Object { "Fn::Split": Array [ "||", Object { - "Ref": "AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3VersionKeyA793E728", + "Ref": "AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3VersionKeyD7CA3182", }, ], }, @@ -2462,30 +2429,23 @@ Object { ], }, }, - "DeadLetterConfig": Object { - "TargetArn": Object { - "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionDeadLetterQueueFE498555", - "Arn", - ], - }, - }, - "Description": "Periodically query npm.js index for new Constructs", + "Description": "Periodically query npm.js index for new construct libraries", "Environment": Object { "Variables": Object { + "BUCKET_NAME": Object { + "Ref": "ConstructHubDiscoveryStagingBucket1F2F7AE8", + }, "QUEUE_URL": Object { "Ref": "ConstructHubIngestionQueue637E4740", }, - "STAGING_BUCKET_NAME": Object { - "Ref": "ConstructHubStagingBucket29942A98", - }, }, }, "Handler": "index.handler", "MemorySize": 10240, + "ReservedConcurrentExecutions": 1, "Role": Object { "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionServiceRoleD371794F", + "ConstructHubDiscoveryServiceRole1B3CFF96", "Arn", ], }, @@ -2494,44 +2454,44 @@ Object { }, "Type": "AWS::Lambda::Function", }, - "ConstructHubDiscoveryFunctionScheduleRuleAllowEventRuleTestConstructHubDiscoveryFunctionC36D63A5E62E305C": Object { + "ConstructHubDiscoveryScheduleRule90EE2E2A": Object { + "Properties": Object { + "ScheduleExpression": "rate(15 minutes)", + "State": "ENABLED", + "Targets": Array [ + Object { + "Arn": Object { + "Fn::GetAtt": Array [ + "ConstructHubDiscoveryD6EEC2B8", + "Arn", + ], + }, + "Id": "Target0", + }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "ConstructHubDiscoveryScheduleRuleAllowEventRuleTestConstructHubDiscovery5714D5BB9D860B10": Object { "Properties": Object { "Action": "lambda:InvokeFunction", "FunctionName": Object { "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionF5E139FD", + "ConstructHubDiscoveryD6EEC2B8", "Arn", ], }, "Principal": "events.amazonaws.com", "SourceArn": Object { "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionScheduleRuleE32FE899", + "ConstructHubDiscoveryScheduleRule90EE2E2A", "Arn", ], }, }, "Type": "AWS::Lambda::Permission", }, - "ConstructHubDiscoveryFunctionScheduleRuleE32FE899": Object { - "Properties": Object { - "ScheduleExpression": "rate(5 minutes)", - "State": "ENABLED", - "Targets": Array [ - Object { - "Arn": Object { - "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionF5E139FD", - "Arn", - ], - }, - "Id": "Target0", - }, - ], - }, - "Type": "AWS::Events::Rule", - }, - "ConstructHubDiscoveryFunctionServiceRoleD371794F": Object { + "ConstructHubDiscoveryServiceRole1B3CFF96": Object { "Properties": Object { "AssumeRolePolicyDocument": Object { "Statement": Array [ @@ -2562,20 +2522,10 @@ Object { }, "Type": "AWS::IAM::Role", }, - "ConstructHubDiscoveryFunctionServiceRoleDefaultPolicyA09546EC": Object { + "ConstructHubDiscoveryServiceRoleDefaultPolicy9D5F91B3": Object { "Properties": Object { "PolicyDocument": Object { "Statement": Array [ - Object { - "Action": "sqs:SendMessage", - "Effect": "Allow", - "Resource": Object { - "Fn::GetAtt": Array [ - "ConstructHubDiscoveryFunctionDeadLetterQueueFE498555", - "Arn", - ], - }, - }, Object { "Action": Array [ "s3:GetObject*", @@ -2589,7 +2539,7 @@ Object { "Resource": Array [ Object { "Fn::GetAtt": Array [ - "ConstructHubStagingBucket29942A98", + "ConstructHubDiscoveryStagingBucket1F2F7AE8", "Arn", ], }, @@ -2599,7 +2549,7 @@ Object { Array [ Object { "Fn::GetAtt": Array [ - "ConstructHubStagingBucket29942A98", + "ConstructHubDiscoveryStagingBucket1F2F7AE8", "Arn", ], }, @@ -2626,15 +2576,31 @@ Object { ], "Version": "2012-10-17", }, - "PolicyName": "ConstructHubDiscoveryFunctionServiceRoleDefaultPolicyA09546EC", + "PolicyName": "ConstructHubDiscoveryServiceRoleDefaultPolicy9D5F91B3", "Roles": Array [ Object { - "Ref": "ConstructHubDiscoveryFunctionServiceRoleD371794F", + "Ref": "ConstructHubDiscoveryServiceRole1B3CFF96", }, ], }, "Type": "AWS::IAM::Policy", }, + "ConstructHubDiscoveryStagingBucket1F2F7AE8": Object { + "DeletionPolicy": "Retain", + "Properties": Object { + "LifecycleConfiguration": Object { + "Rules": Array [ + Object { + "ExpirationInDays": 30, + "Prefix": "staged/", + "Status": "Enabled", + }, + ], + }, + }, + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubIngestionQueue637E4740": Object { "DeletionPolicy": "Delete", "Type": "AWS::SQS::Queue", @@ -3001,22 +2967,6 @@ Object { }, "Type": "Custom::S3BucketNotifications", }, - "ConstructHubStagingBucket29942A98": Object { - "DeletionPolicy": "Retain", - "Properties": Object { - "LifecycleConfiguration": Object { - "Rules": Array [ - Object { - "ExpirationInDays": 30, - "Prefix": "packages", - "Status": "Enabled", - }, - ], - }, - }, - "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Retain", - }, "ConstructHubTransliterator9C48708A": Object { "DependsOn": Array [ "ConstructHubTransliteratorServiceRoleDefaultPolicyB9C4BE06", @@ -3328,7 +3278,7 @@ Object { }, "SourceBucketNames": Array [ Object { - "Ref": "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217", + "Ref": "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787", }, ], "SourceObjectKeys": Array [ @@ -3343,7 +3293,7 @@ Object { "Fn::Split": Array [ "||", Object { - "Ref": "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3VersionKeyA48E92EB", + "Ref": "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3VersionKey4270971F", }, ], }, @@ -3356,7 +3306,7 @@ Object { "Fn::Split": Array [ "||", Object { - "Ref": "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3VersionKeyA48E92EB", + "Ref": "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3VersionKey4270971F", }, ], }, @@ -3628,7 +3578,7 @@ Object { }, ":s3:::", Object { - "Ref": "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217", + "Ref": "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787", }, ], ], @@ -3643,7 +3593,7 @@ Object { }, ":s3:::", Object { - "Ref": "AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217", + "Ref": "AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787", }, "/*", ], diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 506ed4bbc..7a58648bf 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -209,21 +209,21 @@ Resources: Fn::GetAtt: - ConstructHubPackageDataDC5EF35E - Arn - ConstructHubStagingBucket29942A98: + ConstructHubIngestionQueue637E4740: + Type: AWS::SQS::Queue + UpdateReplacePolicy: Delete + DeletionPolicy: Delete + ConstructHubDiscoveryStagingBucket1F2F7AE8: Type: AWS::S3::Bucket Properties: LifecycleConfiguration: Rules: - ExpirationInDays: 30 - Prefix: packages + Prefix: staged/ Status: Enabled UpdateReplacePolicy: Retain DeletionPolicy: Retain - ConstructHubIngestionQueue637E4740: - Type: AWS::SQS::Queue - UpdateReplacePolicy: Delete - DeletionPolicy: Delete - ConstructHubDiscoveryFunctionServiceRoleD371794F: + ConstructHubDiscoveryServiceRole1B3CFF96: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: @@ -239,17 +239,11 @@ Resources: - - "arn:" - Ref: AWS::Partition - :iam::aws:policy/service-role/AWSLambdaBasicExecutionRole - ConstructHubDiscoveryFunctionServiceRoleDefaultPolicyA09546EC: + ConstructHubDiscoveryServiceRoleDefaultPolicy9D5F91B3: Type: AWS::IAM::Policy Properties: PolicyDocument: Statement: - - Action: sqs:SendMessage - Effect: Allow - Resource: - Fn::GetAtt: - - ConstructHubDiscoveryFunctionDeadLetterQueueFE498555 - - Arn - Action: - s3:GetObject* - s3:GetBucket* @@ -260,12 +254,12 @@ Resources: Effect: Allow Resource: - Fn::GetAtt: - - ConstructHubStagingBucket29942A98 + - ConstructHubDiscoveryStagingBucket1F2F7AE8 - Arn - Fn::Join: - "" - - Fn::GetAtt: - - ConstructHubStagingBucket29942A98 + - ConstructHubDiscoveryStagingBucket1F2F7AE8 - Arn - /* - Action: @@ -278,21 +272,15 @@ Resources: - ConstructHubIngestionQueue637E4740 - Arn Version: 2012-10-17 - PolicyName: ConstructHubDiscoveryFunctionServiceRoleDefaultPolicyA09546EC + PolicyName: ConstructHubDiscoveryServiceRoleDefaultPolicy9D5F91B3 Roles: - - Ref: ConstructHubDiscoveryFunctionServiceRoleD371794F - ConstructHubDiscoveryFunctionDeadLetterQueueFE498555: - Type: AWS::SQS::Queue - Properties: - MessageRetentionPeriod: 1209600 - UpdateReplacePolicy: Delete - DeletionPolicy: Delete - ConstructHubDiscoveryFunctionF5E139FD: + - Ref: ConstructHubDiscoveryServiceRole1B3CFF96 + ConstructHubDiscoveryD6EEC2B8: Type: AWS::Lambda::Function Properties: Code: S3Bucket: - Ref: AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3BucketE20F2CD6 + Ref: AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3Bucket91D6685B S3Key: Fn::Join: - "" @@ -300,58 +288,54 @@ Resources: - 0 - Fn::Split: - "||" - - Ref: AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3VersionKeyA793E728 + - Ref: AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3VersionKeyD7CA3182 - Fn::Select: - 1 - Fn::Split: - "||" - - Ref: AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3VersionKeyA793E728 + - Ref: AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3VersionKeyD7CA3182 Role: Fn::GetAtt: - - ConstructHubDiscoveryFunctionServiceRoleD371794F + - ConstructHubDiscoveryServiceRole1B3CFF96 - Arn - DeadLetterConfig: - TargetArn: - Fn::GetAtt: - - ConstructHubDiscoveryFunctionDeadLetterQueueFE498555 - - Arn - Description: Periodically query npm.js index for new Constructs + Description: Periodically query npm.js index for new construct libraries Environment: Variables: - STAGING_BUCKET_NAME: - Ref: ConstructHubStagingBucket29942A98 + BUCKET_NAME: + Ref: ConstructHubDiscoveryStagingBucket1F2F7AE8 QUEUE_URL: Ref: ConstructHubIngestionQueue637E4740 Handler: index.handler MemorySize: 10240 + ReservedConcurrentExecutions: 1 Runtime: nodejs14.x Timeout: 900 DependsOn: - - ConstructHubDiscoveryFunctionServiceRoleDefaultPolicyA09546EC - - ConstructHubDiscoveryFunctionServiceRoleD371794F - ConstructHubDiscoveryFunctionScheduleRuleE32FE899: + - ConstructHubDiscoveryServiceRoleDefaultPolicy9D5F91B3 + - ConstructHubDiscoveryServiceRole1B3CFF96 + ConstructHubDiscoveryScheduleRule90EE2E2A: Type: AWS::Events::Rule Properties: - ScheduleExpression: rate(5 minutes) + ScheduleExpression: rate(15 minutes) State: ENABLED Targets: - Arn: Fn::GetAtt: - - ConstructHubDiscoveryFunctionF5E139FD + - ConstructHubDiscoveryD6EEC2B8 - Arn Id: Target0 - ConstructHubDiscoveryFunctionScheduleRuleAllowEventRuledevConstructHubDiscoveryFunction9006704D902BB631: + ConstructHubDiscoveryScheduleRuleAllowEventRuledevConstructHubDiscovery67F2A54169E41C36: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction FunctionName: Fn::GetAtt: - - ConstructHubDiscoveryFunctionF5E139FD + - ConstructHubDiscoveryD6EEC2B8 - Arn Principal: events.amazonaws.com SourceArn: Fn::GetAtt: - - ConstructHubDiscoveryFunctionScheduleRuleE32FE899 + - ConstructHubDiscoveryScheduleRule90EE2E2A - Arn ConstructHubTransliteratorServiceRole0F8A20C8: Type: AWS::IAM::Role @@ -668,7 +652,7 @@ Resources: - CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C81C01536 - Arn SourceBucketNames: - - Ref: AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217 + - Ref: AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787 SourceObjectKeys: - Fn::Join: - "" @@ -676,12 +660,12 @@ Resources: - 0 - Fn::Split: - "||" - - Ref: AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3VersionKeyA48E92EB + - Ref: AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3VersionKey4270971F - Fn::Select: - 1 - Fn::Split: - "||" - - Ref: AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3VersionKeyA48E92EB + - Ref: AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3VersionKey4270971F DestinationBucketName: Ref: ConstructHubWebAppWebsiteBucket4B2B9DB2 Prune: true @@ -889,13 +873,13 @@ Resources: - - "arn:" - Ref: AWS::Partition - ":s3:::" - - Ref: AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217 + - Ref: AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787 - Fn::Join: - "" - - "arn:" - Ref: AWS::Partition - ":s3:::" - - Ref: AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217 + - Ref: AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787 - /* - Action: - s3:GetObject* @@ -972,18 +956,18 @@ Outputs: Export: Name: ConstructHubDomainName Parameters: - AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3BucketE20F2CD6: + AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3Bucket91D6685B: Type: String Description: S3 bucket for asset - "7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132" - AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132S3VersionKeyA793E728: + "f9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52" + AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52S3VersionKeyD7CA3182: Type: String Description: S3 key for asset version - "7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132" - AssetParameters7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132ArtifactHashA47619D7: + "f9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52" + AssetParametersf9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52ArtifactHash1E5261D1: Type: String Description: Artifact hash for asset - "7db26aca26e95fa59a924b510eb2138ce673540b66774f948ac7a5a04158d132" + "f9858316fe38c4505937630579f80e6167f3c2877cb62d785652ef8f68670c52" AssetParametersda254f731d96f448a42b847d5e631a6a426b230b48eab5b0862307875334e305S3BucketCED66570: Type: String Description: S3 bucket for asset @@ -1044,18 +1028,18 @@ Parameters: Type: String Description: Artifact hash for asset "c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf" - AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3Bucket265D5217: + AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3BucketBFB05787: Type: String Description: S3 bucket for asset - "c9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524" - AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524S3VersionKeyA48E92EB: + "ac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7" + AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7S3VersionKey4270971F: Type: String Description: S3 key for asset version - "c9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524" - AssetParametersc9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524ArtifactHashABFB44E7: + "ac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7" + AssetParametersac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7ArtifactHashC15CEDBA: Type: String Description: Artifact hash for asset - "c9cd5cf7cd05639608564a28ce3c6ce732dcfba9df2045d125302637301dc524" + "ac07adba47a066f4139e473ffb8c677dda20b946934af3d78d4aed0d904713c7" AssetParametersdcc427ad1a9c4d2a63344256acec911f0ef035359b814054fe1485aeb59e789dS3Bucket8C070E54: Type: String Description: S3 bucket for asset diff --git a/src/backend/discovery/discovery.lambda.ts b/src/backend/discovery/discovery.lambda.ts index 266915287..e2e5d8d3a 100644 --- a/src/backend/discovery/discovery.lambda.ts +++ b/src/backend/discovery/discovery.lambda.ts @@ -1,30 +1,18 @@ import * as console from 'console'; import * as https from 'https'; -import * as process from 'process'; import { URL } from 'url'; // eslint-disable-next-line import/no-unresolved -import type { Context } from 'aws-lambda'; -import * as AWS from 'aws-sdk'; -import * as Nano from 'nano'; -import { IngestionInput, integrity } from '../shared'; +import type { Context, ScheduledEvent } from 'aws-lambda'; +// eslint-disable-next-line @typescript-eslint/no-require-imports +import Nano = require('nano'); +import { aws, IngestionInput, integrity, requireEnv } from '../shared'; const TIMEOUT_MILLISECONDS = 10_000; const CONSTRUCT_KEYWORDS: ReadonlySet = new Set(['cdk', 'aws-cdk', 'cdk8s', 'cdktf']); const MARKER_FILE_NAME = 'couchdb-last-transaction-id'; const NPM_REPLICA_REGISTRY_URL = 'https://replicate.npmjs.com/'; -/** - * The S3 prefix for storing change that we failed processing - */ -const FAILED_CHANGE_PREFIX = 'failed'; - -let s3: AWS.S3 | undefined; -let sqs: AWS.SQS | undefined; - -let stagingBucket: string; -let queueUrl: string; - /** * This function triggers on a fixed schedule and reads a stream of changes frm npmjs couchdb _changes endpoint. * Upon invocation the function starts reading from a sequence stored in an s3 object - the `marker`. @@ -37,215 +25,284 @@ let queueUrl: string; * npm registry API docs: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md * @param context a Lambda execution context */ -export async function handler(_request: unknown, context: Context) { - if (s3 == null) { - s3 = new AWS.S3(); - } - if (sqs == null) { - sqs = new AWS.SQS(); - } +export async function handler(event: ScheduledEvent, context: Context) { + console.log(`Event: ${JSON.stringify(event, null, 2)}`); - if (!process.env.STAGING_BUCKET_NAME) { - throw new Error('The STAGING_BUCKET_NAME environment variable is not set'); - } - stagingBucket = process.env.STAGING_BUCKET_NAME; + const stagingBucket = requireEnv('BUCKET_NAME'); + const queueUrl = requireEnv('QUEUE_URL'); - if (!process.env.QUEUE_URL) { - throw new Error('The QUEUE_URL environment variable is not set'); - } - queueUrl = process.env.QUEUE_URL; - - // load marker from the staging bucket - let marker; - try { - await s3.getObject({ - Bucket: stagingBucket, - Key: MARKER_FILE_NAME, - }).promise().then(function(data) { - marker = data.Body?.toString('utf-8'); - }); - } catch (error) { - throw new Error(`Failed to load marker for bucket: ${stagingBucket}, exiting`); - } - - console.log(`Starting changes stream read from ${marker}`); + const initialMarker = await loadLastTransactionMarker(139369); const config: Nano.ChangesReaderOptions = { includeDocs: true, // pause the changes reader after each request wait: true, - since: marker, + since: initialMarker.toFixed(), // `changesReader.get` stops once a response with zero changes is received, however it waits too long // since we want to terminate the Lambda function we define a timeout shorter than the default timeout: TIMEOUT_MILLISECONDS, + // Only items with a name + selector: { + name: { $gt: null }, + }, }; const nano = Nano(NPM_REPLICA_REGISTRY_URL); const db = nano.db.use('registry'); - db.changesReader.get(config) - .on('batch', async (batch: readonly Change[]) => { - console.log(`Received a batch of ${batch.length} element(s)`); - const batchPromises = batch - // ignores changes which are not package update, see https://github.com/cdklabs/construct-hub/issues/3#issuecomment-858246275 - .filter((change) => change.doc.name) - .map((change) => - processPackageUpdate(change, context) - .catch((err) => - s3!.putObject({ - Bucket: stagingBucket, - Key: `${FAILED_CHANGE_PREFIX}/${change.id}`, - Body: JSON.stringify({ ...change, _construct_hub_failure_reason: err }, null, 2), - Metadata: { - 'Lambda-Log-Group': context.logGroupName, - 'Lambda-Log-Stream': context.logStreamName, - 'Lambda-Run-Id': context.awsRequestId, - }, - }).promise().then(() => (void undefined)))); - - await Promise.all(batchPromises).then(function () { - // write the last sequence to the marker file in S3 - const lastSequence = batch[batch.length - 1].seq; - return writeMarkerToS3(lastSequence, context); - }).then( - // resume reader - () => (db.changesReader as any).resume(), - ).catch((error) => { - db.changesReader.stop(); - return Promise.reject(new Error(`Error while processing batch, marker will not be updated, exiting.\n${error}`)); - }); - }) - .on('end', () => { - console.log('Changes feed monitoring has stopped'); - }); -} + // We need to make an explicit Promise here, because otherwise Lambda won't + // know when it's done... + return new Promise((ok, ko) => { + let updatedMarker = initialMarker; -async function writeMarkerToS3(sequence: Number, context: Context) { - if (process.env.SKIP_MARKER) { - return; - } - await s3!.putObject({ - Bucket: stagingBucket, - Key: MARKER_FILE_NAME, - Body: sequence.toString(), - Metadata: { - 'Lambda-Log-Group': context.logGroupName, - 'Lambda-Log-Stream': context.logStreamName, - 'Lambda-Run-Id': context.awsRequestId, - }, - }).promise(); -} + db.changesReader.get(config) + .on('batch', async (batch: readonly Change[]) => { + try { + console.log(`Received a batch of ${batch.length} element(s)`); + const lastSeq = Math.max(...batch.map((change) => change.seq)); -async function processPackageUpdate(change: Change, context: Context) { - console.log(`Processing transaction: ${change.seq}`); - if (Object.keys(change.doc.versions).length === 0) { - console.log(`Ignoring document ${change.id}, as it contains no versions`); - return; + // Filter out all elements that don't have a "name" in the document, as + // these are schemas, which are not relevant to our business here. + batch = batch.filter((item) => item.doc.name); + console.log(`Identified ${batch.length} package update element(s)`); + + // Obtain the modified package version from the update event, and filter + // out packages that are not of interest to us (not construct libraries). + const versionInfos = getRelevantVersionInfos(batch); + console.log(`Identified ${versionInfos.length} relevant package version update(s)`); + + // Process all remaining updates + await Promise.all(versionInfos.map(processUpdatedVersion)); + + // Update the transaction marker in S3. + await saveLastTransactionMarker(lastSeq); + updatedMarker = lastSeq; + + // If we have enough time left before timeout, proceed with the next batch, otherwise we're done here. + if (context.getRemainingTimeInMillis() >= 120_000 /* 2 minutes */) { + console.log('There is still time, requesting the next batch...'); + // Note: the `resume` function is missing from the `nano` type definitions, but is there... + (db.changesReader as any).resume(); + } else { + console.log('We are almost out of time, so stopping here.'); + db.changesReader.stop(); + ok({ initialMarker, updatedMarker }); + } + } catch (err) { + // An exception bubbled out, which means this Lambda execution has failed. + console.error(`Unexpected error: ${err}`); + db.changesReader.stop(); + ko(err); + } + }) + .once('end', () => { + console.log('No more updates to process, exiting.'); + ok({ initialMarker, updatedMarker }); + }); + }); + + //#region Last transaction marker + /** + * Loads the last transaction marker from S3. + * + * @param defaultValue the value to return in case the marker does not exist + * + * @returns the value of the last transaction marker. + */ + async function loadLastTransactionMarker(defaultValue: number): Promise { + try { + const response = await aws.s3().getObject({ + Bucket: stagingBucket, + Key: MARKER_FILE_NAME, + }).promise(); + const marker = Number.parseInt(response.Body!.toString('utf-8'), 10); + console.log(`Read last transaction marker: ${marker}`); + return marker; + } catch (error) { + if (error.code !== 'NoSuchKey') { + throw error; + } + console.log(`Marker object (s3://${stagingBucket}/${MARKER_FILE_NAME}) does not exist, starting from the default (${defaultValue})`); + return defaultValue; + } } - const [latestVersion, publishTime] = getLatestVersion(change); - if (latestVersion == null || !isJsiiModule(latestVersion) || !isConstruct(latestVersion)) { - console.log(`Ignoring document ${change.id}, as it is not relevant`); - return; + /** + * Updates the last transaction marker in S3. + * + * @param sequence the last transaction marker value + */ + async function saveLastTransactionMarker(sequence: Number) { + console.log(`Updating last transaction marker to ${sequence}`); + return putObject(MARKER_FILE_NAME, sequence.toFixed(), { ContentType: 'text/plain' }); } + //#endregion - // change.dist.tarball => https://registry.npmjs.org/<@scope>//-/-.tgz - // staging bucket key => packages/<@scope>//-/-.tgz - const key = `packages/${new URL(latestVersion.dist.tarball).pathname}`; - const tarball = await copyPackageToS3(latestVersion, key, context); - await sendSqsMessage(latestVersion, tarball, publishTime, change.seq, key); -} + //#region Business Logic + async function processUpdatedVersion({ infos, modified, seq }: UpdatedVersion): Promise { + try { + // Download the tarball + const tarball = await httpGet(infos.dist.tarball); -/** - * Copy the tarball from the npm registry to S3 -*/ -async function copyPackageToS3(versionInfo: VersionInfo, key: string, context: Context): Promise { - console.log(`uploading tarball to s3, key: ${key}`); - return new Promise((ok, ko) => { - https.get(versionInfo.dist.tarball, (response) => { - const buffer = Buffer.from(response); - s3!.putObject({ - Bucket: stagingBucket, - Key: key, - Body: buffer, + // Store the tarball into the staging bucket + // - infos.dist.tarball => https://registry.npmjs.org/<@scope>//-/-.tgz + // - stagingKey => staged/<@scope>//-/-.tgz + const stagingKey = `staged/${new URL(infos.dist.tarball).pathname}`; + await putObject(stagingKey, tarball, { ContentType: 'application/x-gtar', Metadata: { - 'Lambda-Log-Group': context.logGroupName, - 'Lambda-Log-Stream': context.logStreamName, - 'Lambda-Run-Id': context.awsRequestId, - 'Origin-Uri': versionInfo.dist.tarball, + 'Modified-At': modified.toISOString(), + 'Origin-Integrity': infos.dist.shasum, + 'Origin-URI': infos.dist.tarball, + 'Sequence': seq.toFixed(), }, - }).promise().then(() => ok(buffer), ko); - }).on('error', (error) => { - console.error(`Error attempting to stage tarball in S3: ${error}`); - ko(`Failed downloading file from ${versionInfo.dist.tarball}`); - }); - }); -} + }); -/** - * Send an SQS message to notify that a new version was published - * @param versionInfo the version info - * @param key the s3 key to which the tarball was uploaded - */ -async function sendSqsMessage(versionInfo: VersionInfo, tarball: Buffer, publishTime: Date, transactionId: number, key: string): Promise { - console.log(`Posting sqs message for ${versionInfo.packageName}`); - const messageBase = { - tarballUri: `s3://${stagingBucket}/${key}`, - metadata: { tid: transactionId.toFixed() }, - time: publishTime.toUTCString(), - }; - const message: IngestionInput = { - ...messageBase, - integrity: integrity(messageBase, tarball), - }; + // Prepare SQS message for ingestion + const messageBase = { + tarballUri: `s3://${stagingBucket}/${stagingKey}`, + metadata: { + dist: infos.dist.tarball, + seq: seq.toFixed(), + }, + time: modified.toUTCString(), + }; + const message: IngestionInput = { + ...messageBase, + integrity: integrity(messageBase, tarball), + }; - await sqs!.sendMessage({ - MessageBody: JSON.stringify(message, null, 2), - QueueUrl: queueUrl, - }).promise(); -} + // Send the SQS message out + await aws.sqs().sendMessage({ + MessageBody: JSON.stringify(message, null, 2), + QueueUrl: queueUrl, + }).promise(); + } catch (err) { + // Something failed, store the payload in the problem prefix, and move on. + console.error(`[${seq}] Failed processing ${infos.name}@${infos.version}: ${err}`); + await putObject(`failed/${seq}`, JSON.stringify(infos, null, 2), { + ContentType: 'text/json', + Metadata: { + // User-defined metadata is limited to 2KB in size, in total. So we + // cap the error text to 1KB maximum, allowing up to 1KB for other + // attributes (which should be sufficient). + 'Error': `${err.stack ?? err}`.substring(0, 1_024), + 'Modified-At': modified.toISOString(), + }, + }); + } + } + //#endregion -/** - * @returns returns true if the package.json contains a jsii clause, false otherwise - */ -function isJsiiModule(versionInfo: VersionInfo): boolean { - return versionInfo.jsii != null; + //#region Asynchronous Primitives + /** + * Makes an HTTP GET request, and returns the resulting payload. + * + * @param url the URL to get. + * + * @returns a Buffer containing the received data. + */ + function httpGet(url: string) { + return new Promise((ok, ko) => { + https.get(url, (response) => { + if (response.statusCode !== 200) { + throw new Error(`Unsuccessful GET: ${response.statusCode} - ${response.statusMessage}`); + } + + let body = Buffer.alloc(0); + response.on('data', (chunk) => body = Buffer.concat([body, Buffer.from(chunk)])); + response.once('close', () => ok(body)); + response.once('error', ko); + }); + }); + } + + /** + * Puts an object in the staging bucket, with standardized object metadata. + * + * @param key the key for the object to be put. + * @param body the body of the object to be put. + * @param opts any other options to use when sending the S3 request. + * + * @returns the result of the S3 request. + */ + function putObject(key: string, body: AWS.S3.Body, opts: Omit = {}) { + return aws.s3().putObject({ + Bucket: stagingBucket, + Key: key, + Body: body, + Metadata: { + 'Lambda-Log-Group': context.logGroupName, + 'Lambda-Log-Stream': context.logStreamName, + 'Lambda-Run-Id': context.awsRequestId, + ...opts.Metadata, + }, + ...opts, + }).promise(); + } + //#endregion } /** - * Retrieves the version information that was updated in a specific document - * change. This assumes only one version was updated in a given change, and that - * the `times` entry in the `Change` object reflects that correctly. + * Obtains the `VersionInfo` corresponding to the modified version(s) in the + * provided `Change` objects, ensures they are relevant (construct libraries), + * and returns those only. * - * @param update the `Change` entry from which a version info is to be extracted. + * @param changes the changes to be processed. * - * @returns the version from the versions array with the most recent - * modification timestamp. + * @returns a list of `VersionInfo` objects */ -function getLatestVersion(update: Change): [VersionInfo | undefined, Date] { - const [lastModifiedVersion] = Object.entries(update.doc.time) - // Ignore created & modified timestamps - .filter(([key]) => key !== 'created' && key !== 'modified') - // Sort by timestamp, descending - .sort(([, ld], [, rd]) => new Date(rd).getTime() - new Date(ld).getTime()) - // First entry is most recently changed version - [0]; - - return [ - update.doc.versions[lastModifiedVersion], - new Date(update.doc.time[lastModifiedVersion]), - ]; -} +function getRelevantVersionInfos(changes: readonly Change[]): readonly UpdatedVersion[] { + const result = new Array(); + for (const change of changes) { + // Sometimes, there are no versions in the document. We skip those. + if (Object.keys(change.doc.versions).length === 0) { + console.error(`[${change.seq}] Changed document contains no 'versions': ${JSON.stringify(change, null, 2)}`); + continue; + } -/** - * This method applies different heuristics to check if this is a Construct - * @param pkgJason - * @returns - */ -function isConstruct(versionInfo: VersionInfo): boolean { - // currently we only check for specific keywords - return versionInfo.keywords?.some((k) => CONSTRUCT_KEYWORDS.has(k)); + // Sometimes, there is no 'time' entry in the document. We skip those. + if (change.doc.time == null) { + console.error(`[${change.seq}] Changed document contains no 'time': ${JSON.stringify(change, null, 2)}`); + continue; + } + + // Get the last modification date from the change + const sortedUpdates = Object.entries(change.doc.time) + // Ignore the "created" and "modified" keys here + .filter(([key]) => key !== 'created' && key !== 'modified') + // Parse all the dates to ensure they are comparable + .map(([version, isoDate]) => [version, new Date(isoDate)] as const) + // Sort by date, descending + .sort(([, l], [, r]) => r.getTime() - l.getTime()); + + let latestModified: Date | undefined; + for (const [version, modified] of sortedUpdates) { + if (latestModified == null || latestModified.getTime() === modified.getTime()) { + const infos = change.doc.versions[version]; + if (infos == null) { + // Could be the version in question was un-published. + console.log(`[${change.seq}] Could not find info for "${change.doc.name}@${version}". Was it un-published?`); + } else if (isRelevantPackageVersion(infos)) { + result.push({ infos, modified, seq: change.seq }); + } else { + console.log(`[${change.seq}] Ignoring "${change.doc.name}@${version}" as it is not a construct library.`); + } + latestModified = modified; + } + } + } + return result; + + function isRelevantPackageVersion(infos: VersionInfo): boolean { + if (infos.jsii == null) { + return false; + } + return infos.name === 'construct' + || infos.name === 'aws-cdk-lib' + || infos.name.startsWith('@aws-cdk') + || infos.keywords?.some((kw) => CONSTRUCT_KEYWORDS.has(kw)); + } } /** @@ -266,6 +323,23 @@ interface VersionInfo { readonly version: string; } +interface UpdatedVersion { + /** + * The `VersionInfo` for the modified package version. + */ + readonly infos: VersionInfo; + + /** + * The time at which the `VersionInfo` was last modified. + */ + readonly modified: Date; + + /** + * The CouchDB transaction number for the update. + */ + readonly seq: number; +} + interface Document { /** diff --git a/src/backend/discovery/index.ts b/src/backend/discovery/index.ts index 1e026cd7d..1f777cbff 100644 --- a/src/backend/discovery/index.ts +++ b/src/backend/discovery/index.ts @@ -1,16 +1,13 @@ import { Rule, Schedule } from '@aws-cdk/aws-events'; import { LambdaFunction } from '@aws-cdk/aws-events-targets'; import { RetentionDays } from '@aws-cdk/aws-logs'; -import { Bucket } from '@aws-cdk/aws-s3'; +import { Bucket, IBucket } from '@aws-cdk/aws-s3'; import { Queue } from '@aws-cdk/aws-sqs'; import { Construct, Duration } from '@aws-cdk/core'; import { Discovery as Handler } from './discovery'; export interface DiscoveryFunctionProps { - - readonly stagingBucket: Bucket; - /** * The queue to post package updated messages to */ @@ -23,26 +20,39 @@ export interface DiscoveryFunctionProps { readonly logRetention?: RetentionDays; } -export class DiscoveryFunction extends Construct { +export class Discovery extends Construct { + public readonly bucket: IBucket; + public constructor(scope: Construct, id: string, props: DiscoveryFunctionProps) { super(scope, id); + this.bucket = new Bucket(this, 'StagingBucket', { + lifecycleRules: [ + { + prefix: 'staged/', // delete the staged tarball after 30 days + expiration: Duration.days(30), + }, + ], + }); + + // Note: the handler is designed to stop processing more batches about 2 minutes ahead of the timeout. + const timeout = Duration.minutes(15); const lambda = new Handler(this, 'Default', { - description: 'Periodically query npm.js index for new Constructs', - deadLetterQueueEnabled: true, + description: 'Periodically query npm.js index for new construct libraries', memorySize: 10_240, - timeout: Duration.minutes(15), + reservedConcurrentExecutions: 1, // Only one execution (avoids race conditions on the S3 marker object) + timeout, environment: { - STAGING_BUCKET_NAME: props.stagingBucket.bucketName, + BUCKET_NAME: this.bucket.bucketName, QUEUE_URL: props.queue.queueUrl, }, }); - props.stagingBucket.grantReadWrite(lambda); + this.bucket.grantReadWrite(lambda); props.queue.grantSendMessages(lambda); new Rule(this, 'ScheduleRule', { - schedule: Schedule.rate(Duration.minutes(5)), + schedule: Schedule.rate(timeout), targets: [new LambdaFunction(lambda)], }); } diff --git a/src/backend/shared/client.lambda-shared.ts b/src/backend/shared/client.lambda-shared.ts new file mode 100644 index 000000000..0573fdcb0 --- /dev/null +++ b/src/backend/shared/client.lambda-shared.ts @@ -0,0 +1,26 @@ +import * as AWS from 'aws-sdk'; + +let _s3: AWS.S3 | undefined; +let _sqs: AWS.SQS | undefined; + +export function s3(): AWS.S3 { + if (_s3 == null) { + _s3 = new AWS.S3(); + } + return _s3; +} + +export function sqs(): AWS.SQS { + if (_sqs == null) { + _sqs = new AWS.SQS(); + } + return _sqs; +} + +/** + * Resets all clients vended by this module. This is useful in unit tests when + * `aws-sdk-mocks` is used, so that new mocks are injected as intended. + */ +export function reset(): void { + _s3 = _sqs = undefined; +} diff --git a/src/backend/shared/env.lambda-shared.ts b/src/backend/shared/env.lambda-shared.ts new file mode 100644 index 000000000..2ec30db28 --- /dev/null +++ b/src/backend/shared/env.lambda-shared.ts @@ -0,0 +1,9 @@ +import { env } from 'process'; + +export function requireEnv(name: string): string { + const result = env[name]; + if (!result) { + throw new Error(`No value specified for required environment variable: ${name}`); + } + return result; +} diff --git a/src/backend/shared/index.ts b/src/backend/shared/index.ts index dbd0b3ff7..ca9a6cde7 100644 --- a/src/backend/shared/index.ts +++ b/src/backend/shared/index.ts @@ -1,2 +1,4 @@ -export * from './ingestion-input'; -export * from './integrity'; +export * as aws from './client.lambda-shared'; +export * from './env.lambda-shared'; +export * from './ingestion-input.lambda-shared'; +export * from './integrity.lambda-shared'; diff --git a/src/backend/shared/ingestion-input.ts b/src/backend/shared/ingestion-input.lambda-shared.ts similarity index 100% rename from src/backend/shared/ingestion-input.ts rename to src/backend/shared/ingestion-input.lambda-shared.ts diff --git a/src/backend/shared/integrity.ts b/src/backend/shared/integrity.lambda-shared.ts similarity index 93% rename from src/backend/shared/integrity.ts rename to src/backend/shared/integrity.lambda-shared.ts index ab052ace8..9926a4fbc 100644 --- a/src/backend/shared/integrity.ts +++ b/src/backend/shared/integrity.lambda-shared.ts @@ -1,5 +1,5 @@ import { createHash } from 'crypto'; -import type { IngestionInput } from './ingestion-input'; +import type { IngestionInput } from './ingestion-input.lambda-shared'; /** * Computes an integrity checksum for the provided `IngestionInput`. diff --git a/src/construct-hub.ts b/src/construct-hub.ts index 2feec2a32..409665294 100644 --- a/src/construct-hub.ts +++ b/src/construct-hub.ts @@ -4,7 +4,7 @@ import { Queue } from '@aws-cdk/aws-sqs'; import { Construct as CoreConstruct, Duration } from '@aws-cdk/core'; import { Construct } from 'constructs'; import { AlarmActions, Domain } from './api'; -import { CatalogBuilder, DiscoveryFunction, Transliterator } from './backend'; +import { CatalogBuilder, Discovery, Transliterator } from './backend'; import { Monitoring } from './monitoring'; import { WebApp } from './webapp'; @@ -57,20 +57,10 @@ export class ConstructHub extends CoreConstruct { versioned: true, }); - const stagingBucket = new s3.Bucket(this, 'StagingBucket', { - lifecycleRules: [ - { - prefix: 'packages', // delete the staged tarball after 30 days - expiration: Duration.days(30), - }, - ], - }); - const ingestionQueue = new Queue(this, 'IngestionQueue'); - new DiscoveryFunction(this, 'DiscoveryFunction', { + new Discovery(this, 'Discovery', { queue: ingestionQueue, - stagingBucket, }); new Transliterator(this, 'Transliterator', { diff --git a/yarn.lock b/yarn.lock index 7337fbe9b..d0e1142cd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5256,11 +5256,6 @@ character-reference-invalid@^1.0.0: resolved "https://registry.yarnpkg.com/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz#083329cda0eae272ab3dbbf37e9a382c13af1560" integrity sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg== -chardet@^0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" - integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== - charenc@0.0.2: version "0.0.2" resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" @@ -5377,13 +5372,6 @@ cli-color@~0.1.6: dependencies: es5-ext "0.8.x" -cli-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" - integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== - dependencies: - restore-cursor "^3.1.0" - cli-table@^0.3.6: version "0.3.6" resolved "https://registry.yarnpkg.com/cli-table/-/cli-table-0.3.6.tgz#e9d6aa859c7fe636981fd3787378c2a20bce92fc" @@ -5391,11 +5379,6 @@ cli-table@^0.3.6: dependencies: colors "1.0.3" -cli-width@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-3.0.0.tgz#a2f48437a2caa9a22436e794bf071ec9e61cedf6" - integrity sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw== - cliui@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/cliui/-/cliui-5.0.0.tgz#deefcfdb2e800784aa34f46fa08e06851c7bbbc5" @@ -7588,15 +7571,6 @@ extend@^3.0.0, extend@~3.0.2: resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== -external-editor@^3.0.3: - version "3.1.0" - resolved "https://registry.yarnpkg.com/external-editor/-/external-editor-3.1.0.tgz#cb03f740befae03ea4d283caed2741a83f335495" - integrity sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew== - dependencies: - chardet "^0.7.0" - iconv-lite "^0.4.24" - tmp "^0.0.33" - extglob@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/extglob/-/extglob-2.0.4.tgz#ad00fe4dc612a9232e8718711dc5cb5ab0285543" @@ -7674,7 +7648,7 @@ figgy-pudding@^3.5.1: resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e" integrity sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw== -figures@^3.0.0, figures@^3.1.0: +figures@^3.1.0: version "3.2.0" resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== @@ -8789,7 +8763,7 @@ humanize-ms@^1.2.1: dependencies: ms "^2.0.0" -iconv-lite@0.4.24, iconv-lite@^0.4.24: +iconv-lite@0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== @@ -8970,25 +8944,6 @@ inline-style-parser@0.1.1: resolved "https://registry.yarnpkg.com/inline-style-parser/-/inline-style-parser-0.1.1.tgz#ec8a3b429274e9c0a1f1c4ffa9453a7fef72cea1" integrity sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q== -inquirer@^7.3.3: - version "7.3.3" - resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-7.3.3.tgz#04d176b2af04afc157a83fd7c100e98ee0aad003" - integrity sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA== - dependencies: - ansi-escapes "^4.2.1" - chalk "^4.1.0" - cli-cursor "^3.1.0" - cli-width "^3.0.0" - external-editor "^3.0.3" - figures "^3.0.0" - lodash "^4.17.19" - mute-stream "0.0.8" - run-async "^2.4.0" - rxjs "^6.6.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - through "^2.3.6" - internal-ip@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/internal-ip/-/internal-ip-4.3.0.tgz#845452baad9d2ca3b69c635a137acb9a0dad0907" @@ -11225,7 +11180,7 @@ multicast-dns@^6.0.1: dns-packet "^1.3.1" thunky "^1.0.2" -mute-stream@0.0.8, mute-stream@~0.0.4: +mute-stream@~0.0.4: version "0.0.8" resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d" integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA== @@ -11814,11 +11769,6 @@ os-browserify@^0.3.0: resolved "https://registry.yarnpkg.com/os-browserify/-/os-browserify-0.3.0.tgz#854373c7f5c2315914fc9bfc6bd8238fdda1ec27" integrity sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc= -os-tmpdir@~1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" - integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= - p-cancelable@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc" @@ -13032,10 +12982,10 @@ progress@^2.0.0, progress@^2.0.3: resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== -projen@^0.22.15: - version "0.22.15" - resolved "https://registry.yarnpkg.com/projen/-/projen-0.22.15.tgz#5a77df27140ec2c723570fd34b46a27d1a5394da" - integrity sha512-dk7vqIOZTBJeoCII35oPGUt4hku5iwXP6wUVq0TtIExw+XsHQqGrDA/s529KDbkC74no66LGKL9kBofo18KpSg== +projen@^0.24.7: + version "0.24.8" + resolved "https://registry.yarnpkg.com/projen/-/projen-0.24.8.tgz#d78e2bd5efbbb9434e9766dc1936095dc669a181" + integrity sha512-knaMmBnHygKqPzxAGyI5ouPKE2p7NZpK8Tb9TvdO0l0ZkZgsmKHTHQAABnCWFNWci/6DD0lnTEkw4oEAmICD5w== dependencies: "@iarna/toml" "^2.2.5" chalk "^4.1.1" @@ -13043,7 +12993,6 @@ projen@^0.22.15: fs-extra "^9.1.0" glob "^7" ini "^2.0.0" - inquirer "^7.3.3" semver "^7.3.5" shx "^0.3.3" xmlbuilder2 "^2.4.1" @@ -14036,14 +13985,6 @@ responselike@^2.0.0: dependencies: lowercase-keys "^2.0.0" -restore-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" - integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== - dependencies: - onetime "^5.1.0" - signal-exit "^3.0.2" - ret@~0.1.10: version "0.1.15" resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" @@ -14149,11 +14090,6 @@ rsvp@^4.8.4: resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-4.8.5.tgz#c8f155311d167f68f21e168df71ec5b083113734" integrity sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA== -run-async@^2.4.0: - version "2.4.1" - resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455" - integrity sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ== - run-parallel@^1.1.9: version "1.2.0" resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" @@ -14168,13 +14104,6 @@ run-queue@^1.0.0, run-queue@^1.0.3: dependencies: aproba "^1.1.1" -rxjs@^6.6.0: - version "6.6.7" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.7.tgz#90ac018acabf491bf65044235d5863c4dab804c9" - integrity sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ== - dependencies: - tslib "^1.9.0" - safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" @@ -15341,7 +15270,7 @@ through2@^4.0.0: dependencies: readable-stream "3" -through@2, "through@>=2.2.7 <3", through@^2.3.6: +through@2, "through@>=2.2.7 <3": version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= @@ -15378,13 +15307,6 @@ tinycolor2@1.4.2: resolved "https://registry.yarnpkg.com/tinycolor2/-/tinycolor2-1.4.2.tgz#3f6a4d1071ad07676d7fa472e1fac40a719d8803" integrity sha512-vJhccZPs965sV/L2sU4oRQVAos0pQXwsvTLkWYdqJ+a8Q5kPFzJTuOFwy7UniPli44NKQGAglksjvOcpo95aZA== -tmp@^0.0.33: - version "0.0.33" - resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" - integrity sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw== - dependencies: - os-tmpdir "~1.0.2" - tmpl@1.0.x: version "1.0.4" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1" @@ -15516,7 +15438,7 @@ tsconfig-paths@^3.9.0: minimist "^1.2.0" strip-bom "^3.0.0" -tslib@^1.0.0, tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3: +tslib@^1.0.0, tslib@^1.8.1, tslib@^1.9.3: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==