From 2a05791cae27ed553acaf225237c81ffbcb809f0 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Wed, 18 Sep 2024 09:58:35 -0700 Subject: [PATCH 1/4] Switch primaryDatasource -> datasources --- src/commands/dataconnect-services-list.ts | 7 ++--- src/dataconnect/load.ts | 8 +++--- src/dataconnect/schemaMigration.ts | 30 ++++++++++++++-------- src/dataconnect/types.ts | 2 +- src/deploy/dataconnect/deploy.ts | 31 ++++++++++++----------- src/init/features/dataconnect/index.ts | 7 ++--- 6 files changed, 47 insertions(+), 38 deletions(-) diff --git a/src/commands/dataconnect-services-list.ts b/src/commands/dataconnect-services-list.ts index 29b2de9075c..969e60f6a79 100644 --- a/src/commands/dataconnect-services-list.ts +++ b/src/commands/dataconnect-services-list.ts @@ -34,14 +34,15 @@ export const command = new Command("dataconnect:services:list") for (const service of services) { const schema = (await client.getSchema(service.name)) ?? { name: "", - primaryDatasource: {}, + datasources: [{}], source: { files: [] }, }; const connectors = await client.listConnectors(service.name); const serviceName = names.parseServiceName(service.name); - const instanceName = schema?.primaryDatasource.postgresql?.cloudSql.instance ?? ""; + const primaryDatasource = schema?.datasources.find((d) => d.postgresql); + const instanceName = primaryDatasource?.postgresql?.cloudSql.instance ?? ""; const instanceId = instanceName.split("/").pop(); - const dbId = schema?.primaryDatasource.postgresql?.database ?? ""; + const dbId = primaryDatasource?.postgresql?.database ?? ""; const dbName = `CloudSQL Instance: ${instanceId}\nDatabase: ${dbId}`; table.push([ serviceName.serviceId, diff --git a/src/dataconnect/load.ts b/src/dataconnect/load.ts index 6f84d7cba8e..8365260a8fb 100644 --- a/src/dataconnect/load.ts +++ b/src/dataconnect/load.ts @@ -39,11 +39,9 @@ export async function load( sourceDirectory: resolvedDir, schema: { name: `${serviceName}/schemas/${SCHEMA_ID}`, - primaryDatasource: toDatasource( - projectId, - dataConnectYaml.location, - dataConnectYaml.schema.datasource, - ), + datasources: [ + toDatasource(projectId, dataConnectYaml.location, dataConnectYaml.schema.datasource), + ], source: { files: schemaGQLs, }, diff --git a/src/dataconnect/schemaMigration.ts b/src/dataconnect/schemaMigration.ts index cb72ba29258..ce186a8437f 100644 --- a/src/dataconnect/schemaMigration.ts +++ b/src/dataconnect/schemaMigration.ts @@ -237,8 +237,11 @@ function diffsEqual(x: Diff[], y: Diff[]): boolean { } function setSchemaValidationMode(schema: Schema, schemaValidation: SchemaValidation) { - if (experiments.isEnabled("fdccompatiblemode") && schema.primaryDatasource.postgresql) { - schema.primaryDatasource.postgresql.schemaValidation = schemaValidation; + if (experiments.isEnabled("fdccompatiblemode")) { + const primaryDatasource = schema.datasources.find((d) => d.postgresql); + if (primaryDatasource?.postgresql) { + primaryDatasource.postgresql.schemaValidation = schemaValidation; + } } } @@ -248,13 +251,14 @@ function getIdentifiers(schema: Schema): { databaseId: string; serviceName: string; } { - const databaseId = schema.primaryDatasource.postgresql?.database; + const primaryDatasource = schema.datasources.find((d) => d.postgresql); + const databaseId = primaryDatasource?.postgresql?.database; if (!databaseId) { throw new FirebaseError( "Schema is missing primaryDatasource.postgresql?.database, cannot migrate", ); } - const instanceName = schema.primaryDatasource.postgresql?.cloudSql.instance; + const instanceName = primaryDatasource?.postgresql?.cloudSql.instance; if (!instanceName) { throw new FirebaseError( "tried to migrate schema but instance name was not provided in dataconnect.yaml", @@ -509,17 +513,21 @@ async function ensureServiceIsConnectedToCloudSql( source: { files: [], }, - primaryDatasource: { - postgresql: { - database: databaseId, - cloudSql: { - instance: instanceId, + datasources: [ + { + postgresql: { + database: databaseId, + cloudSql: { + instance: instanceId, + }, }, }, - }, + ], }; } - const postgresql = currentSchema.primaryDatasource.postgresql; + + const primaryDatasource = currentSchema.datasources.find((d) => d.postgresql); + const postgresql = primaryDatasource?.postgresql; if (postgresql?.cloudSql.instance !== instanceId) { logLabeledWarning( "dataconnect", diff --git a/src/dataconnect/types.ts b/src/dataconnect/types.ts index 2cb3f7f07ff..553cc2736b7 100644 --- a/src/dataconnect/types.ts +++ b/src/dataconnect/types.ts @@ -16,7 +16,7 @@ export interface Service extends BaseResource { export interface Schema extends BaseResource { name: string; - primaryDatasource: Datasource; + datasources: Datasource[]; source: Source; } diff --git a/src/deploy/dataconnect/deploy.ts b/src/deploy/dataconnect/deploy.ts index ab064cb4dd7..d40fc6e09ca 100644 --- a/src/deploy/dataconnect/deploy.ts +++ b/src/deploy/dataconnect/deploy.ts @@ -88,22 +88,23 @@ export default async function ( return !filters || filters?.some((f) => si.dataConnectYaml.serviceId === f.serviceId); }) .map(async (s) => { - const instanceId = s.schema.primaryDatasource.postgresql?.cloudSql.instance - .split("/") - .pop(); - const databaseId = s.schema.primaryDatasource.postgresql?.database; - if (!instanceId || !databaseId) { - return Promise.resolve(); + const primaryDatasource = s.schema.datasources.find((d) => d.postgresql); + if (primaryDatasource) { + const instanceId = primaryDatasource.postgresql?.cloudSql.instance.split("/").pop(); + const databaseId = primaryDatasource.postgresql?.database; + if (!instanceId || !databaseId) { + return Promise.resolve(); + } + const enableGoogleMlIntegration = requiresVector(s.deploymentMetadata); + return provisionCloudSql({ + projectId, + locationId: parseServiceName(s.serviceName).location, + instanceId, + databaseId, + enableGoogleMlIntegration, + waitForCreation: true, + }); } - const enableGoogleMlIntegration = requiresVector(s.deploymentMetadata); - return provisionCloudSql({ - projectId, - locationId: parseServiceName(s.serviceName).location, - instanceId, - databaseId, - enableGoogleMlIntegration, - waitForCreation: true, - }); }), ); return; diff --git a/src/init/features/dataconnect/index.ts b/src/init/features/dataconnect/index.ts index 6e83b0b3ce4..acfad6a143e 100644 --- a/src/init/features/dataconnect/index.ts +++ b/src/init/features/dataconnect/index.ts @@ -262,16 +262,17 @@ async function promptForService( info.serviceId = serviceName.serviceId; info.locationId = serviceName.location; if (choice.schema) { - if (choice.schema.primaryDatasource.postgresql?.cloudSql.instance) { + const primaryDatasource = choice.schema.datasources.find((d) => d.postgresql); + if (primaryDatasource?.postgresql?.cloudSql.instance) { const instanceName = parseCloudSQLInstanceName( - choice.schema.primaryDatasource.postgresql?.cloudSql.instance, + primaryDatasource.postgresql.cloudSql.instance, ); info.cloudSqlInstanceId = instanceName.instanceId; } if (choice.schema.source.files) { info.schemaGql = choice.schema.source.files; } - info.cloudSqlDatabase = choice.schema.primaryDatasource.postgresql?.database ?? ""; + info.cloudSqlDatabase = primaryDatasource?.postgresql?.database ?? ""; const connectors = await listConnectors(choice.service.name, [ "connectors.name", "connectors.source.files", From ded0dc7c7a75100700c87ef96017e9ffbd62ff59 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Wed, 18 Sep 2024 10:32:02 -0700 Subject: [PATCH 2/4] ?. --- scripts/dataconnect-test/tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dataconnect-test/tests.ts b/scripts/dataconnect-test/tests.ts index d972f07dff3..256e40d6888 100644 --- a/scripts/dataconnect-test/tests.ts +++ b/scripts/dataconnect-test/tests.ts @@ -173,7 +173,7 @@ describe("firebase deploy", () => { serviceId, databaseId, service["schemaUpdateTime"], - service["connectors"][0]["connectorLastUpdated"], + service["connectors"]?.[0]?.["connectorLastUpdated"], ), ); } From 70ffa6a7a36b660b0b68bfa64b420adbc0ab1320 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Wed, 18 Sep 2024 12:17:24 -0700 Subject: [PATCH 3/4] primaryDatasource -> postgresDatasource --- src/commands/dataconnect-services-list.ts | 6 +++--- src/dataconnect/schemaMigration.ts | 18 +++++++++--------- src/deploy/dataconnect/deploy.ts | 10 ++++++---- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/commands/dataconnect-services-list.ts b/src/commands/dataconnect-services-list.ts index 969e60f6a79..b0e5a3c2da4 100644 --- a/src/commands/dataconnect-services-list.ts +++ b/src/commands/dataconnect-services-list.ts @@ -39,10 +39,10 @@ export const command = new Command("dataconnect:services:list") }; const connectors = await client.listConnectors(service.name); const serviceName = names.parseServiceName(service.name); - const primaryDatasource = schema?.datasources.find((d) => d.postgresql); - const instanceName = primaryDatasource?.postgresql?.cloudSql.instance ?? ""; + const postgresDatasource = schema?.datasources.find((d) => d.postgresql); + const instanceName = postgresDatasource?.postgresql?.cloudSql.instance ?? ""; const instanceId = instanceName.split("/").pop(); - const dbId = primaryDatasource?.postgresql?.database ?? ""; + const dbId = postgresDatasource?.postgresql?.database ?? ""; const dbName = `CloudSQL Instance: ${instanceId}\nDatabase: ${dbId}`; table.push([ serviceName.serviceId, diff --git a/src/dataconnect/schemaMigration.ts b/src/dataconnect/schemaMigration.ts index ce186a8437f..8bda3a72935 100644 --- a/src/dataconnect/schemaMigration.ts +++ b/src/dataconnect/schemaMigration.ts @@ -238,9 +238,9 @@ function diffsEqual(x: Diff[], y: Diff[]): boolean { function setSchemaValidationMode(schema: Schema, schemaValidation: SchemaValidation) { if (experiments.isEnabled("fdccompatiblemode")) { - const primaryDatasource = schema.datasources.find((d) => d.postgresql); - if (primaryDatasource?.postgresql) { - primaryDatasource.postgresql.schemaValidation = schemaValidation; + const postgresDatasource = schema.datasources.find((d) => d.postgresql); + if (postgresDatasource?.postgresql) { + postgresDatasource.postgresql.schemaValidation = schemaValidation; } } } @@ -251,14 +251,14 @@ function getIdentifiers(schema: Schema): { databaseId: string; serviceName: string; } { - const primaryDatasource = schema.datasources.find((d) => d.postgresql); - const databaseId = primaryDatasource?.postgresql?.database; + const postgresDatasource = schema.datasources.find((d) => d.postgresql); + const databaseId = postgresDatasource?.postgresql?.database; if (!databaseId) { throw new FirebaseError( - "Schema is missing primaryDatasource.postgresql?.database, cannot migrate", + "Service does not have a postgres datasource, cannot migrate", ); } - const instanceName = primaryDatasource?.postgresql?.cloudSql.instance; + const instanceName = postgresDatasource?.postgresql?.cloudSql.instance; if (!instanceName) { throw new FirebaseError( "tried to migrate schema but instance name was not provided in dataconnect.yaml", @@ -526,8 +526,8 @@ async function ensureServiceIsConnectedToCloudSql( }; } - const primaryDatasource = currentSchema.datasources.find((d) => d.postgresql); - const postgresql = primaryDatasource?.postgresql; + const postgresDatasource = currentSchema.datasources.find((d) => d.postgresql); + const postgresql = postgresDatasource?.postgresql; if (postgresql?.cloudSql.instance !== instanceId) { logLabeledWarning( "dataconnect", diff --git a/src/deploy/dataconnect/deploy.ts b/src/deploy/dataconnect/deploy.ts index d40fc6e09ca..e2b942539dc 100644 --- a/src/deploy/dataconnect/deploy.ts +++ b/src/deploy/dataconnect/deploy.ts @@ -88,10 +88,12 @@ export default async function ( return !filters || filters?.some((f) => si.dataConnectYaml.serviceId === f.serviceId); }) .map(async (s) => { - const primaryDatasource = s.schema.datasources.find((d) => d.postgresql); - if (primaryDatasource) { - const instanceId = primaryDatasource.postgresql?.cloudSql.instance.split("/").pop(); - const databaseId = primaryDatasource.postgresql?.database; + const postgresDatasource = s.schema.datasources.find(d => d.postgresql); + if (postgresDatasource) { + const instanceId = postgresDatasource.postgresql?.cloudSql.instance + .split("/") + .pop(); + const databaseId = postgresDatasource.postgresql?.database; if (!instanceId || !databaseId) { return Promise.resolve(); } From fb6b7464ec283cb5bcd3e766c19bc4b3fd276e6d Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Wed, 18 Sep 2024 12:23:04 -0700 Subject: [PATCH 4/4] formats --- src/dataconnect/schemaMigration.ts | 4 +--- src/deploy/dataconnect/deploy.ts | 6 ++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/dataconnect/schemaMigration.ts b/src/dataconnect/schemaMigration.ts index 8bda3a72935..4537b75772e 100644 --- a/src/dataconnect/schemaMigration.ts +++ b/src/dataconnect/schemaMigration.ts @@ -254,9 +254,7 @@ function getIdentifiers(schema: Schema): { const postgresDatasource = schema.datasources.find((d) => d.postgresql); const databaseId = postgresDatasource?.postgresql?.database; if (!databaseId) { - throw new FirebaseError( - "Service does not have a postgres datasource, cannot migrate", - ); + throw new FirebaseError("Service does not have a postgres datasource, cannot migrate"); } const instanceName = postgresDatasource?.postgresql?.cloudSql.instance; if (!instanceName) { diff --git a/src/deploy/dataconnect/deploy.ts b/src/deploy/dataconnect/deploy.ts index e2b942539dc..bc58e762ba9 100644 --- a/src/deploy/dataconnect/deploy.ts +++ b/src/deploy/dataconnect/deploy.ts @@ -88,11 +88,9 @@ export default async function ( return !filters || filters?.some((f) => si.dataConnectYaml.serviceId === f.serviceId); }) .map(async (s) => { - const postgresDatasource = s.schema.datasources.find(d => d.postgresql); + const postgresDatasource = s.schema.datasources.find((d) => d.postgresql); if (postgresDatasource) { - const instanceId = postgresDatasource.postgresql?.cloudSql.instance - .split("/") - .pop(); + const instanceId = postgresDatasource.postgresql?.cloudSql.instance.split("/").pop(); const databaseId = postgresDatasource.postgresql?.database; if (!instanceId || !databaseId) { return Promise.resolve();