From d2e38e044555f15f5981e5831263627fdf59b5f1 Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Fri, 2 Aug 2024 17:31:02 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A8=20switch=20to=20client=20side=20UU?= =?UTF-8?q?ID=20v7=20with=20text=20serialization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteServer/apiRouter.ts | 3 +- db/db.ts | 10 ------- .../1719842654592-AddChartConfigsTable.ts | 29 ++++++++++++------- db/tests/basic.test.ts | 4 +-- package.json | 1 + .../types/src/dbTypes/ChartConfigs.ts | 3 +- .../types/src/dbTypes/Charts.ts | 2 +- yarn.lock | 10 +++++++ 8 files changed, 36 insertions(+), 26 deletions(-) diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index bf1c70db1ba..e15d9426a0c 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -84,6 +84,7 @@ import { FlatTagGraph, DbRawChartConfig, } from "@ourworldindata/types" +import { uuidv7 } from "uuidv7" import { getVariableDataRoute, getVariableMetadataRoute, @@ -374,7 +375,7 @@ const saveGrapher = async ( [now, user.id, chartId] ) } else { - const configId = await db.getBinaryUUID(knex) + const configId = uuidv7() await db.knexRaw( knex, `-- sql diff --git a/db/db.ts b/db/db.ts index fb3aa5aac69..71726c286c4 100644 --- a/db/db.ts +++ b/db/db.ts @@ -668,13 +668,3 @@ export async function getLinkedIndicatorSlugs({ .then((gdocs) => gdocs.flatMap((gdoc) => gdoc.linkedKeyIndicatorSlugs)) .then((slugs) => new Set(slugs)) } - -export const getBinaryUUID = async ( - knex: KnexReadonlyTransaction -): Promise => { - const { id } = (await knexRawFirst<{ id: Buffer }>( - knex, - `SELECT UUID_TO_BIN(UUID(), 1) AS id` - ))! - return id -} diff --git a/db/migration/1719842654592-AddChartConfigsTable.ts b/db/migration/1719842654592-AddChartConfigsTable.ts index 8cc14a4e8c2..b2e84bfe643 100644 --- a/db/migration/1719842654592-AddChartConfigsTable.ts +++ b/db/migration/1719842654592-AddChartConfigsTable.ts @@ -1,13 +1,12 @@ import { MigrationInterface, QueryRunner } from "typeorm" - +import { uuidv7 } from "uuidv7" export class AddChartConfigsTable1719842654592 implements MigrationInterface { private async createChartConfigsTable( queryRunner: QueryRunner ): Promise { await queryRunner.query(`-- sql CREATE TABLE chart_configs ( - id binary(16) NOT NULL DEFAULT (UUID_TO_BIN(UUID(), 1)) PRIMARY KEY, - uuid varchar(36) GENERATED ALWAYS AS (BIN_TO_UUID(id, 1)) VIRTUAL, + id char(36) NOT NULL PRIMARY KEY, patch json NOT NULL, full json NOT NULL, slug varchar(255) GENERATED ALWAYS AS (JSON_UNQUOTE(JSON_EXTRACT(full, '$.slug'))) STORED, @@ -25,7 +24,7 @@ export class AddChartConfigsTable1719842654592 implements MigrationInterface { // that points to the `chart_configs` table await queryRunner.query(`-- sql ALTER TABLE charts - ADD COLUMN configId binary(16) UNIQUE AFTER type, + ADD COLUMN configId char(36) UNIQUE AFTER type, ADD CONSTRAINT charts_configId FOREIGN KEY (configId) REFERENCES chart_configs (id) @@ -44,11 +43,21 @@ export class AddChartConfigsTable1719842654592 implements MigrationInterface { WHERE id != config ->> "$.id"; `) - // insert all the configs into the `chart_configs` table - await queryRunner.query(`-- sql - INSERT INTO chart_configs (patch, full) - SELECT config, config FROM charts - `) + const chartConfigs: { config: string }[] = + await queryRunner.query(`-- sql + select config from charts`) + + // I tried to write this as a chunked builk insert of 500 at a time but + // failed to get it to work without doing strange things. We only run this once + // for ~5000 items so it's not too bad to do it one insert at a time + for (const chartConfig of chartConfigs) { + await queryRunner.query( + `-- sql + INSERT INTO chart_configs (id, patch, full) + VALUES (?, ?, ?)`, + [uuidv7(), chartConfig.config, chartConfig.config] + ) + } // update the `configId` column in the `charts` table await queryRunner.query(`-- sql @@ -61,7 +70,7 @@ export class AddChartConfigsTable1719842654592 implements MigrationInterface { // now that the `configId` column is filled, make it NOT NULL await queryRunner.query(`-- sql ALTER TABLE charts - MODIFY COLUMN configId binary(16) NOT NULL; + MODIFY COLUMN configId char(36) NOT NULL; `) // update `createdAt` and `updatedAt` of the chart_configs table diff --git a/db/tests/basic.test.ts b/db/tests/basic.test.ts index e3b56667408..e50e413edd1 100644 --- a/db/tests/basic.test.ts +++ b/db/tests/basic.test.ts @@ -9,9 +9,9 @@ import { knexRawFirst, knexReadonlyTransaction, TransactionCloseMode, - getBinaryUUID, } from "../db.js" import { deleteUser, insertUser, updateUser } from "../model/User.js" +import { uuidv7 } from "uuidv7" import { ChartsTableName, ChartConfigsTableName, @@ -71,7 +71,7 @@ test("timestamps are automatically created and updated", async () => { .first() expect(user).toBeTruthy() expect(user.email).toBe("admin@example.com") - const configId = await getBinaryUUID(trx) + const configId = uuidv7() const chartConfig: DbInsertChartConfig = { id: configId, patch: "{}", diff --git a/package.json b/package.json index e1cc7f51dbb..8ab030f444c 100644 --- a/package.json +++ b/package.json @@ -165,6 +165,7 @@ "url-parse": "^1.5.10", "url-slug": "^3.0.2", "usehooks-ts": "^3.1.0", + "uuidv7": "^1.0.1", "webfontloader": "^1.6.28", "workerpool": "^6.2.0", "yaml": "^2.4.2" diff --git a/packages/@ourworldindata/types/src/dbTypes/ChartConfigs.ts b/packages/@ourworldindata/types/src/dbTypes/ChartConfigs.ts index d6b889f9a87..24c98ee6b6f 100644 --- a/packages/@ourworldindata/types/src/dbTypes/ChartConfigs.ts +++ b/packages/@ourworldindata/types/src/dbTypes/ChartConfigs.ts @@ -3,8 +3,7 @@ import { GrapherInterface } from "../grapherTypes/GrapherTypes.js" export const ChartConfigsTableName = "chart_configs" export interface DbInsertChartConfig { - id: Buffer - uuid?: string + id: string patch: JsonString full: JsonString slug?: string | null diff --git a/packages/@ourworldindata/types/src/dbTypes/Charts.ts b/packages/@ourworldindata/types/src/dbTypes/Charts.ts index 98276fbe7f4..baa75642201 100644 --- a/packages/@ourworldindata/types/src/dbTypes/Charts.ts +++ b/packages/@ourworldindata/types/src/dbTypes/Charts.ts @@ -1,6 +1,6 @@ export const ChartsTableName = "charts" export interface DbInsertChart { - configId: Buffer + configId: string createdAt?: Date id?: number isIndexable?: number diff --git a/yarn.lock b/yarn.lock index 933ce200e2f..51f73fce493 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11075,6 +11075,7 @@ __metadata: url-parse: "npm:^1.5.10" url-slug: "npm:^3.0.2" usehooks-ts: "npm:^3.1.0" + uuidv7: "npm:^1.0.1" vite: "npm:^5.3.4" vite-plugin-checker: "npm:^0.7.2" webfontloader: "npm:^1.6.28" @@ -20025,6 +20026,15 @@ __metadata: languageName: node linkType: hard +"uuidv7@npm:^1.0.1": + version: 1.0.1 + resolution: "uuidv7@npm:1.0.1" + bin: + uuidv7: cli.js + checksum: 10/24f835d067ee69ee46b1734b1da33e6f751f04ccfa5aa83703fc84693b8628d0ae0b4cd7f31c940ec933268971c338931bf5b57b09e882e14df47dfebba2b760 + languageName: node + linkType: hard + "v8-to-istanbul@npm:^9.0.1": version: 9.0.1 resolution: "v8-to-istanbul@npm:9.0.1"