🔨 (db) add chart_configs table / TAS-563 (#3767)

## Summary Adds a `chart_configs` table to the database, adds a `configId` column to the `charts` table that points to a `chart_configs` row, and removes the `config` column. Shouldn't result in any functionality change. ## Details - All configs are copied from the `charts` table to the `chart_configs` table (for now, `patch config = full config` since inheritance hasn't been implemented yet) - Configs have a UUID, but it's not yet used (the id of the chart is still the sequential primary key of the charts table) ## Testing - [x] Bakes locally - [x] Bakes staging site Site: - [x] http://staging-site-db-chart-configs/ - [x] http://staging-site-db-chart-configs/grapher/life-expectancy - [x] http://staging-site-db-chart-configs/explorers/coronavirus-data-explorer - [x] http://staging-site-db-chart-configs/explorers/democracy - [x] http://staging-site-db-chart-configs/explorers/conflict-data - [x] http://staging-site-db-chart-configs/poverty - [x] http://staging-site-db-chart-configs/financing-education - [x] http://staging-site-db-chart-configs/part-two-how-many-people-die-from-extreme-temperatures-and-how-could-this-change-in-the-future - [x] http://staging-site-db-chart-configs/covid-hospitalizations - [x] http://staging-site-db-chart-configs/country/germany - [x] http://staging-site-db-chart-configs/energy/country/india - [x] http://staging-site-db-chart-configs/data-insights - [x] http://staging-site-db-chart-configs/collection/top-charts - [x] http://staging-site-db-chart-configs/sdgs - [x] http://staging-site-db-chart-configs/sdgs/no-poverty Admin: - [x] Chart editor: - [x] Create new chart - [x] Save as new chart - [x] Update existing chart - [x] Publish chart - [x] Unpublish chart - [x] Delete chart - [x] Bulk chart editor - [x] Chart test page - [x] Variable page (chart list) ## On the ETL side Mojmir opened a PR for changes to the ETL: owid/etl#2957 ## To do - [x] ~Is it safe to rewrite the config's id?~ I think it's okay - [x] I don't understand why we manually insert `createdAt` and `updatedAt` since the db inserts values on insert/update? - [x] Checklist for updating the db: https://www.notion.so/owid/Database-access-in-Grapher-d9db343c2bfb4ae0b14b3dec72f686c6?pvs=4#dd5b51bab6f84630839f4173b59feba2 - [x] Let Mojmir know so that he can update the ETL db schema (wait for the table structure to be signed off on) ## To do after merging - [ ] ETL: owid/etl#2957 - [ ] Datasette: owid/analytics#138 - [ ] Schema validation: owid/automation#12
owid · Sep 3, 2024 · 6c3057a · 6c3057a
1 parent 1f3c2d9
commit 6c3057a
Show file tree

Hide file tree

Showing 24 changed files with 776 additions and 360 deletions.
diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts
diff --git a/adminSiteServer/testPageRouter.tsx b/adminSiteServer/testPageRouter.tsx
@@ -24,13 +24,13 @@ import {
 import { grapherToSVG } from "../baker/GrapherImageBaker.js"
 import {
     ChartTypeName,
-    ChartsTableName,
     ColorSchemeName,
-    DbRawChart,
+    DbRawChartConfig,
+    DbPlainChart,
     EntitySelectionMode,
     GrapherTabOption,
     StackMode,
-    parseChartsRow,
+    parseChartConfig,
 } from "@ourworldindata/types"
 import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.js"
 import { GIT_CMS_DIR } from "../gitCms/GitCmsConstants.js"
@@ -140,27 +140,28 @@ async function propsFromQueryParams(
 
     let query = knex
         .table("charts")
-        .whereRaw("publishedAt IS NOT NULL")
-        .orderBy("id", "DESC")
+        .join({ cc: "chart_configs" }, "charts.configId", "cc.id")
+        .whereRaw("charts.publishedAt IS NOT NULL")
+        .orderBy("charts.id", "DESC")
     console.error(query.toSQL())
 
     let tab = params.tab
 
     if (params.type) {
         if (params.type === ChartTypeName.WorldMap) {
-            query = query.andWhereRaw(`config->>"$.hasMapTab" = "true"`)
+            query = query.andWhereRaw(`cc.full->>"$.hasMapTab" = "true"`)
             tab = tab || GrapherTabOption.map
         } else {
             if (params.type === "LineChart") {
                 query = query.andWhereRaw(
                     `(
-                        config->"$.type" = "LineChart"
-                        OR config->"$.type" IS NULL
-                    ) AND COALESCE(config->>"$.hasChartTab", "true") = "true"`
+                        cc.full->"$.type" = "LineChart"
+                        OR cc.full->"$.type" IS NULL
+                    ) AND COALESCE(cc.full->>"$.hasChartTab", "true") = "true"`
                 )
             } else {
                 query = query.andWhereRaw(
-                    `config->"$.type" = :type AND COALESCE(config->>"$.hasChartTab", "true") = "true"`,
+                    `cc.full->"$.type" = :type AND COALESCE(cc.full->>"$.hasChartTab", "true") = "true"`,
                     { type: params.type }
                 )
             }
@@ -170,27 +171,27 @@ async function propsFromQueryParams(
 
     if (params.logLinear) {
         query = query.andWhereRaw(
-            `config->>'$.yAxis.canChangeScaleType' = "true" OR config->>'$.xAxis.canChangeScaleType'  = "true"`
+            `cc.full->>'$.yAxis.canChangeScaleType' = "true" OR cc.full->>'$.xAxis.canChangeScaleType'  = "true"`
         )
         tab = GrapherTabOption.chart
     }
 
     if (params.comparisonLines) {
         query = query.andWhereRaw(
-            `config->'$.comparisonLines[0].yEquals' != ''`
+            `cc.full->'$.comparisonLines[0].yEquals' != ''`
         )
         tab = GrapherTabOption.chart
     }
 
     if (params.stackMode) {
-        query = query.andWhereRaw(`config->'$.stackMode' = :stackMode`, {
+        query = query.andWhereRaw(`cc.full->'$.stackMode' = :stackMode`, {
             stackMode: params.stackMode,
         })
         tab = GrapherTabOption.chart
     }
 
     if (params.relativeToggle) {
-        query = query.andWhereRaw(`config->>'$.hideRelativeToggle' = "false"`)
+        query = query.andWhereRaw(`cc.full->>'$.hideRelativeToggle' = "false"`)
         tab = GrapherTabOption.chart
     }
 
@@ -199,7 +200,7 @@ async function propsFromQueryParams(
         // have a visible categorial legend, and can leave out some that have one.
         // But in practice it seems to work reasonably well.
         query = query.andWhereRaw(
-            `json_length(config->'$.map.colorScale.customCategoryColors') > 1`
+            `json_length(cc.full->'$.map.colorScale.customCategoryColors') > 1`
         )
         tab = GrapherTabOption.map
     }
@@ -225,13 +226,13 @@ async function propsFromQueryParams(
         const mode = params.addCountryMode
         if (mode === EntitySelectionMode.MultipleEntities) {
             query = query.andWhereRaw(
-                `config->'$.addCountryMode' IS NULL OR config->'$.addCountryMode' = :mode`,
+                `cc.full->'$.addCountryMode' IS NULL OR cc.full->'$.addCountryMode' = :mode`,
                 {
                     mode: EntitySelectionMode.MultipleEntities,
                 }
             )
         } else {
-            query = query.andWhereRaw(`config->'$.addCountryMode' = :mode`, {
+            query = query.andWhereRaw(`cc.full->'$.addCountryMode' = :mode`, {
                 mode,
             })
         }
@@ -242,10 +243,10 @@ async function propsFromQueryParams(
     }
 
     if (tab === GrapherTabOption.map) {
-        query = query.andWhereRaw(`config->>"$.hasMapTab" = "true"`)
+        query = query.andWhereRaw(`cc.full->>"$.hasMapTab" = "true"`)
     } else if (tab === GrapherTabOption.chart) {
         query = query.andWhereRaw(
-            `COALESCE(config->>"$.hasChartTab", "true") = "true"`
+            `COALESCE(cc.full->>"$.hasChartTab", "true") = "true"`
         )
     }
 
@@ -283,7 +284,7 @@ async function propsFromQueryParams(
 
     const chartsQuery = query
         .clone()
-        .select("id", "slug")
+        .select(knex.raw("charts.id, cc.slug"))
         .limit(perPage)
         .offset(perPage * (page - 1))
 
@@ -473,13 +474,26 @@ getPlainRouteWithROTransaction(
     "/embeds/:id",
     async (req, res, trx) => {
         const id = req.params.id
-        const chartRaw: DbRawChart = await trx
-            .table(ChartsTableName)
-            .where({ id: id })
-            .first()
-        const chartEnriched = parseChartsRow(chartRaw)
-        const viewProps = await getViewPropsFromQueryParams(req.query)
-        if (chartEnriched) {
+        const chartRaw = await db.knexRawFirst<
+            Pick<DbPlainChart, "id"> & { config: DbRawChartConfig["full"] }
+        >(
+            trx,
+            `--sql
+                select ca.id, cc.full as config
+                from charts ca
+                join chart_configs cc
+                on ca.configId = cc.id
+                where ca.id = ?
+            `,
+            [id]
+        )
+
+        if (chartRaw) {
+            const chartEnriched = {
+                ...chartRaw,
+                config: parseChartConfig(chartRaw.config),
+            }
+            const viewProps = await getViewPropsFromQueryParams(req.query)
             const charts = [
                 {
                     id: chartEnriched.id,
@@ -727,9 +741,15 @@ getPlainRouteWithROTransaction(
     testPageRouter,
     "/previews",
     async (req, res, trx) => {
-        const rows = await db.knexRaw(
+        const rows = await db.knexRaw<{ config: DbRawChartConfig["full"] }>(
             trx,
-            `SELECT config FROM charts LIMIT 200`
+            `--sql
+                SELECT cc.full as config
+                FROM charts ca
+                JOIN chart_configs cc
+                ON ca.configId = cc.id
+                LIMIT 200
+            `
         )
         const charts = rows.map((row: any) => JSON.parse(row.config))
 
@@ -741,9 +761,15 @@ getPlainRouteWithROTransaction(
     testPageRouter,
     "/embedVariants",
     async (req, res, trx) => {
-        const rows = await db.knexRaw(
+        const rows = await db.knexRaw<{ config: DbRawChartConfig["full"] }>(
             trx,
-            `SELECT config FROM charts WHERE id=64`
+            `--sql
+                SELECT cc.full as config
+                FROM charts ca
+                JOIN chart_configs cc
+                ON ca.configId = cc.id
+                WHERE ca.id=64
+            `
         )
         const charts = rows.map((row: any) => JSON.parse(row.config))
         const viewProps = getViewPropsFromQueryParams(req.query)

diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx
@@ -38,6 +38,8 @@ import {
     OwidChartDimensionInterface,
     FaqEntryData,
     ImageMetadata,
+    DbPlainChart,
+    DbRawChartConfig,
 } from "@ourworldindata/types"
 import ProgressBar from "progress"
 import {
@@ -399,16 +401,24 @@ export const bakeSingleGrapherChart = async (
 export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers =
     // TODO: this transaction is only RW because somewhere inside it we fetch images
     async (bakedSiteDir: string, knex: db.KnexReadWriteTransaction) => {
-        const chartsToBake: { id: number; config: string; slug: string }[] =
-            await knexRaw(
-                knex,
-                `-- sql
-                SELECT
-                    id, config, config->>'$.slug' as slug
-                FROM charts WHERE JSON_EXTRACT(config, "$.isPublished")=true
-                ORDER BY JSON_EXTRACT(config, "$.slug") ASC
+        const chartsToBake = await knexRaw<
+            Pick<DbPlainChart, "id"> & {
+                config: DbRawChartConfig["full"]
+                slug: string
+            }
+        >(
+            knex,
+            `-- sql
+                    SELECT
+                        c.id,
+                        cc.full as config,
+                        cc.slug
+                    FROM charts c
+                    JOIN chart_configs cc ON c.configId = cc.id
+                    WHERE JSON_EXTRACT(cc.full, "$.isPublished")=true
+                    ORDER BY cc.slug ASC
                 `
-            )
+        )
 
         const newSlugs = chartsToBake.map((row) => row.slug)
         await fs.mkdirp(bakedSiteDir + "/grapher")

diff --git a/baker/GrapherBakingUtils.ts b/baker/GrapherBakingUtils.ts
@@ -82,7 +82,12 @@ export const bakeGrapherUrls = async (
 
         const rows = await db.knexRaw<{ version: number }>(
             knex,
-            `SELECT charts.config->>"$.version" AS version FROM charts WHERE charts.id=?`,
+            `-- sql
+                SELECT cc.full->>"$.version" AS version
+                FROM charts c
+                JOIN chart_configs cc ON c.configId = cc.id
+                WHERE c.id=?
+            `,
             [chartId]
         )
         if (!rows.length) {

diff --git a/baker/GrapherImageBaker.tsx b/baker/GrapherImageBaker.tsx
@@ -1,7 +1,8 @@
 import {
     DbPlainChartSlugRedirect,
-    DbRawChart,
+    DbPlainChart,
     GrapherInterface,
+    DbRawChartConfig,
 } from "@ourworldindata/types"
 import { Grapher, GrapherProgrammaticInterface } from "@ourworldindata/grapher"
 import { MultipleOwidVariableDataDimensionsMap } from "@ourworldindata/utils"
@@ -78,9 +79,16 @@ export async function getPublishedGraphersBySlug(
     const graphersById: Map<number, GrapherInterface> = new Map()
 
     // Select all graphers that are published
-    const sql = `SELECT id, config FROM charts WHERE config->>"$.isPublished" = "true"`
-
-    const query = db.knexRaw<Pick<DbRawChart, "id" | "config">>(knex, sql)
+    const sql = `-- sql
+        SELECT c.id, cc.full as config
+        FROM charts c
+        JOIN chart_configs cc ON c.configId = cc.id
+        WHERE cc.full ->> "$.isPublished" = 'true'
+    `
+
+    const query = db.knexRaw<
+        Pick<DbPlainChart, "id"> & { config: DbRawChartConfig["full"] }
+    >(knex, sql)
     for (const row of await query) {
         const grapher = JSON.parse(row.config)
 

diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx
@@ -724,19 +724,21 @@ export class SiteBaker {
                 knex,
                 `-- sql
                 SELECT
-                    config ->> '$.slug' as slug,
-                    config ->> '$.subtitle' as subtitle,
-                    config ->> '$.note' as note
+                    cc.slug,
+                    cc.full ->> '$.subtitle' as subtitle,
+                    cc.full ->> '$.note' as note
                 FROM
-                    charts
+                    charts c
+                JOIN
+                    chart_configs cc ON c.configId = cc.id
                 WHERE
-                    JSON_EXTRACT(config, "$.isPublished") = true
+                    JSON_EXTRACT(cc.full, "$.isPublished") = true
                 AND (
-                    JSON_EXTRACT(config, "$.subtitle") LIKE "%#dod:%"
-                    OR JSON_EXTRACT(config, "$.note") LIKE "%#dod:%"
+                    JSON_EXTRACT(cc.full, "$.subtitle") LIKE "%#dod:%"
+                    OR JSON_EXTRACT(cc.full, "$.note") LIKE "%#dod:%"
                 )
                 ORDER BY
-                    JSON_EXTRACT(config, "$.slug") ASC
+                    cc.slug ASC
             `
             )
 

diff --git a/baker/algolia/indexChartsToAlgolia.ts b/baker/algolia/indexChartsToAlgolia.ts
@@ -123,19 +123,20 @@ const getChartsRecords = async (
         `-- sql
         WITH indexable_charts_with_entity_names AS (
             SELECT c.id,
-                   config ->> "$.slug"                    AS slug,
-                   config ->> "$.title"                   AS title,
-                   config ->> "$.variantName"             AS variantName,
-                   config ->> "$.subtitle"                AS subtitle,
-                   JSON_LENGTH(config ->> "$.dimensions") AS numDimensions,
+                   cc.slug,
+                   cc.full ->> "$.title"                   AS title,
+                   cc.full ->> "$.variantName"             AS variantName,
+                   cc.full ->> "$.subtitle"                AS subtitle,
+                   JSON_LENGTH(cc.full ->> "$.dimensions") AS numDimensions,
                    c.publishedAt,
                    c.updatedAt,
                    JSON_ARRAYAGG(e.name)                  AS entityNames
             FROM charts c
+                     LEFT JOIN chart_configs cc ON c.configId = cc.id
                      LEFT JOIN charts_x_entities ce ON c.id = ce.chartId
                      LEFT JOIN entities e ON ce.entityId = e.id
-            WHERE config ->> "$.isPublished" = 'true'
-              AND isIndexable IS TRUE
+            WHERE cc.full ->> "$.isPublished" = 'true'
+                AND c.isIndexable IS TRUE
             GROUP BY c.id
         )
         SELECT c.id,

diff --git a/baker/countryProfiles.tsx b/baker/countryProfiles.tsx
@@ -7,6 +7,8 @@ import {
     DbEnrichedVariable,
     VariablesTableName,
     parseVariablesRow,
+    DbRawChartConfig,
+    parseChartConfig,
 } from "@ourworldindata/types"
 import * as lodash from "lodash"
 import {
@@ -45,13 +47,20 @@ const countryIndicatorGraphers = async (
     trx: db.KnexReadonlyTransaction
 ): Promise<GrapherInterface[]> =>
     bakeCache(countryIndicatorGraphers, async () => {
-        const graphers = (
-            await trx
-                .table("charts")
-                .whereRaw(
-                    "publishedAt is not null and config->>'$.isPublished' = 'true' and isIndexable is true"
-                )
-        ).map((c: any) => JSON.parse(c.config)) as GrapherInterface[]
+        const configs = await db.knexRaw<{ config: DbRawChartConfig["full"] }>(
+            trx,
+            `-- sql
+                SELECT cc.full as config
+                FROM charts c
+                JOIN chart_configs cc ON cc.id = c.configId
+                WHERE
+                    c.publishedAt is not null
+                    AND cc.full->>'$.isPublished' = 'true'
+                    AND c.isIndexable is true
+            `
+        )
+
+        const graphers = configs.map((c: any) => parseChartConfig(c.config))
 
         return graphers.filter(checkShouldShowIndicator)
     })

diff --git a/baker/redirects.ts b/baker/redirects.ts
@@ -97,9 +97,11 @@ export const getGrapherRedirectsMap = async (
     }>(
         knex,
         `-- sql
-        SELECT chart_slug_redirects.slug as oldSlug, charts.config ->> "$.slug" as newSlug
-        FROM chart_slug_redirects INNER JOIN charts ON charts.id=chart_id
-    `
+            SELECT chart_slug_redirects.slug as oldSlug, chart_configs.slug as newSlug
+            FROM chart_slug_redirects
+            INNER JOIN charts ON charts.id=chart_id
+            INNER JOIN chart_configs ON chart_configs.id=charts.configId
+        `
     )) as Array<{ oldSlug: string; newSlug: string }>
 
     return new Map(