From f4b68525f2769e604e02cf5b5fdcabec6ce34c3c Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 2 May 2025 15:00:45 +0000 Subject: [PATCH 01/47] file-server: refactor code to use image file... --- src/packages/file-server/package.json | 16 ++- src/packages/file-server/zfs/config.ts | 23 +++-- src/packages/file-server/zfs/create.ts | 51 +++++----- src/packages/file-server/zfs/db.ts | 3 - src/packages/file-server/zfs/names.ts | 22 ++++- src/packages/file-server/zfs/pools.ts | 70 +++++++++++-- src/packages/file-server/zfs/pull.ts | 24 ++--- .../file-server/zfs/test/pull.test.ts | 98 +++++++++---------- src/packages/file-server/zfs/test/util.ts | 78 +++------------ src/packages/package.json | 7 +- src/packages/pnpm-lock.yaml | 65 ++++++++---- 11 files changed, 264 insertions(+), 193 deletions(-) diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index 0665a655dc..e0fe5f9580 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -12,9 +12,17 @@ "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --runInBand" }, - "files": ["dist/**", "README.md", "package.json"], + "files": [ + "dist/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["utilities", "nats", "cocalc"], + "keywords": [ + "utilities", + "nats", + "cocalc" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", @@ -22,11 +30,11 @@ "@cocalc/nats": "workspace:*", "@cocalc/util": "workspace:*", "awaiting": "^3.0.0", - "better-sqlite3": "^11.8.1", + "better-sqlite3": "^11.9.1", "lodash": "^4.17.21" }, "devDependencies": { - "@types/better-sqlite3": "^7.6.12", + "@types/better-sqlite3": "^7.6.13", "@types/lodash": "^4.14.202" }, "repository": { diff --git a/src/packages/file-server/zfs/config.ts b/src/packages/file-server/zfs/config.ts index 96b0df9d74..6893bd4535 100644 --- a/src/packages/file-server/zfs/config.ts +++ b/src/packages/file-server/zfs/config.ts @@ -1,17 +1,20 @@ import { join } from "path"; import { databaseFilename } from "./names"; -// we ONLY put filesystems on pools whose name have this prefix. -// all other pools are ignored. We also mount everything in /{PREFIX} on the filesystem. -const PREFIX = process.env.COCALC_TEST_MODE ? "cocalcfs-test" : "cocalcfs"; +// names of all pools we create/manage will start with this prefix: +const PREFIX = "cocalcfs"; -const DATA = `/${PREFIX}`; +// this is where all data is stored or mounted on the filesystem +const DATA = process.env.COCALC_TEST_MODE ? "/data/zfs-test" : "/data/zfs"; const SQLITE3_DATABASE_FILE = databaseFilename(DATA); -// Directory on server where filesystems get mounted (so NFS can serve them) +// Directory where filesystems get mounted (so NFS can serve them) const FILESYSTEMS = join(DATA, "filesystems"); +// Directory where sparse image files are stored +const IMAGES = join(DATA, "images"); + // Directory on server where zfs send streams (and tar?) are stored const ARCHIVES = join(DATA, "archives"); @@ -19,7 +22,7 @@ const ARCHIVES = join(DATA, "archives"); // E.g., this keeps around copies of the sqlite state database of each remote. const PULL = join(DATA, "pull"); -// Directory for bup +// Directory for bup backups const BUP = join(DATA, "bup"); export const context = { @@ -29,6 +32,7 @@ export const context = { SQLITE3_DATABASE_FILE, FILESYSTEMS, ARCHIVES, + IMAGES, PULL, BUP, }; @@ -39,17 +43,20 @@ export const context = { // changing the context out from under it would lead to nonsense and corruption. export function setContext({ namespace, + data, prefix, }: { namespace?: string; + data?: string; prefix?: string; }) { context.namespace = namespace ?? process.env.NAMESPACE ?? "default"; context.PREFIX = prefix ?? PREFIX; - context.DATA = `/${context.PREFIX}`; + context.DATA = data ?? DATA; context.SQLITE3_DATABASE_FILE = databaseFilename(context.DATA); context.FILESYSTEMS = join(context.DATA, "filesystems"); context.ARCHIVES = join(context.DATA, "archives"); + context.IMAGES = join(context.DATA, "images"); context.PULL = join(context.DATA, "pull"); context.BUP = join(context.DATA, "bup"); } @@ -57,6 +64,8 @@ export function setContext({ // Every filesystem has at least this much quota (?) export const MIN_QUOTA = 1024 * 1024 * 1; // 1MB +export const DEFAULT_POOL_SIZE = "100G"; + // We periodically do "zpool list" to find out what pools are available // and how much space they have left. This info is cached for this long // to avoid excessive calls: diff --git a/src/packages/file-server/zfs/create.ts b/src/packages/file-server/zfs/create.ts index 7be3005470..8c57120c61 100644 --- a/src/packages/file-server/zfs/create.ts +++ b/src/packages/file-server/zfs/create.ts @@ -1,3 +1,20 @@ +/* + +DEVELOPMENT: + +If you're using Docker, make sure the DATA directory in config.ts is inside a folder +that is bind mounted from the host. + +Start node. + +a = require('@cocalc/file-server/zfs/create') + +fs = await a.createFilesystem({namespace:'default', owner_type:'project', owner_id:'6b851643-360e-435e-b87e-f9a6ab64a8b1', name:'home'}) + +await a.deleteFilesystem({namespace:'default', owner_type:'project', owner_id:'6b851643-360e-435e-b87e-f9a6ab64a8b1', name:'home'}) + +*/ + import { create, get, getDb, deleteFromDb, filesystemExists } from "./db"; import { exec } from "./util"; import { @@ -5,8 +22,9 @@ import { bupFilesystemMountpoint, filesystemDataset, filesystemMountpoint, + poolName, } from "./names"; -import { getPools, initializePool } from "./pools"; +import { initializePool } from "./pools"; import { dearchiveFilesystem } from "./archive"; import { UID, GID } from "./config"; import { createSnapshot } from "./snapshots"; @@ -31,7 +49,7 @@ export async function createFilesystem( let pool: undefined | string = undefined; if (source != null) { - // use same pool as source filesystem. (we could use zfs send/recv but that's much slower and not a clone) + // For clone, we use same pool as source filesystem. (we could use zfs send/recv but that's much slower and not a clone) pool = source.pool; } else { if (affinity) { @@ -44,33 +62,8 @@ export async function createFilesystem( pool = x?.pool; } if (!pool) { - // assign one with *least* filesystems - const x = db - .prepare( - "SELECT pool, COUNT(pool) AS cnt FROM filesystems GROUP BY pool ORDER by cnt ASC", - ) - .all() as any; - const pools = await getPools(); - if (Object.keys(pools).length > x.length) { - // rare case: there exists a pool that isn't used yet, so not - // represented in above query at all; use it - const v = new Set(); - for (const { pool } of x) { - v.add(pool); - } - for (const name in pools) { - if (!v.has(name)) { - pool = name; - break; - } - } - } else { - if (x.length == 0) { - throw Error("cannot create filesystem -- no available pools"); - } - // just use the least crowded - pool = x[0].pool; - } + // create new pool + pool = poolName(pk); } } if (!pool) { diff --git a/src/packages/file-server/zfs/db.ts b/src/packages/file-server/zfs/db.ts index 9b33d202dc..e54dc4cb5a 100644 --- a/src/packages/file-server/zfs/db.ts +++ b/src/packages/file-server/zfs/db.ts @@ -209,9 +209,6 @@ export function create( affinity?: string; }, ) { - if (!obj.pool.startsWith(context.PREFIX)) { - throw Error(`pool must start with ${context.PREFIX} - ${obj.pool}`); - } getDb() .prepare( "INSERT INTO filesystems(namespace, owner_type, owner_id, name, pool, affinity, last_edited) VALUES(?,?,?,?,?,?,?)", diff --git a/src/packages/file-server/zfs/names.ts b/src/packages/file-server/zfs/names.ts index aee7b41cd5..8aa4df808f 100644 --- a/src/packages/file-server/zfs/names.ts +++ b/src/packages/file-server/zfs/names.ts @@ -116,7 +116,10 @@ export function filesystemsPath({ namespace }) { export function filesystemMountpoint(fs: PrimaryKey) { const pk = primaryKey(fs); - return join(filesystemsPath(pk), pk.owner_type, pk.owner_id, pk.name); + return join( + context.FILESYSTEMS, + `${pk.owner_type}-${pk.owner_id}-${pk.name}`, + ); } export function filesystemSnapshotMountpoint( @@ -153,6 +156,23 @@ export function filesystemDataset({ return `${filesystemsDataset({ pool, namespace: namespace })}/${owner_type}-${owner_id}-${name}`; } +// A unique name for the pool that will store this fs. Other fs with +// same affinity and clones could also be stored on this pool. +export function poolName(fs): string { + // NOTE: imageDirectory below assumes the pool name starts with "${context.PREFIX}-{namespace}-" + const { namespace, owner_type, owner_id, name } = primaryKey(fs); + return `${context.PREFIX}-${namespace}-${owner_type}-${owner_id}-${name}`; +} + +export function poolImageDirectory({ pool }: { pool: string }): string { + const namespace = pool.slice(context.PREFIX.length + 1).split("-")[0]; + return join(context.IMAGES, namespace, pool); +} + +export function poolImageFile({ pool }: { pool: string }): string { + return join(poolImageDirectory({ pool }), "0.img"); +} + export function tempDataset({ pool, namespace, diff --git a/src/packages/file-server/zfs/pools.ts b/src/packages/file-server/zfs/pools.ts index b71bdc45f2..ca2cceb50c 100644 --- a/src/packages/file-server/zfs/pools.ts +++ b/src/packages/file-server/zfs/pools.ts @@ -14,7 +14,7 @@ OPERATIONS: */ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { context, POOLS_CACHE_MS } from "./config"; +import { context, DEFAULT_POOL_SIZE, POOLS_CACHE_MS } from "./config"; import { exec } from "./util"; import { archivesDataset, @@ -25,6 +25,8 @@ import { bupDataset, bupMountpoint, tempDataset, + poolImageDirectory, + poolImageFile, } from "./names"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { getNamespacesAndPools } from "./db"; @@ -52,8 +54,8 @@ let poolsCache: { [prefix: string]: Pools } = {}; export const getPools = reuseInFlight( async ({ noCache }: { noCache?: boolean } = {}): Promise => { - if (!noCache && poolsCache[context.PREFIX]) { - return poolsCache[context.PREFIX]; + if (!noCache && poolsCache[context.DATA]) { + return poolsCache[context.DATA]; } const { stdout } = await exec({ verbose: true, @@ -92,11 +94,65 @@ export const initializePool = reuseInFlight( namespace?: string; pool: string; }) => { - if (!pool.startsWith(context.PREFIX)) { - throw Error( - `pool (="${pool}") must start with the prefix '${context.PREFIX}'`, - ); + const image = poolImageFile({ pool }); + if (!(await exists(image))) { + const dir = poolImageDirectory({ pool }); + + await exec({ + verbose: true, + command: "sudo", + args: ["mkdir", "-p", dir], + }); + + await exec({ + verbose: true, + command: "sudo", + args: ["truncate", "-s", DEFAULT_POOL_SIZE, image], + what: { pool, desc: "create sparse image file" }, + }); + + // create the pool + await exec({ + verbose: true, + command: "sudo", + args: [ + "zpool", + "create", + "-o", + "feature@fast_dedup=enabled", + "-m", + "none", + pool, + image, + ], + what: { + pool, + desc: `create the zpool ${pool} using the device ${image}`, + }, + }); + } else { + // make sure pool is imported + try { + await exec({ + verbose: true, + command: "zpool", + args: ["list", pool], + what: { pool, desc: `check if ${pool} needs to be imported` }, + }); + } catch { + const dir = poolImageDirectory({ pool }); + await exec({ + verbose: true, + command: "sudo", + args: ["zpool", "import", pool, "-d", dir], + what: { + pool, + desc: `import the zpool ${pool} from ${dir}`, + }, + }); + } } + // archives and filesystems for each namespace are in this dataset await ensureDatasetExists({ name: namespaceDataset({ namespace, pool }), diff --git a/src/packages/file-server/zfs/pull.ts b/src/packages/file-server/zfs/pull.ts index a0a0236814..ca1087d854 100644 --- a/src/packages/file-server/zfs/pull.ts +++ b/src/packages/file-server/zfs/pull.ts @@ -1,5 +1,5 @@ /* -Use zfs replication over ssh to pull recent filesystems from +Use zfs replication over ssh to pull recent filesystems from one file-server to another one. This will be used for: @@ -59,17 +59,19 @@ export const SYNCED_FIELDS = [ interface Remote { // remote = user@hostname that you can ssh to remote: string; - // filesystem prefix of the remote server, so {prefix}/database.sqlite3 has the + // filesystem location on the remote server, so {data}/database.sqlite3 has the // database that defines the state of the remote server. - prefix: string; + data: string; } +// [ ] TODO: this is very likely broken due to prefix --> data change + // Copy from remote to here every filesystem that has changed since cutoff. export async function pull({ cutoff, filesystem, remote, - prefix, + data, deleteFilesystemCutoff, deleteSnapshots, dryRun, @@ -93,9 +95,9 @@ export async function pull({ toUpdate: { remoteFs: Filesystem; localFs?: Filesystem }[]; toDelete: RawFilesystem[]; }> { - logger.debug("pull: from ", { remote, prefix, cutoff, filesystem }); - if (prefix.startsWith("/")) { - throw Error("prefix should not start with /"); + logger.debug("pull: from ", { remote, data, cutoff, filesystem }); + if (data.startsWith("/")) { + throw Error("data should start with /"); } if (cutoff == null) { cutoff = new Date(Date.now() - 1000 * 60 * 60 * 24 * 7); @@ -104,12 +106,12 @@ export async function pull({ await exec({ command: "mkdir", args: ["-p", context.PULL] }); const remoteDatabase = join( context.PULL, - `${remote}:${prefix}---${new Date().toISOString()}.sqlite3`, + `${remote}:${data}---${new Date().toISOString()}.sqlite3`, ); - // delete all but the most recent remote database files for this remote/prefix (?). + // delete all but the most recent remote database files for this remote/data (?). const oldDbFiles = (await readdir(context.PULL)) .sort() - .filter((x) => x.startsWith(`${remote}:${prefix}---`)) + .filter((x) => x.startsWith(`${remote}:${data}---`)) .slice(0, -NUM_DB_TO_KEEP); for (const path of oldDbFiles) { await unlink(join(context.PULL, path)); @@ -117,7 +119,7 @@ export async function pull({ await exec({ command: "scp", - args: [`${remote}:/${databaseFilename(prefix)}`, remoteDatabase], + args: [`${remote}:/${databaseFilename(data)}`, remoteDatabase], }); logger.debug("pull: compare state"); diff --git a/src/packages/file-server/zfs/test/pull.test.ts b/src/packages/file-server/zfs/test/pull.test.ts index a2689d7270..06a5c4af4c 100644 --- a/src/packages/file-server/zfs/test/pull.test.ts +++ b/src/packages/file-server/zfs/test/pull.test.ts @@ -2,10 +2,10 @@ DEVELOPMENT: This tests pull replication by setting up two separate file-servers on disk locally -and doing pulls from one to the other over ssh. This involves password-less ssh +and doing pulls from one to the other over ssh. This involves password-less ssh to root on localhost, and creating multiple pools, so use with caution and don't -expect this to work unless you really know what you're doing. -Also, these tests are going to take a while. +expect this to work unless you really know what you're doing. +Also, these tests are going to take a while. Efficient powerful backup isn't trivial and is very valuable, so its' worth the wait! @@ -33,20 +33,20 @@ import { SYNCED_FIELDS } from "../pull"; describe("create two separate file servers, then do pulls to sync one to the other under various conditions", () => { let one: any = null, two: any = null; - const prefix1 = context.PREFIX + ".1"; - const prefix2 = context.PREFIX + ".2"; + const data1 = context.DATA + ".1"; + const data2 = context.DATA + ".2"; const remote = "root@localhost"; beforeAll(async () => { - one = await createTestPools({ count: 1, size: "1G", prefix: prefix1 }); - setContext({ prefix: prefix1 }); + one = await createTestPools({ count: 1, size: "1G", data: data1 }); + setContext({ data: data1 }); await init(); two = await createTestPools({ count: 1, size: "1G", - prefix: prefix2, + data: data2, }); - setContext({ prefix: prefix2 }); + setContext({ data: data2 }); await init(); }); @@ -56,7 +56,7 @@ describe("create two separate file servers, then do pulls to sync one to the oth }); it("creates a filesystem in pool one, writes a file and takes a snapshot", async () => { - setContext({ prefix: prefix1 }); + setContext({ data: data1 }); const fs = await createFilesystem({ project_id: "00000000-0000-0000-0000-000000000001", }); @@ -66,7 +66,7 @@ describe("create two separate file servers, then do pulls to sync one to the oth }); it("pulls filesystem one to filesystem two, and confirms the fs and file were indeed sync'd", async () => { - setContext({ prefix: prefix2 }); + setContext({ data: data2 }); expect( await filesystemExists({ project_id: "00000000-0000-0000-0000-000000000001", @@ -76,7 +76,7 @@ describe("create two separate file servers, then do pulls to sync one to the oth // first dryRun const { toUpdate, toDelete } = await pull({ remote, - prefix: prefix1, + data: data1, dryRun: true, }); expect(toDelete.length).toBe(0); @@ -89,7 +89,7 @@ describe("create two separate file servers, then do pulls to sync one to the oth // now for real const { toUpdate: toUpdate1, toDelete: toDelete1 } = await pull({ remote, - prefix: prefix1, + data: data1, }); expect(toDelete1).toEqual(toDelete); @@ -103,22 +103,22 @@ describe("create two separate file servers, then do pulls to sync one to the oth // nothing if we sync again: const { toUpdate: toUpdate2, toDelete: toDelete2 } = await pull({ remote, - prefix: prefix1, + data: data1, }); expect(toDelete2.length).toBe(0); expect(toUpdate2.length).toBe(0); }); it("creates another file in our filesystem, creates another snapshot, syncs again, and sees that the sync worked", async () => { - setContext({ prefix: prefix1 }); + setContext({ data: data1 }); const fs = { project_id: "00000000-0000-0000-0000-000000000001" }; await writeFile(join(filesystemMountpoint(fs), "b.txt"), "cocalc"); await createSnapshot({ ...fs, force: true }); const { snapshots } = get(fs); expect(snapshots.length).toBe(2); - setContext({ prefix: prefix2 }); - await pull({ remote, prefix: prefix1 }); + setContext({ data: data2 }); + await pull({ remote, data: data1 }); expect( (await readFile(join(filesystemMountpoint(fs), "b.txt"))).toString(), @@ -127,17 +127,17 @@ describe("create two separate file servers, then do pulls to sync one to the oth it("archives the project, does sync, and see the other one got archived", async () => { const fs = { project_id: "00000000-0000-0000-0000-000000000001" }; - setContext({ prefix: prefix2 }); + setContext({ data: data2 }); const project2before = get(fs); expect(project2before.archived).toBe(false); - setContext({ prefix: prefix1 }); + setContext({ data: data1 }); await archiveFilesystem(fs); const project1 = get(fs); expect(project1.archived).toBe(true); - setContext({ prefix: prefix2 }); - await pull({ remote, prefix: prefix1 }); + setContext({ data: data2 }); + await pull({ remote, data: data1 }); const project2 = get(fs); expect(project2.archived).toBe(true); expect(project1.last_edited).toEqual(project2.last_edited); @@ -145,29 +145,29 @@ describe("create two separate file servers, then do pulls to sync one to the oth it("dearchives, does sync, then sees the other gets dearchived; this just tests that sync de-archives, but works even if there are no new snapshots", async () => { const fs = { project_id: "00000000-0000-0000-0000-000000000001" }; - setContext({ prefix: prefix1 }); + setContext({ data: data1 }); await dearchiveFilesystem(fs); const project1 = get(fs); expect(project1.archived).toBe(false); - setContext({ prefix: prefix2 }); - await pull({ remote, prefix: prefix1 }); + setContext({ data: data2 }); + await pull({ remote, data: data1 }); const project2 = get(fs); expect(project2.archived).toBe(false); }); it("archives project, does sync, de-archives project, adds another snapshot, then does sync, thus testing that sync both de-archives *and* pulls latest snapshot", async () => { const fs = { project_id: "00000000-0000-0000-0000-000000000001" }; - setContext({ prefix: prefix1 }); + setContext({ data: data1 }); expect(get(fs).archived).toBe(false); await archiveFilesystem(fs); expect(get(fs).archived).toBe(true); - setContext({ prefix: prefix2 }); - await pull({ remote, prefix: prefix1 }); + setContext({ data: data2 }); + await pull({ remote, data: data1 }); expect(get(fs).archived).toBe(true); // now dearchive - setContext({ prefix: prefix1 }); + setContext({ data: data1 }); await dearchiveFilesystem(fs); // write content await writeFile(join(filesystemMountpoint(fs), "d.txt"), "hello"); @@ -175,8 +175,8 @@ describe("create two separate file servers, then do pulls to sync one to the oth await createSnapshot({ ...fs, force: true }); const project1 = get(fs); - setContext({ prefix: prefix2 }); - await pull({ remote, prefix: prefix1 }); + setContext({ data: data2 }); + await pull({ remote, data: data1 }); const project2 = get(fs); expect(project2.snapshots).toEqual(project1.snapshots); expect(project2.archived).toBe(false); @@ -184,26 +184,26 @@ describe("create two separate file servers, then do pulls to sync one to the oth it("deletes project, does sync, then sees the other does NOT gets deleted without passing the deleteFilesystemCutoff option, and also with deleteFilesystemCutoff an hour ago, but does get deleted with it now", async () => { const fs = { project_id: "00000000-0000-0000-0000-000000000001" }; - setContext({ prefix: prefix1 }); + setContext({ data: data1 }); expect(await filesystemExists(fs)).toEqual(true); await deleteFilesystem(fs); expect(await filesystemExists(fs)).toEqual(false); - setContext({ prefix: prefix2 }); + setContext({ data: data2 }); expect(await filesystemExists(fs)).toEqual(true); - await pull({ remote, prefix: prefix1 }); + await pull({ remote, data: data1 }); expect(await filesystemExists(fs)).toEqual(true); await pull({ remote, - prefix: prefix1, + data: data1, deleteFilesystemCutoff: new Date(Date.now() - 1000 * 60 * 60), }); expect(await filesystemExists(fs)).toEqual(true); await pull({ remote, - prefix: prefix1, + data: data1, deleteFilesystemCutoff: new Date(), }); expect(await filesystemExists(fs)).toEqual(false); @@ -224,7 +224,7 @@ describe("create two separate file servers, then do pulls to sync one to the oth }, ]; it("creates 3 filesystems in 2 different namespaces, and confirms sync works", async () => { - setContext({ prefix: prefix1 }); + setContext({ data: data1 }); for (const fs of v) { await createFilesystem(fs); } @@ -237,8 +237,8 @@ describe("create two separate file servers, then do pulls to sync one to the oth const p = v.map((x) => get(x)); // do the sync - setContext({ prefix: prefix2 }); - await pull({ remote, prefix: prefix1 }); + setContext({ data: data2 }); + await pull({ remote, data: data1 }); // verify that we have everything for (const fs of v) { @@ -259,16 +259,16 @@ describe("create two separate file servers, then do pulls to sync one to the oth it("edits some files on one of the above filesystems, snapshots, sync's, goes back and deletes a snapshot, edits more files, sync's, and notices that snapshots on sync target properly match snapshots on source.", async () => { // edits some files on one of the above filesystems, snapshots: - setContext({ prefix: prefix1 }); + setContext({ data: data1 }); await writeFile(join(filesystemMountpoint(v[1]), "a2.txt"), "hello2"); await createSnapshot({ ...v[1], force: true }); // sync's - setContext({ prefix: prefix2 }); - await pull({ remote, prefix: prefix1 }); + setContext({ data: data2 }); + await pull({ remote, data: data1 }); // delete snapshot - setContext({ prefix: prefix1 }); + setContext({ data: data1 }); const fs1 = get(v[1]); await deleteSnapshot({ ...v[1], snapshot: fs1.snapshots[0] }); @@ -278,29 +278,29 @@ describe("create two separate file servers, then do pulls to sync one to the oth const snapshots1 = get(v[1]).snapshots; // sync - setContext({ prefix: prefix2 }); - await pull({ remote, prefix: prefix1 }); + setContext({ data: data2 }); + await pull({ remote, data: data1 }); // snapshots do NOT initially match, since we didn't enable snapshot deleting! let snapshots2 = get(v[1]).snapshots; expect(snapshots1).not.toEqual(snapshots2); - await pull({ remote, prefix: prefix1, deleteSnapshots: true }); + await pull({ remote, data: data1, deleteSnapshots: true }); // now snapshots should match exactly! snapshots2 = get(v[1]).snapshots; expect(snapshots1).toEqual(snapshots2); }); it("test directly pulling one filesystem, rather than doing a full sync", async () => { - setContext({ prefix: prefix1 }); + setContext({ data: data1 }); await writeFile(join(filesystemMountpoint(v[1]), "a3.txt"), "hello2"); await createSnapshot({ ...v[1], force: true }); await writeFile(join(filesystemMountpoint(v[2]), "a4.txt"), "hello"); await createSnapshot({ ...v[2], force: true }); const p = v.map((x) => get(x)); - setContext({ prefix: prefix2 }); - await pull({ remote, prefix: prefix1, filesystem: v[1] }); + setContext({ data: data2 }); + await pull({ remote, data: data1, filesystem: v[1] }); const p2 = v.map((x) => get(x)); // now filesystem 1 should match, but not filesystem 2 @@ -308,7 +308,7 @@ describe("create two separate file servers, then do pulls to sync one to the oth expect(p[2].snapshots).not.toEqual(p2[2].snapshots); // finally a full sync will get filesystem 2 - await pull({ remote, prefix: prefix1 }); + await pull({ remote, data: data1 }); const p2b = v.map((x) => get(x)); expect(p[2].snapshots).toEqual(p2b[2].snapshots); }); diff --git a/src/packages/file-server/zfs/test/util.ts b/src/packages/file-server/zfs/test/util.ts index 6d4b978297..34bacecad4 100644 --- a/src/packages/file-server/zfs/test/util.ts +++ b/src/packages/file-server/zfs/test/util.ts @@ -1,13 +1,11 @@ // application/typescript text import { context, setContext } from "@cocalc/file-server/zfs/config"; -import { mkdtemp, rm } from "fs/promises"; +import { mkdtemp } from "fs/promises"; import { tmpdir } from "os"; import { join } from "path"; import { executeCode } from "@cocalc/backend/execute-code"; import { initDataDir } from "@cocalc/file-server/zfs/util"; import { resetDb } from "@cocalc/file-server/zfs/db"; -import { getPools } from "@cocalc/file-server/zfs/pools"; -import { map as asyncMap } from "awaiting"; // export "describe" from here that is a no-op unless TEST_ZFS is set @@ -26,48 +24,20 @@ export async function init() { export async function createTestPools({ size = "10G", count = 1, - prefix, + data, }: { size?: string; count?: number; - prefix?: string; -}): Promise<{ tempDir: string; pools: string[]; prefix?: string }> { - setContext({ prefix }); - if (!context.PREFIX.includes("test")) { - throw Error(`context.PREFIX=${context.PREFIX} must contain 'test'`); + data?: string; +}): Promise<{ tempDir: string; data?: string }> { + console.log("TODO:", { size, count }); + setContext({ data }); + if (!context.DATA.includes("test")) { + throw Error(`context.DATA=${context.DATA} must contain 'test'`); } // Create temp directory const tempDir = await mkdtemp(join(tmpdir(), "test-")); - const pools: string[] = []; - // in case pools left from a failing test: - for (const pool of Object.keys(await getPools())) { - try { - await executeCode({ - command: "sudo", - args: ["zpool", "destroy", pool], - }); - } catch {} - } - for (let n = 0; n < count; n++) { - const image = join(tempDir, `${n}`, "0.img"); - await executeCode({ - command: "mkdir", - args: [join(tempDir, `${n}`)], - }); - await executeCode({ - command: "truncate", - args: ["-s", size, image], - }); - const pool = `${context.PREFIX}-${n}`; - pools.push(pool); - await executeCode({ - command: "sudo", - args: ["zpool", "create", pool, image], - }); - } - // ensure pool cache is cleared: - await getPools({ noCache: true }); - return { tempDir, pools, prefix }; + return { tempDir, data }; } // Even after setting sharefnfs=off, it can be a while (a minute?) until NFS @@ -80,32 +50,14 @@ export async function restartNfsServer() { }); } -export async function deleteTestPools(x?: { - tempDir: string; - pools: string[]; - prefix?: string; -}) { +export async function deleteTestPools(x?: { tempDir: string; data?: string }) { if (!x) { return; } - const { tempDir, pools, prefix } = x; - setContext({ prefix }); - if (!context.PREFIX.includes("test")) { - throw Error("context.PREFIX must contain 'test'"); + const { data } = x; + setContext({ data }); + if (!context.DATA.includes("test")) { + throw Error("context.DATA must contain 'test'"); } - - const f = async (pool) => { - try { - await executeCode({ - command: "sudo", - args: ["zpool", "destroy", pool], - }); - } catch (err) { - // if (!`$err}`.includes("no such pool")) { - // console.log(err); - // } - } - }; - await asyncMap(pools, pools.length, f); - await rm(tempDir, { recursive: true }); + throw Error("not implemented"); } diff --git a/src/packages/package.json b/src/packages/package.json index ce79fc3b40..44669a7658 100644 --- a/src/packages/package.json +++ b/src/packages/package.json @@ -28,6 +28,11 @@ "nanoid@<3.3.8": "^3.3.8", "tar-fs@2.1.1": "2.1.2" }, - "onlyBuiltDependencies": ["websocket-sftp", "websocketfs", "zeromq"] + "onlyBuiltDependencies": [ + "websocket-sftp", + "websocketfs", + "zeromq", + "better-sqlite3" + ] } } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index a6ff16dda5..44796cb094 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -273,14 +273,14 @@ importers: specifier: ^3.0.0 version: 3.0.0 better-sqlite3: - specifier: ^11.8.1 + specifier: ^11.9.1 version: 11.9.1 lodash: specifier: ^4.17.21 version: 4.17.21 devDependencies: '@types/better-sqlite3': - specifier: ^7.6.12 + specifier: ^7.6.13 version: 7.6.13 '@types/lodash': specifier: ^4.14.202 @@ -4524,6 +4524,9 @@ packages: '@types/node@18.19.86': resolution: {integrity: sha512-fifKayi175wLyKyc5qUfyENhQ1dCNI1UNjp653d8kuYcPQN5JhX3dGuP/XmvPTg/xRBn1VTLpbmi+H/Mr7tLfQ==} + '@types/node@22.15.3': + resolution: {integrity: sha512-lX7HFZeHf4QG/J7tBZqrCAXwz9J5RD56Y6MpP0eJkka8p+K0RY/yBTW7CYFJ4VGCclxqOLKmiGP5juQc6MKgcw==} + '@types/node@9.6.61': resolution: {integrity: sha512-/aKAdg5c8n468cYLy2eQrcR5k6chlbNwZNGUj3TboyPa2hcO2QAJcfymlqPzMiRj8B6nYKXjzQz36minFE0RwQ==} @@ -6369,6 +6372,10 @@ packages: resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==} engines: {node: '>=8'} + detect-libc@2.0.4: + resolution: {integrity: sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==} + engines: {node: '>=8'} + detect-newline@3.1.0: resolution: {integrity: sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==} engines: {node: '>=8'} @@ -6640,8 +6647,8 @@ packages: resolution: {integrity: sha512-uDn+FE1yrDzyC0pCo961B2IHbdM8y/ACZsKD4dG6WqrjV53BADjwa7D+1aom2rsNVfLyDgU/eigvlJGJ08OQ4w==} engines: {node: '>= 0.4'} - es-module-lexer@1.6.0: - resolution: {integrity: sha512-qqnD1yMU6tk/jnaMosogGySTZP8YtUgAffA9nMN+E/rjxcfRQ6IEk7IiozUjgxKoFHBGjTLnrHB/YC45r/59EQ==} + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} es-object-atoms@1.1.1: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} @@ -9095,8 +9102,8 @@ packages: no-case@3.0.4: resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} - node-abi@3.74.0: - resolution: {integrity: sha512-c5XK0MjkGBrQPGYG24GBADZud0NCbznxNx0ZkS+ebUTrmV1qTDxPxSL8zEAPURXSbLRWVexxmP4986BziahL5w==} + node-abi@3.75.0: + resolution: {integrity: sha512-OhYaY5sDsIka7H7AtijtI9jwGYLyl29eQn/W623DiN/MIv5sUqc4g7BIDThX+gb7di9f6xK02nkp8sdfFWZLTg==} engines: {node: '>=10'} node-addon-api@6.1.0: @@ -10731,6 +10738,10 @@ packages: resolution: {integrity: sha512-Gf9qqc58SpCA/xdziiHz35F4GNIWYWZrEshUc/G/r5BnLph6xpKuLeoJoQuj5WfBIx/eQLf+hmVPYHaxJu7V2g==} engines: {node: '>= 10.13.0'} + schema-utils@4.3.2: + resolution: {integrity: sha512-Gn/JaSk/Mt9gYubxTtSn/QCV4em9mpAPiR1rqy/Ocu19u/G9J5WWdNoUT4SiV6mFC3y6cxyFcFwdzPM3FgxGAQ==} + engines: {node: '>= 10.13.0'} + script-loader@0.7.2: resolution: {integrity: sha512-UMNLEvgOAQuzK8ji8qIscM3GIrRCWN6MmMXGD4SD5l6cSycgGsCo0tX5xRnfQcoghqct0tjHjcykgI1PyBE2aA==} @@ -11604,6 +11615,9 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + unicorn-magic@0.1.0: resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==} engines: {node: '>=18'} @@ -14653,7 +14667,7 @@ snapshots: '@types/better-sqlite3@7.6.13': dependencies: - '@types/node': 18.19.86 + '@types/node': 22.15.3 '@types/body-parser@1.19.5': dependencies: @@ -14972,6 +14986,10 @@ snapshots: dependencies: undici-types: 5.26.5 + '@types/node@22.15.3': + dependencies: + undici-types: 6.21.0 + '@types/node@9.6.61': {} '@types/nodemailer@6.4.17': @@ -17062,6 +17080,8 @@ snapshots: detect-libc@2.0.3: {} + detect-libc@2.0.4: {} + detect-newline@3.1.0: {} detect-node@2.1.0: {} @@ -17401,7 +17421,7 @@ snapshots: iterator.prototype: 1.1.5 safe-array-concat: 1.1.3 - es-module-lexer@1.6.0: {} + es-module-lexer@1.7.0: {} es-object-atoms@1.1.1: dependencies: @@ -19540,7 +19560,7 @@ snapshots: jest-worker@27.5.1: dependencies: - '@types/node': 18.19.86 + '@types/node': 22.15.3 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -20485,7 +20505,7 @@ snapshots: lower-case: 2.0.2 tslib: 2.8.1 - node-abi@3.74.0: + node-abi@3.75.0: dependencies: semver: 7.7.1 @@ -21306,13 +21326,13 @@ snapshots: prebuild-install@7.1.3: dependencies: - detect-libc: 2.0.3 + detect-libc: 2.0.4 expand-template: 2.0.3 github-from-package: 0.0.0 minimist: 1.2.8 mkdirp-classic: 0.5.3 napi-build-utils: 2.0.0 - node-abi: 3.74.0 + node-abi: 3.75.0 pump: 3.0.2 rc: 1.2.8 simple-get: 4.0.1 @@ -22442,6 +22462,13 @@ snapshots: ajv-formats: 2.1.1(ajv@8.17.1) ajv-keywords: 5.1.0(ajv@8.17.1) + schema-utils@4.3.2: + dependencies: + '@types/json-schema': 7.0.15 + ajv: 8.17.1 + ajv-formats: 2.1.1(ajv@8.17.1) + ajv-keywords: 5.1.0(ajv@8.17.1) + script-loader@0.7.2: dependencies: raw-loader: 0.5.1 @@ -23106,7 +23133,7 @@ snapshots: dependencies: '@jridgewell/trace-mapping': 0.3.25 jest-worker: 27.5.1 - schema-utils: 4.3.0 + schema-utils: 4.3.2 serialize-javascript: 6.0.2 terser: 5.39.0 webpack: 5.99.5(uglify-js@3.19.3) @@ -23117,7 +23144,7 @@ snapshots: dependencies: '@jridgewell/trace-mapping': 0.3.25 jest-worker: 27.5.1 - schema-utils: 4.3.0 + schema-utils: 4.3.2 serialize-javascript: 6.0.2 terser: 5.39.0 webpack: 5.99.5 @@ -23403,6 +23430,8 @@ snapshots: undici-types@5.26.5: {} + undici-types@6.21.0: {} + unicorn-magic@0.1.0: {} unified@10.1.2: @@ -23807,7 +23836,7 @@ snapshots: browserslist: 4.24.4 chrome-trace-event: 1.0.4 enhanced-resolve: 5.18.1 - es-module-lexer: 1.6.0 + es-module-lexer: 1.7.0 eslint-scope: 5.1.1 events: 3.3.0 glob-to-regexp: 0.4.1 @@ -23816,7 +23845,7 @@ snapshots: loader-runner: 4.3.0 mime-types: 2.1.35 neo-async: 2.6.2 - schema-utils: 4.3.0 + schema-utils: 4.3.2 tapable: 2.2.1 terser-webpack-plugin: 5.3.14(webpack@5.99.5) watchpack: 2.4.2 @@ -23837,7 +23866,7 @@ snapshots: browserslist: 4.24.4 chrome-trace-event: 1.0.4 enhanced-resolve: 5.18.1 - es-module-lexer: 1.6.0 + es-module-lexer: 1.7.0 eslint-scope: 5.1.1 events: 3.3.0 glob-to-regexp: 0.4.1 @@ -23846,7 +23875,7 @@ snapshots: loader-runner: 4.3.0 mime-types: 2.1.35 neo-async: 2.6.2 - schema-utils: 4.3.0 + schema-utils: 4.3.2 tapable: 2.2.1 terser-webpack-plugin: 5.3.14(uglify-js@3.19.3)(webpack@5.99.5(uglify-js@3.19.3)) watchpack: 2.4.2 From d1b79290f029d5cf2286b9a24efba1ebc5d55a6f Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 2 May 2025 17:18:45 +0000 Subject: [PATCH 02/47] file server: start a new cleaner compositional approach --- src/packages/file-server/package.json | 16 +-- src/packages/file-server/storage/README.md | 23 +++ src/packages/file-server/storage/index.ts | 1 + src/packages/file-server/storage/pool.ts | 158 +++++++++++++++++++++ src/packages/file-server/storage/pools.ts | 62 ++++++++ src/packages/file-server/storage/util.ts | 40 ++++++ 6 files changed, 289 insertions(+), 11 deletions(-) create mode 100644 src/packages/file-server/storage/README.md create mode 100644 src/packages/file-server/storage/index.ts create mode 100644 src/packages/file-server/storage/pool.ts create mode 100644 src/packages/file-server/storage/pools.ts create mode 100644 src/packages/file-server/storage/util.ts diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index e0fe5f9580..0293b9e353 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -4,7 +4,9 @@ "description": "CoCalc File Server", "exports": { "./zfs": "./dist/zfs/index.js", - "./zfs/*": "./dist/zfs/*.js" + "./zfs/*": "./dist/zfs/*.js", + "./storage": "./dist/storage/index.js", + "./storage/*": "./dist/storage/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", @@ -12,17 +14,9 @@ "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --runInBand" }, - "files": [ - "dist/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "nats", - "cocalc" - ], + "keywords": ["utilities", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", diff --git a/src/packages/file-server/storage/README.md b/src/packages/file-server/storage/README.md new file mode 100644 index 0000000000..3e2c3bdc25 --- /dev/null +++ b/src/packages/file-server/storage/README.md @@ -0,0 +1,23 @@ +This module allows one to: + +Parameters: + +- Images path +- Mount path + +What it does: + +- Create a pool + +- Create a named filesystem in a pool. + +- Set properties of a filesystem + - quota + - dedup + +- Manages the underlying storage: + - trim + - snapshot + - backup + - replicate + - archive to cloud storage diff --git a/src/packages/file-server/storage/index.ts b/src/packages/file-server/storage/index.ts new file mode 100644 index 0000000000..ec45ac575f --- /dev/null +++ b/src/packages/file-server/storage/index.ts @@ -0,0 +1 @@ +export { pools } from "./pools"; diff --git a/src/packages/file-server/storage/pool.ts b/src/packages/file-server/storage/pool.ts new file mode 100644 index 0000000000..2d3148f8d7 --- /dev/null +++ b/src/packages/file-server/storage/pool.ts @@ -0,0 +1,158 @@ +import refCache from "@cocalc/util/refcache"; +import { chmod, exec, exists, mkdirp } from "./util"; +import { join } from "path"; + +const DEFAULT_SIZE = "10G"; +const POOL_NAME_REGEXP = /^(?!-)(?!(\.{1,2})$)[A-Za-z0-9_.:-]{1,255}$/; + +export interface Options { + // where to store its image file(s) + images: string; + // where to mount its filesystems + mount: string; + // the name of the pool + name: string; +} + +export class Pool { + private opts: Options; + private image: string; + + constructor(opts: Options) { + if (!POOL_NAME_REGEXP.test(opts.name)) { + throw Error(`invalid ZFS pool name '${opts.name}'`); + } + this.opts = opts; + this.image = join(opts.images, "0.img"); + } + + exists = async () => { + return await exists(this.image); + }; + + destroy = async () => { + if (!(await this.exists())) { + return; + } + try { + await exec({ + command: "sudo", + args: ["zpool", "destroy", "-f", this.opts.name], + }); + } catch (err) { + if (!`${err}`.includes("no such pool")) { + throw err; + } + } + await exec({ command: "sudo", args: ["rm", this.image] }); + await exec({ command: "sudo", args: ["rmdir", this.opts.images] }); + }; + + create = async () => { + if (await this.exists()) { + // already exists + return; + } + await mkdirp([this.opts.images, this.opts.mount]); + await chmod(["a+rx", this.opts.mount]); + await exec({ + command: "sudo", + args: ["truncate", "-s", DEFAULT_SIZE, this.image], + }); + await exec({ + command: "sudo", + args: [ + "zpool", + "create", + "-o", + "feature@fast_dedup=enabled", + "-m", + this.opts.mount, + this.opts.name, + this.image, + ], + desc: `create the pool ${this.opts.name} using the device ${this.image}`, + }); + await exec({ + command: "sudo", + args: [ + "zfs", + "set", + "-o", + "compression=lz4", + "-o", + "dedup=on", + this.opts.name, + ], + }); + }; + + list = async (): Promise => { + const { stdout } = await exec({ + command: "zpool", + args: ["list", "-j", "--json-int", this.opts.name], + }); + const x = JSON.parse(stdout); + const y = x.pools[this.opts.name]; + for (const a in y.properties) { + y.properties[a] = y.properties[a].value; + } + y.properties.dedupratio = parseFloat(y.properties.dedupratio); + return y; + }; + + trim = async () => { + await exec({ + command: "sudo", + args: ["zpool", "trim", "-w", this.opts.name], + }); + }; + + // bytes of disk used by image + bytes = async (): Promise => { + const { stdout } = await exec({ + command: "sudo", + args: ["ls", "-s", this.image], + }); + return parseFloat(stdout.split(" ")[0]); + }; + + close = () => { + // nothing, yet + }; +} + +const cache = refCache({ + name: "zfs-pool", + createObject: async (options: Options) => { + return new Pool(options); + }, +}); + +export async function pool( + options: Options & { noCache?: boolean }, +): Promise { + return await cache(options); +} + +interface PoolListOutput { + name: string; + type: "POOL"; + state: "ONLINE" | string; // todo + pool_guid: number; + txg: number; + spa_version: number; + zpl_version: number; + properties: { + size: number; + allocated: number; + free: number; + checkpoint: string; + expandsize: string; + fragmentation: number; + capacity: number; + dedupratio: number; + health: "ONLINE" | string; // todo + altroot: string; + }; +} diff --git a/src/packages/file-server/storage/pools.ts b/src/packages/file-server/storage/pools.ts new file mode 100644 index 0000000000..d423a65cbb --- /dev/null +++ b/src/packages/file-server/storage/pools.ts @@ -0,0 +1,62 @@ +/* +DEVELOPMENT: + +Start node, then: + +a = require('@cocalc/file-server/storage') +pools = await a.pools({images:'/data/zfs/images', mount:'/data/zfs/mnt'}) + +x = await pools.pool({name:'x'}) +await x.create() + +t = await x.list() + + +*/ + +import refCache from "@cocalc/util/refcache"; +import { join } from "path"; +import { mkdirp } from "./util"; +import { pool } from "./pool"; + +export interface Options { + images: string; + mount: string; +} + +export class Pools { + private opts: Options; + + constructor(opts: Options) { + this.opts = opts; + } + + init = async () => { + await mkdirp([this.opts.images, this.opts.mount]); + }; + + close = () => { + // nothing, yet + }; + + pool = async ({ name }: { name: string }) => { + const images = join(this.opts.images, name); + const mount = join(this.opts.mount, name); + return await pool({ images, mount, name }); + }; +} + +const cache = refCache({ + name: "zfs-pools", + createObject: async (options: Options) => { + const pools = new Pools(options); + await pools.init(); + return pools; + }, +}); + +export async function pools( + options: Options & { noCache?: boolean }, +): Promise { + return await cache(options); +} diff --git a/src/packages/file-server/storage/util.ts b/src/packages/file-server/storage/util.ts new file mode 100644 index 0000000000..3f76d319dd --- /dev/null +++ b/src/packages/file-server/storage/util.ts @@ -0,0 +1,40 @@ +import { + type ExecuteCodeOptions, + type ExecuteCodeOutput, +} from "@cocalc/util/types/execute-code"; +import { executeCode } from "@cocalc/backend/execute-code"; +import getLogger from "@cocalc/backend/logger"; + +const DEFAULT_EXEC_TIMEOUT_MS = 60 * 1000; + +const logger = getLogger("file-server:storage:util"); + +export async function exists(path: string) { + try { + await exec({ command: "sudo", args: ["ls", path] }); + return true; + } catch { + return false; + } +} + +export async function mkdirp(paths: string[]) { + await exec({ command: "sudo", args: ["mkdir", "-p", ...paths] }); +} + +export async function chmod(args: string[]) { + await exec({ command: "sudo", args: ["chmod", ...args] }); +} + +export async function exec( + opts: ExecuteCodeOptions & { desc?: string }, +): Promise { + if (opts.verbose !== false && opts.desc) { + logger.debug("exec", opts.desc); + } + return await executeCode({ + verbose: true, + timeout: DEFAULT_EXEC_TIMEOUT_MS / 1000, + ...opts, + }); +} From 73cc777a167bdfc0ef3771f8b16f517ef45a1d51 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 2 May 2025 17:42:15 +0000 Subject: [PATCH 03/47] fileserver storage -- add filesystem --- .../file-server/storage/filesystem.ts | 71 +++++++++++++++++++ src/packages/file-server/storage/pool.ts | 27 ++++--- src/packages/file-server/storage/pools.ts | 3 + 3 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 src/packages/file-server/storage/filesystem.ts diff --git a/src/packages/file-server/storage/filesystem.ts b/src/packages/file-server/storage/filesystem.ts new file mode 100644 index 0000000000..ce1594db45 --- /dev/null +++ b/src/packages/file-server/storage/filesystem.ts @@ -0,0 +1,71 @@ +import refCache from "@cocalc/util/refcache"; +import { exec } from "./util"; + +const FILESYSTEM_NAME_REGEXP = /^(?!-)(?!(\.{1,2})$)[A-Za-z0-9_.:-]{1,255}$/; + +export interface Options { + // name of pool + pool: string; + // name of filesystem + name: string; +} + +export class Filesystem { + private dataset: string; + + constructor(opts: Options) { + if (!FILESYSTEM_NAME_REGEXP.test(opts.name)) { + throw Error(`invalid ZFS filesystem name '${opts.name}'`); + } + this.dataset = `${opts.pool}/${opts.name}`; + } + + exists = async () => { + try { + await this.list(); + return true; + } catch { + return false; + } + }; + + create = async () => { + if (await this.exists()) { + return; + } + await exec({ + command: "sudo", + args: ["zfs", "create", this.dataset], + }); + }; + + list = async (): Promise => { + const { stdout } = await exec({ + command: "zfs", + args: ["list", "-j", "--json-int", this.dataset], + }); + const x = JSON.parse(stdout); + const y = x.datasets[this.dataset]; + for (const a in y.properties) { + y.properties[a] = y.properties[a].value; + } + return y; + }; + + close = () => { + // nothing, yet + }; +} + +const cache = refCache({ + name: "zfs-filesystem", + createObject: async (options: Options) => { + return new Filesystem(options); + }, +}); + +export async function filesystem( + options: Options & { noCache?: boolean }, +): Promise { + return await cache(options); +} diff --git a/src/packages/file-server/storage/pool.ts b/src/packages/file-server/storage/pool.ts index 2d3148f8d7..581bb90a6e 100644 --- a/src/packages/file-server/storage/pool.ts +++ b/src/packages/file-server/storage/pool.ts @@ -1,6 +1,7 @@ import refCache from "@cocalc/util/refcache"; import { chmod, exec, exists, mkdirp } from "./util"; import { join } from "path"; +import { filesystem } from "./filesystem"; const DEFAULT_SIZE = "10G"; const POOL_NAME_REGEXP = /^(?!-)(?!(\.{1,2})$)[A-Za-z0-9_.:-]{1,255}$/; @@ -46,6 +47,22 @@ export class Pool { } await exec({ command: "sudo", args: ["rm", this.image] }); await exec({ command: "sudo", args: ["rmdir", this.opts.images] }); + await exec({ command: "sudo", args: ["rmdir", this.opts.mount] }); + }; + + filesystem = async ({ name }: { name: string }) => { + return await filesystem({ pool: this.opts.name, name }); + }; + + import = async () => { + if (!(await this.exists())) { + await this.create(); + return; + } + await exec({ + command: "sudo", + args: ["zpool", "import", this.opts.name, "-d", this.opts.images], + }); }; create = async () => { @@ -75,15 +92,7 @@ export class Pool { }); await exec({ command: "sudo", - args: [ - "zfs", - "set", - "-o", - "compression=lz4", - "-o", - "dedup=on", - this.opts.name, - ], + args: ["zfs", "set", "compression=lz4", "dedup=on", this.opts.name], }); }; diff --git a/src/packages/file-server/storage/pools.ts b/src/packages/file-server/storage/pools.ts index d423a65cbb..854dd410de 100644 --- a/src/packages/file-server/storage/pools.ts +++ b/src/packages/file-server/storage/pools.ts @@ -11,7 +11,10 @@ await x.create() t = await x.list() +p = await x.filesystem({name:'puppa'}) +await p.create() +await p.list() */ import refCache from "@cocalc/util/refcache"; From 117328b3cc2c9609efb1d6b8858baf9b05c75471 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 2 May 2025 19:16:25 +0000 Subject: [PATCH 04/47] more making file storage nice --- .../file-server/storage/filesystem.ts | 32 +++++-- src/packages/file-server/storage/pool.ts | 90 +++++++++++-------- src/packages/file-server/storage/pools.ts | 16 +++- src/packages/file-server/storage/util.ts | 26 +++++- 4 files changed, 117 insertions(+), 47 deletions(-) diff --git a/src/packages/file-server/storage/filesystem.ts b/src/packages/file-server/storage/filesystem.ts index ce1594db45..a89c9d4ea8 100644 --- a/src/packages/file-server/storage/filesystem.ts +++ b/src/packages/file-server/storage/filesystem.ts @@ -1,5 +1,5 @@ import refCache from "@cocalc/util/refcache"; -import { exec } from "./util"; +import { sudo } from "./util"; const FILESYSTEM_NAME_REGEXP = /^(?!-)(?!(\.{1,2})$)[A-Za-z0-9_.:-]{1,255}$/; @@ -22,25 +22,41 @@ export class Filesystem { exists = async () => { try { - await this.list(); + await this.list0(); return true; } catch { return false; } }; + private async ensureExists(f: () => Promise): Promise { + try { + return await f(); + } catch (err) { + if (`${err}`.includes("dataset does not exist")) { + await this.create(); + return await f(); + } + } + throw Error("bug"); + } + create = async () => { if (await this.exists()) { return; } - await exec({ - command: "sudo", - args: ["zfs", "create", this.dataset], + await sudo({ + command: "zfs", + args: ["create", this.dataset], }); }; - list = async (): Promise => { - const { stdout } = await exec({ + list = async (): Promise => { + return await this.ensureExists(this.list0); + }; + + private list0 = async (): Promise => { + const { stdout } = await sudo({ command: "zfs", args: ["list", "-j", "--json-int", this.dataset], }); @@ -57,6 +73,8 @@ export class Filesystem { }; } +interface FilesystemListOutput {} + const cache = refCache({ name: "zfs-filesystem", createObject: async (options: Options) => { diff --git a/src/packages/file-server/storage/pool.ts b/src/packages/file-server/storage/pool.ts index 581bb90a6e..1064a604f6 100644 --- a/src/packages/file-server/storage/pool.ts +++ b/src/packages/file-server/storage/pool.ts @@ -1,5 +1,5 @@ import refCache from "@cocalc/util/refcache"; -import { chmod, exec, exists, mkdirp } from "./util"; +import { chmod, sudo, exists, mkdirp, rm, rmdir, listdir } from "./util"; import { join } from "path"; import { filesystem } from "./filesystem"; @@ -36,21 +36,26 @@ export class Pool { return; } try { - await exec({ - command: "sudo", - args: ["zpool", "destroy", "-f", this.opts.name], + await sudo({ + command: "zpool", + args: ["destroy", "-f", this.opts.name], }); } catch (err) { if (!`${err}`.includes("no such pool")) { throw err; } } - await exec({ command: "sudo", args: ["rm", this.image] }); - await exec({ command: "sudo", args: ["rmdir", this.opts.images] }); - await exec({ command: "sudo", args: ["rmdir", this.opts.mount] }); + await rm([this.image]); + await rmdir([this.opts.images]); + if (await exists(this.opts.mount)) { + await rmdir(await listdir(this.opts.mount)); + await rmdir([this.opts.mount]); + } }; filesystem = async ({ name }: { name: string }) => { + // ensure available + await this.list(); return await filesystem({ pool: this.opts.name, name }); }; @@ -59,9 +64,9 @@ export class Pool { await this.create(); return; } - await exec({ - command: "sudo", - args: ["zpool", "import", this.opts.name, "-d", this.opts.images], + await sudo({ + command: "zpool", + args: ["import", this.opts.name, "-d", this.opts.images], }); }; @@ -72,14 +77,13 @@ export class Pool { } await mkdirp([this.opts.images, this.opts.mount]); await chmod(["a+rx", this.opts.mount]); - await exec({ - command: "sudo", - args: ["truncate", "-s", DEFAULT_SIZE, this.image], + await sudo({ + command: "truncate", + args: ["-s", DEFAULT_SIZE, this.image], }); - await exec({ - command: "sudo", + await sudo({ + command: "zpool", args: [ - "zpool", "create", "-o", "feature@fast_dedup=enabled", @@ -90,38 +94,54 @@ export class Pool { ], desc: `create the pool ${this.opts.name} using the device ${this.image}`, }); - await exec({ - command: "sudo", - args: ["zfs", "set", "compression=lz4", "dedup=on", this.opts.name], + await sudo({ + command: "zfs", + args: ["set", "compression=lz4", "dedup=on", this.opts.name], }); }; + private async ensureExists(f: () => Promise): Promise { + try { + return await f(); + } catch (err) { + if (`${err}`.includes("no such pool")) { + await this.import(); + return await f(); + } + } + throw Error("bug"); + } + list = async (): Promise => { - const { stdout } = await exec({ - command: "zpool", - args: ["list", "-j", "--json-int", this.opts.name], + return await this.ensureExists(async () => { + const { stdout } = await sudo({ + command: "zpool", + args: ["list", "-j", "--json-int", this.opts.name], + }); + const x = JSON.parse(stdout); + const y = x.pools[this.opts.name]; + for (const a in y.properties) { + y.properties[a] = y.properties[a].value; + } + y.properties.dedupratio = parseFloat(y.properties.dedupratio); + return y; }); - const x = JSON.parse(stdout); - const y = x.pools[this.opts.name]; - for (const a in y.properties) { - y.properties[a] = y.properties[a].value; - } - y.properties.dedupratio = parseFloat(y.properties.dedupratio); - return y; }; trim = async () => { - await exec({ - command: "sudo", - args: ["zpool", "trim", "-w", this.opts.name], + return await this.ensureExists(async () => { + await sudo({ + command: "zpool", + args: ["trim", "-w", this.opts.name], + }); }); }; // bytes of disk used by image bytes = async (): Promise => { - const { stdout } = await exec({ - command: "sudo", - args: ["ls", "-s", this.image], + const { stdout } = await sudo({ + command: "ls", + args: ["-s", this.image], }); return parseFloat(stdout.split(" ")[0]); }; diff --git a/src/packages/file-server/storage/pools.ts b/src/packages/file-server/storage/pools.ts index 854dd410de..6f33f44f52 100644 --- a/src/packages/file-server/storage/pools.ts +++ b/src/packages/file-server/storage/pools.ts @@ -15,11 +15,21 @@ p = await x.filesystem({name:'puppa'}) await p.create() await p.list() + +// around 10 seconds: + +t = Date.now(); for(let i=0; i<100; i++) { await (await pools.pool({name:'x'+i})).create() }; Date.now() - t + +// around 5 seconds: + +t = Date.now(); for(let i=0; i<100; i++) { await (await x.filesystem({name:'x'+i})).create() }; Date.now() - t + + */ import refCache from "@cocalc/util/refcache"; import { join } from "path"; -import { mkdirp } from "./util"; +import { listdir, mkdirp } from "./util"; import { pool } from "./pool"; export interface Options { @@ -47,6 +57,10 @@ export class Pools { const mount = join(this.opts.mount, name); return await pool({ images, mount, name }); }; + + list = async (): Promise => { + return await listdir(this.opts.images); + }; } const cache = refCache({ diff --git a/src/packages/file-server/storage/util.ts b/src/packages/file-server/storage/util.ts index 3f76d319dd..90504ecea0 100644 --- a/src/packages/file-server/storage/util.ts +++ b/src/packages/file-server/storage/util.ts @@ -11,7 +11,7 @@ const logger = getLogger("file-server:storage:util"); export async function exists(path: string) { try { - await exec({ command: "sudo", args: ["ls", path] }); + await sudo({ command: "ls", args: [path] }); return true; } catch { return false; @@ -19,14 +19,15 @@ export async function exists(path: string) { } export async function mkdirp(paths: string[]) { - await exec({ command: "sudo", args: ["mkdir", "-p", ...paths] }); + if (paths.length == 0) return; + await sudo({ command: "mkdir", args: ["-p", ...paths] }); } export async function chmod(args: string[]) { - await exec({ command: "sudo", args: ["chmod", ...args] }); + await sudo({ command: "chmod", args: args }); } -export async function exec( +export async function sudo( opts: ExecuteCodeOptions & { desc?: string }, ): Promise { if (opts.verbose !== false && opts.desc) { @@ -36,5 +37,22 @@ export async function exec( verbose: true, timeout: DEFAULT_EXEC_TIMEOUT_MS / 1000, ...opts, + command: "sudo", + args: [opts.command, ...(opts.args ?? [])], }); } + +export async function rm(paths: string[]) { + if (paths.length == 0) return; + await sudo({ command: "rm", args: paths }); +} + +export async function rmdir(paths: string[]) { + if (paths.length == 0) return; + await sudo({ command: "rmdir", args: paths }); +} + +export async function listdir(path: string) { + const { stdout } = await sudo({ command: "ls", args: ["-1", path] }); + return stdout.split("\n").filter((x) => x); +} From 2c1677959d1aa267fc29786193dff975cf6f6da6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 2 May 2025 19:54:49 +0000 Subject: [PATCH 05/47] zfs filesystems: more support -- get, set, cloning --- .../file-server/storage/filesystem.ts | 79 +++++++++++++++++-- src/packages/file-server/storage/pool.ts | 4 +- src/packages/file-server/storage/pools.ts | 8 +- 3 files changed, 83 insertions(+), 8 deletions(-) diff --git a/src/packages/file-server/storage/filesystem.ts b/src/packages/file-server/storage/filesystem.ts index a89c9d4ea8..f13d61c974 100644 --- a/src/packages/file-server/storage/filesystem.ts +++ b/src/packages/file-server/storage/filesystem.ts @@ -8,15 +8,20 @@ export interface Options { pool: string; // name of filesystem name: string; + // if given when creating the current filesystem, it will be made as a clone + // of "clone". This only needs to be set when the filesystem is created. + clone?: string; } export class Filesystem { private dataset: string; + private opts: Options; constructor(opts: Options) { if (!FILESYSTEM_NAME_REGEXP.test(opts.name)) { throw Error(`invalid ZFS filesystem name '${opts.name}'`); } + this.opts = opts; this.dataset = `${opts.pool}/${opts.name}`; } @@ -45,10 +50,31 @@ export class Filesystem { if (await this.exists()) { return; } - await sudo({ - command: "zfs", - args: ["create", this.dataset], - }); + if (this.opts.clone) { + const snapshot = `${this.opts.pool}/${this.opts.clone}@clone-${this.opts.name}`; + await sudo({ + command: "zfs", + args: ["snapshot", snapshot], + }); + try { + // create as a clone + await sudo({ + command: "zfs", + args: ["clone", snapshot, this.dataset], + }); + } catch (err) { + // we only delete the snapshot on error, since it can't be deleted as + // long as the clone exists: + await sudo({ command: "zfs", args: ["destroy", snapshot] }); + throw err; + } + } else { + // non-clone + await sudo({ + command: "zfs", + args: ["create", this.dataset], + }); + } }; list = async (): Promise => { @@ -68,12 +94,55 @@ export class Filesystem { return y; }; + get = async (property: string) => { + return await this.ensureExists(async () => { + const { stdout } = await sudo({ + command: "zfs", + args: ["get", "-j", "--json-int", property, this.dataset], + }); + const x = JSON.parse(stdout); + const { value } = x.datasets[this.dataset].properties[property]; + if (/^-?\d+(\.\d+)?$/.test(value)) { + return parseFloat(value); + } else { + return value; + } + }); + }; + + set = async (props: { [property: string]: any }) => { + return await this.ensureExists(async () => { + const v: string[] = []; + for (const p in props) { + v.push(`${p}=${props[p]}`); + } + if (v.length == 0) { + return; + } + await sudo({ + command: "zfs", + args: ["set", ...v, this.dataset], + }); + }); + }; + close = () => { // nothing, yet }; } -interface FilesystemListOutput {} +interface FilesystemListOutput { + name: string; + type: "FILESYSTEM"; + pool: string; + createtxg: number; + properties: { + used: number; + available: number; + referenced: number; + mountpoint: string; + }; +} const cache = refCache({ name: "zfs-filesystem", diff --git a/src/packages/file-server/storage/pool.ts b/src/packages/file-server/storage/pool.ts index 1064a604f6..17ce5a6592 100644 --- a/src/packages/file-server/storage/pool.ts +++ b/src/packages/file-server/storage/pool.ts @@ -53,10 +53,10 @@ export class Pool { } }; - filesystem = async ({ name }: { name: string }) => { + filesystem = async ({ name, clone }: { name: string; clone?: string }) => { // ensure available await this.list(); - return await filesystem({ pool: this.opts.name, name }); + return await filesystem({ pool: this.opts.name, name, clone }); }; import = async () => { diff --git a/src/packages/file-server/storage/pools.ts b/src/packages/file-server/storage/pools.ts index 6f33f44f52..6a3842861c 100644 --- a/src/packages/file-server/storage/pools.ts +++ b/src/packages/file-server/storage/pools.ts @@ -11,11 +11,17 @@ await x.create() t = await x.list() -p = await x.filesystem({name:'puppa'}) +p = await x.filesystem({name:'1'}) await p.create() +await p.get('compressratio') + await p.list() +q = await x.filesystem({name:'c', clone:'1'}) +await q.create() +await q.get('origin') // --> 'x/1@clone-c' + // around 10 seconds: t = Date.now(); for(let i=0; i<100; i++) { await (await pools.pool({name:'x'+i})).create() }; Date.now() - t From 1dbe50bcda103afa551f54c99dec94941fe949cf Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 2 May 2025 21:21:30 +0000 Subject: [PATCH 06/47] fileserver -- rolling snapshots --- .../file-server/storage/filesystem.ts | 107 ++++++++++++++-- src/packages/file-server/storage/pool.ts | 44 ++++++- src/packages/file-server/storage/snapshots.ts | 115 ++++++++++++++++++ src/packages/file-server/storage/util.ts | 4 +- 4 files changed, 260 insertions(+), 10 deletions(-) create mode 100644 src/packages/file-server/storage/snapshots.ts diff --git a/src/packages/file-server/storage/filesystem.ts b/src/packages/file-server/storage/filesystem.ts index f13d61c974..ff1f79d210 100644 --- a/src/packages/file-server/storage/filesystem.ts +++ b/src/packages/file-server/storage/filesystem.ts @@ -1,5 +1,9 @@ import refCache from "@cocalc/util/refcache"; import { sudo } from "./util"; +import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("file-server:storage:filesystem"); const FILESYSTEM_NAME_REGEXP = /^(?!-)(?!(\.{1,2})$)[A-Za-z0-9_.:-]{1,255}$/; @@ -14,20 +18,20 @@ export interface Options { } export class Filesystem { - private dataset: string; + public readonly dataset: string; private opts: Options; constructor(opts: Options) { - if (!FILESYSTEM_NAME_REGEXP.test(opts.name)) { + if (opts.name !== "" && !FILESYSTEM_NAME_REGEXP.test(opts.name)) { throw Error(`invalid ZFS filesystem name '${opts.name}'`); } this.opts = opts; - this.dataset = `${opts.pool}/${opts.name}`; + this.dataset = opts.name ? `${opts.pool}/${opts.name}` : opts.pool; } exists = async () => { try { - await this.list0(); + await this._info(); return true; } catch { return false; @@ -42,6 +46,7 @@ export class Filesystem { await this.create(); return await f(); } + throw err; } throw Error("bug"); } @@ -51,6 +56,10 @@ export class Filesystem { return; } if (this.opts.clone) { + logger.debug("create clone", { + dataset: this.dataset, + clone: this.opts.clone, + }); const snapshot = `${this.opts.pool}/${this.opts.clone}@clone-${this.opts.name}`; await sudo({ command: "zfs", @@ -69,6 +78,9 @@ export class Filesystem { throw err; } } else { + logger.debug("create dataset", { + dataset: this.dataset, + }); // non-clone await sudo({ command: "zfs", @@ -77,11 +89,11 @@ export class Filesystem { } }; - list = async (): Promise => { - return await this.ensureExists(this.list0); + info = async (): Promise => { + return await this.ensureExists(this._info); }; - private list0 = async (): Promise => { + private _info = async (): Promise => { const { stdout } = await sudo({ command: "zfs", args: ["list", "-j", "--json-int", this.dataset], @@ -129,6 +141,70 @@ export class Filesystem { close = () => { // nothing, yet }; + + createSnapshot = async (name: string) => { + logger.debug("createSnapshot", { name, dataset: this.dataset }); + await this.ensureExists(async () => { + await sudo({ + command: "zfs", + args: ["snapshot", `${this.dataset}@${name}`], + }); + }); + }; + + snapshots = async (): Promise => { + return await this.ensureExists(async () => { + const { stdout } = await sudo({ + command: "zfs", + args: [ + "list", + "-j", + "--json-int", + "-r", + "-d", + "1", + "-t", + "snapshot", + `${this.dataset}`, + ], + }); + const { datasets } = JSON.parse(stdout); + for (const name in datasets) { + const y = datasets[name]; + for (const a in y.properties) { + y.properties[a] = y.properties[a].value; + } + } + return datasets; + }); + }; + + destroySnapshot = async (name) => { + logger.debug("destroySnapshot", { name, dataset: this.dataset }); + await this.ensureExists(async () => { + await sudo({ + command: "zfs", + args: ["destroy", `${this.dataset}@${name}`], + }); + }); + }; + + updateRollingSnapshots = async (counts?: Partial) => { + return await this.ensureExists(async () => { + return await updateRollingSnapshots({ filesystem: this, counts }); + }); + }; + + // number of newly written bytes in filesystem since last snapshot + writtenSinceLastSnapshot = async (): Promise => { + return await this.ensureExists(async () => { + const { stdout } = await sudo({ + command: "zfs", + args: ["list", "-Hpo", "written", this.dataset], + }); + return parseInt(stdout); + }); + }; } interface FilesystemListOutput { @@ -144,6 +220,23 @@ interface FilesystemListOutput { }; } +interface Snapshot { + name: string; + type: "SNAPSHOT"; + pool: string; + createtxg: number; + dataset: string; + snapshot_name: string; + properties: { + used: number; + available: string | number; + referenced: number; + mountpoint: string; // '-' if not mounted + }; +} + +export type Snapshots = { [name: string]: Snapshot }; + const cache = refCache({ name: "zfs-filesystem", createObject: async (options: Options) => { diff --git a/src/packages/file-server/storage/pool.ts b/src/packages/file-server/storage/pool.ts index 17ce5a6592..582226c4e7 100644 --- a/src/packages/file-server/storage/pool.ts +++ b/src/packages/file-server/storage/pool.ts @@ -108,11 +108,12 @@ export class Pool { await this.import(); return await f(); } + throw err; } throw Error("bug"); } - list = async (): Promise => { + info = async (): Promise => { return await this.ensureExists(async () => { const { stdout } = await sudo({ command: "zpool", @@ -128,6 +129,17 @@ export class Pool { }); }; + status = async (): Promise => { + return await this.ensureExists(async () => { + const { stdout } = await sudo({ + command: "zpool", + args: ["status", "-j", "--json-int", this.opts.name], + }); + const x = JSON.parse(stdout); + return x.pools[this.opts.name]; + }); + }; + trim = async () => { return await this.ensureExists(async () => { await sudo({ @@ -146,11 +158,41 @@ export class Pool { return parseFloat(stdout.split(" ")[0]); }; + list = async (): Promise<{ [dataset: string]: Dataset }> => { + return await this.ensureExists<{ [dataset: string]: Dataset }>(async () => { + const { stdout } = await sudo({ + command: "zfs", + args: ["list", "-j", "--json-int", "-r", this.opts.name], + }); + const { datasets } = JSON.parse(stdout); + for (const name in datasets) { + const y = datasets[name]; + for (const a in y.properties) { + y.properties[a] = y.properties[a].value; + } + } + return datasets; + }); + }; + close = () => { // nothing, yet }; } +interface Dataset { + name: string; + type: "FILESYSTEM"; + pool: string; + createtxg: number; + properties: { + used: number; + available: number; + referenced: number; + mountpoint: string; + }; +} + const cache = refCache({ name: "zfs-pool", createObject: async (options: Options) => { diff --git a/src/packages/file-server/storage/snapshots.ts b/src/packages/file-server/storage/snapshots.ts new file mode 100644 index 0000000000..11f28a890e --- /dev/null +++ b/src/packages/file-server/storage/snapshots.ts @@ -0,0 +1,115 @@ +import { type Filesystem } from "./filesystem"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("file-server:storage:snapshots"); + +const DATE_REGEXP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; + +// Lengths of time in minutes to keep snapshots +// (code below assumes these are listed in ORDER from shortest to longest) +export const SNAPSHOT_INTERVALS_MS = { + frequent: 15 * 1000 * 60, + daily: 60 * 24 * 1000 * 60, + weekly: 60 * 24 * 7 * 1000 * 60, + monthly: 60 * 24 * 7 * 4 * 1000 * 60, +}; + +// How many of each type of snapshot to retain +export const DEFAULT_SNAPSHOT_COUNTS = { + frequent: 24, + daily: 14, + weekly: 7, + monthly: 4, +} as SnapshotCounts; + +export interface SnapshotCounts { + frequent: number; + daily: number; + weekly: number; + monthly: number; +} + +export async function updateRollingSnapshots({ + filesystem, + counts, +}: { + filesystem: Filesystem; + counts?: Partial; +}) { + counts = { ...DEFAULT_SNAPSHOT_COUNTS, ...counts }; + + const written = await filesystem.writtenSinceLastSnapshot(); + logger.debug("updateRollingSnapshots", { + dataset: filesystem.dataset, + counts, + written, + }); + if (written == 0) { + // definitely no data written since most recent snapshot, so nothing to do + return; + } + + // get exactly the iso timestamp snapshot names: + const snapshots = Object.values(await filesystem.snapshots()) + .map((z) => z.snapshot_name) + .filter((x) => DATE_REGEXP.test(x)); + snapshots.sort(); + if (snapshots.length > 0) { + const age = Date.now() - new Date(snapshots.slice(-1)[0]).valueOf(); + for (const key in SNAPSHOT_INTERVALS_MS) { + if (counts[key]) { + if (age < SNAPSHOT_INTERVALS_MS[key]) { + // no need to snapshot since there is already a sufficiently recent snapshot + logger.debug("updateRollingSnapshots: no need to snapshot", { + dataset: filesystem.dataset, + }); + return; + } + // counts[key] nonzero and snapshot is old enough so we'll be making a snapshot + break; + } + } + } + + // make a new snapshot + const snapshot = new Date().toISOString(); + await filesystem.createSnapshot(snapshot); + // delete extra snapshots + snapshots.push(snapshot); + const toDelete = snapshotsToDelete({ counts, snapshots }); + for (const snapshot of toDelete) { + await filesystem.destroySnapshot(snapshot); + } +} + +function snapshotsToDelete({ counts, snapshots }): string[] { + if (snapshots.length == 0) { + // nothing to do + return []; + } + + // sorted from BIGGEST to smallest + const times = snapshots.map((x) => new Date(x).valueOf()); + times.reverse(); + const save = new Set(); + for (const type in counts) { + const count = counts[type]; + const length_ms = SNAPSHOT_INTERVALS_MS[type]; + + // Pick the first count newest snapshots at intervals of length + // length_ms milliseconds. + let n = 0, + i = 0, + last_tm = 0; + while (n < count && i < times.length) { + const tm = times[i]; + if (!last_tm || tm <= last_tm - length_ms) { + save.add(tm); + last_tm = tm; + n += 1; // found one more + } + i += 1; // move to next snapshot + } + } + return snapshots.filter((x) => !save.has(new Date(x).valueOf())); +} diff --git a/src/packages/file-server/storage/util.ts b/src/packages/file-server/storage/util.ts index 90504ecea0..4d95dcaf48 100644 --- a/src/packages/file-server/storage/util.ts +++ b/src/packages/file-server/storage/util.ts @@ -5,10 +5,10 @@ import { import { executeCode } from "@cocalc/backend/execute-code"; import getLogger from "@cocalc/backend/logger"; -const DEFAULT_EXEC_TIMEOUT_MS = 60 * 1000; - const logger = getLogger("file-server:storage:util"); +const DEFAULT_EXEC_TIMEOUT_MS = 60 * 1000; + export async function exists(path: string) { try { await sudo({ command: "ls", args: [path] }); From d25cde430cdd4e7cc1b5ba82bf34010c8a67c991 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 2 May 2025 21:38:04 +0000 Subject: [PATCH 07/47] fileserver -- expand pool --- src/packages/file-server/storage/pool.ts | 39 +++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/packages/file-server/storage/pool.ts b/src/packages/file-server/storage/pool.ts index 582226c4e7..7e163d5b6e 100644 --- a/src/packages/file-server/storage/pool.ts +++ b/src/packages/file-server/storage/pool.ts @@ -2,8 +2,12 @@ import refCache from "@cocalc/util/refcache"; import { chmod, sudo, exists, mkdirp, rm, rmdir, listdir } from "./util"; import { join } from "path"; import { filesystem } from "./filesystem"; +import getLogger from "@cocalc/backend/logger"; +import { executeCode } from "@cocalc/backend/execute-code"; -const DEFAULT_SIZE = "10G"; +const logger = getLogger("file-server:storage:pool"); + +const DEFAULT_SIZE = "1G"; const POOL_NAME_REGEXP = /^(?!-)(?!(\.{1,2})$)[A-Za-z0-9_.:-]{1,255}$/; export interface Options { @@ -53,6 +57,39 @@ export class Pool { } }; + expand = async (size: string | number) => { + logger.debug(`expand to ${size}`); + if (typeof size == "string") { + // convert to bytes + const { stdout } = await executeCode({ + command: "numfmt", + args: ["--from=iec", size], + }); + size = parseFloat(stdout); + } + if (!(await exists(this.image))) { + await this.create(); + } + const { stdout } = await sudo({ + command: "stat", + args: ["--format=%s", this.image], + }); + const bytes = parseFloat(stdout); + if (size < bytes) { + throw Error(`size must be at least ${bytes}`); + } + if (size == bytes) { + return; + } + await this.ensureExists(async () => { + await sudo({ command: "truncate", args: ["-s", size, this.image] }); + await sudo({ + command: "zpool", + args: ["online", "-e", this.opts.name, this.image], + }); + }); + }; + filesystem = async ({ name, clone }: { name: string; clone?: string }) => { // ensure available await this.list(); From 3035217f3f28835f3f6008e1e9d9c39c9f78a793 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 2 May 2025 22:33:34 +0000 Subject: [PATCH 08/47] filesystem: implemented shrink --- src/packages/file-server/storage/pool.ts | 138 +++++++++++++++++++--- src/packages/file-server/storage/pools.ts | 4 + 2 files changed, 126 insertions(+), 16 deletions(-) diff --git a/src/packages/file-server/storage/pool.ts b/src/packages/file-server/storage/pool.ts index 7e163d5b6e..824f3d5073 100644 --- a/src/packages/file-server/storage/pool.ts +++ b/src/packages/file-server/storage/pool.ts @@ -4,6 +4,7 @@ import { join } from "path"; import { filesystem } from "./filesystem"; import getLogger from "@cocalc/backend/logger"; import { executeCode } from "@cocalc/backend/execute-code"; +import { randomId } from "@cocalc/nats/names"; const logger = getLogger("file-server:storage:pool"); @@ -36,9 +37,6 @@ export class Pool { }; destroy = async () => { - if (!(await this.exists())) { - return; - } try { await sudo({ command: "zpool", @@ -49,23 +47,26 @@ export class Pool { throw err; } } - await rm([this.image]); - await rmdir([this.opts.images]); + if (await exists(this.image)) { + await rm([this.image]); + } + if (await exists(this.opts.images)) { + await rmdir([this.opts.images]); + } if (await exists(this.opts.mount)) { - await rmdir(await listdir(this.opts.mount)); + const v = await listdir(this.opts.mount); + await rmdir(v.map((x) => join(this.opts.mount, x))); await rmdir([this.opts.mount]); } }; - expand = async (size: string | number) => { - logger.debug(`expand to ${size}`); - if (typeof size == "string") { - // convert to bytes - const { stdout } = await executeCode({ - command: "numfmt", - args: ["--from=iec", size], - }); - size = parseFloat(stdout); + // enlarge pool to have given size (which can be a string like '1G' or + // a number of bytes). This is very fast/CHEAP and can be done live. + enlarge = async (size: string | number) => { + logger.debug(`enlarge to ${size}`); + size = await sizeToBytes(size); + if (typeof size != "number") { + throw Error("bug"); } if (!(await exists(this.image))) { await this.create(); @@ -82,7 +83,7 @@ export class Pool { return; } await this.ensureExists(async () => { - await sudo({ command: "truncate", args: ["-s", size, this.image] }); + await sudo({ command: "truncate", args: ["-s", `${size}`, this.image] }); await sudo({ command: "zpool", args: ["online", "-e", this.opts.name, this.image], @@ -90,6 +91,86 @@ export class Pool { }); }; + // shrink pool to have given size (which can be a string like '1G' or + // a number of bytes). This is EXPENSIVE, requiring rewriting everything, and + // the pool must be unmounted. + shrink = async (size: string | number) => { + // TODO: this is so dangerous, so make sure there is a backup first, once + // backups are implemented + logger.debug(`shrink to ${size}`); + logger.debug("shrink -- 0. size checks"); + size = await sizeToBytes(size); + if (typeof size != "number") { + throw Error("bug"); + } + if (size < (await sizeToBytes(DEFAULT_SIZE))) { + throw Error(`size must be at least ${DEFAULT_SIZE}`); + } + const info = await this.info(); + // TOOD: this is made up + const min_alloc = info.properties.allocated * 1.25 + 1000000; + if (size <= min_alloc) { + throw Error( + `size must be at least as big as currently allocated space ${min_alloc}`, + ); + } + if (size >= info.properties.size) { + logger.debug("shrink -- it's already smaller than the shrink goal."); + return; + } + logger.debug("shrink -- 1. unmount all datasets"); + for (const dataset of Object.keys(await this.list())) { + try { + await sudo({ command: "zfs", args: ["unmount", dataset] }); + } catch (err) { + if (`${err}`.includes("not currently mounted")) { + // that's fine + continue; + } + throw err; + } + } + logger.debug("shrink -- 2. make new smaller temporary pool"); + const id = "-" + randomId(); + const name = this.opts.name + id; + const images = this.opts.images + id; + const mount = this.opts.mount + id; + const temp = await pool({ images, mount, name }); + await temp.create(); + await temp.enlarge(size); + const snapshot = `${this.opts.name}@shrink${id}`; + logger.debug("shrink -- 3. replicate data to target"); + await sudo({ command: "zfs", args: ["snapshot", "-r", snapshot] }); + try { + await executeCode({ + command: `sudo zfs send -c -R ${snapshot} | sudo zfs recv -F ${name}`, + }); + } catch (err) { + await temp.destroy(); + throw err; + } + await temp.export(); + + logger.debug("shrink -- 4. destroy original pool"); + await this.destroy(); + logger.debug("shrink -- 5. rename temporary pool"); + await sudo({ + command: "zpool", + args: ["import", "-d", images, name, this.opts.name], + }); + await sudo({ + command: "zpool", + args: ["export", this.opts.name], + }); + logger.debug("shrink -- 6. move image file"); + await mkdirp([this.opts.images, this.opts.mount]); + await sudo({ command: "mv", args: [temp.image, this.image] }); + logger.debug("shrink -- 7. destroy temp files"); + await temp.destroy(); + logger.debug("shrink -- 8. Import our new pool"); + await this.import(); + }; + filesystem = async ({ name, clone }: { name: string; clone?: string }) => { // ensure available await this.list(); @@ -107,6 +188,20 @@ export class Pool { }); }; + export = async () => { + try { + await sudo({ + command: "zpool", + args: ["export", this.opts.name], + }); + } catch (err) { + if (`${err}`.includes("no such pool")) { + return; + } + throw err; + } + }; + create = async () => { if (await this.exists()) { // already exists @@ -264,3 +359,14 @@ interface PoolListOutput { altroot: string; }; } + +async function sizeToBytes(size: number | string): Promise { + if (typeof size == "number") { + return size; + } + const { stdout } = await executeCode({ + command: "numfmt", + args: ["--from=iec", size], + }); + return parseFloat(stdout); +} diff --git a/src/packages/file-server/storage/pools.ts b/src/packages/file-server/storage/pools.ts index 6a3842861c..72062dda4d 100644 --- a/src/packages/file-server/storage/pools.ts +++ b/src/packages/file-server/storage/pools.ts @@ -14,6 +14,10 @@ t = await x.list() p = await x.filesystem({name:'1'}) await p.create() +await x.enlarge('1T') + +await x.shrink('3G') + await p.get('compressratio') await p.list() From 08544bc91e18045ce495ff9be18071dec6a4bb13 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 2 May 2025 23:44:50 +0000 Subject: [PATCH 09/47] fs: implement rsync with automounting --- src/packages/file-server/storage/pool.ts | 22 +++++++++----- src/packages/file-server/storage/pools.ts | 36 +++++++++++++++++++---- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/packages/file-server/storage/pool.ts b/src/packages/file-server/storage/pool.ts index 824f3d5073..245f5a8df5 100644 --- a/src/packages/file-server/storage/pool.ts +++ b/src/packages/file-server/storage/pool.ts @@ -171,9 +171,8 @@ export class Pool { await this.import(); }; - filesystem = async ({ name, clone }: { name: string; clone?: string }) => { - // ensure available - await this.list(); + filesystem = async (name, { clone }: { clone?: string } = {}) => { + await this.import(); return await filesystem({ pool: this.opts.name, name, clone }); }; @@ -182,10 +181,19 @@ export class Pool { await this.create(); return; } - await sudo({ - command: "zpool", - args: ["import", this.opts.name, "-d", this.opts.images], - }); + try { + await sudo({ + command: "zpool", + args: ["import", this.opts.name, "-d", this.opts.images], + verbose: false, + }); + } catch (err) { + if (`${err}`.includes("pool with that name already exists")) { + // already imported + return; + } + throw err; + } }; export = async () => { diff --git a/src/packages/file-server/storage/pools.ts b/src/packages/file-server/storage/pools.ts index 72062dda4d..7c5bd8b834 100644 --- a/src/packages/file-server/storage/pools.ts +++ b/src/packages/file-server/storage/pools.ts @@ -6,12 +6,11 @@ Start node, then: a = require('@cocalc/file-server/storage') pools = await a.pools({images:'/data/zfs/images', mount:'/data/zfs/mnt'}) -x = await pools.pool({name:'x'}) -await x.create() +x = await pools.pool('x') t = await x.list() -p = await x.filesystem({name:'1'}) +p = await x.filesystem('1') await p.create() await x.enlarge('1T') @@ -22,7 +21,7 @@ await p.get('compressratio') await p.list() -q = await x.filesystem({name:'c', clone:'1'}) +q = await x.filesystem('c', {clone:'1'}) await q.create() await q.get('origin') // --> 'x/1@clone-c' @@ -39,7 +38,7 @@ t = Date.now(); for(let i=0; i<100; i++) { await (await x.filesystem({name:'x'+i import refCache from "@cocalc/util/refcache"; import { join } from "path"; -import { listdir, mkdirp } from "./util"; +import { listdir, mkdirp, sudo } from "./util"; import { pool } from "./pool"; export interface Options { @@ -62,7 +61,7 @@ export class Pools { // nothing, yet }; - pool = async ({ name }: { name: string }) => { + pool = async (name: string) => { const images = join(this.opts.images, name); const mount = join(this.opts.mount, name); return await pool({ images, mount, name }); @@ -71,6 +70,31 @@ export class Pools { list = async (): Promise => { return await listdir(this.opts.images); }; + + rsync = async ({ + src, + target, + args = ["-axH"], + timeout = 5 * 60 * 1000, + }: { + src: string; + target: string; + args?: string[]; + timeout?: number; + }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { + const srcPool = await this.pool(src.split("/")[0]); + await srcPool.import(); + const targetPool = await this.pool(target.split("/")[0]); + await targetPool.import(); + const srcPath = join(this.opts.mount, src); + const targetPath = join(this.opts.mount, target); + return await sudo({ + command: "rsync", + args: [...args, srcPath, targetPath], + err_on_exit: false, + timeout: timeout / 1000, + }); + }; } const cache = refCache({ From a0fc4e10d85c7958705f626eba14b643de417fb5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 2 May 2025 23:55:18 +0000 Subject: [PATCH 10/47] fs: clone function --- src/packages/file-server/storage/pool.ts | 10 ++++++++-- src/packages/file-server/storage/pools.ts | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/packages/file-server/storage/pool.ts b/src/packages/file-server/storage/pool.ts index 245f5a8df5..1df3220844 100644 --- a/src/packages/file-server/storage/pool.ts +++ b/src/packages/file-server/storage/pool.ts @@ -171,9 +171,15 @@ export class Pool { await this.import(); }; - filesystem = async (name, { clone }: { clone?: string } = {}) => { + filesystem = async (name) => { await this.import(); - return await filesystem({ pool: this.opts.name, name, clone }); + return await filesystem({ pool: this.opts.name, name }); + }; + + // create a lightweight clone callend name of the given filesystem source. + clone = async (name: string, source: string) => { + await this.import(); + return await filesystem({ pool: this.opts.name, name, clone: source }); }; import = async () => { diff --git a/src/packages/file-server/storage/pools.ts b/src/packages/file-server/storage/pools.ts index 7c5bd8b834..b766fb7b6d 100644 --- a/src/packages/file-server/storage/pools.ts +++ b/src/packages/file-server/storage/pools.ts @@ -21,17 +21,17 @@ await p.get('compressratio') await p.list() -q = await x.filesystem('c', {clone:'1'}) +q = await x.clone('c', '1') await q.create() await q.get('origin') // --> 'x/1@clone-c' // around 10 seconds: -t = Date.now(); for(let i=0; i<100; i++) { await (await pools.pool({name:'x'+i})).create() }; Date.now() - t +t = Date.now(); for(let i=0; i<100; i++) { await (await pools.pool('x'+i)).create() }; Date.now() - t // around 5 seconds: -t = Date.now(); for(let i=0; i<100; i++) { await (await x.filesystem({name:'x'+i})).create() }; Date.now() - t +t = Date.now(); for(let i=0; i<100; i++) { await (await x.filesystem('x'+i)).create() }; Date.now() - t */ From 4815c9877236af5157e6d92ec661978594d17acd Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 4 May 2025 14:43:10 +0000 Subject: [PATCH 11/47] start btrfs version of storage --- src/packages/file-server/package.json | 6 +- .../file-server/storage-btrfs/filesystem.ts | 182 ++++++++++++++++++ .../file-server/storage-btrfs/index.ts | 1 + .../file-server/storage-btrfs/subvolume.ts | 67 +++++++ .../{storage => storage-zfs}/README.md | 0 .../{storage => storage-zfs}/filesystem.ts | 0 .../{storage => storage-zfs}/index.ts | 0 .../{storage => storage-zfs}/pool.ts | 0 .../{storage => storage-zfs}/pools.ts | 2 +- .../{storage => storage-zfs}/snapshots.ts | 0 .../{storage => storage-zfs}/util.ts | 0 11 files changed, 255 insertions(+), 3 deletions(-) create mode 100644 src/packages/file-server/storage-btrfs/filesystem.ts create mode 100644 src/packages/file-server/storage-btrfs/index.ts create mode 100644 src/packages/file-server/storage-btrfs/subvolume.ts rename src/packages/file-server/{storage => storage-zfs}/README.md (100%) rename src/packages/file-server/{storage => storage-zfs}/filesystem.ts (100%) rename src/packages/file-server/{storage => storage-zfs}/index.ts (100%) rename src/packages/file-server/{storage => storage-zfs}/pool.ts (100%) rename src/packages/file-server/{storage => storage-zfs}/pools.ts (98%) rename src/packages/file-server/{storage => storage-zfs}/snapshots.ts (100%) rename src/packages/file-server/{storage => storage-zfs}/util.ts (100%) diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index 0293b9e353..b70f098333 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -5,8 +5,10 @@ "exports": { "./zfs": "./dist/zfs/index.js", "./zfs/*": "./dist/zfs/*.js", - "./storage": "./dist/storage/index.js", - "./storage/*": "./dist/storage/*.js" + "./storage-zfs": "./dist/storage-zfs/index.js", + "./storage-zfs/*": "./dist/storage-zfs/*.js", + "./storage-btrfs": "./dist/storage-btrfs/index.js", + "./storage-btrfs/*": "./dist/storage-btrfs/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/storage-btrfs/filesystem.ts new file mode 100644 index 0000000000..c5bd779203 --- /dev/null +++ b/src/packages/file-server/storage-btrfs/filesystem.ts @@ -0,0 +1,182 @@ +/* +A BTRFS Filesystem + +DEVELOPMENT: + +Start node, then: + +a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', format:true, mount:'/mnt/btrfs', uid:238309483}) + +*/ + +import refCache from "@cocalc/util/refcache"; +import { exists, mkdirp, sudo } from "@cocalc/file-server/storage-zfs/util"; +import { subvolume } from "./subvolume"; +import { join } from "path"; + +// default size of btrfs filesystem if creating an image file. +const DEFAULT_SIZE = "10G"; + +const MOUNT_ERROR = "wrong fs type, bad option, bad superblock"; + +export interface Options { + // the underlying block device. + // If this is a file (or filename) ending in .img, then it's a sparse file mounted as a loopback device. + // If this starts with "/dev" then it is a raw block device. + device: string; + // if true, format the device or image, if it doesn't mount with an error containing "wrong fs type, bad option, bad superblock" + format?: boolean; + // where the btrfs filesystem is mounted + mount: string; + // path where btrfs send streams of subvolumes are stored (using "btrfs send") + streams?: string; + // path where bup backups of subvolumes are stored + bup?: string; + + // all subvolumes will have this owner + uid?: number; +} + +export class Filesystem { + public readonly opts: Options; + + constructor(opts: Options) { + this.opts = opts; + } + + init = async () => { + await mkdirp( + [this.opts.mount, this.opts.streams, this.opts.bup].filter( + (x) => x, + ) as string[], + ); + await this.initDevice(); + await this.mountFilesystem(); + await sudo({ command: "chmod", args: ["a+rx", this.opts.mount] }); + }; + + private initDevice = async () => { + if (!isImageFile(this.opts.device)) { + // raw block device -- nothing to do + return; + } + if (!(await exists(this.opts.device))) { + await sudo({ + command: "truncate", + args: ["-s", DEFAULT_SIZE, this.opts.device], + }); + } + }; + + info = async () => { + return await sudo({ + command: "btrfs", + args: ["subvolume", "show", this.opts.mount], + }); + }; + + // + private mountFilesystem = async () => { + try { + await this.info(); + // already mounted + return; + } catch {} + const { stderr, exit_code } = await this._mountFilesystem(); + if (exit_code) { + if (stderr.includes(MOUNT_ERROR)) { + if (this.opts.format) { + await this.formatDevice(); + const { stderr, exit_code } = await this._mountFilesystem(); + if (exit_code) { + throw Error(stderr); + } else { + return; + } + } + } + throw Error(stderr); + } + }; + + private _mountFilesystem = async () => { + const args: string[] = isImageFile(this.opts.device) ? ["-o", "loop"] : []; + args.push(this.opts.device); + args.push("-t"); + args.push("btrfs"); + args.push(this.opts.mount); + return await sudo({ + command: "mount", + args, + err_on_exit: false, + }); + }; + + private formatDevice = async () => { + await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); + }; + + close = () => { + // nothing, yet + }; + + subvolume = async (name: string) => { + return await subvolume({ filesystem: this, name }); + }; + + deleteSubvolume = async (name: string) => { + await sudo({ command: "btrfs", args: ["subvolume", "delete", name] }); + }; + + list = async (): Promise => { + const { stdout } = await sudo({ + command: "btrfs", + args: ["subvolume", "list", this.opts.mount], + }); + return stdout.split("\n").map((x) => x.split(" ").slice(-1)[0]); + }; + + rsync = async ({ + src, + target, + args = ["-axH"], + timeout = 5 * 60 * 1000, + }: { + src: string; + target: string; + args?: string[]; + timeout?: number; + }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { + const srcPath = join(this.opts.mount, src); + const targetPath = join(this.opts.mount, target); + return await sudo({ + command: "rsync", + args: [...args, srcPath, targetPath], + err_on_exit: false, + timeout: timeout / 1000, + }); + }; +} + +function isImageFile(name: string) { + if (name.startsWith("/dev")) { + return false; + } + // TODO: could probably check os for a device with given name? + return name.endsWith(".img"); +} + +const cache = refCache({ + name: "btrfs-filesystems", + createObject: async (options: Options) => { + const filesystem = new Filesystem(options); + await filesystem.init(); + return filesystem; + }, +}); + +export async function filesystem( + options: Options & { noCache?: boolean }, +): Promise { + return await cache(options); +} diff --git a/src/packages/file-server/storage-btrfs/index.ts b/src/packages/file-server/storage-btrfs/index.ts new file mode 100644 index 0000000000..edfd27c3a9 --- /dev/null +++ b/src/packages/file-server/storage-btrfs/index.ts @@ -0,0 +1 @@ +export { filesystem } from "./filesystem"; diff --git a/src/packages/file-server/storage-btrfs/subvolume.ts b/src/packages/file-server/storage-btrfs/subvolume.ts new file mode 100644 index 0000000000..cfbfbd81ff --- /dev/null +++ b/src/packages/file-server/storage-btrfs/subvolume.ts @@ -0,0 +1,67 @@ +/* +A subvolume +*/ + +import { type Filesystem } from "./filesystem"; +import refCache from "@cocalc/util/refcache"; +import { exists, sudo } from "@cocalc/file-server/storage-zfs/util"; +import { join } from "path"; + +interface Options { + filesystem: Filesystem; + name: string; +} + +export class Subvolume { + private filesystem: Filesystem; + private name: string; + private path: string; + + constructor({ filesystem, name }: Options) { + this.filesystem = filesystem; + this.name = name; + this.path = join(filesystem.opts.mount, name); + } + + init = async () => { + if (!(await exists(this.path))) { + await sudo({ + command: "btrfs", + args: ["subvolume", "create", this.path], + }); + if (this.filesystem.opts.uid) { + await sudo({ + command: "chown", + args: [ + `${this.filesystem.opts.uid}:${this.filesystem.opts.uid}`, + this.path, + ], + }); + } + } + }; + + close = () => { + // @ts-ignore + delete this.filesystem; + // @ts-ignore + delete this.name; + // @ts-ignore + delete this.path; + }; +} + +const cache = refCache({ + name: "btrfs-subvolumes", + createObject: async (options: Options) => { + const subvolume = new Subvolume(options); + await subvolume.init(); + return subvolume; + }, +}); + +export async function subvolume( + options: Options & { noCache?: boolean }, +): Promise { + return await cache(options); +} diff --git a/src/packages/file-server/storage/README.md b/src/packages/file-server/storage-zfs/README.md similarity index 100% rename from src/packages/file-server/storage/README.md rename to src/packages/file-server/storage-zfs/README.md diff --git a/src/packages/file-server/storage/filesystem.ts b/src/packages/file-server/storage-zfs/filesystem.ts similarity index 100% rename from src/packages/file-server/storage/filesystem.ts rename to src/packages/file-server/storage-zfs/filesystem.ts diff --git a/src/packages/file-server/storage/index.ts b/src/packages/file-server/storage-zfs/index.ts similarity index 100% rename from src/packages/file-server/storage/index.ts rename to src/packages/file-server/storage-zfs/index.ts diff --git a/src/packages/file-server/storage/pool.ts b/src/packages/file-server/storage-zfs/pool.ts similarity index 100% rename from src/packages/file-server/storage/pool.ts rename to src/packages/file-server/storage-zfs/pool.ts diff --git a/src/packages/file-server/storage/pools.ts b/src/packages/file-server/storage-zfs/pools.ts similarity index 98% rename from src/packages/file-server/storage/pools.ts rename to src/packages/file-server/storage-zfs/pools.ts index b766fb7b6d..7268c47cef 100644 --- a/src/packages/file-server/storage/pools.ts +++ b/src/packages/file-server/storage-zfs/pools.ts @@ -3,7 +3,7 @@ DEVELOPMENT: Start node, then: -a = require('@cocalc/file-server/storage') +a = require('@cocalc/file-server/storage-zfs') pools = await a.pools({images:'/data/zfs/images', mount:'/data/zfs/mnt'}) x = await pools.pool('x') diff --git a/src/packages/file-server/storage/snapshots.ts b/src/packages/file-server/storage-zfs/snapshots.ts similarity index 100% rename from src/packages/file-server/storage/snapshots.ts rename to src/packages/file-server/storage-zfs/snapshots.ts diff --git a/src/packages/file-server/storage/util.ts b/src/packages/file-server/storage-zfs/util.ts similarity index 100% rename from src/packages/file-server/storage/util.ts rename to src/packages/file-server/storage-zfs/util.ts From 1893f87da8c5bda9f7f6ab65431c5565fca4f082 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 4 May 2025 15:13:03 +0000 Subject: [PATCH 12/47] btrfs: working on quotas --- .../file-server/storage-btrfs/filesystem.ts | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/storage-btrfs/filesystem.ts index c5bd779203..faa0b5b84a 100644 --- a/src/packages/file-server/storage-btrfs/filesystem.ts +++ b/src/packages/file-server/storage-btrfs/filesystem.ts @@ -5,7 +5,7 @@ DEVELOPMENT: Start node, then: -a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', format:true, mount:'/mnt/btrfs', uid:238309483}) +a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', formatIfNeeded:true, mount:'/mnt/btrfs', uid:238309483}) */ @@ -24,8 +24,11 @@ export interface Options { // If this is a file (or filename) ending in .img, then it's a sparse file mounted as a loopback device. // If this starts with "/dev" then it is a raw block device. device: string; - // if true, format the device or image, if it doesn't mount with an error containing "wrong fs type, bad option, bad superblock" - format?: boolean; + // formatIfNeeded -- DANGEROUS! if true, format the device or image, + // if it doesn't mount with an error containing "wrong fs type, + // bad option, bad superblock". Never use this in production. Useful + // for testing and dev. + formatIfNeeded?: boolean; // where the btrfs filesystem is mounted mount: string; // path where btrfs send streams of subvolumes are stored (using "btrfs send") @@ -53,6 +56,10 @@ export class Filesystem { await this.initDevice(); await this.mountFilesystem(); await sudo({ command: "chmod", args: ["a+rx", this.opts.mount] }); + await sudo({ + command: "btrfs", + args: ["quota", "enable", "--simple", this.opts.mount], + }); }; private initDevice = async () => { @@ -85,7 +92,7 @@ export class Filesystem { const { stderr, exit_code } = await this._mountFilesystem(); if (exit_code) { if (stderr.includes(MOUNT_ERROR)) { - if (this.opts.format) { + if (this.opts.formatIfNeeded) { await this.formatDevice(); const { stderr, exit_code } = await this._mountFilesystem(); if (exit_code) { @@ -101,10 +108,20 @@ export class Filesystem { private _mountFilesystem = async () => { const args: string[] = isImageFile(this.opts.device) ? ["-o", "loop"] : []; - args.push(this.opts.device); - args.push("-t"); - args.push("btrfs"); - args.push(this.opts.mount); + args.push( + "-o", + "compress=zstd", + "-o", + "noatime", + "-o", + "space_cache=v2", + "-o", + "autodefrag", + this.opts.device, + "-t", + "btrfs", + this.opts.mount, + ); return await sudo({ command: "mount", args, From 591295fb3f5fee4e5ad6f0837110e55376abf6d9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 4 May 2025 20:45:40 +0000 Subject: [PATCH 13/47] btrfs: quota --- .../file-server/storage-btrfs/filesystem.ts | 7 ++++-- .../file-server/storage-btrfs/subvolume.ts | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/storage-btrfs/filesystem.ts index faa0b5b84a..e33e9f7a2a 100644 --- a/src/packages/file-server/storage-btrfs/filesystem.ts +++ b/src/packages/file-server/storage-btrfs/filesystem.ts @@ -5,7 +5,7 @@ DEVELOPMENT: Start node, then: -a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', formatIfNeeded:true, mount:'/mnt/btrfs', uid:238309483}) +a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', formatIfNeeded:true, mount:'/mnt/btrfs', uid:293597964}) */ @@ -150,7 +150,10 @@ export class Filesystem { command: "btrfs", args: ["subvolume", "list", this.opts.mount], }); - return stdout.split("\n").map((x) => x.split(" ").slice(-1)[0]); + return stdout + .split("\n") + .map((x) => x.split(" ").slice(-1)[0]) + .filter((x) => x); }; rsync = async ({ diff --git a/src/packages/file-server/storage-btrfs/subvolume.ts b/src/packages/file-server/storage-btrfs/subvolume.ts index cfbfbd81ff..bc7da1e30d 100644 --- a/src/packages/file-server/storage-btrfs/subvolume.ts +++ b/src/packages/file-server/storage-btrfs/subvolume.ts @@ -41,6 +41,28 @@ export class Subvolume { } }; + private quotaInfo = async () => { + const { stdout } = await sudo({ + verbose: false, + command: "btrfs", + args: ["--format=json", "qgroup", "show", "-reF", this.path], + }); + const x = JSON.parse(stdout); + return x["qgroup-show"][0]; + }; + + setSize = async (size: string | number) => { + await sudo({ + command: "btrfs", + args: ["qgroup", "limit", size, this.path], + }); + }; + + getUsage = async (): Promise<{ size: number; usage: number }> => { + const { max_referenced: size, referenced: usage } = await this.quotaInfo(); + return { usage, size }; + }; + close = () => { // @ts-ignore delete this.filesystem; From 5d1ff98ad6f18f42243703ac243247a60a970d5c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 4 May 2025 21:45:09 +0000 Subject: [PATCH 14/47] btrfs: rolling snapshots --- .../file-server/storage-btrfs/filesystem.ts | 16 ++- .../file-server/storage-btrfs/snapshots.ts | 115 ++++++++++++++++ .../file-server/storage-btrfs/subvolume.ts | 125 +++++++++++++++--- 3 files changed, 233 insertions(+), 23 deletions(-) create mode 100644 src/packages/file-server/storage-btrfs/snapshots.ts diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/storage-btrfs/filesystem.ts index e33e9f7a2a..a38d5ebcfb 100644 --- a/src/packages/file-server/storage-btrfs/filesystem.ts +++ b/src/packages/file-server/storage-btrfs/filesystem.ts @@ -15,7 +15,10 @@ import { subvolume } from "./subvolume"; import { join } from "path"; // default size of btrfs filesystem if creating an image file. -const DEFAULT_SIZE = "10G"; +const DEFAULT_FILESYSTEM_SIZE = "10G"; + +// default for newly created subvolumes +export const DEFAULT_SUBVOLUME_SIZE = "1G"; const MOUNT_ERROR = "wrong fs type, bad option, bad superblock"; @@ -38,12 +41,21 @@ export interface Options { // all subvolumes will have this owner uid?: number; + + // default size of newly created subvolumes + defaultSize?: string | number; + defaultFilesystemSize?: string | number; } export class Filesystem { public readonly opts: Options; constructor(opts: Options) { + opts = { + defaultSize: DEFAULT_SUBVOLUME_SIZE, + defaultFilesystemSize: DEFAULT_FILESYSTEM_SIZE, + ...opts, + }; this.opts = opts; } @@ -70,7 +82,7 @@ export class Filesystem { if (!(await exists(this.opts.device))) { await sudo({ command: "truncate", - args: ["-s", DEFAULT_SIZE, this.opts.device], + args: ["-s", `${this.opts.defaultFilesystemSize}`, this.opts.device], }); } }; diff --git a/src/packages/file-server/storage-btrfs/snapshots.ts b/src/packages/file-server/storage-btrfs/snapshots.ts new file mode 100644 index 0000000000..735a450324 --- /dev/null +++ b/src/packages/file-server/storage-btrfs/snapshots.ts @@ -0,0 +1,115 @@ +import { type Subvolume } from "./subvolume"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("file-server:storage-btrfs:snapshots"); + +const DATE_REGEXP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; + +// Lengths of time in minutes to keep snapshots +// (code below assumes these are listed in ORDER from shortest to longest) +export const SNAPSHOT_INTERVALS_MS = { + frequent: 15 * 1000 * 60, + daily: 60 * 24 * 1000 * 60, + weekly: 60 * 24 * 7 * 1000 * 60, + monthly: 60 * 24 * 7 * 4 * 1000 * 60, +}; + +// How many of each type of snapshot to retain +export const DEFAULT_SNAPSHOT_COUNTS = { + frequent: 24, + daily: 14, + weekly: 7, + monthly: 4, +} as SnapshotCounts; + +export interface SnapshotCounts { + frequent: number; + daily: number; + weekly: number; + monthly: number; +} + +export async function updateRollingSnapshots({ + subvolume, + counts, +}: { + subvolume: Subvolume; + counts?: Partial; +}) { + counts = { ...DEFAULT_SNAPSHOT_COUNTS, ...counts }; + + const changed = await subvolume.hasUnsavedChanges(); + logger.debug("updateRollingSnapshots", { + name: subvolume.name, + counts, + changed, + }); + if (!changed) { + // definitely no data written since most recent snapshot, so nothing to do + return; + } + + // get exactly the iso timestamp snapshot names: + const snapshots = (await subvolume.snapshots()).filter((x) => + DATE_REGEXP.test(x), + ); + snapshots.sort(); + if (snapshots.length > 0) { + const age = Date.now() - new Date(snapshots.slice(-1)[0]).valueOf(); + for (const key in SNAPSHOT_INTERVALS_MS) { + if (counts[key]) { + if (age < SNAPSHOT_INTERVALS_MS[key]) { + // no need to snapshot since there is already a sufficiently recent snapshot + logger.debug("updateRollingSnapshots: no need to snapshot", { + name: subvolume.name, + }); + return; + } + // counts[key] nonzero and snapshot is old enough so we'll be making a snapshot + break; + } + } + } + + // make a new snapshot + const snapshot = new Date().toISOString(); + await subvolume.createSnapshot(snapshot); + // delete extra snapshots + snapshots.push(snapshot); + const toDelete = snapshotsToDelete({ counts, snapshots }); + for (const snapshot of toDelete) { + await subvolume.deleteSnapshot(snapshot); + } +} + +function snapshotsToDelete({ counts, snapshots }): string[] { + if (snapshots.length == 0) { + // nothing to do + return []; + } + + // sorted from BIGGEST to smallest + const times = snapshots.map((x) => new Date(x).valueOf()); + times.reverse(); + const save = new Set(); + for (const type in counts) { + const count = counts[type]; + const length_ms = SNAPSHOT_INTERVALS_MS[type]; + + // Pick the first count newest snapshots at intervals of length + // length_ms milliseconds. + let n = 0, + i = 0, + last_tm = 0; + while (n < count && i < times.length) { + const tm = times[i]; + if (!last_tm || tm <= last_tm - length_ms) { + save.add(tm); + last_tm = tm; + n += 1; // found one more + } + i += 1; // move to next snapshot + } + } + return snapshots.filter((x) => !save.has(new Date(x).valueOf())); +} diff --git a/src/packages/file-server/storage-btrfs/subvolume.ts b/src/packages/file-server/storage-btrfs/subvolume.ts index bc7da1e30d..22b119468e 100644 --- a/src/packages/file-server/storage-btrfs/subvolume.ts +++ b/src/packages/file-server/storage-btrfs/subvolume.ts @@ -2,10 +2,20 @@ A subvolume */ -import { type Filesystem } from "./filesystem"; +import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; -import { exists, sudo } from "@cocalc/file-server/storage-zfs/util"; +import { + exists, + listdir, + mkdirp, + sudo, +} from "@cocalc/file-server/storage-zfs/util"; import { join } from "path"; +import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; + +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("file-server:storage-btrfs:subvolume"); interface Options { filesystem: Filesystem; @@ -14,13 +24,15 @@ interface Options { export class Subvolume { private filesystem: Filesystem; - private name: string; - private path: string; + public readonly name: string; + public readonly path: string; + public readonly snapshotsDir: string; constructor({ filesystem, name }: Options) { this.filesystem = filesystem; this.name = name; this.path = join(filesystem.opts.mount, name); + this.snapshotsDir = join(this.path, ".snapshots"); } init = async () => { @@ -29,16 +41,33 @@ export class Subvolume { command: "btrfs", args: ["subvolume", "create", this.path], }); - if (this.filesystem.opts.uid) { - await sudo({ - command: "chown", - args: [ - `${this.filesystem.opts.uid}:${this.filesystem.opts.uid}`, - this.path, - ], - }); - } + await this.makeSnapshotsDir(); + await this.chown(this.path); + await this.setSize( + this.filesystem.opts.defaultSize ?? DEFAULT_SUBVOLUME_SIZE, + ); + } + }; + + close = () => { + // @ts-ignore + delete this.filesystem; + // @ts-ignore + delete this.name; + // @ts-ignore + delete this.path; + // @ts-ignore + delete this.snapshotsDir; + }; + + private chown = async (path: string) => { + if (!this.filesystem.opts.uid) { + return; } + await sudo({ + command: "chown", + args: [`${this.filesystem.opts.uid}:${this.filesystem.opts.uid}`, path], + }); }; private quotaInfo = async () => { @@ -54,7 +83,7 @@ export class Subvolume { setSize = async (size: string | number) => { await sudo({ command: "btrfs", - args: ["qgroup", "limit", size, this.path], + args: ["qgroup", "limit", `${size}`, this.path], }); }; @@ -63,14 +92,68 @@ export class Subvolume { return { usage, size }; }; - close = () => { - // @ts-ignore - delete this.filesystem; - // @ts-ignore - delete this.name; - // @ts-ignore - delete this.path; + private makeSnapshotsDir = async () => { + if (await exists(this.snapshotsDir)) { + return; + } + await mkdirp([this.snapshotsDir]); + await this.chown(this.snapshotsDir); + await sudo({ command: "chmod", args: ["a-w", this.snapshotsDir] }); + }; + + createSnapshot = async (name: string) => { + logger.debug("createSnapshot", { name, subvolume: this.name }); + await this.makeSnapshotsDir(); + await sudo({ + command: "btrfs", + args: [ + "subvolume", + "snapshot", + "-r", + this.path, + join(this.snapshotsDir, name), + ], + }); + }; + + snapshots = async (): Promise => { + return (await listdir(this.snapshotsDir)).sort(); + }; + + deleteSnapshot = async (name) => { + await sudo({ + command: "btrfs", + args: ["subvolume", "delete", join(this.snapshotsDir, name)], + }); + }; + + updateRollingSnapshots = async (counts?: Partial) => { + return await updateRollingSnapshots({ subvolume: this, counts }); }; + + // has newly written changes since last snapshot + hasUnsavedChanges = async (): Promise => { + const s = await this.snapshots(); + if (s.length == 0) { + // more than just the .snapshots directory? + return (await listdir(this.path)).length > 1; + } + const pathGen = await getGeneration(this.path); + const snapGen = await getGeneration( + join(this.snapshotsDir, s[s.length - 1]), + ); + console.log({ pathGen, snapGen }); + return snapGen < pathGen; + }; +} + +async function getGeneration(path: string): Promise { + const { stdout } = await sudo({ + command: "btrfs", + args: ["subvolume", "show", path], + verbose: false, + }); + return parseInt(stdout.split("Generation:")[1].split("\n")[0].trim()); } const cache = refCache({ From cd087ba5269790b6e6c67ee5d44f78a73bdec0e5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 4 May 2025 22:55:07 +0000 Subject: [PATCH 15/47] btrfs: cloning subvolumes --- .../file-server/storage-btrfs/filesystem.ts | 44 ++++++++++++++++++- .../file-server/storage-btrfs/subvolume.ts | 32 +++++++++++--- 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/storage-btrfs/filesystem.ts index a38d5ebcfb..337789da1b 100644 --- a/src/packages/file-server/storage-btrfs/filesystem.ts +++ b/src/packages/file-server/storage-btrfs/filesystem.ts @@ -10,8 +10,14 @@ a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({devic */ import refCache from "@cocalc/util/refcache"; -import { exists, mkdirp, sudo } from "@cocalc/file-server/storage-zfs/util"; -import { subvolume } from "./subvolume"; +import { + exists, + listdir, + mkdirp, + rmdir, + sudo, +} from "@cocalc/file-server/storage-zfs/util"; +import { subvolume, SNAPSHOTS } from "./subvolume"; import { join } from "path"; // default size of btrfs filesystem if creating an image file. @@ -153,6 +159,40 @@ export class Filesystem { return await subvolume({ filesystem: this, name }); }; + // create a subvolume by cloning an existing one. + cloneSubvolume = async (source: string, name: string) => { + if (!(await exists(join(this.opts.mount, source)))) { + throw Error(`subvolume ${source} does not exist`); + } + if (await exists(join(this.opts.mount, name))) { + throw Error(`subvolume ${name} already exists`); + } + await sudo({ + command: "btrfs", + args: [ + "subvolume", + "snapshot", + join(this.opts.mount, source), + join(this.opts.mount, source, name), + ], + }); + await sudo({ + command: "mv", + args: [join(this.opts.mount, source, name), join(this.opts.mount, name)], + }); + const snapshots = await listdir(join(this.opts.mount, name, SNAPSHOTS)); + await rmdir( + snapshots.map((x) => join(this.opts.mount, name, SNAPSHOTS, x)), + ); + const src = await this.subvolume(source); + const vol = await this.subvolume(name); + const { size } = await src.getUsage(); + if (size) { + await vol.setSize(size); + } + return vol; + }; + deleteSubvolume = async (name: string) => { await sudo({ command: "btrfs", args: ["subvolume", "delete", name] }); }; diff --git a/src/packages/file-server/storage-btrfs/subvolume.ts b/src/packages/file-server/storage-btrfs/subvolume.ts index 22b119468e..99480816b3 100644 --- a/src/packages/file-server/storage-btrfs/subvolume.ts +++ b/src/packages/file-server/storage-btrfs/subvolume.ts @@ -12,6 +12,9 @@ import { } from "@cocalc/file-server/storage-zfs/util"; import { join } from "path"; import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; +import { human_readable_size } from "@cocalc/util/misc"; + +export const SNAPSHOTS = ".snapshots"; import getLogger from "@cocalc/backend/logger"; @@ -32,7 +35,7 @@ export class Subvolume { this.filesystem = filesystem; this.name = name; this.path = join(filesystem.opts.mount, name); - this.snapshotsDir = join(this.path, ".snapshots"); + this.snapshotsDir = join(this.path, SNAPSHOTS); } init = async () => { @@ -87,9 +90,23 @@ export class Subvolume { }); }; - getUsage = async (): Promise<{ size: number; usage: number }> => { - const { max_referenced: size, referenced: usage } = await this.quotaInfo(); - return { usage, size }; + getUsage = async (): Promise<{ + size: number; + usage: number; + human: { size: string; usage: string }; + }> => { + let { max_referenced: size, referenced: usage } = await this.quotaInfo(); + if (size == "none") { + size = null; + } + return { + usage, + size, + human: { + usage: human_readable_size(usage), + size: size != null ? human_readable_size(size) : size, + }, + }; }; private makeSnapshotsDir = async () => { @@ -135,7 +152,7 @@ export class Subvolume { hasUnsavedChanges = async (): Promise => { const s = await this.snapshots(); if (s.length == 0) { - // more than just the .snapshots directory? + // more than just the SNAPSHOTS directory? return (await listdir(this.path)).length > 1; } const pathGen = await getGeneration(this.path); @@ -145,6 +162,11 @@ export class Subvolume { console.log({ pathGen, snapGen }); return snapGen < pathGen; }; + + // // create a new bup save + // bup = async () => { + + // } } async function getGeneration(path: string): Promise { From 92b9821e581b6437d5b8e103867321be63ff2a03 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 4 May 2025 23:44:19 +0000 Subject: [PATCH 16/47] btrfs: bup snapshots --- .../file-server/storage-btrfs/filesystem.ts | 24 +++++++---- .../file-server/storage-btrfs/subvolume.ts | 40 +++++++++++++++++-- src/packages/file-server/storage-zfs/util.ts | 12 +++++- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/storage-btrfs/filesystem.ts index 337789da1b..c47709bcdd 100644 --- a/src/packages/file-server/storage-btrfs/filesystem.ts +++ b/src/packages/file-server/storage-btrfs/filesystem.ts @@ -28,6 +28,8 @@ export const DEFAULT_SUBVOLUME_SIZE = "1G"; const MOUNT_ERROR = "wrong fs type, bad option, bad superblock"; +const RESERVED = new Set(["bup", "streams", SNAPSHOTS]); + export interface Options { // the underlying block device. // If this is a file (or filename) ending in .img, then it's a sparse file mounted as a loopback device. @@ -40,10 +42,6 @@ export interface Options { formatIfNeeded?: boolean; // where the btrfs filesystem is mounted mount: string; - // path where btrfs send streams of subvolumes are stored (using "btrfs send") - streams?: string; - // path where bup backups of subvolumes are stored - bup?: string; // all subvolumes will have this owner uid?: number; @@ -55,6 +53,8 @@ export interface Options { export class Filesystem { public readonly opts: Options; + public readonly bup: string; + public readonly streams: string; constructor(opts: Options) { opts = { @@ -63,13 +63,13 @@ export class Filesystem { ...opts, }; this.opts = opts; + this.bup = join(this.opts.mount, "bup"); + this.streams = join(this.opts.mount, "streams"); } init = async () => { await mkdirp( - [this.opts.mount, this.opts.streams, this.opts.bup].filter( - (x) => x, - ) as string[], + [this.opts.mount, this.streams, this.bup].filter((x) => x) as string[], ); await this.initDevice(); await this.mountFilesystem(); @@ -78,6 +78,10 @@ export class Filesystem { command: "btrfs", args: ["quota", "enable", "--simple", this.opts.mount], }); + await sudo({ + bash: true, + command: `BUP_DIR=${this.bup} bup init`, + }); }; private initDevice = async () => { @@ -156,11 +160,17 @@ export class Filesystem { }; subvolume = async (name: string) => { + if (RESERVED.has(name)) { + throw Error(`${name} is reserved`); + } return await subvolume({ filesystem: this, name }); }; // create a subvolume by cloning an existing one. cloneSubvolume = async (source: string, name: string) => { + if (RESERVED.has(name)) { + throw Error(`${name} is reserved`); + } if (!(await exists(join(this.opts.mount, source)))) { throw Error(`subvolume ${source} does not exist`); } diff --git a/src/packages/file-server/storage-btrfs/subvolume.ts b/src/packages/file-server/storage-btrfs/subvolume.ts index 99480816b3..ca9afa06a3 100644 --- a/src/packages/file-server/storage-btrfs/subvolume.ts +++ b/src/packages/file-server/storage-btrfs/subvolume.ts @@ -159,14 +159,46 @@ export class Subvolume { const snapGen = await getGeneration( join(this.snapshotsDir, s[s.length - 1]), ); - console.log({ pathGen, snapGen }); return snapGen < pathGen; }; - // // create a new bup save - // bup = async () => { + // create a new bup snapshot + createBupSnapshot = async () => { + await sudo({ + command: "bup", + args: [ + "-d", + this.filesystem.bup, + "index", + "--exclude=", + join(this.path, SNAPSHOTS), + this.path, + ], + }); + await sudo({ + command: "bup", + args: [ + "-d", + this.filesystem.bup, + "save", + "--strip", + "-n", + this.name, + this.path, + ], + }); + }; - // } + bupSnapshots = async (): Promise => { + const { stdout } = await sudo({ + command: "bup", + args: ["-d", this.filesystem.bup, "ls", this.name], + }); + return stdout + .split("\n") + .map((x) => x.split(" ").slice(-1)[0]) + .filter((x) => x); + }; } async function getGeneration(path: string): Promise { diff --git a/src/packages/file-server/storage-zfs/util.ts b/src/packages/file-server/storage-zfs/util.ts index 4d95dcaf48..6969e6f28e 100644 --- a/src/packages/file-server/storage-zfs/util.ts +++ b/src/packages/file-server/storage-zfs/util.ts @@ -33,12 +33,20 @@ export async function sudo( if (opts.verbose !== false && opts.desc) { logger.debug("exec", opts.desc); } + let command, args; + if (opts.bash) { + command = `sudo ${opts.command}`; + args = undefined; + } else { + command = "sudo"; + args = [opts.command, ...(opts.args ?? [])]; + } return await executeCode({ verbose: true, timeout: DEFAULT_EXEC_TIMEOUT_MS / 1000, ...opts, - command: "sudo", - args: [opts.command, ...(opts.args ?? [])], + command, + args, }); } From a61b545106ea1a9e3e3ccd9a25567a6ce78792d4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 5 May 2025 00:18:05 +0000 Subject: [PATCH 17/47] btrfs -- bup prune --- .../file-server/storage-btrfs/subvolume.ts | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/packages/file-server/storage-btrfs/subvolume.ts b/src/packages/file-server/storage-btrfs/subvolume.ts index ca9afa06a3..fadda0d142 100644 --- a/src/packages/file-server/storage-btrfs/subvolume.ts +++ b/src/packages/file-server/storage-btrfs/subvolume.ts @@ -162,8 +162,8 @@ export class Subvolume { return snapGen < pathGen; }; - // create a new bup snapshot - createBupSnapshot = async () => { + // create a new bup backup + createBupBackup = async () => { await sudo({ command: "bup", args: [ @@ -189,7 +189,7 @@ export class Subvolume { }); }; - bupSnapshots = async (): Promise => { + bupBackups = async (): Promise => { const { stdout } = await sudo({ command: "bup", args: ["-d", this.filesystem.bup, "ls", this.name], @@ -199,6 +199,26 @@ export class Subvolume { .map((x) => x.split(" ").slice(-1)[0]) .filter((x) => x); }; + + bupPrune = async ({ + dailies = "1w", + monthlies = "4m", + all = "3d", + }: { dailies?: string; monthlies?: string; all?: string } = {}) => { + await sudo({ + command: "bup", + args: [ + "-d", + this.filesystem.bup, + "prune-older", + `--keep-dailies-for=${dailies}`, + `--keep-monthlies-for=${monthlies}`, + `--keep-all-for=${all}`, + "--unsafe", + this.name, + ], + }); + }; } async function getGeneration(path: string): Promise { From 706f4d232e1ca4061cb8848f0b045c340cab4500 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 5 May 2025 00:50:43 +0000 Subject: [PATCH 18/47] btrfs: implement send --- .../file-server/storage-btrfs/subvolume.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/packages/file-server/storage-btrfs/subvolume.ts b/src/packages/file-server/storage-btrfs/subvolume.ts index fadda0d142..b6df250363 100644 --- a/src/packages/file-server/storage-btrfs/subvolume.ts +++ b/src/packages/file-server/storage-btrfs/subvolume.ts @@ -15,6 +15,9 @@ import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; import { human_readable_size } from "@cocalc/util/misc"; export const SNAPSHOTS = ".snapshots"; +const SEND_SNAPSHOT_PREFIX = "send-"; + +const PAD = 4; import getLogger from "@cocalc/backend/logger"; @@ -219,6 +222,50 @@ export class Subvolume { ], }); }; + + send = async () => { + await mkdirp([join(this.filesystem.streams, this.name)]); + const streams = new Set( + await listdir(join(this.filesystem.streams, this.name)), + ); + const allSnapshots = await this.snapshots(); + const snapshots = allSnapshots.filter( + (x) => x.startsWith(SEND_SNAPSHOT_PREFIX) && streams.has(x), + ); + const nums = snapshots.map((x) => + parseInt(x.slice(SEND_SNAPSHOT_PREFIX.length)), + ); + nums.sort(); + const last = nums.slice(-1)[0]; + let seq, parent; + if (last) { + seq = `${last + 1}`.padStart(PAD, "0"); + const l = `${last}`.padStart(PAD, "0"); + parent = `${SEND_SNAPSHOT_PREFIX}${l}`; + } else { + seq = "1".padStart(PAD, "0"); + parent = ""; + } + const send = `${SEND_SNAPSHOT_PREFIX}${seq}`; + if (allSnapshots.includes(send)) { + await this.deleteSnapshot(send); + } + await this.createSnapshot(send); + await sudo({ + command: "btrfs", + args: [ + "send", + "--compressed-data", + join(this.snapshotsDir, send), + ...(last ? ["-p", join(this.snapshotsDir, parent)] : []), + "-f", + join(this.filesystem.streams, this.name, send), + ], + }); + if (parent) { + await this.deleteSnapshot(parent); + } + }; } async function getGeneration(path: string): Promise { From 9a80b44c4e15e0c33930641e42c0fcd75d1f8a43 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 5 May 2025 02:02:49 +0000 Subject: [PATCH 19/47] btrfs: send/recv - thinking about it --- .../file-server/storage-btrfs/filesystem.ts | 6 +-- .../file-server/storage-btrfs/snapshots.ts | 6 ++- .../file-server/storage-btrfs/subvolume.ts | 47 +++++++++++++++++-- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/storage-btrfs/filesystem.ts index c47709bcdd..5b365af77a 100644 --- a/src/packages/file-server/storage-btrfs/filesystem.ts +++ b/src/packages/file-server/storage-btrfs/filesystem.ts @@ -28,7 +28,7 @@ export const DEFAULT_SUBVOLUME_SIZE = "1G"; const MOUNT_ERROR = "wrong fs type, bad option, bad superblock"; -const RESERVED = new Set(["bup", "streams", SNAPSHOTS]); +const RESERVED = new Set(["bup", "recv", "streams", SNAPSHOTS]); export interface Options { // the underlying block device. @@ -196,9 +196,9 @@ export class Filesystem { ); const src = await this.subvolume(source); const vol = await this.subvolume(name); - const { size } = await src.getUsage(); + const { size } = await src.usage(); if (size) { - await vol.setSize(size); + await vol.size(size); } return vol; }; diff --git a/src/packages/file-server/storage-btrfs/snapshots.ts b/src/packages/file-server/storage-btrfs/snapshots.ts index 735a450324..f0dd6dff7a 100644 --- a/src/packages/file-server/storage-btrfs/snapshots.ts +++ b/src/packages/file-server/storage-btrfs/snapshots.ts @@ -78,7 +78,11 @@ export async function updateRollingSnapshots({ snapshots.push(snapshot); const toDelete = snapshotsToDelete({ counts, snapshots }); for (const snapshot of toDelete) { - await subvolume.deleteSnapshot(snapshot); + try { + await subvolume.deleteSnapshot(snapshot); + } catch { + // some snapshots can't be deleted, e.g., they were used for the last send. + } } } diff --git a/src/packages/file-server/storage-btrfs/subvolume.ts b/src/packages/file-server/storage-btrfs/subvolume.ts index b6df250363..0c54d0e0bc 100644 --- a/src/packages/file-server/storage-btrfs/subvolume.ts +++ b/src/packages/file-server/storage-btrfs/subvolume.ts @@ -49,7 +49,7 @@ export class Subvolume { }); await this.makeSnapshotsDir(); await this.chown(this.path); - await this.setSize( + await this.size( this.filesystem.opts.defaultSize ?? DEFAULT_SUBVOLUME_SIZE, ); } @@ -86,14 +86,17 @@ export class Subvolume { return x["qgroup-show"][0]; }; - setSize = async (size: string | number) => { + size = async (size: string | number) => { + if (!size) { + throw Error("size must be specified"); + } await sudo({ command: "btrfs", args: ["qgroup", "limit", `${size}`, this.path], }); }; - getUsage = async (): Promise<{ + usage = async (): Promise<{ size: number; usage: number; human: { size: string; usage: string }; @@ -140,7 +143,28 @@ export class Subvolume { return (await listdir(this.snapshotsDir)).sort(); }; + lockSnapshot = async (name) => { + if (await exists(join(this.snapshotsDir, name))) { + await sudo({ + command: "touch", + args: [join(this.snapshotsDir, `.${name}.lock`)], + }); + } else { + throw Error(`snapshot ${name} does not exist`); + } + }; + + unlockSnapshot = async (name) => { + await sudo({ + command: "rm", + args: ["-f", join(this.snapshotsDir, `.${name}.lock`)], + }); + }; + deleteSnapshot = async (name) => { + if (await exists(join(this.snapshotsDir, `.${name}.lock`))) { + throw Error(`snapshot ${name} is locked`); + } await sudo({ command: "btrfs", args: ["subvolume", "delete", join(this.snapshotsDir, name)], @@ -223,6 +247,11 @@ export class Subvolume { }); }; + // this was just a quick proof of concept -- I don't like it. Should switch to using + // timestamps and a lock. + // To recover these, doing recv for each in order does work. Then you have to + // snapshot all of the results to move them. It's awkward, but efficient + // and works fine. send = async () => { await mkdirp([join(this.filesystem.streams, this.name)]); const streams = new Set( @@ -266,6 +295,18 @@ export class Subvolume { await this.deleteSnapshot(parent); } }; + + // recv = async (target: string) => { + // const streamsDir = join(this.filesystem.streams, this.name); + // const streams = await listdir(streamsDir); + // streams.sort(); + // for (const stream of streams) { + // await sudo({ + // command: "btrfs", + // args: ["recv", "-f", join(streamsDir, stream)], + // }); + // } + // }; } async function getGeneration(path: string): Promise { From 0fa4dae7a04e6bbcc5030dc06f8bfc8af0c3b0f7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 5 May 2025 02:23:07 +0000 Subject: [PATCH 20/47] btrfs -- fix an rsync issue --- .../file-server/storage-btrfs/filesystem.ts | 19 ++++++++++++++++--- src/packages/file-server/storage-zfs/util.ts | 5 +++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/storage-btrfs/filesystem.ts index 5b365af77a..a9b0982d24 100644 --- a/src/packages/file-server/storage-btrfs/filesystem.ts +++ b/src/packages/file-server/storage-btrfs/filesystem.ts @@ -12,13 +12,14 @@ a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({devic import refCache from "@cocalc/util/refcache"; import { exists, + isdir, listdir, mkdirp, rmdir, sudo, } from "@cocalc/file-server/storage-zfs/util"; import { subvolume, SNAPSHOTS } from "./subvolume"; -import { join } from "path"; +import { join, normalize } from "path"; // default size of btrfs filesystem if creating an image file. const DEFAULT_FILESYSTEM_SIZE = "10G"; @@ -229,8 +230,20 @@ export class Filesystem { args?: string[]; timeout?: number; }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - const srcPath = join(this.opts.mount, src); - const targetPath = join(this.opts.mount, target); + let srcPath = normalize(join(this.opts.mount, src)); + if (!srcPath.startsWith(this.opts.mount)) { + throw Error("suspicious source"); + } + let targetPath = normalize(join(this.opts.mount, target)); + if (!targetPath.startsWith(this.opts.mount)) { + throw Error("suspicious target"); + } + if (!srcPath.endsWith("/") && (await isdir(srcPath))) { + srcPath += "/"; + if (!targetPath.endsWith("/")) { + targetPath += "/"; + } + } return await sudo({ command: "rsync", args: [...args, srcPath, targetPath], diff --git a/src/packages/file-server/storage-zfs/util.ts b/src/packages/file-server/storage-zfs/util.ts index 6969e6f28e..a69255b443 100644 --- a/src/packages/file-server/storage-zfs/util.ts +++ b/src/packages/file-server/storage-zfs/util.ts @@ -64,3 +64,8 @@ export async function listdir(path: string) { const { stdout } = await sudo({ command: "ls", args: ["-1", path] }); return stdout.split("\n").filter((x) => x); } + +export async function isdir(path: string) { + const { stdout } = await sudo({ command: "stat", args: ["-c", "%F", path] }); + return stdout.trim() == "directory"; +} From b309dd62b3db9ee3c6ad6d49ff540e7b4854bea3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 7 May 2025 04:42:23 +0000 Subject: [PATCH 21/47] improve bup support in new btrfs filesystem --- .../file-server/storage-btrfs/filesystem.ts | 2 + .../file-server/storage-btrfs/subvolume.ts | 76 +++++++++++++------ src/packages/file-server/storage-zfs/util.ts | 2 +- 3 files changed, 55 insertions(+), 25 deletions(-) diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/storage-btrfs/filesystem.ts index a9b0982d24..acd02d6105 100644 --- a/src/packages/file-server/storage-btrfs/filesystem.ts +++ b/src/packages/file-server/storage-btrfs/filesystem.ts @@ -5,6 +5,8 @@ DEVELOPMENT: Start node, then: +DEBUG="cocalc:*file-server*" DEBUG_CONSOLE=yes node + a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', formatIfNeeded:true, mount:'/mnt/btrfs', uid:293597964}) */ diff --git a/src/packages/file-server/storage-btrfs/subvolume.ts b/src/packages/file-server/storage-btrfs/subvolume.ts index 0c54d0e0bc..7b55dd09b7 100644 --- a/src/packages/file-server/storage-btrfs/subvolume.ts +++ b/src/packages/file-server/storage-btrfs/subvolume.ts @@ -17,6 +17,8 @@ import { human_readable_size } from "@cocalc/util/misc"; export const SNAPSHOTS = ".snapshots"; const SEND_SNAPSHOT_PREFIX = "send-"; +const BUP_SNAPSHOT = "temp-bup-snapshot"; + const PAD = 4; import getLogger from "@cocalc/backend/logger"; @@ -161,6 +163,10 @@ export class Subvolume { }); }; + snapshotExists = async (name: string) => { + return await exists(join(this.snapshotsDir, name)); + }; + deleteSnapshot = async (name) => { if (await exists(join(this.snapshotsDir, `.${name}.lock`))) { throw Error(`snapshot ${name} is locked`); @@ -190,30 +196,52 @@ export class Subvolume { }; // create a new bup backup - createBupBackup = async () => { - await sudo({ - command: "bup", - args: [ - "-d", - this.filesystem.bup, - "index", - "--exclude=", - join(this.path, SNAPSHOTS), - this.path, - ], - }); - await sudo({ - command: "bup", - args: [ - "-d", - this.filesystem.bup, - "save", - "--strip", - "-n", - this.name, - this.path, - ], - }); + createBupBackup = async ({ + // timeout used for bup index and bup save commands + timeout = 30 * 60 * 1000, + }: { timeout?: number } = {}) => { + if (await this.snapshotExists(BUP_SNAPSHOT)) { + logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`); + await this.deleteSnapshot(BUP_SNAPSHOT); + } + try { + logger.debug( + `createBupBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, + ); + await this.createSnapshot(BUP_SNAPSHOT); + const target = join(this.snapshotsDir, BUP_SNAPSHOT); + logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`); + await sudo({ + command: "bup", + args: [ + "-d", + this.filesystem.bup, + "index", + "--exclude", + join(target, ".snapshots"), + "-x", + target, + ], + timeout, + }); + logger.debug(`createBupBackup: saving ${BUP_SNAPSHOT}`); + await sudo({ + command: "bup", + args: [ + "-d", + this.filesystem.bup, + "save", + "--strip", + "-n", + this.name, + target, + ], + timeout, + }); + } finally { + logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`); + await this.deleteSnapshot(BUP_SNAPSHOT); + } }; bupBackups = async (): Promise => { diff --git a/src/packages/file-server/storage-zfs/util.ts b/src/packages/file-server/storage-zfs/util.ts index a69255b443..bc986e968e 100644 --- a/src/packages/file-server/storage-zfs/util.ts +++ b/src/packages/file-server/storage-zfs/util.ts @@ -11,7 +11,7 @@ const DEFAULT_EXEC_TIMEOUT_MS = 60 * 1000; export async function exists(path: string) { try { - await sudo({ command: "ls", args: [path] }); + await sudo({ command: "ls", args: [path], verbose: false }); return true; } catch { return false; From eee67939551bea8b285101d1729c13f9cca29565 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 23 Jun 2025 04:41:07 +0000 Subject: [PATCH 22/47] fixes for things noticed when building --- src/packages/file-server/storage-zfs/pool.ts | 2 +- src/packages/pnpm-lock.yaml | 142 ++----------------- 2 files changed, 15 insertions(+), 129 deletions(-) diff --git a/src/packages/file-server/storage-zfs/pool.ts b/src/packages/file-server/storage-zfs/pool.ts index 1df3220844..f462a57e5e 100644 --- a/src/packages/file-server/storage-zfs/pool.ts +++ b/src/packages/file-server/storage-zfs/pool.ts @@ -4,7 +4,7 @@ import { join } from "path"; import { filesystem } from "./filesystem"; import getLogger from "@cocalc/backend/logger"; import { executeCode } from "@cocalc/backend/execute-code"; -import { randomId } from "@cocalc/nats/names"; +import { randomId } from "@cocalc/conat/names"; const logger = getLogger("file-server:storage:pool"); diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 98db2edb58..685fc262ef 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -33,10 +33,10 @@ importers: version: 5.0.0 jest: specifier: ^29.7.0 - version: 29.7.0(@types/node@24.0.1) + version: 29.7.0(@types/node@18.19.111) ts-jest: specifier: ^29.2.3 - version: 29.4.0(@babel/core@7.26.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.9))(jest-util@29.7.0)(jest@29.7.0(@types/node@24.0.1))(typescript@5.8.3) + version: 29.4.0(@babel/core@7.26.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.9))(jest-util@29.7.0)(jest@29.7.0(@types/node@18.19.111))(typescript@5.8.3) typescript: specifier: ^5.7.3 version: 5.8.3 @@ -4064,9 +4064,6 @@ packages: '@types/node@18.19.111': resolution: {integrity: sha512-90sGdgA+QLJr1F9X79tQuEut0gEYIfkX9pydI4XGRgvFo9g2JWswefI+WUSUHPYVBHYSEfTEqBxA5hQvAZB3Mw==} - '@types/node@24.0.1': - resolution: {integrity: sha512-MX4Zioh39chHlDJbKmEgydJDS3tspMP/lnQC67G3SWsTnb9NeYVWOjkxpOSy4oMfPs4StcWHwBrvUb4ybfnuaw==} - '@types/nodemailer@6.4.17': resolution: {integrity: sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww==} @@ -5895,10 +5892,6 @@ packages: engines: {node: '>=0.10'} hasBin: true - detect-libc@2.0.3: - resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==} - engines: {node: '>=8'} - detect-libc@2.0.4: resolution: {integrity: sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==} engines: {node: '>=8'} @@ -10992,9 +10985,6 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@7.8.0: - resolution: {integrity: sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==} - unicorn-magic@0.1.0: resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==} engines: {node: '>=18'} @@ -12482,7 +12472,7 @@ snapshots: '@jest/console@29.7.0': dependencies: '@jest/types': 29.6.3 - '@types/node': 24.0.1 + '@types/node': 18.19.111 chalk: 4.1.2 jest-message-util: 29.7.0 jest-util: 29.7.0 @@ -12495,14 +12485,14 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.0.1 + '@types/node': 18.19.111 ansi-escapes: 4.3.2 chalk: 4.1.2 ci-info: 3.9.0 exit: 0.1.2 graceful-fs: 4.2.11 jest-changed-files: 29.7.0 - jest-config: 29.7.0(@types/node@24.0.1) + jest-config: 29.7.0(@types/node@18.19.111) jest-haste-map: 29.7.0 jest-message-util: 29.7.0 jest-regex-util: 29.6.3 @@ -12567,7 +12557,7 @@ snapshots: '@jest/transform': 29.7.0 '@jest/types': 29.6.3 '@jridgewell/trace-mapping': 0.3.25 - '@types/node': 24.0.1 + '@types/node': 18.19.111 chalk: 4.1.2 collect-v8-coverage: 1.0.2 exit: 0.1.2 @@ -12645,7 +12635,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.1 - '@types/node': 24.0.1 + '@types/node': 18.19.111 '@types/yargs': 17.0.24 chalk: 4.1.2 @@ -13872,7 +13862,7 @@ snapshots: '@types/better-sqlite3@7.6.13': dependencies: - '@types/node': 22.15.3 + '@types/node': 18.19.111 '@types/body-parser@1.19.5': dependencies: @@ -14211,10 +14201,6 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@24.0.1': - dependencies: - undici-types: 7.8.0 - '@types/nodemailer@6.4.17': dependencies: '@types/node': 18.19.111 @@ -15736,21 +15722,6 @@ snapshots: - supports-color - ts-node - create-jest@29.7.0(@types/node@24.0.1): - dependencies: - '@jest/types': 29.6.3 - chalk: 4.1.2 - exit: 0.1.2 - graceful-fs: 4.2.11 - jest-config: 29.7.0(@types/node@24.0.1) - jest-util: 29.7.0 - prompts: 2.4.2 - transitivePeerDependencies: - - '@types/node' - - babel-plugin-macros - - supports-color - - ts-node - create-react-class@15.7.0: dependencies: loose-envify: 1.4.0 @@ -16301,8 +16272,6 @@ snapshots: detect-libc@1.0.3: optional: true - detect-libc@2.0.3: {} - detect-libc@2.0.4: {} detect-newline@3.1.0: {} @@ -18492,25 +18461,6 @@ snapshots: - supports-color - ts-node - jest-cli@29.7.0(@types/node@24.0.1): - dependencies: - '@jest/core': 29.7.0 - '@jest/test-result': 29.7.0 - '@jest/types': 29.6.3 - chalk: 4.1.2 - create-jest: 29.7.0(@types/node@24.0.1) - exit: 0.1.2 - import-local: 3.2.0 - jest-config: 29.7.0(@types/node@24.0.1) - jest-util: 29.7.0 - jest-validate: 29.7.0 - yargs: 17.7.2 - transitivePeerDependencies: - - '@types/node' - - babel-plugin-macros - - supports-color - - ts-node - jest-config@29.7.0(@types/node@18.19.111): dependencies: '@babel/core': 7.26.9 @@ -18541,36 +18491,6 @@ snapshots: - babel-plugin-macros - supports-color - jest-config@29.7.0(@types/node@24.0.1): - dependencies: - '@babel/core': 7.26.9 - '@jest/test-sequencer': 29.7.0 - '@jest/types': 29.6.3 - babel-jest: 29.7.0(@babel/core@7.26.9) - chalk: 4.1.2 - ci-info: 3.9.0 - deepmerge: 4.3.1 - glob: 7.2.3 - graceful-fs: 4.2.11 - jest-circus: 29.7.0 - jest-environment-node: 29.7.0 - jest-get-type: 29.6.3 - jest-regex-util: 29.6.3 - jest-resolve: 29.7.0 - jest-runner: 29.7.0 - jest-util: 29.7.0 - jest-validate: 29.7.0 - micromatch: 4.0.8 - parse-json: 5.2.0 - pretty-format: 29.7.0 - slash: 3.0.0 - strip-json-comments: 3.1.1 - optionalDependencies: - '@types/node': 24.0.1 - transitivePeerDependencies: - - babel-plugin-macros - - supports-color - jest-diff@26.6.2: dependencies: chalk: 4.1.2 @@ -18614,7 +18534,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@types/graceful-fs': 4.1.9 - '@types/node': 24.0.1 + '@types/node': 18.19.111 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -18709,7 +18629,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.0.1 + '@types/node': 18.19.111 chalk: 4.1.2 emittery: 0.13.1 graceful-fs: 4.2.11 @@ -18737,7 +18657,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.0.1 + '@types/node': 18.19.111 chalk: 4.1.2 cjs-module-lexer: 1.2.3 collect-v8-coverage: 1.0.2 @@ -18783,7 +18703,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 24.0.1 + '@types/node': 18.19.111 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -18802,7 +18722,7 @@ snapshots: dependencies: '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.0.1 + '@types/node': 18.19.111 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.13.1 @@ -18811,7 +18731,7 @@ snapshots: jest-worker@27.5.1: dependencies: - '@types/node': 22.15.3 + '@types/node': 18.19.111 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -18834,18 +18754,6 @@ snapshots: - supports-color - ts-node - jest@29.7.0(@types/node@24.0.1): - dependencies: - '@jest/core': 29.7.0 - '@jest/types': 29.6.3 - import-local: 3.2.0 - jest-cli: 29.7.0(@types/node@24.0.1) - transitivePeerDependencies: - - '@types/node' - - babel-plugin-macros - - supports-color - - ts-node - jquery-focus-exit@1.0.1(jquery@3.7.1): dependencies: jquery: 3.7.1 @@ -22428,26 +22336,6 @@ snapshots: babel-jest: 29.7.0(@babel/core@7.26.9) jest-util: 29.7.0 - ts-jest@29.4.0(@babel/core@7.26.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.9))(jest-util@29.7.0)(jest@29.7.0(@types/node@24.0.1))(typescript@5.8.3): - dependencies: - bs-logger: 0.2.6 - ejs: 3.1.10 - fast-json-stable-stringify: 2.1.0 - jest: 29.7.0(@types/node@24.0.1) - json5: 2.2.3 - lodash.memoize: 4.1.2 - make-error: 1.3.6 - semver: 7.7.2 - type-fest: 4.41.0 - typescript: 5.8.3 - yargs-parser: 21.1.1 - optionalDependencies: - '@babel/core': 7.26.9 - '@jest/transform': 29.7.0 - '@jest/types': 29.6.3 - babel-jest: 29.7.0(@babel/core@7.26.9) - jest-util: 29.7.0 - tsd@0.22.0: dependencies: '@tsd/typescript': 4.7.4 @@ -22573,8 +22461,6 @@ snapshots: undici-types@5.26.5: {} - undici-types@7.8.0: {} - unicorn-magic@0.1.0: {} unified@11.0.5: From 101e86e292d9640c3d278d53f3da45080124fa60 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 13 Jul 2025 23:03:58 +0000 Subject: [PATCH 23/47] basic btrfs testing started --- src/packages/file-server/package.json | 14 +++++++++++-- .../file-server/storage-btrfs/filesystem.ts | 8 ++++++++ .../storage-btrfs/test/basics.test.ts | 13 ++++++++++++ .../file-server/storage-btrfs/test/setup.ts | 20 +++++++++++++++++++ src/packages/file-server/tsconfig.json | 6 +++++- src/packages/pnpm-lock.yaml | 3 +++ 6 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 src/packages/file-server/storage-btrfs/test/basics.test.ts create mode 100644 src/packages/file-server/storage-btrfs/test/setup.ts diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index c3b43ffd04..b62bb6754b 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -17,15 +17,25 @@ "test": "pnpm exec jest --runInBand", "depcheck": "pnpx depcheck" }, - "files": ["dist/**", "README.md", "package.json"], + "files": [ + "dist/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["utilities", "btrfs", "zfs", "cocalc"], + "keywords": [ + "utilities", + "btrfs", + "zfs", + "cocalc" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/file-server": "workspace:*", "@cocalc/util": "workspace:*", + "@types/node": "^18.16.14", "awaiting": "^3.0.0", "better-sqlite3": "^11.10.0", "lodash": "^4.17.21" diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/storage-btrfs/filesystem.ts index acd02d6105..46667bd33e 100644 --- a/src/packages/file-server/storage-btrfs/filesystem.ts +++ b/src/packages/file-server/storage-btrfs/filesystem.ts @@ -154,6 +154,14 @@ export class Filesystem { }); }; + unmount = async () => { + await sudo({ + command: "umount", + args: [this.opts.mount], + err_on_exit: true, + }); + }; + private formatDevice = async () => { await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); }; diff --git a/src/packages/file-server/storage-btrfs/test/basics.test.ts b/src/packages/file-server/storage-btrfs/test/basics.test.ts new file mode 100644 index 0000000000..7b90267844 --- /dev/null +++ b/src/packages/file-server/storage-btrfs/test/basics.test.ts @@ -0,0 +1,13 @@ +import { before, after, fs } from "./setup"; + +beforeAll(before); + +describe("some basic tests", () => { + it("gets basic info", async () => { + const info = await fs.info(); + //console.log(info); + expect(info).not.toEqual(null); + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/storage-btrfs/test/setup.ts b/src/packages/file-server/storage-btrfs/test/setup.ts new file mode 100644 index 0000000000..c342381dbf --- /dev/null +++ b/src/packages/file-server/storage-btrfs/test/setup.ts @@ -0,0 +1,20 @@ +import { + filesystem, + type Filesystem, +} from "@cocalc/file-server/storage-btrfs/filesystem"; +import process from "node:process"; + +export let fs: Filesystem; + +export async function before() { + fs = await filesystem({ + device: "/tmp/btrfs.img", + formatIfNeeded: true, + mount: "/mnt/btrfs", + uid: process.getuid?.(), + }); +} + +export async function after() { + await fs.unmount(); +} diff --git a/src/packages/file-server/tsconfig.json b/src/packages/file-server/tsconfig.json index c2fcccc371..2e97efad86 100644 --- a/src/packages/file-server/tsconfig.json +++ b/src/packages/file-server/tsconfig.json @@ -1,9 +1,13 @@ { "extends": "../tsconfig.json", "compilerOptions": { + "types": ["node", "jest"], + "lib": ["es7"], "rootDir": "./", "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../util", "path": "../conat", "path": "../backend" }] + "references": [ + { "path": "../util", "path": "../conat", "path": "../backend" } + ] } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index e9ef7d834b..8a747aad76 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -298,6 +298,9 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util + '@types/node': + specifier: ^18.16.14 + version: 18.19.118 awaiting: specifier: ^3.0.0 version: 3.0.0 From 38eec7cad85ab5c4ec8ad13f79a195844fbe42eb Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 14 Jul 2025 04:42:15 +0000 Subject: [PATCH 24/47] adding some btrfs unit tests --- .../file-server/storage-btrfs/filesystem.ts | 23 +++++-- .../storage-btrfs/test/basics.test.ts | 63 ++++++++++++++++++- .../file-server/storage-btrfs/test/setup.ts | 20 +++++- 3 files changed, 96 insertions(+), 10 deletions(-) diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/storage-btrfs/filesystem.ts index 46667bd33e..d06905840c 100644 --- a/src/packages/file-server/storage-btrfs/filesystem.ts +++ b/src/packages/file-server/storage-btrfs/filesystem.ts @@ -20,7 +20,7 @@ import { rmdir, sudo, } from "@cocalc/file-server/storage-zfs/util"; -import { subvolume, SNAPSHOTS } from "./subvolume"; +import { subvolume, SNAPSHOTS, type Subvolume } from "./subvolume"; import { join, normalize } from "path"; // default size of btrfs filesystem if creating an image file. @@ -100,11 +100,18 @@ export class Filesystem { } }; - info = async () => { - return await sudo({ + info = async (): Promise<{ [field: string]: string }> => { + const { stdout } = await sudo({ command: "btrfs", args: ["subvolume", "show", this.opts.mount], }); + const obj: { [field: string]: string } = {}; + for (const x of stdout.split("\n")) { + const i = x.indexOf(":"); + if (i == -1) continue; + obj[x.slice(0, i).trim()] = x.slice(i + 1).trim(); + } + return obj; }; // @@ -170,7 +177,7 @@ export class Filesystem { // nothing, yet }; - subvolume = async (name: string) => { + subvolume = async (name: string): Promise => { if (RESERVED.has(name)) { throw Error(`${name} is reserved`); } @@ -215,7 +222,10 @@ export class Filesystem { }; deleteSubvolume = async (name: string) => { - await sudo({ command: "btrfs", args: ["subvolume", "delete", name] }); + await sudo({ + command: "btrfs", + args: ["subvolume", "delete", join(this.opts.mount, name)], + }); }; list = async (): Promise => { @@ -226,7 +236,8 @@ export class Filesystem { return stdout .split("\n") .map((x) => x.split(" ").slice(-1)[0]) - .filter((x) => x); + .filter((x) => x) + .sort(); }; rsync = async ({ diff --git a/src/packages/file-server/storage-btrfs/test/basics.test.ts b/src/packages/file-server/storage-btrfs/test/basics.test.ts index 7b90267844..933e4a36e1 100644 --- a/src/packages/file-server/storage-btrfs/test/basics.test.ts +++ b/src/packages/file-server/storage-btrfs/test/basics.test.ts @@ -1,13 +1,74 @@ import { before, after, fs } from "./setup"; +import { isValidUUID } from "@cocalc/util/misc"; +// import { readFile, writeFile } from "fs/promises"; +// import { join } from "path"; beforeAll(before); describe("some basic tests", () => { it("gets basic info", async () => { const info = await fs.info(); - //console.log(info); expect(info).not.toEqual(null); + expect(info.Name).toBe(""); + expect(isValidUUID(info.UUID)).toBe(true); + const creation = new Date(info["Creation time"]); + expect(Math.abs(creation.valueOf() - Date.now())).toBeLessThan(15000); + expect(info["Snapshot(s)"]).toBe(""); }); + + it("lists the subvolumes (there are none)", async () => { + expect(await fs.list()).toEqual([]); + }); +}); + +describe("operations with subvolumes", () => { + it("can't use a reserved subvolume name", async () => { + expect(async () => { + await fs.subvolume("bup"); + }).rejects.toThrow("is reserved"); + }); + + it("creates a subvolume", async () => { + const vol = await fs.subvolume("cocalc"); + expect(vol.name).toBe("cocalc"); + // it has no snapshots + expect(await vol.snapshots()).toEqual([]); + }); + + it("our subvolume is in the list", async () => { + expect(await fs.list()).toEqual(["cocalc"]); + }); + + it("create another two subvolumes", async () => { + await fs.subvolume("sagemath"); + await fs.subvolume("a-math"); + // list is sorted: + expect(await fs.list()).toEqual(["a-math", "cocalc", "sagemath"]); + }); + + it("delete a subvolume", async () => { + await fs.deleteSubvolume("a-math"); + expect(await fs.list()).toEqual(["cocalc", "sagemath"]); + }); + + it("clone a subvolume", async () => { + await fs.cloneSubvolume("sagemath", "cython"); + expect(await fs.list()).toEqual(["cocalc", "cython", "sagemath"]); + }); + + it("rsync from one volume to another", async () => { + await fs.rsync({ src: "sagemath", target: "cython" }); + }); + + // it("rsync with an actual file", async () => { + // const sagemath = await fs.subvolume("sagemath"); + // const cython = await fs.subvolume("cython"); + // await writeFile(join(sagemath.path, "README.md"), "hi"); + // await fs.rsync({ src: "sagemath", target: "cython" }); + // expect((await readFile(join(sagemath.path, "README.md")), "utf8")).toEqual( + // "hi", + // ); + // }); }); afterAll(after); diff --git a/src/packages/file-server/storage-btrfs/test/setup.ts b/src/packages/file-server/storage-btrfs/test/setup.ts index c342381dbf..51a7e1c9e8 100644 --- a/src/packages/file-server/storage-btrfs/test/setup.ts +++ b/src/packages/file-server/storage-btrfs/test/setup.ts @@ -3,18 +3,32 @@ import { type Filesystem, } from "@cocalc/file-server/storage-btrfs/filesystem"; import process from "node:process"; +import { chmod, mkdtemp, mkdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; export let fs: Filesystem; +let tempDir; export async function before() { + tempDir = await mkdtemp(join(tmpdir(), "cocalc-test-btrfs-")); + // Set world read/write/execute + await chmod(tempDir, 0o777); + const mount = join(tempDir, "mnt"); + await mkdir(mount); + await chmod(mount, 0o777); fs = await filesystem({ - device: "/tmp/btrfs.img", + device: join(tempDir, "btrfs.img"), formatIfNeeded: true, - mount: "/mnt/btrfs", + mount: join(tempDir, "mnt"), uid: process.getuid?.(), }); } export async function after() { - await fs.unmount(); + try { + await fs.unmount(); + } catch {} + console.log("deleting ", tempDir); + await rm(tempDir, { force: true, recursive: true }); } From b99c9099bfccc60c525ba0bfc2faa1afc1db850d Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 14 Jul 2025 20:04:55 +0000 Subject: [PATCH 25/47] add missing package --- src/packages/file-server/package.json | 1 + src/packages/pnpm-lock.yaml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index b62bb6754b..90285d0ec7 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -35,6 +35,7 @@ "@cocalc/conat": "workspace:*", "@cocalc/file-server": "workspace:*", "@cocalc/util": "workspace:*", + "@types/jest": "^29.5.14", "@types/node": "^18.16.14", "awaiting": "^3.0.0", "better-sqlite3": "^11.10.0", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 8a747aad76..04316cec4f 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -298,6 +298,9 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 '@types/node': specifier: ^18.16.14 version: 18.19.118 From 20a2444e6557f6aa28e54944a2d462312faa4e82 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 14 Jul 2025 22:14:47 +0000 Subject: [PATCH 26/47] more subvolume tests --- .../storage-btrfs/test/basics.test.ts | 28 +++++++++++-------- .../file-server/storage-btrfs/test/setup.ts | 1 - 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/packages/file-server/storage-btrfs/test/basics.test.ts b/src/packages/file-server/storage-btrfs/test/basics.test.ts index 933e4a36e1..17d181f307 100644 --- a/src/packages/file-server/storage-btrfs/test/basics.test.ts +++ b/src/packages/file-server/storage-btrfs/test/basics.test.ts @@ -1,7 +1,7 @@ import { before, after, fs } from "./setup"; import { isValidUUID } from "@cocalc/util/misc"; -// import { readFile, writeFile } from "fs/promises"; -// import { join } from "path"; +import { readFile, writeFile } from "fs/promises"; +import { join } from "path"; beforeAll(before); @@ -60,15 +60,21 @@ describe("operations with subvolumes", () => { await fs.rsync({ src: "sagemath", target: "cython" }); }); - // it("rsync with an actual file", async () => { - // const sagemath = await fs.subvolume("sagemath"); - // const cython = await fs.subvolume("cython"); - // await writeFile(join(sagemath.path, "README.md"), "hi"); - // await fs.rsync({ src: "sagemath", target: "cython" }); - // expect((await readFile(join(sagemath.path, "README.md")), "utf8")).toEqual( - // "hi", - // ); - // }); + it("rsync an actual file", async () => { + const sagemath = await fs.subvolume("sagemath"); + const cython = await fs.subvolume("cython"); + await writeFile(join(sagemath.path, "README.md"), "hi"); + await fs.rsync({ src: "sagemath", target: "cython" }); + const copy = await readFile(join(cython.path, "README.md"), "utf8"); + expect(copy).toEqual("hi"); + }); + + it("clone a subvolume with contents", async () => { + await fs.cloneSubvolume("cython", "pyrex"); + const pyrex = await fs.subvolume("pyrex"); + const clone = await readFile(join(pyrex.path, "README.md"), "utf8"); + expect(clone).toEqual("hi"); + }); }); afterAll(after); diff --git a/src/packages/file-server/storage-btrfs/test/setup.ts b/src/packages/file-server/storage-btrfs/test/setup.ts index 51a7e1c9e8..c0cd036d17 100644 --- a/src/packages/file-server/storage-btrfs/test/setup.ts +++ b/src/packages/file-server/storage-btrfs/test/setup.ts @@ -29,6 +29,5 @@ export async function after() { try { await fs.unmount(); } catch {} - console.log("deleting ", tempDir); await rm(tempDir, { force: true, recursive: true }); } From 92c74be82059b6965f359bf5259c15ef7733f4d9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 14 Jul 2025 22:21:46 +0000 Subject: [PATCH 27/47] delete all the zfs code - that's two weeks work that never got used for anything. it was pretty cool too and complicated/powerful. Deleted! --- .../{storage-btrfs => btrfs}/filesystem.ts | 9 +- .../{storage-btrfs => btrfs}/index.ts | 0 .../{storage-btrfs => btrfs}/snapshots.ts | 0 .../{storage-btrfs => btrfs}/subvolume.ts | 2 +- .../test/filesystem.test.ts} | 0 .../{storage-btrfs => btrfs}/test/setup.ts | 2 +- src/packages/file-server/package.json | 24 +- .../file-server/storage-zfs/README.md | 23 -- .../file-server/storage-zfs/filesystem.ts | 251 ------------ src/packages/file-server/storage-zfs/index.ts | 1 - src/packages/file-server/storage-zfs/pool.ts | 386 ------------------ src/packages/file-server/storage-zfs/pools.ts | 113 ----- .../file-server/storage-zfs/snapshots.ts | 115 ------ src/packages/file-server/storage-zfs/util.ts | 71 ---- src/packages/file-server/zfs/archive.ts | 236 ----------- src/packages/file-server/zfs/backup.ts | 176 -------- src/packages/file-server/zfs/config.ts | 107 ----- src/packages/file-server/zfs/copy.ts | 14 - src/packages/file-server/zfs/create.ts | 201 --------- src/packages/file-server/zfs/db.ts | 274 ------------- src/packages/file-server/zfs/index.ts | 29 -- src/packages/file-server/zfs/names.ts | 196 --------- src/packages/file-server/zfs/nfs.ts | 111 ----- src/packages/file-server/zfs/pools.ts | 284 ------------- src/packages/file-server/zfs/properties.ts | 127 ------ src/packages/file-server/zfs/pull.ts | 303 -------------- src/packages/file-server/zfs/snapshots.ts | 315 -------------- src/packages/file-server/zfs/streams.ts | 253 ------------ .../file-server/zfs/test/archive.test.ts | 105 ----- .../file-server/zfs/test/create-types.test.ts | 78 ---- .../file-server/zfs/test/create.test.ts | 256 ------------ src/packages/file-server/zfs/test/nfs.test.ts | 118 ------ .../file-server/zfs/test/pull.test.ts | 315 -------------- src/packages/file-server/zfs/test/util.ts | 63 --- src/packages/file-server/zfs/types.ts | 195 --------- src/packages/file-server/zfs/util.ts | 29 -- src/packages/pnpm-lock.yaml | 20 +- 37 files changed, 11 insertions(+), 4791 deletions(-) rename src/packages/file-server/{storage-btrfs => btrfs}/filesystem.ts (98%) rename src/packages/file-server/{storage-btrfs => btrfs}/index.ts (100%) rename src/packages/file-server/{storage-btrfs => btrfs}/snapshots.ts (100%) rename src/packages/file-server/{storage-btrfs => btrfs}/subvolume.ts (99%) rename src/packages/file-server/{storage-btrfs/test/basics.test.ts => btrfs/test/filesystem.test.ts} (100%) rename src/packages/file-server/{storage-btrfs => btrfs}/test/setup.ts (93%) delete mode 100644 src/packages/file-server/storage-zfs/README.md delete mode 100644 src/packages/file-server/storage-zfs/filesystem.ts delete mode 100644 src/packages/file-server/storage-zfs/index.ts delete mode 100644 src/packages/file-server/storage-zfs/pool.ts delete mode 100644 src/packages/file-server/storage-zfs/pools.ts delete mode 100644 src/packages/file-server/storage-zfs/snapshots.ts delete mode 100644 src/packages/file-server/storage-zfs/util.ts delete mode 100644 src/packages/file-server/zfs/archive.ts delete mode 100644 src/packages/file-server/zfs/backup.ts delete mode 100644 src/packages/file-server/zfs/config.ts delete mode 100644 src/packages/file-server/zfs/copy.ts delete mode 100644 src/packages/file-server/zfs/create.ts delete mode 100644 src/packages/file-server/zfs/db.ts delete mode 100644 src/packages/file-server/zfs/index.ts delete mode 100644 src/packages/file-server/zfs/names.ts delete mode 100644 src/packages/file-server/zfs/nfs.ts delete mode 100644 src/packages/file-server/zfs/pools.ts delete mode 100644 src/packages/file-server/zfs/properties.ts delete mode 100644 src/packages/file-server/zfs/pull.ts delete mode 100644 src/packages/file-server/zfs/snapshots.ts delete mode 100644 src/packages/file-server/zfs/streams.ts delete mode 100644 src/packages/file-server/zfs/test/archive.test.ts delete mode 100644 src/packages/file-server/zfs/test/create-types.test.ts delete mode 100644 src/packages/file-server/zfs/test/create.test.ts delete mode 100644 src/packages/file-server/zfs/test/nfs.test.ts delete mode 100644 src/packages/file-server/zfs/test/pull.test.ts delete mode 100644 src/packages/file-server/zfs/test/util.ts delete mode 100644 src/packages/file-server/zfs/types.ts delete mode 100644 src/packages/file-server/zfs/util.ts diff --git a/src/packages/file-server/storage-btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts similarity index 98% rename from src/packages/file-server/storage-btrfs/filesystem.ts rename to src/packages/file-server/btrfs/filesystem.ts index d06905840c..79e1f19785 100644 --- a/src/packages/file-server/storage-btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -12,14 +12,7 @@ a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({devic */ import refCache from "@cocalc/util/refcache"; -import { - exists, - isdir, - listdir, - mkdirp, - rmdir, - sudo, -} from "@cocalc/file-server/storage-zfs/util"; +import { exists, isdir, listdir, mkdirp, rmdir, sudo } from "./util"; import { subvolume, SNAPSHOTS, type Subvolume } from "./subvolume"; import { join, normalize } from "path"; diff --git a/src/packages/file-server/storage-btrfs/index.ts b/src/packages/file-server/btrfs/index.ts similarity index 100% rename from src/packages/file-server/storage-btrfs/index.ts rename to src/packages/file-server/btrfs/index.ts diff --git a/src/packages/file-server/storage-btrfs/snapshots.ts b/src/packages/file-server/btrfs/snapshots.ts similarity index 100% rename from src/packages/file-server/storage-btrfs/snapshots.ts rename to src/packages/file-server/btrfs/snapshots.ts diff --git a/src/packages/file-server/storage-btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts similarity index 99% rename from src/packages/file-server/storage-btrfs/subvolume.ts rename to src/packages/file-server/btrfs/subvolume.ts index 7b55dd09b7..768fa87bee 100644 --- a/src/packages/file-server/storage-btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -9,7 +9,7 @@ import { listdir, mkdirp, sudo, -} from "@cocalc/file-server/storage-zfs/util"; +} from "./util"; import { join } from "path"; import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; import { human_readable_size } from "@cocalc/util/misc"; diff --git a/src/packages/file-server/storage-btrfs/test/basics.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts similarity index 100% rename from src/packages/file-server/storage-btrfs/test/basics.test.ts rename to src/packages/file-server/btrfs/test/filesystem.test.ts diff --git a/src/packages/file-server/storage-btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts similarity index 93% rename from src/packages/file-server/storage-btrfs/test/setup.ts rename to src/packages/file-server/btrfs/test/setup.ts index c0cd036d17..9487833385 100644 --- a/src/packages/file-server/storage-btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -1,7 +1,7 @@ import { filesystem, type Filesystem, -} from "@cocalc/file-server/storage-btrfs/filesystem"; +} from "@cocalc/file-server/btrfs/filesystem"; import process from "node:process"; import { chmod, mkdtemp, mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index 90285d0ec7..d6480a39bf 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -3,19 +3,16 @@ "version": "1.0.0", "description": "CoCalc File Server", "exports": { - "./zfs": "./dist/zfs/index.js", - "./zfs/*": "./dist/zfs/*.js", - "./storage-zfs": "./dist/storage-zfs/index.js", - "./storage-zfs/*": "./dist/storage-zfs/*.js", - "./storage-btrfs": "./dist/storage-btrfs/index.js", - "./storage-btrfs/*": "./dist/storage-btrfs/*.js" + "./btrfs": "./dist/btrfs/index.js", + "./btrfs/*": "./dist/btrfs/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --runInBand", - "depcheck": "pnpx depcheck" + "depcheck": "pnpx depcheck", + "clean": "rm -rf node_modules dist" }, "files": [ "dist/**", @@ -26,24 +23,17 @@ "keywords": [ "utilities", "btrfs", - "zfs", "cocalc" ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", - "@cocalc/conat": "workspace:*", "@cocalc/file-server": "workspace:*", - "@cocalc/util": "workspace:*", - "@types/jest": "^29.5.14", - "@types/node": "^18.16.14", - "awaiting": "^3.0.0", - "better-sqlite3": "^11.10.0", - "lodash": "^4.17.21" + "@cocalc/util": "workspace:*" }, "devDependencies": { - "@types/better-sqlite3": "^7.6.13", - "@types/lodash": "^4.14.202" + "@types/jest": "^29.5.14", + "@types/node": "^18.16.14" }, "repository": { "type": "git", diff --git a/src/packages/file-server/storage-zfs/README.md b/src/packages/file-server/storage-zfs/README.md deleted file mode 100644 index 3e2c3bdc25..0000000000 --- a/src/packages/file-server/storage-zfs/README.md +++ /dev/null @@ -1,23 +0,0 @@ -This module allows one to: - -Parameters: - -- Images path -- Mount path - -What it does: - -- Create a pool - -- Create a named filesystem in a pool. - -- Set properties of a filesystem - - quota - - dedup - -- Manages the underlying storage: - - trim - - snapshot - - backup - - replicate - - archive to cloud storage diff --git a/src/packages/file-server/storage-zfs/filesystem.ts b/src/packages/file-server/storage-zfs/filesystem.ts deleted file mode 100644 index ff1f79d210..0000000000 --- a/src/packages/file-server/storage-zfs/filesystem.ts +++ /dev/null @@ -1,251 +0,0 @@ -import refCache from "@cocalc/util/refcache"; -import { sudo } from "./util"; -import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; -import getLogger from "@cocalc/backend/logger"; - -const logger = getLogger("file-server:storage:filesystem"); - -const FILESYSTEM_NAME_REGEXP = /^(?!-)(?!(\.{1,2})$)[A-Za-z0-9_.:-]{1,255}$/; - -export interface Options { - // name of pool - pool: string; - // name of filesystem - name: string; - // if given when creating the current filesystem, it will be made as a clone - // of "clone". This only needs to be set when the filesystem is created. - clone?: string; -} - -export class Filesystem { - public readonly dataset: string; - private opts: Options; - - constructor(opts: Options) { - if (opts.name !== "" && !FILESYSTEM_NAME_REGEXP.test(opts.name)) { - throw Error(`invalid ZFS filesystem name '${opts.name}'`); - } - this.opts = opts; - this.dataset = opts.name ? `${opts.pool}/${opts.name}` : opts.pool; - } - - exists = async () => { - try { - await this._info(); - return true; - } catch { - return false; - } - }; - - private async ensureExists(f: () => Promise): Promise { - try { - return await f(); - } catch (err) { - if (`${err}`.includes("dataset does not exist")) { - await this.create(); - return await f(); - } - throw err; - } - throw Error("bug"); - } - - create = async () => { - if (await this.exists()) { - return; - } - if (this.opts.clone) { - logger.debug("create clone", { - dataset: this.dataset, - clone: this.opts.clone, - }); - const snapshot = `${this.opts.pool}/${this.opts.clone}@clone-${this.opts.name}`; - await sudo({ - command: "zfs", - args: ["snapshot", snapshot], - }); - try { - // create as a clone - await sudo({ - command: "zfs", - args: ["clone", snapshot, this.dataset], - }); - } catch (err) { - // we only delete the snapshot on error, since it can't be deleted as - // long as the clone exists: - await sudo({ command: "zfs", args: ["destroy", snapshot] }); - throw err; - } - } else { - logger.debug("create dataset", { - dataset: this.dataset, - }); - // non-clone - await sudo({ - command: "zfs", - args: ["create", this.dataset], - }); - } - }; - - info = async (): Promise => { - return await this.ensureExists(this._info); - }; - - private _info = async (): Promise => { - const { stdout } = await sudo({ - command: "zfs", - args: ["list", "-j", "--json-int", this.dataset], - }); - const x = JSON.parse(stdout); - const y = x.datasets[this.dataset]; - for (const a in y.properties) { - y.properties[a] = y.properties[a].value; - } - return y; - }; - - get = async (property: string) => { - return await this.ensureExists(async () => { - const { stdout } = await sudo({ - command: "zfs", - args: ["get", "-j", "--json-int", property, this.dataset], - }); - const x = JSON.parse(stdout); - const { value } = x.datasets[this.dataset].properties[property]; - if (/^-?\d+(\.\d+)?$/.test(value)) { - return parseFloat(value); - } else { - return value; - } - }); - }; - - set = async (props: { [property: string]: any }) => { - return await this.ensureExists(async () => { - const v: string[] = []; - for (const p in props) { - v.push(`${p}=${props[p]}`); - } - if (v.length == 0) { - return; - } - await sudo({ - command: "zfs", - args: ["set", ...v, this.dataset], - }); - }); - }; - - close = () => { - // nothing, yet - }; - - createSnapshot = async (name: string) => { - logger.debug("createSnapshot", { name, dataset: this.dataset }); - await this.ensureExists(async () => { - await sudo({ - command: "zfs", - args: ["snapshot", `${this.dataset}@${name}`], - }); - }); - }; - - snapshots = async (): Promise => { - return await this.ensureExists(async () => { - const { stdout } = await sudo({ - command: "zfs", - args: [ - "list", - "-j", - "--json-int", - "-r", - "-d", - "1", - "-t", - "snapshot", - `${this.dataset}`, - ], - }); - const { datasets } = JSON.parse(stdout); - for (const name in datasets) { - const y = datasets[name]; - for (const a in y.properties) { - y.properties[a] = y.properties[a].value; - } - } - return datasets; - }); - }; - - destroySnapshot = async (name) => { - logger.debug("destroySnapshot", { name, dataset: this.dataset }); - await this.ensureExists(async () => { - await sudo({ - command: "zfs", - args: ["destroy", `${this.dataset}@${name}`], - }); - }); - }; - - updateRollingSnapshots = async (counts?: Partial) => { - return await this.ensureExists(async () => { - return await updateRollingSnapshots({ filesystem: this, counts }); - }); - }; - - // number of newly written bytes in filesystem since last snapshot - writtenSinceLastSnapshot = async (): Promise => { - return await this.ensureExists(async () => { - const { stdout } = await sudo({ - command: "zfs", - args: ["list", "-Hpo", "written", this.dataset], - }); - return parseInt(stdout); - }); - }; -} - -interface FilesystemListOutput { - name: string; - type: "FILESYSTEM"; - pool: string; - createtxg: number; - properties: { - used: number; - available: number; - referenced: number; - mountpoint: string; - }; -} - -interface Snapshot { - name: string; - type: "SNAPSHOT"; - pool: string; - createtxg: number; - dataset: string; - snapshot_name: string; - properties: { - used: number; - available: string | number; - referenced: number; - mountpoint: string; // '-' if not mounted - }; -} - -export type Snapshots = { [name: string]: Snapshot }; - -const cache = refCache({ - name: "zfs-filesystem", - createObject: async (options: Options) => { - return new Filesystem(options); - }, -}); - -export async function filesystem( - options: Options & { noCache?: boolean }, -): Promise { - return await cache(options); -} diff --git a/src/packages/file-server/storage-zfs/index.ts b/src/packages/file-server/storage-zfs/index.ts deleted file mode 100644 index ec45ac575f..0000000000 --- a/src/packages/file-server/storage-zfs/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { pools } from "./pools"; diff --git a/src/packages/file-server/storage-zfs/pool.ts b/src/packages/file-server/storage-zfs/pool.ts deleted file mode 100644 index f462a57e5e..0000000000 --- a/src/packages/file-server/storage-zfs/pool.ts +++ /dev/null @@ -1,386 +0,0 @@ -import refCache from "@cocalc/util/refcache"; -import { chmod, sudo, exists, mkdirp, rm, rmdir, listdir } from "./util"; -import { join } from "path"; -import { filesystem } from "./filesystem"; -import getLogger from "@cocalc/backend/logger"; -import { executeCode } from "@cocalc/backend/execute-code"; -import { randomId } from "@cocalc/conat/names"; - -const logger = getLogger("file-server:storage:pool"); - -const DEFAULT_SIZE = "1G"; -const POOL_NAME_REGEXP = /^(?!-)(?!(\.{1,2})$)[A-Za-z0-9_.:-]{1,255}$/; - -export interface Options { - // where to store its image file(s) - images: string; - // where to mount its filesystems - mount: string; - // the name of the pool - name: string; -} - -export class Pool { - private opts: Options; - private image: string; - - constructor(opts: Options) { - if (!POOL_NAME_REGEXP.test(opts.name)) { - throw Error(`invalid ZFS pool name '${opts.name}'`); - } - this.opts = opts; - this.image = join(opts.images, "0.img"); - } - - exists = async () => { - return await exists(this.image); - }; - - destroy = async () => { - try { - await sudo({ - command: "zpool", - args: ["destroy", "-f", this.opts.name], - }); - } catch (err) { - if (!`${err}`.includes("no such pool")) { - throw err; - } - } - if (await exists(this.image)) { - await rm([this.image]); - } - if (await exists(this.opts.images)) { - await rmdir([this.opts.images]); - } - if (await exists(this.opts.mount)) { - const v = await listdir(this.opts.mount); - await rmdir(v.map((x) => join(this.opts.mount, x))); - await rmdir([this.opts.mount]); - } - }; - - // enlarge pool to have given size (which can be a string like '1G' or - // a number of bytes). This is very fast/CHEAP and can be done live. - enlarge = async (size: string | number) => { - logger.debug(`enlarge to ${size}`); - size = await sizeToBytes(size); - if (typeof size != "number") { - throw Error("bug"); - } - if (!(await exists(this.image))) { - await this.create(); - } - const { stdout } = await sudo({ - command: "stat", - args: ["--format=%s", this.image], - }); - const bytes = parseFloat(stdout); - if (size < bytes) { - throw Error(`size must be at least ${bytes}`); - } - if (size == bytes) { - return; - } - await this.ensureExists(async () => { - await sudo({ command: "truncate", args: ["-s", `${size}`, this.image] }); - await sudo({ - command: "zpool", - args: ["online", "-e", this.opts.name, this.image], - }); - }); - }; - - // shrink pool to have given size (which can be a string like '1G' or - // a number of bytes). This is EXPENSIVE, requiring rewriting everything, and - // the pool must be unmounted. - shrink = async (size: string | number) => { - // TODO: this is so dangerous, so make sure there is a backup first, once - // backups are implemented - logger.debug(`shrink to ${size}`); - logger.debug("shrink -- 0. size checks"); - size = await sizeToBytes(size); - if (typeof size != "number") { - throw Error("bug"); - } - if (size < (await sizeToBytes(DEFAULT_SIZE))) { - throw Error(`size must be at least ${DEFAULT_SIZE}`); - } - const info = await this.info(); - // TOOD: this is made up - const min_alloc = info.properties.allocated * 1.25 + 1000000; - if (size <= min_alloc) { - throw Error( - `size must be at least as big as currently allocated space ${min_alloc}`, - ); - } - if (size >= info.properties.size) { - logger.debug("shrink -- it's already smaller than the shrink goal."); - return; - } - logger.debug("shrink -- 1. unmount all datasets"); - for (const dataset of Object.keys(await this.list())) { - try { - await sudo({ command: "zfs", args: ["unmount", dataset] }); - } catch (err) { - if (`${err}`.includes("not currently mounted")) { - // that's fine - continue; - } - throw err; - } - } - logger.debug("shrink -- 2. make new smaller temporary pool"); - const id = "-" + randomId(); - const name = this.opts.name + id; - const images = this.opts.images + id; - const mount = this.opts.mount + id; - const temp = await pool({ images, mount, name }); - await temp.create(); - await temp.enlarge(size); - const snapshot = `${this.opts.name}@shrink${id}`; - logger.debug("shrink -- 3. replicate data to target"); - await sudo({ command: "zfs", args: ["snapshot", "-r", snapshot] }); - try { - await executeCode({ - command: `sudo zfs send -c -R ${snapshot} | sudo zfs recv -F ${name}`, - }); - } catch (err) { - await temp.destroy(); - throw err; - } - await temp.export(); - - logger.debug("shrink -- 4. destroy original pool"); - await this.destroy(); - logger.debug("shrink -- 5. rename temporary pool"); - await sudo({ - command: "zpool", - args: ["import", "-d", images, name, this.opts.name], - }); - await sudo({ - command: "zpool", - args: ["export", this.opts.name], - }); - logger.debug("shrink -- 6. move image file"); - await mkdirp([this.opts.images, this.opts.mount]); - await sudo({ command: "mv", args: [temp.image, this.image] }); - logger.debug("shrink -- 7. destroy temp files"); - await temp.destroy(); - logger.debug("shrink -- 8. Import our new pool"); - await this.import(); - }; - - filesystem = async (name) => { - await this.import(); - return await filesystem({ pool: this.opts.name, name }); - }; - - // create a lightweight clone callend name of the given filesystem source. - clone = async (name: string, source: string) => { - await this.import(); - return await filesystem({ pool: this.opts.name, name, clone: source }); - }; - - import = async () => { - if (!(await this.exists())) { - await this.create(); - return; - } - try { - await sudo({ - command: "zpool", - args: ["import", this.opts.name, "-d", this.opts.images], - verbose: false, - }); - } catch (err) { - if (`${err}`.includes("pool with that name already exists")) { - // already imported - return; - } - throw err; - } - }; - - export = async () => { - try { - await sudo({ - command: "zpool", - args: ["export", this.opts.name], - }); - } catch (err) { - if (`${err}`.includes("no such pool")) { - return; - } - throw err; - } - }; - - create = async () => { - if (await this.exists()) { - // already exists - return; - } - await mkdirp([this.opts.images, this.opts.mount]); - await chmod(["a+rx", this.opts.mount]); - await sudo({ - command: "truncate", - args: ["-s", DEFAULT_SIZE, this.image], - }); - await sudo({ - command: "zpool", - args: [ - "create", - "-o", - "feature@fast_dedup=enabled", - "-m", - this.opts.mount, - this.opts.name, - this.image, - ], - desc: `create the pool ${this.opts.name} using the device ${this.image}`, - }); - await sudo({ - command: "zfs", - args: ["set", "compression=lz4", "dedup=on", this.opts.name], - }); - }; - - private async ensureExists(f: () => Promise): Promise { - try { - return await f(); - } catch (err) { - if (`${err}`.includes("no such pool")) { - await this.import(); - return await f(); - } - throw err; - } - throw Error("bug"); - } - - info = async (): Promise => { - return await this.ensureExists(async () => { - const { stdout } = await sudo({ - command: "zpool", - args: ["list", "-j", "--json-int", this.opts.name], - }); - const x = JSON.parse(stdout); - const y = x.pools[this.opts.name]; - for (const a in y.properties) { - y.properties[a] = y.properties[a].value; - } - y.properties.dedupratio = parseFloat(y.properties.dedupratio); - return y; - }); - }; - - status = async (): Promise => { - return await this.ensureExists(async () => { - const { stdout } = await sudo({ - command: "zpool", - args: ["status", "-j", "--json-int", this.opts.name], - }); - const x = JSON.parse(stdout); - return x.pools[this.opts.name]; - }); - }; - - trim = async () => { - return await this.ensureExists(async () => { - await sudo({ - command: "zpool", - args: ["trim", "-w", this.opts.name], - }); - }); - }; - - // bytes of disk used by image - bytes = async (): Promise => { - const { stdout } = await sudo({ - command: "ls", - args: ["-s", this.image], - }); - return parseFloat(stdout.split(" ")[0]); - }; - - list = async (): Promise<{ [dataset: string]: Dataset }> => { - return await this.ensureExists<{ [dataset: string]: Dataset }>(async () => { - const { stdout } = await sudo({ - command: "zfs", - args: ["list", "-j", "--json-int", "-r", this.opts.name], - }); - const { datasets } = JSON.parse(stdout); - for (const name in datasets) { - const y = datasets[name]; - for (const a in y.properties) { - y.properties[a] = y.properties[a].value; - } - } - return datasets; - }); - }; - - close = () => { - // nothing, yet - }; -} - -interface Dataset { - name: string; - type: "FILESYSTEM"; - pool: string; - createtxg: number; - properties: { - used: number; - available: number; - referenced: number; - mountpoint: string; - }; -} - -const cache = refCache({ - name: "zfs-pool", - createObject: async (options: Options) => { - return new Pool(options); - }, -}); - -export async function pool( - options: Options & { noCache?: boolean }, -): Promise { - return await cache(options); -} - -interface PoolListOutput { - name: string; - type: "POOL"; - state: "ONLINE" | string; // todo - pool_guid: number; - txg: number; - spa_version: number; - zpl_version: number; - properties: { - size: number; - allocated: number; - free: number; - checkpoint: string; - expandsize: string; - fragmentation: number; - capacity: number; - dedupratio: number; - health: "ONLINE" | string; // todo - altroot: string; - }; -} - -async function sizeToBytes(size: number | string): Promise { - if (typeof size == "number") { - return size; - } - const { stdout } = await executeCode({ - command: "numfmt", - args: ["--from=iec", size], - }); - return parseFloat(stdout); -} diff --git a/src/packages/file-server/storage-zfs/pools.ts b/src/packages/file-server/storage-zfs/pools.ts deleted file mode 100644 index 7268c47cef..0000000000 --- a/src/packages/file-server/storage-zfs/pools.ts +++ /dev/null @@ -1,113 +0,0 @@ -/* -DEVELOPMENT: - -Start node, then: - -a = require('@cocalc/file-server/storage-zfs') -pools = await a.pools({images:'/data/zfs/images', mount:'/data/zfs/mnt'}) - -x = await pools.pool('x') - -t = await x.list() - -p = await x.filesystem('1') -await p.create() - -await x.enlarge('1T') - -await x.shrink('3G') - -await p.get('compressratio') - -await p.list() - -q = await x.clone('c', '1') -await q.create() -await q.get('origin') // --> 'x/1@clone-c' - -// around 10 seconds: - -t = Date.now(); for(let i=0; i<100; i++) { await (await pools.pool('x'+i)).create() }; Date.now() - t - -// around 5 seconds: - -t = Date.now(); for(let i=0; i<100; i++) { await (await x.filesystem('x'+i)).create() }; Date.now() - t - - -*/ - -import refCache from "@cocalc/util/refcache"; -import { join } from "path"; -import { listdir, mkdirp, sudo } from "./util"; -import { pool } from "./pool"; - -export interface Options { - images: string; - mount: string; -} - -export class Pools { - private opts: Options; - - constructor(opts: Options) { - this.opts = opts; - } - - init = async () => { - await mkdirp([this.opts.images, this.opts.mount]); - }; - - close = () => { - // nothing, yet - }; - - pool = async (name: string) => { - const images = join(this.opts.images, name); - const mount = join(this.opts.mount, name); - return await pool({ images, mount, name }); - }; - - list = async (): Promise => { - return await listdir(this.opts.images); - }; - - rsync = async ({ - src, - target, - args = ["-axH"], - timeout = 5 * 60 * 1000, - }: { - src: string; - target: string; - args?: string[]; - timeout?: number; - }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - const srcPool = await this.pool(src.split("/")[0]); - await srcPool.import(); - const targetPool = await this.pool(target.split("/")[0]); - await targetPool.import(); - const srcPath = join(this.opts.mount, src); - const targetPath = join(this.opts.mount, target); - return await sudo({ - command: "rsync", - args: [...args, srcPath, targetPath], - err_on_exit: false, - timeout: timeout / 1000, - }); - }; -} - -const cache = refCache({ - name: "zfs-pools", - createObject: async (options: Options) => { - const pools = new Pools(options); - await pools.init(); - return pools; - }, -}); - -export async function pools( - options: Options & { noCache?: boolean }, -): Promise { - return await cache(options); -} diff --git a/src/packages/file-server/storage-zfs/snapshots.ts b/src/packages/file-server/storage-zfs/snapshots.ts deleted file mode 100644 index 11f28a890e..0000000000 --- a/src/packages/file-server/storage-zfs/snapshots.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { type Filesystem } from "./filesystem"; -import getLogger from "@cocalc/backend/logger"; - -const logger = getLogger("file-server:storage:snapshots"); - -const DATE_REGEXP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; - -// Lengths of time in minutes to keep snapshots -// (code below assumes these are listed in ORDER from shortest to longest) -export const SNAPSHOT_INTERVALS_MS = { - frequent: 15 * 1000 * 60, - daily: 60 * 24 * 1000 * 60, - weekly: 60 * 24 * 7 * 1000 * 60, - monthly: 60 * 24 * 7 * 4 * 1000 * 60, -}; - -// How many of each type of snapshot to retain -export const DEFAULT_SNAPSHOT_COUNTS = { - frequent: 24, - daily: 14, - weekly: 7, - monthly: 4, -} as SnapshotCounts; - -export interface SnapshotCounts { - frequent: number; - daily: number; - weekly: number; - monthly: number; -} - -export async function updateRollingSnapshots({ - filesystem, - counts, -}: { - filesystem: Filesystem; - counts?: Partial; -}) { - counts = { ...DEFAULT_SNAPSHOT_COUNTS, ...counts }; - - const written = await filesystem.writtenSinceLastSnapshot(); - logger.debug("updateRollingSnapshots", { - dataset: filesystem.dataset, - counts, - written, - }); - if (written == 0) { - // definitely no data written since most recent snapshot, so nothing to do - return; - } - - // get exactly the iso timestamp snapshot names: - const snapshots = Object.values(await filesystem.snapshots()) - .map((z) => z.snapshot_name) - .filter((x) => DATE_REGEXP.test(x)); - snapshots.sort(); - if (snapshots.length > 0) { - const age = Date.now() - new Date(snapshots.slice(-1)[0]).valueOf(); - for (const key in SNAPSHOT_INTERVALS_MS) { - if (counts[key]) { - if (age < SNAPSHOT_INTERVALS_MS[key]) { - // no need to snapshot since there is already a sufficiently recent snapshot - logger.debug("updateRollingSnapshots: no need to snapshot", { - dataset: filesystem.dataset, - }); - return; - } - // counts[key] nonzero and snapshot is old enough so we'll be making a snapshot - break; - } - } - } - - // make a new snapshot - const snapshot = new Date().toISOString(); - await filesystem.createSnapshot(snapshot); - // delete extra snapshots - snapshots.push(snapshot); - const toDelete = snapshotsToDelete({ counts, snapshots }); - for (const snapshot of toDelete) { - await filesystem.destroySnapshot(snapshot); - } -} - -function snapshotsToDelete({ counts, snapshots }): string[] { - if (snapshots.length == 0) { - // nothing to do - return []; - } - - // sorted from BIGGEST to smallest - const times = snapshots.map((x) => new Date(x).valueOf()); - times.reverse(); - const save = new Set(); - for (const type in counts) { - const count = counts[type]; - const length_ms = SNAPSHOT_INTERVALS_MS[type]; - - // Pick the first count newest snapshots at intervals of length - // length_ms milliseconds. - let n = 0, - i = 0, - last_tm = 0; - while (n < count && i < times.length) { - const tm = times[i]; - if (!last_tm || tm <= last_tm - length_ms) { - save.add(tm); - last_tm = tm; - n += 1; // found one more - } - i += 1; // move to next snapshot - } - } - return snapshots.filter((x) => !save.has(new Date(x).valueOf())); -} diff --git a/src/packages/file-server/storage-zfs/util.ts b/src/packages/file-server/storage-zfs/util.ts deleted file mode 100644 index bc986e968e..0000000000 --- a/src/packages/file-server/storage-zfs/util.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { - type ExecuteCodeOptions, - type ExecuteCodeOutput, -} from "@cocalc/util/types/execute-code"; -import { executeCode } from "@cocalc/backend/execute-code"; -import getLogger from "@cocalc/backend/logger"; - -const logger = getLogger("file-server:storage:util"); - -const DEFAULT_EXEC_TIMEOUT_MS = 60 * 1000; - -export async function exists(path: string) { - try { - await sudo({ command: "ls", args: [path], verbose: false }); - return true; - } catch { - return false; - } -} - -export async function mkdirp(paths: string[]) { - if (paths.length == 0) return; - await sudo({ command: "mkdir", args: ["-p", ...paths] }); -} - -export async function chmod(args: string[]) { - await sudo({ command: "chmod", args: args }); -} - -export async function sudo( - opts: ExecuteCodeOptions & { desc?: string }, -): Promise { - if (opts.verbose !== false && opts.desc) { - logger.debug("exec", opts.desc); - } - let command, args; - if (opts.bash) { - command = `sudo ${opts.command}`; - args = undefined; - } else { - command = "sudo"; - args = [opts.command, ...(opts.args ?? [])]; - } - return await executeCode({ - verbose: true, - timeout: DEFAULT_EXEC_TIMEOUT_MS / 1000, - ...opts, - command, - args, - }); -} - -export async function rm(paths: string[]) { - if (paths.length == 0) return; - await sudo({ command: "rm", args: paths }); -} - -export async function rmdir(paths: string[]) { - if (paths.length == 0) return; - await sudo({ command: "rmdir", args: paths }); -} - -export async function listdir(path: string) { - const { stdout } = await sudo({ command: "ls", args: ["-1", path] }); - return stdout.split("\n").filter((x) => x); -} - -export async function isdir(path: string) { - const { stdout } = await sudo({ command: "stat", args: ["-c", "%F", path] }); - return stdout.trim() == "directory"; -} diff --git a/src/packages/file-server/zfs/archive.ts b/src/packages/file-server/zfs/archive.ts deleted file mode 100644 index 70347e14c6..0000000000 --- a/src/packages/file-server/zfs/archive.ts +++ /dev/null @@ -1,236 +0,0 @@ -/* -Archiving and restore filesystems -*/ - -import { get, set } from "./db"; -import { createSnapshot, zfsGetSnapshots } from "./snapshots"; -import { - filesystemDataset, - filesystemArchivePath, - filesystemArchiveFilename, - filesystemDatasetTemp, - filesystemMountpoint, -} from "./names"; -import { exec } from "./util"; -import { - mountFilesystem, - unmountFilesystem, - zfsGetProperties, -} from "./properties"; -import { delay } from "awaiting"; -import { primaryKey, type PrimaryKey } from "./types"; -import { isEqual } from "lodash"; - -export async function dearchiveFilesystem( - opts: PrimaryKey & { - // called during dearchive with status updates: - progress?: (status: { - // a number between 0 and 100 indicating progress - progress: number; - // estimated number of seconds remaining - seconds_remaining?: number; - // how much of the total data we have de-archived - read?: number; - // total amount of data to de-archive - total?: number; - }) => void; - }, -) { - const start = Date.now(); - opts.progress?.({ progress: 0 }); - const pk = primaryKey(opts); - const filesystem = get(pk); - if (!filesystem.archived) { - throw Error("filesystem is not archived"); - } - const { used_by_dataset, used_by_snapshots } = filesystem; - const total = (used_by_dataset ?? 0) + (used_by_snapshots ?? 0); - const dataset = filesystemDataset(filesystem); - let done = false; - let progress = 0; - if (opts.progress && total > 0) { - (async () => { - const t0 = Date.now(); - let lastProgress = 0; - while (!done) { - await delay(750); - let x; - try { - x = await zfsGetProperties(dataset); - } catch { - // this is expected to fail, e.g., if filesystem doesn't exist yet. - } - if (done) { - return; - } - const read = x.used_by_dataset + x.used_by_snapshots; - progress = Math.min(100, Math.round((read * 100) / total)); - if (progress == lastProgress) { - continue; - } - lastProgress = progress; - let seconds_remaining: number | undefined = undefined; - if (progress > 0) { - const rate = (Date.now() - t0) / progress; - seconds_remaining = Math.ceil((rate * (100 - progress)) / 1000); - } - opts.progress?.({ progress, seconds_remaining, total, read }); - if (progress >= 100) { - break; - } - } - })(); - } - - // now we de-archive it: - const stream = filesystemArchiveFilename(filesystem); - await exec({ - verbose: true, - // have to use sudo sh -c because zfs recv only supports reading from stdin: - command: `sudo sh -c 'cat ${stream} | zfs recv ${dataset}'`, - what: { - ...pk, - desc: "de-archive a filesystem via zfs recv", - }, - }); - done = true; - if (progress < 100) { - opts.progress?.({ - progress: 100, - seconds_remaining: 0, - total, - read: total, - }); - } - await mountFilesystem(filesystem); - // mounting worked so remove the archive - await exec({ - command: "sudo", - args: ["rm", stream], - what: { - ...pk, - desc: "removing the stream during de-archive", - }, - }); - set({ ...pk, archived: false }); - return { milliseconds: Date.now() - start }; -} - -export async function archiveFilesystem(fs: PrimaryKey) { - const start = Date.now(); - const pk = primaryKey(fs); - const filesystem = get(pk); - if (filesystem.archived) { - throw Error("filesystem is already archived"); - } - // create or get most recent snapshot - const snapshot = await createSnapshot({ ...filesystem, ifChanged: true }); - // where archive of this filesystem goes: - const archive = filesystemArchivePath(filesystem); - const stream = filesystemArchiveFilename(filesystem); - await exec({ - command: "sudo", - args: ["mkdir", "-p", archive], - what: { ...pk, desc: "make archive target directory" }, - }); - - await mountFilesystem(filesystem); - const find = await hashFileTree({ - verbose: true, - path: filesystemMountpoint(filesystem), - what: { ...pk, desc: "getting sha1sum of file listing" }, - }); - // mountpoint will be used for test below, and also no point in archiving - // if we can't even unmount filesystem - await unmountFilesystem(filesystem); - - // make *full* zfs send - await exec({ - verbose: true, - // have to use sudo sh -c because zfs send only supports writing to stdout: - command: `sudo sh -c 'zfs send -e -c -R ${filesystemDataset(filesystem)}@${snapshot} > ${stream}'`, - what: { - ...pk, - desc: "zfs send of full filesystem dataset to archive it", - }, - }); - - // verify that the entire send stream is valid - const temp = filesystemDatasetTemp(filesystem); - try { - await exec({ - verbose: true, - // have to use sudo sh -c because zfs send only supports writing to stdout: - command: `sudo sh -c 'cat ${stream} | zfs recv ${temp}'`, - what: { - ...pk, - desc: "verify the archive zfs send is valid", - }, - }); - // inspect the list of all files, and verify that it is identical (has same sha1sum). - // I think this should be not necessary because the above read didn't fail, and there - // are supposed to be checksums. But I also think there are some ways to corrupt a - // stream so it reads in as empty (say), so this will definitely catch that. - const findtest = await hashFileTree({ - verbose: true, - path: filesystemMountpoint(filesystem), // same mountpoint due to being part of recv data - what: { ...pk, desc: "getting sha1sum of file listing" }, - }); - if (findtest != find) { - throw Error( - "files in archived filesystem do not match. Refusing to archive!", - ); - } - // Inspect list of snapshots, and verify they are identical as well. This is another - // good consistency check that the stream works. - const snapshots = await zfsGetSnapshots(temp); - if (!isEqual(snapshots, filesystem.snapshots)) { - throw Error( - "snapshots in archived filesystem do not match. Refusing to archive!", - ); - } - } finally { - // destroy the temporary filesystem - await exec({ - verbose: true, - command: "sudo", - args: ["zfs", "destroy", "-r", temp], - what: { - ...pk, - desc: "destroying temporary filesystem dataset used for testing archive stream", - }, - }); - } - - // destroy dataset - await exec({ - verbose: true, - command: "sudo", - args: ["zfs", "destroy", "-r", filesystemDataset(filesystem)], - what: { ...pk, desc: "destroying filesystem dataset" }, - }); - - // set as archived in database - set({ ...pk, archived: true }); - - return { snapshot, milliseconds: Date.now() - start }; -} - -// Returns a hash of the file tree. This uses the find command to get path names, but -// doesn't actually read the *contents* of any files, so it's reasonbly fast. -async function hashFileTree({ - path, - what, - verbose, -}: { - path: string; - what?; - verbose?; -}): Promise { - const { stdout } = await exec({ - verbose, - command: `sudo sh -c 'cd "${path}" && find . -xdev -printf "%p %s %TY-%Tm-%Td %TH:%TM\n" | sha1sum'`, - what, - }); - return stdout; -} diff --git a/src/packages/file-server/zfs/backup.ts b/src/packages/file-server/zfs/backup.ts deleted file mode 100644 index 5ded8ee4af..0000000000 --- a/src/packages/file-server/zfs/backup.ts +++ /dev/null @@ -1,176 +0,0 @@ -/* -Make backups using bup. -*/ - -import { bupFilesystemMountpoint, filesystemSnapshotMountpoint } from "./names"; -import { get, getRecent, set } from "./db"; -import { exec } from "./util"; -import getLogger from "@cocalc/backend/logger"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { join } from "path"; -import { mountFilesystem } from "./properties"; -import { split } from "@cocalc/util/misc"; -import { BUP_INTERVAL_MS } from "./config"; -import { primaryKey, type PrimaryKey } from "./types"; -import { createSnapshot } from "./snapshots"; - -const logger = getLogger("file-server:zfs:backup"); - -const EXCLUDES = [".conda", ".npm", "cache", ".julia", ".local/share/pnpm"]; - -export async function createBackup( - fs: PrimaryKey, -): Promise<{ BUP_DIR: string }> { - const pk = primaryKey(fs); - logger.debug("createBackup", pk); - const filesystem = get(pk); - await mountFilesystem(pk); - const snapshot = await createSnapshot({ ...filesystem, ifChanged: true }); - const mountpoint = filesystemSnapshotMountpoint({ ...filesystem, snapshot }); - const excludes: string[] = []; - for (const path of EXCLUDES) { - excludes.push("--exclude"); - excludes.push(join(mountpoint, path)); - } - logger.debug("createBackup: index", pk); - const BUP_DIR = bupFilesystemMountpoint(filesystem); - if (!(await exists(BUP_DIR))) { - await exec({ - verbose: true, - command: "sudo", - args: ["mkdir", "-p", BUP_DIR], - what: { ...pk, desc: "make bup repo" }, - }); - await exec({ - verbose: true, - command: "sudo", - args: ["bup", "-d", BUP_DIR, "init"], - what: { ...pk, desc: "bup init" }, - }); - } - await exec({ - verbose: true, - env: { BUP_DIR }, - command: "sudo", - args: [ - "--preserve-env", - "bup", - "index", - ...excludes, - "-x", - mountpoint, - "--no-check-device", - ], - what: { ...pk, desc: "creating bup index" }, - }); - logger.debug("createBackup: save", pk); - await exec({ - verbose: true, - env: { BUP_DIR }, - command: "sudo", - args: [ - "--preserve-env", - "bup", - "save", - "-q", - "--strip", - "-n", - "master", - mountpoint, - ], - what: { ...pk, desc: "save new bup snapshot" }, - }); - - const { stdout } = await exec({ - env: { BUP_DIR }, - command: "sudo", - args: ["--preserve-env", "bup", "ls", "master"], - what: { ...pk, desc: "getting name of backup" }, - }); - const v = split(stdout); - const last_bup_backup = v[v.length - 2]; - logger.debug("createBackup: created ", { last_bup_backup }); - set({ ...pk, last_bup_backup }); - - // prune-older --unsafe --keep-all-for 8d --keep-dailies-for 4w --keep-monthlies-for 6m --keep-yearlies-for 10y - logger.debug("createBackup: prune", pk); - await exec({ - verbose: true, - env: { BUP_DIR }, - command: "sudo", - args: [ - "--preserve-env", - "bup", - "prune-older", - "--unsafe", - "--keep-all-for", - "8d", - "--keep-dailies-for", - "4w", - "--keep-monthlies-for", - "6m", - "--keep-yearlies-for", - "5y", - ], - what: { ...pk, desc: "save new bup snapshot" }, - }); - - return { BUP_DIR }; -} - -// Go through ALL filesystems with last_edited >= cutoff and make a bup -// backup if they are due. -// cutoff = a Date (default = 1 week ago) -export async function maintainBackups(cutoff?: Date) { - logger.debug("backupActiveFilesystems: getting..."); - const v = getRecent({ cutoff }); - logger.debug( - `backupActiveFilesystems: considering ${v.length} filesystems`, - cutoff, - ); - let i = 0; - for (const { archived, last_edited, last_bup_backup, ...pk } of v) { - if (archived || !last_edited) { - continue; - } - const age = - new Date(last_edited).valueOf() - bupToDate(last_bup_backup).valueOf(); - if (age < BUP_INTERVAL_MS) { - // there's a new backup already - continue; - } - try { - await createBackup(pk); - } catch (err) { - logger.debug(`backupActiveFilesystems: error -- ${err}`); - } - i += 1; - if (i % 10 == 0) { - logger.debug(`backupActiveFilesystems: ${i}/${v.length}`); - } - } -} - -function bupToDate(dateString?: string): Date { - if (!dateString) { - return new Date(0); - } - // Extract components using regular expression - const match = dateString.match( - /^(\d{4})-(\d{2})-(\d{2})-(\d{2})(\d{2})(\d{2})$/, - ); - - if (match) { - const [_, year, month, day, hour, minute, second] = match; // Destructure components - return new Date( - parseInt(year), - parseInt(month) - 1, - parseInt(day), - parseInt(hour), - parseInt(minute), - parseInt(second), - ); - } else { - throw Error("Invalid bup date format"); - } -} diff --git a/src/packages/file-server/zfs/config.ts b/src/packages/file-server/zfs/config.ts deleted file mode 100644 index 6893bd4535..0000000000 --- a/src/packages/file-server/zfs/config.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { join } from "path"; -import { databaseFilename } from "./names"; - -// names of all pools we create/manage will start with this prefix: -const PREFIX = "cocalcfs"; - -// this is where all data is stored or mounted on the filesystem -const DATA = process.env.COCALC_TEST_MODE ? "/data/zfs-test" : "/data/zfs"; - -const SQLITE3_DATABASE_FILE = databaseFilename(DATA); - -// Directory where filesystems get mounted (so NFS can serve them) -const FILESYSTEMS = join(DATA, "filesystems"); - -// Directory where sparse image files are stored -const IMAGES = join(DATA, "images"); - -// Directory on server where zfs send streams (and tar?) are stored -const ARCHIVES = join(DATA, "archives"); - -// Directory to store data used in pulling as part of sync. -// E.g., this keeps around copies of the sqlite state database of each remote. -const PULL = join(DATA, "pull"); - -// Directory for bup backups -const BUP = join(DATA, "bup"); - -export const context = { - namespace: process.env.NAMESPACE ?? "default", - PREFIX, - DATA, - SQLITE3_DATABASE_FILE, - FILESYSTEMS, - ARCHIVES, - IMAGES, - PULL, - BUP, -}; - -// WARNING: this "setContext" is global. It's very useful for **UNIT TESTING**, but -// for any other use, you want to set this at most once and never again!!! The reason -// is because with nodejs you could have async code running all over the place, and -// changing the context out from under it would lead to nonsense and corruption. -export function setContext({ - namespace, - data, - prefix, -}: { - namespace?: string; - data?: string; - prefix?: string; -}) { - context.namespace = namespace ?? process.env.NAMESPACE ?? "default"; - context.PREFIX = prefix ?? PREFIX; - context.DATA = data ?? DATA; - context.SQLITE3_DATABASE_FILE = databaseFilename(context.DATA); - context.FILESYSTEMS = join(context.DATA, "filesystems"); - context.ARCHIVES = join(context.DATA, "archives"); - context.IMAGES = join(context.DATA, "images"); - context.PULL = join(context.DATA, "pull"); - context.BUP = join(context.DATA, "bup"); -} - -// Every filesystem has at least this much quota (?) -export const MIN_QUOTA = 1024 * 1024 * 1; // 1MB - -export const DEFAULT_POOL_SIZE = "100G"; - -// We periodically do "zpool list" to find out what pools are available -// and how much space they have left. This info is cached for this long -// to avoid excessive calls: -export const POOLS_CACHE_MS = 15000; - -// two hour default for running any commands (e.g., zfs send/recv) -export const DEFAULT_EXEC_TIMEOUT_MS = 2 * 1000 * 60 * 60; - -// **all** user files for filesystems have this owner and group. -export const UID = 2001; -export const GID = 2001; - -// We make/update snapshots periodically, with this being the minimum interval. -export const SNAPSHOT_INTERVAL_MS = 60 * 30 * 1000; -//export const SNAPSHOT_INTERVAL_MS = 10 * 1000; - -// Lengths of time in minutes to keep these snapshots -export const SNAPSHOT_INTERVALS_MS = { - halfhourly: 30 * 1000 * 60, - daily: 60 * 24 * 1000 * 60, - weekly: 60 * 24 * 7 * 1000 * 60, - monthly: 60 * 24 * 7 * 4 * 1000 * 60, -}; - -// How many of each type of snapshot to retain -export const SNAPSHOT_COUNTS = { - halfhourly: 24, - daily: 14, - weekly: 7, - monthly: 4, -}; - -// Minimal interval for bup backups -export const BUP_INTERVAL_MS = 24 * 1000 * 60 * 60; - -// minimal interval for zfs streams -export const STREAM_INTERVAL_MS = 24 * 1000 * 60 * 60; -// when more than this many streams, we recompact down -export const MAX_STREAMS = 30; diff --git a/src/packages/file-server/zfs/copy.ts b/src/packages/file-server/zfs/copy.ts deleted file mode 100644 index 8248d3061f..0000000000 --- a/src/packages/file-server/zfs/copy.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* -Copy between projects on this server -*/ - -export async function copy(opts: { - source_project_id: string; - target_project_id?: string; - source_path: string; - target_path: string; - rsyncOptions?: string; -}) { - console.log("copy", opts); - throw Error("copy: not implemented"); -} diff --git a/src/packages/file-server/zfs/create.ts b/src/packages/file-server/zfs/create.ts deleted file mode 100644 index 8c57120c61..0000000000 --- a/src/packages/file-server/zfs/create.ts +++ /dev/null @@ -1,201 +0,0 @@ -/* - -DEVELOPMENT: - -If you're using Docker, make sure the DATA directory in config.ts is inside a folder -that is bind mounted from the host. - -Start node. - -a = require('@cocalc/file-server/zfs/create') - -fs = await a.createFilesystem({namespace:'default', owner_type:'project', owner_id:'6b851643-360e-435e-b87e-f9a6ab64a8b1', name:'home'}) - -await a.deleteFilesystem({namespace:'default', owner_type:'project', owner_id:'6b851643-360e-435e-b87e-f9a6ab64a8b1', name:'home'}) - -*/ - -import { create, get, getDb, deleteFromDb, filesystemExists } from "./db"; -import { exec } from "./util"; -import { - filesystemArchivePath, - bupFilesystemMountpoint, - filesystemDataset, - filesystemMountpoint, - poolName, -} from "./names"; -import { initializePool } from "./pools"; -import { dearchiveFilesystem } from "./archive"; -import { UID, GID } from "./config"; -import { createSnapshot } from "./snapshots"; -import { type Filesystem, primaryKey, type PrimaryKey } from "./types"; - -export async function createFilesystem( - opts: PrimaryKey & { - affinity?: string; - clone?: PrimaryKey; - }, -): Promise { - if (filesystemExists(opts)) { - return get(opts); - } - const pk = primaryKey(opts); - const { namespace } = pk; - const { affinity, clone } = opts; - const source = clone ? get(clone) : undefined; - - const db = getDb(); - // select a pool: - let pool: undefined | string = undefined; - - if (source != null) { - // For clone, we use same pool as source filesystem. (we could use zfs send/recv but that's much slower and not a clone) - pool = source.pool; - } else { - if (affinity) { - // if affinity is set, have preference to use same pool as other filesystems with this affinity. - const x = db - .prepare( - "SELECT pool, COUNT(pool) AS cnt FROM filesystems WHERE namespace=? AND affinity=? ORDER by cnt DESC", - ) - .get(namespace, affinity) as { pool: string; cnt: number } | undefined; - pool = x?.pool; - } - if (!pool) { - // create new pool - pool = poolName(pk); - } - } - if (!pool) { - throw Error("bug -- unable to select a pool"); - } - - const { cnt } = db - .prepare( - "SELECT COUNT(pool) AS cnt FROM filesystems WHERE pool=? AND namespace=?", - ) - .get(pool, namespace) as { cnt: number }; - - if (cnt == 0) { - // initialize pool for use in this namespace: - await initializePool({ pool, namespace }); - } - - if (source == null) { - // create filesystem on the selected pool - const mountpoint = filesystemMountpoint(pk); - const dataset = filesystemDataset({ ...pk, pool }); - await exec({ - verbose: true, - command: "sudo", - args: [ - "zfs", - "create", - "-o", - `mountpoint=${mountpoint}`, - "-o", - "compression=lz4", - "-o", - "dedup=on", - dataset, - ], - what: { - ...pk, - desc: `create filesystem ${dataset} for filesystem on the selected pool mounted at ${mountpoint}`, - }, - }); - await exec({ - verbose: true, - command: "sudo", - args: ["chown", "-R", `${UID}:${GID}`, mountpoint], - whate: { - ...pk, - desc: `setting permissions of filesystem mounted at ${mountpoint}`, - }, - }); - } else { - // clone source - // First ensure filesystem isn't archived - // (we might alternatively de-archive to make the clone...?) - if (source.archived) { - await dearchiveFilesystem(source); - } - // Get newest snapshot, or make one if there are none - const snapshot = await createSnapshot({ ...source, ifChanged: true }); - if (!snapshot) { - throw Error("bug -- source should have snapshot"); - } - const source_snapshot = `${filesystemDataset(source)}@${snapshot}`; - await exec({ - verbose: true, - command: "sudo", - args: [ - "zfs", - "clone", - "-o", - `mountpoint=${filesystemMountpoint(pk)}`, - "-o", - "compression=lz4", - "-o", - "dedup=on", - source_snapshot, - filesystemDataset({ ...pk, pool }), - ], - what: { - ...pk, - desc: `clone filesystem from ${source_snapshot}`, - }, - }); - } - - // update database - create({ ...pk, pool, affinity }); - return get(pk); -} - -// delete -- This is very dangerous -- it deletes the filesystem, -// the archive, and any backups and removes knowledge the filesystem from the db. - -export async function deleteFilesystem(fs: PrimaryKey) { - const filesystem = get(fs); - const dataset = filesystemDataset(filesystem); - if (!filesystem.archived) { - await exec({ - verbose: true, - command: "sudo", - args: ["zfs", "destroy", "-r", dataset], - what: { - ...filesystem, - desc: `destroy dataset ${dataset} containing the filesystem`, - }, - }); - } - await exec({ - verbose: true, - command: "sudo", - args: ["rm", "-rf", filesystemMountpoint(filesystem)], - what: { - ...filesystem, - desc: `delete directory '${filesystemMountpoint(filesystem)}' where filesystem was stored`, - }, - }); - await exec({ - verbose: true, - command: "sudo", - args: ["rm", "-rf", bupFilesystemMountpoint(filesystem)], - what: { - ...filesystem, - desc: `delete directory '${bupFilesystemMountpoint(filesystem)}' where backups were stored`, - }, - }); - await exec({ - verbose: true, - command: "sudo", - args: ["rm", "-rf", filesystemArchivePath(filesystem)], - what: { - ...filesystem, - desc: `delete directory '${filesystemArchivePath(filesystem)}' where archives were stored`, - }, - }); - deleteFromDb(filesystem); -} diff --git a/src/packages/file-server/zfs/db.ts b/src/packages/file-server/zfs/db.ts deleted file mode 100644 index e54dc4cb5a..0000000000 --- a/src/packages/file-server/zfs/db.ts +++ /dev/null @@ -1,274 +0,0 @@ -/* -Database -*/ - -import Database from "better-sqlite3"; -import { context } from "./config"; -import { - primaryKey, - type PrimaryKey, - type Filesystem, - type RawFilesystem, - type SetFilesystem, - OWNER_ID_FIELDS, -} from "./types"; -import { is_array, is_date } from "@cocalc/util/misc"; - -let db: { [file: string]: Database.Database } = {}; - -const tableName = "filesystems"; -const schema = { - // this uniquely defines the filesystem (it's the compound primary key) - owner_type: "TEXT", - owner_id: "TEXT", - namespace: "TEXT", - name: "TEXT", - - // data about the filesystem - pool: "TEXT", - archived: "INTEGER", - affinity: "TEXT", - nfs: "TEXT", - snapshots: "TEXT", - last_edited: "TEXT", - last_send_snapshot: "TEXT", - last_bup_backup: "TEXT", - error: "TEXT", - last_error: "TEXT", - used_by_dataset: "INTEGER", - used_by_snapshots: "INTEGER", - quota: "INTEGER", -}; - -const WHERE_PRIMARY_KEY = - "WHERE namespace=? AND owner_type=? AND owner_id=? AND name=?"; -function primaryKeyArgs(fs: PrimaryKey) { - const { namespace, owner_type, owner_id, name } = primaryKey(fs); - return [namespace, owner_type, owner_id, name]; -} - -export function getDb(databaseFile?): Database.Database { - const file = databaseFile ?? context.SQLITE3_DATABASE_FILE; - if (db[file] == null) { - db[file] = new Database(file); - initDb(db[file]); - } - return db[file]!; -} - -function initDb(db) { - const columnDefinitions = Object.entries(schema) - .map(([name, type]) => `${name} ${type}`) - .join(", "); - - // Create table if it doesn't exist - db.prepare( - `CREATE TABLE IF NOT EXISTS ${tableName} ( - ${columnDefinitions}, - PRIMARY KEY (namespace, owner_type, owner_id, name) - )`, - ).run(); - - // Check for missing columns and add them - const existingColumnsStmt = db.prepare(`PRAGMA table_info(${tableName})`); - const existingColumns = existingColumnsStmt.all().map((row) => row.name); - - for (const [name, type] of Object.entries(schema)) { - if (!existingColumns.includes(name)) { - db.prepare(`ALTER TABLE ${tableName} ADD COLUMN ${name} ${type}`).run(); - } - } -} - -// This is extremely dangerous and mainly used for unit testing: -export function resetDb() { - const db = new Database(context.SQLITE3_DATABASE_FILE); - db.prepare("DROP TABLE IF EXISTS filesystems").run(); - initDb(db); -} - -function convertToSqliteType({ value, getFilesystem }) { - if (is_array(value)) { - return value.join(","); - } else if (is_date(value)) { - return value.toISOString(); - } else if (typeof value == "boolean") { - return value ? 1 : 0; - } else if (typeof value == "function") { - const x = value(getFilesystem()); - if (typeof x == "function") { - throw Error("function must not return a function"); - } - // returned value needs to be converted - return convertToSqliteType({ value: x, getFilesystem }); - } - return value; -} - -export function set(obj: SetFilesystem) { - const pk = primaryKey(obj); - const fields: string[] = []; - const values: any[] = []; - let filesystem: null | Filesystem = null; - const getFilesystem = () => { - if (filesystem == null) { - filesystem = get(pk); - } - return filesystem; - }; - for (const field in obj) { - if (pk[field] !== undefined || OWNER_ID_FIELDS.includes(field)) { - continue; - } - fields.push(field); - values.push(convertToSqliteType({ value: obj[field], getFilesystem })); - } - let query = `UPDATE filesystems SET - ${fields.map((field) => `${field}=?`).join(", ")} - ${WHERE_PRIMARY_KEY} - `; - for (const x of primaryKeyArgs(pk)) { - values.push(x); - } - const db = getDb(); - db.prepare(query).run(...values); -} - -// Call this if something that should never happen, does in fact, happen. -// It will set the error state of the filesystem and throw the exception. -// Admins will be regularly notified of all filesystems in an error state. -export function fatalError( - obj: PrimaryKey & { - err: Error; - desc?: string; - }, -) { - set({ - ...primaryKey(obj), - error: `${obj.err}${obj.desc ? " - " + obj.desc : ""}`, - last_error: new Date(), - }); - throw obj.err; -} - -export function clearError(fs: PrimaryKey) { - set({ ...fs, error: null }); -} - -export function clearAllErrors() { - const db = getDb(); - db.prepare("UPDATE filesystems SET error=null").run(); -} - -export function getErrors() { - const db = getDb(); - return db - .prepare("SELECT * FROM filesystems WHERE error!=''") - .all() as RawFilesystem[]; -} - -export function touch(fs: PrimaryKey) { - set({ ...fs, last_edited: new Date() }); -} - -export function filesystemExists( - fs: PrimaryKey, - databaseFile?: string, -): boolean { - const db = getDb(databaseFile); - const x = db - .prepare("SELECT COUNT(*) AS count FROM filesystems " + WHERE_PRIMARY_KEY) - .get(...primaryKeyArgs(fs)); - return (x as any).count > 0; -} - -export function get(fs: PrimaryKey, databaseFile?: string): Filesystem { - const db = getDb(databaseFile); - const filesystem = db - .prepare("SELECT * FROM filesystems " + WHERE_PRIMARY_KEY) - .get(...primaryKeyArgs(fs)) as any; - if (filesystem == null) { - throw Error(`no filesystem ${JSON.stringify(fs)}`); - } - for (const key of ["nfs", "snapshots"]) { - filesystem[key] = sqliteStringToArray(filesystem[key]); - } - filesystem["archived"] = !!filesystem["archived"]; - if (filesystem.last_edited) { - filesystem.last_edited = new Date(filesystem.last_edited); - } - if (filesystem.last_error) { - filesystem.last_error = new Date(filesystem.last_error); - } - return filesystem as Filesystem; -} - -export function create( - obj: PrimaryKey & { - pool: string; - affinity?: string; - }, -) { - getDb() - .prepare( - "INSERT INTO filesystems(namespace, owner_type, owner_id, name, pool, affinity, last_edited) VALUES(?,?,?,?,?,?,?)", - ) - .run( - ...primaryKeyArgs(obj), - obj.pool, - obj.affinity, - new Date().toISOString(), - ); -} - -export function deleteFromDb(fs: PrimaryKey) { - getDb() - .prepare("DELETE FROM filesystems " + WHERE_PRIMARY_KEY) - .run(...primaryKeyArgs(fs)); -} - -export function getAll({ - namespace = context.namespace, -}: { namespace?: string } = {}): RawFilesystem[] { - const db = getDb(); - return db - .prepare("SELECT * FROM filesystems WHERE namespace=?") - .all(namespace) as RawFilesystem[]; -} - -export function getNamespacesAndPools(): { namespace: string; pool: string }[] { - const db = getDb(); - return db - .prepare("SELECT DISTINCT namespace, pool FROM filesystems") - .all() as any; -} - -export function getRecent({ - namespace, - cutoff, - databaseFile, -}: { - namespace?: string; - cutoff?: Date; - databaseFile?: string; -} = {}): RawFilesystem[] { - const db = getDb(databaseFile); - if (cutoff == null) { - cutoff = new Date(Date.now() - 1000 * 60 * 60 * 24 * 7); - } - const query = "SELECT * FROM filesystems WHERE last_edited>=?"; - if (namespace == null) { - return db.prepare(query).all(cutoff.toISOString()) as RawFilesystem[]; - } else { - return db - .prepare(`${query} AND namespace=?`) - .all(cutoff.toISOString(), namespace) as RawFilesystem[]; - } -} - -function sqliteStringToArray(s?: string): string[] { - if (!s) { - return []; - } - return s.split(","); -} diff --git a/src/packages/file-server/zfs/index.ts b/src/packages/file-server/zfs/index.ts deleted file mode 100644 index 8729ea2d04..0000000000 --- a/src/packages/file-server/zfs/index.ts +++ /dev/null @@ -1,29 +0,0 @@ -export { getPools, initializePool, initializeAllPools } from "./pools"; -export { - getModifiedFiles, - deleteSnapshot, - deleteExtraSnapshotsOfActiveFilesystems, - deleteExtraSnapshots, -} from "./snapshots"; -export { - getAll, - getRecent, - get, - set, - clearError, - getErrors, - clearAllErrors, -} from "./db"; -export { shareNFS, unshareNFS } from "./nfs"; -export { createFilesystem, deleteFilesystem } from "./create"; -export { createSnapshot, getSnapshots, maintainSnapshots } from "./snapshots"; -export { - mountFilesystem, - unmountFilesystem, - setQuota, - syncProperties, -} from "./properties"; -export { archiveFilesystem, dearchiveFilesystem } from "./archive"; -export { maintainBackups, createBackup } from "./backup"; -export { recv, send, recompact, maintainStreams } from "./streams"; -export { pull } from "./pull"; diff --git a/src/packages/file-server/zfs/names.ts b/src/packages/file-server/zfs/names.ts deleted file mode 100644 index 1a644478aa..0000000000 --- a/src/packages/file-server/zfs/names.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { join } from "path"; -import { context } from "./config"; -import { primaryKey, type PrimaryKey } from "./types"; -import { randomId } from "@cocalc/conat/names"; - -export function databaseFilename(data: string) { - return join(data, "database.sqlite3"); -} - -export function namespaceDataset({ - pool, - namespace, -}: { - pool: string; - namespace: string; -}) { - return `${pool}/${namespace}`; -} - -// Archives -// There is one single dataset for each namespace/pool pair: All the different -// archives across filesystems are stored in the *same* dataset, since there is no -// point in separating them. -export function archivesDataset({ - pool, - namespace, -}: { - pool: string; - namespace: string; -}) { - return `${namespaceDataset({ pool, namespace })}/archives`; -} - -export function archivesMountpoint({ - pool, - namespace, -}: { - pool: string; - namespace: string; -}) { - return join(context.ARCHIVES, namespace, pool); -} - -export function filesystemArchivePath({ - pool, - ...fs -}: PrimaryKey & { pool: string }) { - const pk = primaryKey(fs); - return join( - archivesMountpoint({ pool, namespace: pk.namespace }), - pk.owner_type, - pk.owner_id, - pk.name, - ); -} - -export function filesystemArchiveFilename(opts: PrimaryKey & { pool: string }) { - const { owner_type, owner_id, name } = primaryKey(opts); - return join( - filesystemArchivePath(opts), - `full-${owner_type}-${owner_id}-${name}.zfs`, - ); -} - -export function filesystemStreamsPath(opts: PrimaryKey & { pool: string }) { - return join(filesystemArchivePath(opts), "streams"); -} - -export function filesystemStreamsFilename({ - snapshot1, - snapshot2, - ...opts -}: PrimaryKey & { snapshot1: string; snapshot2: string; pool: string }) { - return join(filesystemStreamsPath(opts), `${snapshot1}-${snapshot2}.zfs`); -} - -// Bup -export function bupDataset({ - pool, - namespace, -}: { - pool: string; - namespace: string; -}) { - return `${namespaceDataset({ pool, namespace })}/bup`; -} - -export function bupMountpoint({ - pool, - namespace, -}: { - pool: string; - namespace: string; -}) { - return join(context.BUP, namespace, pool); -} - -export function bupFilesystemMountpoint({ - pool, - ...fs -}: PrimaryKey & { pool: string }) { - const pk = primaryKey(fs); - return join( - bupMountpoint({ ...pk, pool }), - pk.owner_type, - pk.owner_id, - pk.name, - ); -} - -// Filesystems - -export function filesystemsPath({ namespace }) { - return join(context.FILESYSTEMS, namespace); -} - -export function filesystemMountpoint(fs: PrimaryKey) { - const pk = primaryKey(fs); - return join( - context.FILESYSTEMS, - `${pk.owner_type}-${pk.owner_id}-${pk.name}`, - ); -} - -export function filesystemSnapshotMountpoint( - opts: PrimaryKey & { snapshot: string }, -) { - return join(filesystemMountpoint(opts), ".zfs", "snapshot", opts.snapshot); -} - -export function filesystemsDataset({ - pool, - namespace, -}: { - pool: string; - namespace: string; -}) { - return `${namespaceDataset({ pool, namespace })}/filesystems`; -} - -// There is one single dataset for each project_id/namespace/pool tripple since it -// is critical to separate each project to properly support snapshots, clones, -// backups, etc. -export function filesystemDataset({ - pool, - ...fs -}: PrimaryKey & { pool: string }) { - const { namespace, owner_type, owner_id, name } = primaryKey(fs); - // NOTE: we *could* use a heirarchy of datasets like this: - // ${owner_type}/${owner_id}/${name} - // However, that greatly increases the raw number of datasets, and there's a huge performance - // penalty. Since the owner_type is a fixed small list, owner_id is a uuid and the name is - // more general, there's no possible overlaps just concating them as below, and this way there's - // only one dataset, rather than three. (We also don't need to worry about deleting parents - // when there are no children...) - return `${filesystemsDataset({ pool, namespace: namespace })}/${owner_type}-${owner_id}-${name}`; -} - -// A unique name for the pool that will store this fs. Other fs with -// same affinity and clones could also be stored on this pool. -export function poolName(fs): string { - // NOTE: imageDirectory below assumes the pool name starts with "${context.PREFIX}-{namespace}-" - const { namespace, owner_type, owner_id, name } = primaryKey(fs); - return `${context.PREFIX}-${namespace}-${owner_type}-${owner_id}-${name}`; -} - -export function poolImageDirectory({ pool }: { pool: string }): string { - const namespace = pool.slice(context.PREFIX.length + 1).split("-")[0]; - return join(context.IMAGES, namespace, pool); -} - -export function poolImageFile({ pool }: { pool: string }): string { - return join(poolImageDirectory({ pool }), "0.img"); -} - -export function tempDataset({ - pool, - namespace, -}: { - pool: string; - namespace: string; -}) { - return `${namespaceDataset({ pool, namespace })}/temp`; -} - -export function filesystemDatasetTemp({ - pool, - ...fs -}: PrimaryKey & { pool: string }) { - const { namespace, owner_type, owner_id, name } = primaryKey(fs); - return `${tempDataset({ pool, namespace })}/${owner_type}-${owner_id}-${name}-${randomId()}`; -} - -// NOTE: We use "join" for actual file paths and explicit -// strings with / for ZFS filesystem names, since in some whacky -// futuristic world maybe this server is running on MS Windows. diff --git a/src/packages/file-server/zfs/nfs.ts b/src/packages/file-server/zfs/nfs.ts deleted file mode 100644 index a3c6f87603..0000000000 --- a/src/packages/file-server/zfs/nfs.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { get, set, touch } from "./db"; -import { exec } from "./util"; -import { filesystemDataset, filesystemMountpoint } from "./names"; -import { primaryKey, type PrimaryKey } from "./types"; - -// Ensure that this filesystem is mounted and setup so that export to the -// given client is allowed. -// Returns the remote that the client should use for NFS mounting, i.e., -// this return s, then type "mount s /mnt/..." to mount the filesystem. -// If client is not given, just sets the share at NFS level -// to what's specified in the database. -export async function shareNFS({ - client, - ...fs -}: PrimaryKey & { client?: string }): Promise { - client = client?.trim(); - const pk = primaryKey(fs); - const { pool, nfs } = get(pk); - let hostname; - if (client) { - hostname = await hostnameFor(client); - if (!nfs.includes(client)) { - nfs.push(client); - // update database which tracks what the share should be. - set({ ...pk, nfs: ({ nfs }) => [...nfs, client] }); - } - } - // actually ensure share is configured. - const name = filesystemDataset({ pool, ...pk }); - const sharenfs = - nfs.length > 0 - ? `${nfs.map((client) => `rw=${client}`).join(",")},no_root_squash,crossmnt,no_subtree_check` - : "off"; - await exec({ - verbose: true, - command: "sudo", - args: ["zfs", "set", `sharenfs=${sharenfs}`, name], - }); - if (nfs.length > 0) { - touch(pk); - } - if (client) { - return `${hostname}:${filesystemMountpoint(pk)}`; - } else { - return ""; - } -} - -// remove given client from nfs sharing -export async function unshareNFS({ - client, - ...fs -}: PrimaryKey & { client: string }) { - const pk = primaryKey(fs); - let { nfs } = get(pk); - if (!nfs.includes(client)) { - // nothing to do - return; - } - nfs = nfs.filter((x) => x != client); - // update database which tracks what the share should be. - set({ ...pk, nfs }); - // update zfs/nfs to no longer share to this client - await shareNFS(pk); -} - -let serverIps: null | string[] = null; -async function hostnameFor(client: string) { - if (serverIps == null) { - const ipRegex = /^(\d{1,3}\.){3}\d{1,3}$/; - const { stdout } = await exec({ - verbose: false, - command: "ifconfig", - }); - let i = stdout.indexOf("inet "); - const v: string[] = []; - while (i != -1) { - let j = stdout.indexOf("\n", i); - if (j == -1) { - break; - } - const x = stdout.slice(i, j).split(" "); - const ip = x[1]; - if (ipRegex.test(ip)) { - v.push(ip); - } - i = stdout.indexOf("inet ", j); - } - if (v.length == 0) { - throw Error("unable to determine server ip address"); - } - serverIps = v; - } - for (const ip of serverIps) { - if (subnetMatch(ip, client)) { - return ip; - } - } - throw Error("found no matching subdomain"); -} - -// a and b are ip addresses. Return true -// if the are on the same subnet, by which -// we mean that the first *TWO* segments match, -// since that's the size of our subnets usually. -// TODO: make configurable (?). -function subnetMatch(a, b) { - const v = a.split("."); - const w = b.split("."); - return v[0] == w[0] && v[1] == w[1]; -} diff --git a/src/packages/file-server/zfs/pools.ts b/src/packages/file-server/zfs/pools.ts deleted file mode 100644 index ca2cceb50c..0000000000 --- a/src/packages/file-server/zfs/pools.ts +++ /dev/null @@ -1,284 +0,0 @@ -/* -This code sets things up for each pool and namespace, e.g., defining datasets, creating directories, -etc. as defined in config and names. - -WARNING: For efficientcy and sanity, it assumes that once something is setup, it stays setup. -If there is a chaos monkey running around breaking things (e.g., screwing up -file permissions, deleting datasets, etc.,) then this code won't help at all. - -OPERATIONS: - -- To add a new pool, just create it using zfs with a name sthat starts with context.PREFIX. - It should automatically start getting used within POOLS_CACHE_MS by newly created filesystems. - -*/ - -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { context, DEFAULT_POOL_SIZE, POOLS_CACHE_MS } from "./config"; -import { exec } from "./util"; -import { - archivesDataset, - archivesMountpoint, - namespaceDataset, - filesystemsDataset, - filesystemsPath, - bupDataset, - bupMountpoint, - tempDataset, - poolImageDirectory, - poolImageFile, -} from "./names"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { getNamespacesAndPools } from "./db"; - -// Make sure all pools and namespaces are initialized for all existing filesystems. -// This should be needed after booting up the server and importing the pools. -export async function initializeAllPools() { - // TODO: maybe import all here? - - for (const { namespace, pool } of getNamespacesAndPools()) { - await initializePool({ namespace, pool }); - } -} - -interface Pool { - name: string; - state: "ONLINE" | "OFFLINE"; - size: number; - allocated: number; - free: number; -} - -type Pools = { [name: string]: Pool }; -let poolsCache: { [prefix: string]: Pools } = {}; - -export const getPools = reuseInFlight( - async ({ noCache }: { noCache?: boolean } = {}): Promise => { - if (!noCache && poolsCache[context.DATA]) { - return poolsCache[context.DATA]; - } - const { stdout } = await exec({ - verbose: true, - command: "zpool", - args: ["list", "-j", "--json-int", "-o", "size,allocated,free"], - }); - const { pools } = JSON.parse(stdout); - const v: { [name: string]: Pool } = {}; - for (const name in pools) { - if (!name.startsWith(context.PREFIX)) { - continue; - } - const pool = pools[name]; - for (const key in pool.properties) { - pool.properties[key] = pool.properties[key].value; - } - v[name] = { name, state: pool.state, ...pool.properties }; - } - poolsCache[context.PREFIX] = v; - if (!process.env.COCALC_TEST_MODE) { - // only clear cache in non-test mode - setTimeout(() => { - delete poolsCache[context.PREFIX]; - }, POOLS_CACHE_MS); - } - return v; - }, -); - -// OK to call this again even if initialized already. -export const initializePool = reuseInFlight( - async ({ - namespace = context.namespace, - pool, - }: { - namespace?: string; - pool: string; - }) => { - const image = poolImageFile({ pool }); - if (!(await exists(image))) { - const dir = poolImageDirectory({ pool }); - - await exec({ - verbose: true, - command: "sudo", - args: ["mkdir", "-p", dir], - }); - - await exec({ - verbose: true, - command: "sudo", - args: ["truncate", "-s", DEFAULT_POOL_SIZE, image], - what: { pool, desc: "create sparse image file" }, - }); - - // create the pool - await exec({ - verbose: true, - command: "sudo", - args: [ - "zpool", - "create", - "-o", - "feature@fast_dedup=enabled", - "-m", - "none", - pool, - image, - ], - what: { - pool, - desc: `create the zpool ${pool} using the device ${image}`, - }, - }); - } else { - // make sure pool is imported - try { - await exec({ - verbose: true, - command: "zpool", - args: ["list", pool], - what: { pool, desc: `check if ${pool} needs to be imported` }, - }); - } catch { - const dir = poolImageDirectory({ pool }); - await exec({ - verbose: true, - command: "sudo", - args: ["zpool", "import", pool, "-d", dir], - what: { - pool, - desc: `import the zpool ${pool} from ${dir}`, - }, - }); - } - } - - // archives and filesystems for each namespace are in this dataset - await ensureDatasetExists({ - name: namespaceDataset({ namespace, pool }), - }); - - // Initialize archives dataset, used for archiving filesystems. - await ensureDatasetExists({ - name: archivesDataset({ pool, namespace }), - mountpoint: archivesMountpoint({ pool, namespace }), - }); - // This sets up the parent filesystem for all filesystems - // and enable compression and dedup. - await ensureDatasetExists({ - name: filesystemsDataset({ namespace, pool }), - }); - await ensureDatasetExists({ - name: tempDataset({ namespace, pool }), - dedup: "off", - }); - // Initialize bup dataset, used for backups. - await ensureDatasetExists({ - name: bupDataset({ pool, namespace }), - mountpoint: bupMountpoint({ pool, namespace }), - compression: "off", - dedup: "off", - }); - - const filesystems = filesystemsPath({ namespace }); - if (!(await exists(filesystems))) { - await exec({ - verbose: true, - command: "sudo", - args: ["mkdir", "-p", filesystems], - }); - await exec({ - verbose: true, - command: "sudo", - args: ["chmod", "a+rx", context.FILESYSTEMS], - }); - await exec({ - verbose: true, - command: "sudo", - args: ["chmod", "a+rx", filesystems], - }); - } - }, -); - -// If a dataset exists, it is assumed to exist henceforth for the life of this process. -// That's fine for *this* application here of initializing pools, since we never delete -// anything here. -const datasetExistsCache = new Set(); -async function datasetExists(name: string): Promise { - if (datasetExistsCache.has(name)) { - return true; - } - try { - await exec({ - verbose: true, - command: "zfs", - args: ["list", name], - }); - datasetExistsCache.add(name); - return true; - } catch { - return false; - } -} - -async function isMounted(dataset): Promise { - const { stdout } = await exec({ - command: "zfs", - args: ["get", "mounted", dataset, "-j"], - }); - const x = JSON.parse(stdout); - return x.datasets[dataset].properties.mounted.value == "yes"; -} - -async function ensureDatasetExists({ - name, - mountpoint, - compression = "lz4", - dedup = "on", -}: { - name: string; - mountpoint?: string; - compression?: "lz4" | "off"; - dedup?: "on" | "off"; -}) { - if (await datasetExists(name)) { - if (mountpoint && !(await isMounted(name))) { - // ensure mounted - await exec({ - verbose: true, - command: "sudo", - args: ["zfs", "mount", name], - }); - } - return; - } - await exec({ - verbose: true, - command: "sudo", - args: [ - "zfs", - "create", - "-o", - `mountpoint=${mountpoint ? mountpoint : "none"}`, - "-o", - `compression=${compression}`, - "-o", - `dedup=${dedup}`, - name, - ], - }); - // make sure it is very hard to accidentally delete the entire dataset - // see https://github.com/openzfs/zfs/issues/4134#issuecomment-2565724994 - const safety = `${name}@safety`; - await exec({ - verbose: true, - command: "sudo", - args: ["zfs", "snapshot", safety], - }); - await exec({ - verbose: true, - command: "sudo", - args: ["zfs", "hold", "safety", safety], - }); -} diff --git a/src/packages/file-server/zfs/properties.ts b/src/packages/file-server/zfs/properties.ts deleted file mode 100644 index 77c228a942..0000000000 --- a/src/packages/file-server/zfs/properties.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { exec } from "./util"; -import { filesystemDataset } from "./names"; -import { get, set } from "./db"; -import { MIN_QUOTA } from "./config"; -import { primaryKey, type PrimaryKey } from "./types"; - -export async function setQuota({ - // quota in **number of bytes**. - // If quota is smaller than actual dataset, then the quota is set to what is - // actually used (plus 10 MB), hopefully allowing user to delete some data. - // The quota is never less than MIN_QUOTA. - // The value stored in database is *also* then set to this amount. - // So this is not some magic fire and forget setting, but something - // that cocalc should regularly call when starting the filesystem. - quota, - noSync, - ...fs -}: { - quota: number; - noSync?: boolean; -} & PrimaryKey) { - const pk = primaryKey(fs); - // this will update current usage in the database - await syncProperties(pk); - const { pool, used_by_dataset } = get(pk); - const used = (used_by_dataset ?? 0) + 10 * 1024; - if (quota < used) { - quota = used!; - } - quota = Math.max(MIN_QUOTA, quota); - try { - await exec({ - verbose: true, - command: "sudo", - args: [ - "zfs", - "set", - // refquota so snapshots don't count against the user - `refquota=${quota}`, - filesystemDataset({ pool, ...pk }), - ], - }); - } finally { - // this sets quota in database in bytes to whatever was just set above. - await syncProperties(pk); - } -} - -// Sync with ZFS the properties for the given filesystem by -// setting the database to what is in ZFS: -// - total space used by snapshots -// - total space used by dataset -// - the quota -export async function syncProperties(fs: PrimaryKey) { - const pk = primaryKey(fs); - const { pool, archived } = get(pk); - if (archived) { - // they can't have changed - return; - } - set({ - ...pk, - ...(await zfsGetProperties(filesystemDataset({ pool, ...pk }))), - }); -} - -export async function zfsGetProperties(dataset: string): Promise<{ - used_by_snapshots: number; - used_by_dataset: number; - quota: number | null; -}> { - const { stdout } = await exec({ - command: "zfs", - args: [ - "list", - dataset, - "-j", - "--json-int", - "-o", - "usedsnap,usedds,refquota", - ], - }); - const x = JSON.parse(stdout); - const { properties } = x.datasets[dataset]; - return { - used_by_snapshots: properties.usedbysnapshots.value, - used_by_dataset: properties.usedbydataset.value, - quota: properties.refquota.value ? properties.refquota.value : null, - }; -} - -export async function mountFilesystem(fs: PrimaryKey) { - const pk = primaryKey(fs); - const { pool } = get(pk); - try { - await exec({ - command: "sudo", - args: ["zfs", "mount", filesystemDataset({ pool, ...pk })], - what: { ...pk, desc: "mount filesystem" }, - }); - } catch (err) { - if (`${err}`.includes("already mounted")) { - // fine - return; - } - throw err; - } -} - -export async function unmountFilesystem(fs: PrimaryKey) { - const pk = primaryKey(fs); - const { pool } = get(pk); - try { - await exec({ - verbose: true, - command: "sudo", - args: ["zfs", "unmount", filesystemDataset({ pool, ...pk })], - what: { ...pk, desc: "unmount filesystem" }, - }); - } catch (err) { - if (`${err}`.includes("not currently mounted")) { - // fine - } else { - throw err; - } - } -} diff --git a/src/packages/file-server/zfs/pull.ts b/src/packages/file-server/zfs/pull.ts deleted file mode 100644 index ca1087d854..0000000000 --- a/src/packages/file-server/zfs/pull.ts +++ /dev/null @@ -1,303 +0,0 @@ -/* -Use zfs replication over ssh to pull recent filesystems from -one file-server to another one. - -This will be used for: - -- backup -- moving a filesystem from one region/cluster to another -*/ - -import { - type Filesystem, - type RawFilesystem, - primaryKey, - PrimaryKey, -} from "./types"; -import { exec } from "./util"; -import { - databaseFilename, - filesystemDataset, - filesystemMountpoint, -} from "./names"; -import { filesystemExists, getRecent, get, set } from "./db"; -import getLogger from "@cocalc/backend/logger"; -import { getSnapshots } from "./snapshots"; -import { createFilesystem, deleteFilesystem } from "./create"; -import { context } from "./config"; -import { archiveFilesystem, dearchiveFilesystem } from "./archive"; -import { deleteSnapshot } from "./snapshots"; -import { isEqual } from "lodash"; -import { join } from "path"; -import { readdir, unlink } from "fs/promises"; - -const logger = getLogger("file-server:zfs:pull"); - -// number of remote backups of db sqlite file to keep. -const NUM_DB_TO_KEEP = 10; - -// This is used for unit testing. It's what fields should match -// after doing a sync, except snapshots where local is a superset, -// unless you pull with deleteSnapshots set to true. -export const SYNCED_FIELDS = [ - // these four fields identify the filesystem, so they better get sync'd: - "namespace", - "owner_type", - "owner_id", - "name", - // snaphots -- reflects that we replicated properly. - "snapshots", - - // last_edited is useful for targetting sync work and making decisions, e.g.., should we delete - "last_edited", - // these just get directly sync'd. They aren't used unless somehow local were to actually server - // data directly. - "affinity", - "nfs", -]; - -interface Remote { - // remote = user@hostname that you can ssh to - remote: string; - // filesystem location on the remote server, so {data}/database.sqlite3 has the - // database that defines the state of the remote server. - data: string; -} - -// [ ] TODO: this is very likely broken due to prefix --> data change - -// Copy from remote to here every filesystem that has changed since cutoff. -export async function pull({ - cutoff, - filesystem, - remote, - data, - deleteFilesystemCutoff, - deleteSnapshots, - dryRun, -}: Remote & { - // pulls everything that's changed with remote last_edited >= cutoff. - cutoff?: Date; - // alternatively -- if given, only pull this filesystem and nothing else: - filesystem?: PrimaryKey; - - // DANGER: if set, any local filesystem with - // cutoff <= last_edited <= deleteFilesystemCutoff - // gets actually deleted. This makes it possible, e.g., to delete every filesystem - // that was deleted on the main server in the last 6 months and deleted at least 1 - // month ago, so we have a bit of time before destroy backups. - deleteFilesystemCutoff?: Date; - // if true, delete local snapshots if they were deleted on the remote. - deleteSnapshots?: boolean; - // just say how much will happen, but don't do anything. - dryRun?: boolean; -}): Promise<{ - toUpdate: { remoteFs: Filesystem; localFs?: Filesystem }[]; - toDelete: RawFilesystem[]; -}> { - logger.debug("pull: from ", { remote, data, cutoff, filesystem }); - if (data.startsWith("/")) { - throw Error("data should start with /"); - } - if (cutoff == null) { - cutoff = new Date(Date.now() - 1000 * 60 * 60 * 24 * 7); - } - logger.debug("pull: get the remote sqlite database"); - await exec({ command: "mkdir", args: ["-p", context.PULL] }); - const remoteDatabase = join( - context.PULL, - `${remote}:${data}---${new Date().toISOString()}.sqlite3`, - ); - // delete all but the most recent remote database files for this remote/data (?). - const oldDbFiles = (await readdir(context.PULL)) - .sort() - .filter((x) => x.startsWith(`${remote}:${data}---`)) - .slice(0, -NUM_DB_TO_KEEP); - for (const path of oldDbFiles) { - await unlink(join(context.PULL, path)); - } - - await exec({ - command: "scp", - args: [`${remote}:/${databaseFilename(data)}`, remoteDatabase], - }); - - logger.debug("pull: compare state"); - const recent = - filesystem != null - ? [get(filesystem, remoteDatabase)] - : getRecent({ cutoff, databaseFile: remoteDatabase }); - const toUpdate: { remoteFs: Filesystem; localFs?: Filesystem }[] = []; - for (const fs of recent) { - const remoteFs = get(fs, remoteDatabase); - if (!filesystemExists(fs)) { - toUpdate.push({ remoteFs }); - } else { - const localFs = get(fs); - if (remoteFs.archived != localFs.archived) { - // different archive state, so needs an update to resolve this (either way) - toUpdate.push({ remoteFs, localFs }); - continue; - } - if (deleteSnapshots) { - // sync if *any* snapshots differ - if (!isEqual(remoteFs.snapshots, localFs.snapshots)) { - toUpdate.push({ remoteFs, localFs }); - } - } else { - // only sync if newest snapshots are different - const newestRemoteSnapshot = - remoteFs.snapshots[remoteFs.snapshots.length - 1]; - if (!newestRemoteSnapshot) { - // no snapshots yet, so nothing to do. - continue; - } - const newestLocalSnapshot = - localFs.snapshots[localFs.snapshots.length - 1]; - if ( - !newestLocalSnapshot || - newestRemoteSnapshot > newestLocalSnapshot - ) { - toUpdate.push({ remoteFs, localFs }); - } - } - } - } - - logger.debug(`pull: toUpdate.length = ${toUpdate.length}`); - if (!dryRun) { - for (const x of toUpdate) { - logger.debug("pull: updating ", x); - await pullOne({ ...x, remote, deleteSnapshots }); - } - } - - const toDelete: RawFilesystem[] = []; - if (deleteFilesystemCutoff) { - for (const fs of getRecent({ cutoff })) { - if (!filesystemExists(fs, remoteDatabase)) { - if (new Date(fs.last_edited ?? 0) <= deleteFilesystemCutoff) { - // it's old enough to delete: - toDelete.push(fs); - } - } - } - } - logger.debug(`pull: toDelete.length = ${toDelete.length}`); - if (!dryRun) { - for (const fs of toDelete) { - logger.debug("pull: deleting", fs); - await deleteFilesystem(fs); - } - } - - return { toUpdate, toDelete }; -} - -async function pullOne({ - remoteFs, - localFs, - remote, - deleteSnapshots, -}: { - remoteFs: Filesystem; - localFs?: Filesystem; - remote?: string; - deleteSnapshots?: boolean; -}) { - logger.debug("pull:", { remoteFs, localFs, remote, deleteSnapshots }); - if (localFs == null) { - localFs = await createFilesystem(remoteFs); - } - - // sync last_edited, affinity and nfs fields in all cases - set({ - ...primaryKey(localFs), - last_edited: remoteFs.last_edited, - affinity: remoteFs.affinity, - nfs: remoteFs.nfs, - }); - - if (localFs.archived && !remoteFs.archived) { - // it's back in use: - await dearchiveFilesystem(localFs); - // don't return -- will then possibly sync more below, in case of new changes - } else if (!localFs.archived && remoteFs.archived) { - // we just archive ours. Note in theory there is a chance - // that our local version is not update-to-date with the remote - // version. However, the point of archiving is it should only happen - // many weeks after a filesystem stopped being used, and by that - // point we should have already pull'd the latest version. - // Don't bother worrying about deleting snapshots. - await archiveFilesystem(localFs); - return; - } - if (localFs.archived && remoteFs.archived) { - // nothing to do - // Also, don't bother worrying about deleting snapshots, since can't. - return; - } - const snapshot = newestCommonSnapshot(localFs.snapshots, remoteFs.snapshots); - const newest_snapshot = remoteFs.snapshots[remoteFs.snapshots.length - 1]; - if (!newest_snapshot || snapshot == newest_snapshot) { - logger.debug("pull: already have the newest snapshot locally"); - } else { - const mountpoint = filesystemMountpoint(localFs); - try { - if (!snapshot) { - // full replication with nothing local - await exec({ - verbose: true, - command: `ssh ${remote} "zfs send -e -c -R ${filesystemDataset(remoteFs)}@${newest_snapshot}" | sudo zfs recv -o mountpoint=${mountpoint} -F ${filesystemDataset(localFs)}`, - what: { - ...localFs, - desc: "pull: doing a full receive from remote", - }, - }); - } else { - // incremental based on the last common snapshot - const force = - localFs.snapshots[localFs.snapshots.length - 1] == snapshot - ? "" - : " -F "; - await exec({ - verbose: true, - command: `ssh ${remote} "zfs send -e -c -I @${snapshot} ${filesystemDataset(remoteFs)}@${newest_snapshot}" | sudo zfs recv -o mountpoint=${mountpoint} -F ${filesystemDataset(localFs)} ${force}`, - what: { - ...localFs, - desc: "pull: doing an incremental replication from remote", - }, - }); - } - } finally { - // even if there was an error, update local snapshots, since we likely have some new - // ones (e.g., even if there was a partial receive, interrupted by a network drop). - await getSnapshots(localFs); - } - } - - if (deleteSnapshots) { - // In general due to snapshot trimming, the - // list of snapshots on local might NOT match remote, but after replication - // local will always have a *supserset* of remote. We thus may have to - // trim some snapshots: - const remoteSnapshots = new Set(remoteFs.snapshots); - const localSnapshots = get(localFs).snapshots; - for (const snapshot of localSnapshots) { - if (!remoteSnapshots.has(snapshot)) { - await deleteSnapshot({ ...localFs, snapshot }); - } - } - } -} - -// s0 and s1 are sorted oldest-to-newest lists of names of snapshots. -// return largest that is in common between the two or undefined if nothing is in common -function newestCommonSnapshot(s0: string[], s1: string[]) { - const t1 = new Set(s1); - for (let i = s0.length - 1; i >= 0; i--) { - if (t1.has(s0[i])) { - return s0[i]; - } - } -} diff --git a/src/packages/file-server/zfs/snapshots.ts b/src/packages/file-server/zfs/snapshots.ts deleted file mode 100644 index c2d3ab4e7f..0000000000 --- a/src/packages/file-server/zfs/snapshots.ts +++ /dev/null @@ -1,315 +0,0 @@ -/* -Manage creating and deleting rolling snapshots of a filesystem. - -We keep track of all state in the sqlite database, so only have to touch -ZFS when we actually need to do something. Keep this in mind though since -if you try to mess with snapshots directly then the sqlite database won't -know you did that. -*/ - -import { exec } from "./util"; -import { get, getRecent, set } from "./db"; -import { filesystemDataset, filesystemMountpoint } from "./names"; -import { splitlines } from "@cocalc/util/misc"; -import getLogger from "@cocalc/backend/logger"; -import { - SNAPSHOT_INTERVAL_MS, - SNAPSHOT_INTERVALS_MS, - SNAPSHOT_COUNTS, -} from "./config"; -import { syncProperties } from "./properties"; -import { primaryKey, type PrimaryKey } from "./types"; -import { isEqual } from "lodash"; - -const logger = getLogger("file-server:zfs/snapshots"); - -export async function maintainSnapshots(cutoff?: Date) { - await deleteExtraSnapshotsOfActiveFilesystems(cutoff); - await snapshotActiveFilesystems(cutoff); -} - -// If there any changes to the filesystem since the last snapshot, -// and there are no snapshots since SNAPSHOT_INTERVAL_MS ms ago, -// make a new one. Always returns the most recent snapshot name. -// Error if filesystem is archived. -export async function createSnapshot({ - force, - ifChanged, - ...fs -}: PrimaryKey & { - force?: boolean; - // note -- ifChanged is VERY fast, but it's not instantaneous... - ifChanged?: boolean; -}): Promise { - logger.debug("createSnapshot: ", fs); - const pk = primaryKey(fs); - const { pool, archived, snapshots } = get(pk); - if (archived) { - throw Error("cannot snapshot an archived filesystem"); - } - if (!force && !ifChanged && snapshots.length > 0) { - // check for sufficiently recent snapshot - const last = new Date(snapshots[snapshots.length - 1]); - if (Date.now() - last.valueOf() < SNAPSHOT_INTERVAL_MS) { - // snapshot sufficiently recent - return snapshots[snapshots.length - 1]; - } - } - - // Check to see if nothing change on disk since last snapshot - if so, don't make a new one: - if (!force && snapshots.length > 0) { - const written = await getWritten(pk); - if (written == 0) { - // for sure definitely nothing written, so no possible - // need to make a snapshot - return snapshots[snapshots.length - 1]; - } - } - - const snapshot = new Date().toISOString(); - await exec({ - verbose: true, - command: "sudo", - args: [ - "zfs", - "snapshot", - `${filesystemDataset({ ...pk, pool })}@${snapshot}`, - ], - what: { ...pk, desc: "creating snapshot of project" }, - }); - set({ - ...pk, - snapshots: ({ snapshots }) => [...snapshots, snapshot], - }); - syncProperties(pk); - return snapshot; -} - -async function getWritten(fs: PrimaryKey) { - const pk = primaryKey(fs); - const { pool } = get(pk); - const { stdout } = await exec({ - verbose: true, - command: "zfs", - args: ["list", "-Hpo", "written", filesystemDataset({ ...pk, pool })], - what: { - ...pk, - desc: "getting amount of newly written data in project since last snapshot", - }, - }); - return parseInt(stdout); -} - -export async function zfsGetSnapshots(dataset: string) { - const { stdout } = await exec({ - command: "zfs", - args: ["list", "-j", "-o", "name", "-r", "-t", "snapshot", dataset], - }); - const snapshots = Object.keys(JSON.parse(stdout).datasets).map( - (name) => name.split("@")[1], - ); - return snapshots; -} - -// gets snapshots from disk via zfs *and* sets the list of snapshots -// in the database to match (and also updates sizes) -export async function getSnapshots(fs: PrimaryKey) { - const pk = primaryKey(fs); - const filesystem = get(fs); - const snapshots = await zfsGetSnapshots(filesystemDataset(filesystem)); - if (!isEqual(snapshots, filesystem.snapshots)) { - set({ ...pk, snapshots }); - syncProperties(fs); - } - return snapshots; -} - -export async function deleteSnapshot({ - snapshot, - ...fs -}: PrimaryKey & { snapshot: string }) { - const pk = primaryKey(fs); - logger.debug("deleteSnapshot: ", pk, snapshot); - const { pool, last_send_snapshot } = get(pk); - if (snapshot == last_send_snapshot) { - throw Error( - "can't delete snapshot since it is the last one used for a zfs send", - ); - } - await exec({ - verbose: true, - command: "sudo", - args: [ - "zfs", - "destroy", - `${filesystemDataset({ ...pk, pool })}@${snapshot}`, - ], - what: { ...pk, desc: "destroying a snapshot of a project" }, - }); - set({ - ...pk, - snapshots: ({ snapshots }) => snapshots.filter((x) => x != snapshot), - }); - syncProperties(pk); -} - -/* -Remove snapshots according to our retention policy, and -never delete last_stream if set. - -Returns names of deleted snapshots. -*/ -export async function deleteExtraSnapshots(fs: PrimaryKey): Promise { - const pk = primaryKey(fs); - logger.debug("deleteExtraSnapshots: ", pk); - const { last_send_snapshot, snapshots } = get(pk); - if (snapshots.length == 0) { - // nothing to do - return []; - } - - // sorted from BIGGEST to smallest - const times = snapshots.map((x) => new Date(x).valueOf()); - times.reverse(); - const save = new Set(); - if (last_send_snapshot) { - save.add(new Date(last_send_snapshot).valueOf()); - } - for (const type in SNAPSHOT_COUNTS) { - const count = SNAPSHOT_COUNTS[type]; - const length_ms = SNAPSHOT_INTERVALS_MS[type]; - - // Pick the first count newest snapshots at intervals of length - // length_ms milliseconds. - let n = 0, - i = 0, - last_tm = 0; - while (n < count && i < times.length) { - const tm = times[i]; - if (!last_tm || tm <= last_tm - length_ms) { - save.add(tm); - last_tm = tm; - n += 1; // found one more - } - i += 1; // move to next snapshot - } - } - const toDelete = snapshots.filter((x) => !save.has(new Date(x).valueOf())); - for (const snapshot of toDelete) { - await deleteSnapshot({ ...pk, snapshot }); - } - return toDelete; -} - -// Go through ALL projects with last_edited >= cutoff stored -// here and run trimActiveFilesystemSnapshots. -export async function deleteExtraSnapshotsOfActiveFilesystems(cutoff?: Date) { - const v = getRecent({ cutoff }); - logger.debug( - `deleteSnapshotsOfActiveFilesystems: considering ${v.length} filesystems`, - ); - let i = 0; - for (const fs of v) { - if (fs.archived) { - continue; - } - try { - await deleteExtraSnapshots(fs); - } catch (err) { - logger.debug(`deleteSnapshotsOfActiveFilesystems: error -- ${err}`); - } - i += 1; - if (i % 10 == 0) { - logger.debug(`deleteSnapshotsOfActiveFilesystems: ${i}/${v.length}`); - } - } -} - -// Go through ALL projects with last_edited >= cutoff and snapshot them -// if they are due a snapshot. -// cutoff = a Date (default = 1 week ago) -export async function snapshotActiveFilesystems(cutoff?: Date) { - logger.debug("snapshotActiveFilesystems: getting..."); - const v = getRecent({ cutoff }); - logger.debug( - `snapshotActiveFilesystems: considering ${v.length} projects`, - cutoff, - ); - let i = 0; - for (const fs of v) { - if (fs.archived) { - continue; - } - try { - await createSnapshot(fs); - } catch (err) { - // error is already logged in error field of database - logger.debug(`snapshotActiveFilesystems: error -- ${err}`); - } - i += 1; - if (i % 10 == 0) { - logger.debug(`snapshotActiveFilesystems: ${i}/${v.length}`); - } - } -} - -/* -Get list of files modified since given snapshot (or last snapshot if not given). - -**There's probably no good reason to ever use this code!** - -The reason is because it's really slow, e.g., I added the -cocalc src directory (5000) files and it takes about 6 seconds -to run this. In contrast. "time find .", which lists EVERYTHING -takes less than 0.074s. You could do that before and after, then -compare them, and it'll be a fraction of a second. -*/ -interface Mod { - time: number; - change: "-" | "+" | "M" | "R"; // remove/create/modify/rename - // see "man zfs diff": - type: "B" | "C" | "/" | ">" | "|" | "@" | "P" | "=" | "F"; - path: string; -} - -export async function getModifiedFiles({ - snapshot, - ...fs -}: PrimaryKey & { snapshot: string }) { - const pk = primaryKey(fs); - logger.debug(`getModifiedFiles: `, pk); - const { pool, snapshots } = get(pk); - if (snapshots.length == 0) { - return []; - } - if (snapshot == null) { - snapshot = snapshots[snapshots.length - 1]; - } - const { stdout } = await exec({ - verbose: true, - command: "sudo", - args: [ - "zfs", - "diff", - "-FHt", - `${filesystemDataset({ ...pk, pool })}@${snapshot}`, - ], - what: { ...pk, desc: "getting files modified since last snapshot" }, - }); - const mnt = filesystemMountpoint(pk) + "/"; - const files: Mod[] = []; - for (const line of splitlines(stdout)) { - const x = line.split(/\t/g); - let path = x[3]; - if (path.startsWith(mnt)) { - path = path.slice(mnt.length); - } - files.push({ - time: parseFloat(x[0]) * 1000, - change: x[1] as any, - type: x[2] as any, - path, - }); - } - return files; -} diff --git a/src/packages/file-server/zfs/streams.ts b/src/packages/file-server/zfs/streams.ts deleted file mode 100644 index 6720c21798..0000000000 --- a/src/packages/file-server/zfs/streams.ts +++ /dev/null @@ -1,253 +0,0 @@ -/* -Send/Receive incremental replication streams of a filesystem. -*/ - -import { type PrimaryKey } from "./types"; -import { get, getRecent, set } from "./db"; -import getLogger from "@cocalc/backend/logger"; -import { - filesystemStreamsPath, - filesystemStreamsFilename, - filesystemDataset, -} from "./names"; -import { exec } from "./util"; -import { split } from "@cocalc/util/misc"; -import { join } from "path"; -import { getSnapshots } from "./snapshots"; -import { STREAM_INTERVAL_MS, MAX_STREAMS } from "./config"; - -const logger = getLogger("file-server:zfs:send"); - -export async function send(fs: PrimaryKey) { - const filesystem = get(fs); - if (filesystem.archived) { - logger.debug("filesystem is archived, so nothing to do", fs); - return; - } - const { snapshots } = filesystem; - const newest_snapshot = snapshots[snapshots.length - 1]; - if (!newest_snapshot) { - logger.debug("no snapshots yet"); - return; - } - if (newest_snapshot == filesystem.last_send_snapshot) { - logger.debug("no new snapshots", fs); - // the most recent snapshot is the same as the last one we used to make - // an archive, so nothing to do. - return; - } - await exec({ - command: "sudo", - args: ["mkdir", "-p", filesystemStreamsPath(filesystem)], - what: { ...filesystem, desc: "make send target directory" }, - }); - - let stream; - if (!filesystem.last_send_snapshot) { - logger.debug("doing first ever send -- a full send"); - stream = filesystemStreamsFilename({ - ...filesystem, - snapshot1: new Date(0).toISOString(), - snapshot2: newest_snapshot, - }); - try { - await exec({ - verbose: true, - command: `sudo sh -c 'zfs send -e -c -R ${filesystemDataset(filesystem)}@${newest_snapshot} > ${stream}.temp'`, - what: { - ...filesystem, - desc: "send: zfs send of full filesystem dataset (first full send)", - }, - }); - } catch (err) { - await exec({ - verbose: true, - command: "sudo", - args: ["rm", `${stream}.temp`], - }); - throw err; - } - } else { - logger.debug("doing incremental send"); - const snapshot1 = filesystem.last_send_snapshot; - const snapshot2 = newest_snapshot; - stream = filesystemStreamsFilename({ - ...filesystem, - snapshot1, - snapshot2, - }); - try { - await exec({ - verbose: true, - command: `sudo sh -c 'zfs send -e -c -I @${snapshot1} ${filesystemDataset(filesystem)}@${snapshot2} > ${stream}.temp'`, - what: { - ...filesystem, - desc: "send: zfs incremental send", - }, - }); - } catch (err) { - await exec({ - verbose: true, - command: "sudo", - args: ["rm", `${stream}.temp`], - }); - throw err; - } - } - await exec({ - verbose: true, - command: "sudo", - args: ["mv", `${stream}.temp`, stream], - }); - set({ ...fs, last_send_snapshot: newest_snapshot }); -} - -async function getStreams(fs: PrimaryKey) { - const filesystem = get(fs); - const streamsPath = filesystemStreamsPath(filesystem); - const { stdout } = await exec({ - command: "sudo", - args: ["ls", streamsPath], - what: { ...filesystem, desc: "getting list of streams" }, - }); - return split(stdout.trim()).filter((path) => path.endsWith(".zfs")); -} - -export async function recv(fs: PrimaryKey) { - const filesystem = get(fs); - if (filesystem.archived) { - throw Error("filesystem must not be archived"); - } - const streams = await getStreams(filesystem); - if (streams.length == 0) { - logger.debug("no streams"); - return; - } - const { snapshots } = filesystem; - const newest_snapshot = snapshots[snapshots.length - 1] ?? ""; - const toRead = streams.filter((snapshot) => snapshot >= newest_snapshot); - if (toRead.length == 0) { - return; - } - const streamsPath = filesystemStreamsPath(filesystem); - try { - for (const stream of toRead) { - await exec({ - verbose: true, - command: `sudo sh -c 'cat ${join(streamsPath, stream)} | zfs recv ${filesystemDataset(filesystem)}'`, - what: { - ...filesystem, - desc: `send: zfs incremental receive`, - }, - }); - } - } finally { - // ensure snapshots and size info in our database is up to date: - await getSnapshots(fs); - } -} - -function getRange(streamName) { - const v = streamName.split("Z-"); - return { snapshot1: v + "Z", snapshot2: v[1].slice(0, -".zfs".length) }; -} - -// Replace older streams so that there are at most maxStreams total streams. -export async function recompact({ - maxStreams, - ...fs -}: PrimaryKey & { maxStreams: number }) { - const filesystem = get(fs); - const { snapshots } = filesystem; - const streams = await getStreams(filesystem); - if (streams.length <= maxStreams) { - // nothing to do - return; - } - if (maxStreams < 1) { - throw Error("maxStreams must be at least 1"); - } - // replace first n streams by one full replication stream - let n = streams.length - maxStreams + 1; - let snapshot2 = getRange(streams[n - 1]).snapshot2; - while (!snapshots.includes(snapshot2) && n < streams.length) { - snapshot2 = getRange(streams[n]).snapshot2; - if (snapshots.includes(snapshot2)) { - break; - } - n += 1; - } - if (!snapshots.includes(snapshot2)) { - throw Error( - "bug -- this can't happen because we never delete the last snapshot used for send", - ); - } - - const stream = filesystemStreamsFilename({ - ...filesystem, - snapshot1: new Date(0).toISOString(), - snapshot2, - }); - try { - await exec({ - verbose: true, - command: `sudo sh -c 'zfs send -e -c -R ${filesystemDataset(filesystem)}@${snapshot2} > ${stream}.temp'`, - what: { - ...filesystem, - desc: "send: zfs send of full filesystem dataset (first full send)", - }, - }); - // if this rm were to fail, then things would be left in a broken state, - // since ${stream}.temp also gets deleted in the catch. But it seems - // highly unlikely this rm of the old streams would ever fail. - const path = filesystemStreamsPath(filesystem); - await exec({ - verbose: true, - command: "sudo", - // full paths to the first n streams: - args: ["rm", "-f", ...streams.slice(0, n).map((x) => join(path, x))], - }); - await exec({ - verbose: true, - command: "sudo", - args: ["mv", `${stream}.temp`, stream], - }); - } catch (err) { - await exec({ - verbose: true, - command: "sudo", - args: ["rm", "-f", `${stream}.temp`], - }); - throw err; - } -} - -// Go through ALL filesystems with last_edited >= cutoff and send a stream if due, -// and also ensure number of streams isn't too large. -export async function maintainStreams(cutoff?: Date) { - logger.debug("backupActiveFilesystems: getting..."); - const v = getRecent({ cutoff }); - logger.debug(`maintainStreams: considering ${v.length} filesystems`, cutoff); - let i = 0; - for (const { archived, last_edited, last_send_snapshot, ...pk } of v) { - if (archived || !last_edited) { - continue; - } - const age = - new Date(last_edited).valueOf() - new Date(last_send_snapshot ?? 0).valueOf(); - if (age < STREAM_INTERVAL_MS) { - // there's a new enough stream already - continue; - } - try { - await send(pk); - await recompact({ ...pk, maxStreams: MAX_STREAMS }); - } catch (err) { - logger.debug(`maintainStreams: error -- ${err}`); - } - i += 1; - if (i % 10 == 0) { - logger.debug(`maintainStreams: ${i}/${v.length}`); - } - } -} diff --git a/src/packages/file-server/zfs/test/archive.test.ts b/src/packages/file-server/zfs/test/archive.test.ts deleted file mode 100644 index dfbb003bdb..0000000000 --- a/src/packages/file-server/zfs/test/archive.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -/* -DEVELOPMENT: - -pnpm exec jest --watch archive.test.ts -*/ - -import { executeCode } from "@cocalc/backend/execute-code"; -import { createTestPools, deleteTestPools, init, describe } from "./util"; -import { - archiveFilesystem, - dearchiveFilesystem, - createFilesystem, - createSnapshot, - getSnapshots, - get, -} from "@cocalc/file-server/zfs"; -import { filesystemMountpoint } from "@cocalc/file-server/zfs/names"; -import { readFile, writeFile } from "fs/promises"; -import { join } from "path"; - -describe("create a project, put in some files/snapshot, archive the project, confirm gone, de-archive it, and confirm files are back as expected", () => { - jest.setTimeout(10000); - let x: any = null; - - beforeAll(async () => { - x = await createTestPools({ count: 1, size: "1G" }); - await init(); - }); - - afterAll(async () => { - if (x != null) { - await deleteTestPools(x); - } - }); - - const project_id = "00000000-0000-0000-0000-000000000001"; - const mnt = filesystemMountpoint({ project_id, namespace: "default" }); - const FILE_CONTENT = "hello"; - const FILENAME = "cocalc.txt"; - it("creates a project and write a file", async () => { - const filesystem = await createFilesystem({ - project_id, - }); - expect(filesystem.owner_type).toBe("project"); - expect(filesystem.owner_id).toBe(project_id); - const path = join(mnt, FILENAME); - await writeFile(path, FILE_CONTENT); - }); - - let snapshot1, snapshot2; - const FILE_CONTENT2 = "hello2"; - const FILENAME2 = "cocalc2.txt"; - - it("create a snapshot and write another file, so there is a nontrivial snapshot to be archived", async () => { - snapshot1 = await createSnapshot({ project_id }); - expect(!!snapshot1).toBe(true); - const path = join(mnt, FILENAME2); - await writeFile(path, FILE_CONTENT2); - snapshot2 = await createSnapshot({ project_id, force: true }); - expect(snapshot2).not.toEqual(snapshot1); - }); - - it("archive the project and checks project is no longer in zfs", async () => { - expect(get({ project_id }).archived).toBe(false); - await archiveFilesystem({ project_id }); - const { stdout } = await executeCode({ - command: "zfs", - args: ["list", x.pools[0]], - }); - expect(stdout).not.toContain(project_id); - expect(get({ project_id }).archived).toBe(true); - }); - - it("archiving an already archived project is an error", async () => { - await expect( - async () => await archiveFilesystem({ project_id }), - ).rejects.toThrow(); - }); - - it("dearchive project and verify zfs filesystem is back, along with files and snapshots", async () => { - let called = false; - await dearchiveFilesystem({ - project_id, - progress: () => { - called = true; - }, - }); - expect(called).toBe(true); - expect(get({ project_id }).archived).toBe(false); - - expect((await readFile(join(mnt, FILENAME))).toString()).toEqual( - FILE_CONTENT, - ); - expect((await readFile(join(mnt, FILENAME2))).toString()).toEqual( - FILE_CONTENT2, - ); - expect(await getSnapshots({ project_id })).toEqual([snapshot1, snapshot2]); - }); - - it("dearchiving an already de-archived project is an error", async () => { - await expect( - async () => await dearchiveFilesystem({ project_id }), - ).rejects.toThrow(); - }); -}); diff --git a/src/packages/file-server/zfs/test/create-types.test.ts b/src/packages/file-server/zfs/test/create-types.test.ts deleted file mode 100644 index 150adda7b5..0000000000 --- a/src/packages/file-server/zfs/test/create-types.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* -DEVELOPMENT: - -pnpm exec jest --watch create-types.test.ts -*/ - -import { createTestPools, deleteTestPools, init, describe } from "./util"; -import { - createFilesystem, -} from "@cocalc/file-server/zfs"; -import type { Filesystem } from "../types"; - -describe("create some account and organization filesystems", () => { - let x: any = null; - - beforeAll(async () => { - x = await createTestPools({ count: 1, size: "1G" }); - await init(); - }); - - afterAll(async () => { - if (x != null) { - await deleteTestPools(x); - } - }); - - // Making these the same intentionally to ensure the filesystem properly - // does not distinguish types based on the owner_id. - const project_id = "00000000-0000-0000-0000-000000000001"; - const account_id = "00000000-0000-0000-0000-000000000001"; - const group_id = "00000000-0000-0000-0000-000000000001"; - const filesystems: Filesystem[] = []; - it("creates filesystems associated to the project, account and group", async () => { - const fs = await createFilesystem({ project_id }); - expect(fs.owner_id).toBe(project_id); - filesystems.push(fs); - const fs2 = await createFilesystem({ account_id, name: "cocalc" }); - expect(fs2.owner_id).toBe(account_id); - filesystems.push(fs2); - const fs3 = await createFilesystem({ group_id, name: "data" }); - expect(fs3.owner_id).toBe(group_id); - filesystems.push(fs3); - }); - - it("tries to create an account and group filesystem with empty name and gets an error", async () => { - expect(async () => { - await createFilesystem({ account_id }); - }).rejects.toThrow("name must be nonempty"); - expect(async () => { - await createFilesystem({ group_id }); - }).rejects.toThrow("name must be nonempty"); - }); - - it('for projects the name defaults to "home"', async () => { - expect(async () => { - await createFilesystem({ project_id, name: "" }); - }).rejects.toThrow("must be nonempty"); - expect(filesystems[0].name).toBe("home"); - }); - - it("name must be less than 64 characters", async () => { - let name = ""; - for (let i = 0; i < 63; i++) { - name += "x"; - } - await createFilesystem({ account_id, name }); - name += 1; - expect(async () => { - await createFilesystem({ account_id, name }); - }).rejects.toThrow("name must be at most 63 characters"); - }); - - it("name must not have 'funny characters'", async () => { - expect(async () => { - await createFilesystem({ account_id, name: "$%@!" }); - }).rejects.toThrow("name must only contain"); - }); -}); diff --git a/src/packages/file-server/zfs/test/create.test.ts b/src/packages/file-server/zfs/test/create.test.ts deleted file mode 100644 index 6204c4683e..0000000000 --- a/src/packages/file-server/zfs/test/create.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -/* -DEVELOPMENT: - -pnpm exec jest --watch create.test.ts - - pnpm exec jest create.test.ts -b -*/ - -// application/typescript text -import { executeCode } from "@cocalc/backend/execute-code"; -import { createTestPools, deleteTestPools, init, describe, describe0 } from "./util"; -import { - createFilesystem, - createBackup, - deleteFilesystem, - getPools, -} from "@cocalc/file-server/zfs"; -import { filesystemMountpoint } from "@cocalc/file-server/zfs/names"; -import { readFile, writeFile } from "fs/promises"; -import { join } from "path"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { uuid } from "@cocalc/util/misc"; -import { map as asyncMap } from "awaiting"; - -describe0("test for zfs", () => { - it("checks for TEST_ZFS", () => { - if (!process.env.TEST_ZFS) { - // make sure people aren't silently overconfident... - console.log( - "WARNing: TEST_ZFS not set, so **SKIPPING ALL ZFS FILE SERVER TESTS!**", - ); - } - }); -}); - -describe("creates project, clone project, delete projects", () => { - let x: any = null; - - beforeAll(async () => { - x = await createTestPools({ count: 1, size: "1G" }); - await init(); - }); - - afterAll(async () => { - if (x != null) { - await deleteTestPools(x); - } - }); - - it("verifies there is a pool", async () => { - const { stdout } = await executeCode({ - command: "zpool", - args: ["list", x.pools[0]], - }); - expect(stdout).toContain(x.pools[0]); - expect(Object.keys(await getPools()).length).toBe(1); - }); - - const project_id = "00000000-0000-0000-0000-000000000001"; - it("creates a project", async () => { - const project = await createFilesystem({ - project_id, - }); - expect(project.owner_id).toBe(project_id); - }); - - it("verify project is in output of zfs list", async () => { - const { stdout } = await executeCode({ - command: "zfs", - args: ["list", "-r", x.pools[0]], - }); - expect(stdout).toContain(project_id); - }); - - const FILE_CONTENT = "hello"; - const FILENAME = "cocalc.txt"; - it("write a file to the project", async () => { - const path = join( - filesystemMountpoint({ project_id, namespace: "default" }), - FILENAME, - ); - await writeFile(path, FILE_CONTENT); - }); - - const project_id2 = "00000000-0000-0000-0000-000000000002"; - it("clones our project to make a second project", async () => { - const project2 = await createFilesystem({ - project_id: project_id2, - clone: { project_id }, - }); - expect(project2.owner_id).toBe(project_id2); - }); - - it("verify clone is in output of zfs list", async () => { - const { stdout } = await executeCode({ - command: "zfs", - args: ["list", "-r", x.pools[0]], - }); - expect(stdout).toContain(project_id2); - }); - - it("read file from the clone", async () => { - const path = join( - filesystemMountpoint({ project_id: project_id2, namespace: "default" }), - FILENAME, - ); - const content = (await readFile(path)).toString(); - expect(content).toEqual(FILE_CONTENT); - }); - - let BUP_DIR; - it("make a backup of project, so can see that it gets deleted below", async () => { - const x = await createBackup({ project_id }); - BUP_DIR = x.BUP_DIR; - expect(await exists(BUP_DIR)).toBe(true); - }); - - it("attempt to delete first project and get error", async () => { - try { - await deleteFilesystem({ project_id }); - throw Error("must throw"); - } catch (err) { - expect(`${err}`).toContain("filesystem has dependent clones"); - } - }); - - it("delete second project, then first project, works", async () => { - await deleteFilesystem({ project_id: project_id2 }); - await deleteFilesystem({ project_id }); - const { stdout } = await executeCode({ - command: "zfs", - args: ["list", "-r", x.pools[0]], - }); - expect(stdout).not.toContain(project_id); - expect(stdout).not.toContain(project_id2); - }); - - it("verifies bup backup is also gone", async () => { - expect(await exists(BUP_DIR)).toBe(false); - }); -}); - -describe("create two projects with the same project_id at the same time, but in different namespaces", () => { - let x: any = null; - - beforeAll(async () => { - x = await createTestPools({ count: 2, size: "1G" }); - await init(); - }); - - afterAll(async () => { - if (x != null) { - await deleteTestPools(x); - } - }); - - it("there are TWO pools this time", async () => { - expect(Object.keys(await getPools()).length).toBe(2); - }); - - const project_id = "00000000-0000-0000-0000-000000000001"; - it("creates two projects", async () => { - const project = await createFilesystem({ - project_id, - namespace: "default", - }); - expect(project.owner_id).toBe(project_id); - - const project2 = await createFilesystem({ - project_id, - namespace: "test", - }); - expect(project2.owner_id).toBe(project_id); - // they are on different pools - expect(project.pool).not.toEqual(project2.pool); - }); - - it("two different entries in zfs list", async () => { - const { stdout: stdout0 } = await executeCode({ - command: "zfs", - args: ["list", "-r", x.pools[0]], - }); - expect(stdout0).toContain(project_id); - const { stdout: stdout1 } = await executeCode({ - command: "zfs", - args: ["list", "-r", x.pools[1]], - }); - expect(stdout1).toContain(project_id); - }); -}); - -describe("test the affinity property when creating projects", () => { - let x: any = null; - - beforeAll(async () => { - x = await createTestPools({ count: 2, size: "1G" }); - await init(); - }); - - afterAll(async () => { - if (x != null) { - await deleteTestPools(x); - } - }); - - const project_id = "00000000-0000-0000-0000-000000000001"; - const project_id2 = "00000000-0000-0000-0000-000000000002"; - const affinity = "math100"; - it("creates two projects with same afinity", async () => { - const project = await createFilesystem({ - project_id, - affinity, - }); - expect(project.owner_id).toBe(project_id); - - const project2 = await createFilesystem({ - project_id: project_id2, - affinity, - }); - expect(project2.owner_id).toBe(project_id2); - // they are on SAME pools, because of affinity - expect(project.pool).toEqual(project2.pool); - }); -}); - -describe("do a stress/race condition test creating a larger number of projects on a larger number of pools", () => { - let x: any = null; - - const count = 3; - const nprojects = 25; - - beforeAll(async () => { - x = await createTestPools({ count, size: "1G" }); - await init(); - }); - - afterAll(async () => { - if (x != null) { - await deleteTestPools(x); - } - }); - - it(`creates ${nprojects} projects in parallel on ${count} pools`, async () => { - const f = async (project_id) => { - await createFilesystem({ project_id }); - }; - const v: string[] = []; - for (let n = 0; n < nprojects; n++) { - v.push(uuid()); - } - // doing these in parallel and having it work is an important stress test, - // since we will get a bid speedup doing this in production, and there we - // will really need it. - await asyncMap(v, nprojects, f); - }); -}); diff --git a/src/packages/file-server/zfs/test/nfs.test.ts b/src/packages/file-server/zfs/test/nfs.test.ts deleted file mode 100644 index 56729b2798..0000000000 --- a/src/packages/file-server/zfs/test/nfs.test.ts +++ /dev/null @@ -1,118 +0,0 @@ -/* -DEVELOPMENT: - -pnpm exec jest --watch nfs.test.ts -*/ - -import { executeCode } from "@cocalc/backend/execute-code"; -import { - createTestPools, - deleteTestPools, - restartNfsServer, - init, - describe, -} from "./util"; -import { - createFilesystem, - createSnapshot, - get, - shareNFS, - unshareNFS, -} from "@cocalc/file-server/zfs"; -import { filesystemMountpoint } from "@cocalc/file-server/zfs/names"; -import { readFile, writeFile } from "fs/promises"; -import { join } from "path"; - -describe("create a project, put in a files, snapshot, another file, then share via NFS, mount and verify it works", () => { - let x: any = null; - const project_id = "00000000-0000-0000-0000-000000000001"; - let nsfMnt = ""; - - beforeAll(async () => { - x = await createTestPools({ count: 1, size: "1G" }); - nsfMnt = join(x.tempDir, project_id); - await init(); - }); - - afterAll(async () => { - if (x != null) { - await deleteTestPools(x); - } - }); - - const mnt = filesystemMountpoint({ project_id, namespace: "default" }); - const FILE_CONTENT = "hello"; - const FILENAME = "cocalc.txt"; - it("creates a project and write a file", async () => { - const project = await createFilesystem({ - project_id, - }); - expect(project.owner_id).toBe(project_id); - const path = join(mnt, FILENAME); - await writeFile(path, FILE_CONTENT); - }); - - let snapshot1, snapshot2; - const FILE_CONTENT2 = "hello2"; - const FILENAME2 = "cocalc2.txt"; - - it("create a snapshot and write another file, so there is a nontrivial snapshot to view through NFS", async () => { - snapshot1 = await createSnapshot({ project_id }); - expect(!!snapshot1).toBe(true); - const path = join(mnt, FILENAME2); - await writeFile(path, FILE_CONTENT2); - snapshot2 = await createSnapshot({ project_id, force: true }); - expect(snapshot2).not.toEqual(snapshot1); - }); - - let host = ""; - const client = "127.0.0.1"; - - const mount = async () => { - await executeCode({ - command: "sudo", - args: ["mkdir", "-p", nsfMnt], - }); - await executeCode({ - command: "sudo", - args: ["mount", host, nsfMnt], - }); - }; - - it("shares the project via NFS, and mounts it", async () => { - host = await shareNFS({ project_id, client }); - const project = get({ project_id }); - expect(project.nfs).toEqual([client]); - await mount(); - }); - - it("confirms our files and snapshots are there as expected", async () => { - const { stdout } = await executeCode({ - command: "sudo", - args: ["ls", nsfMnt], - }); - expect(stdout).toContain(FILENAME); - expect(stdout).toContain(FILENAME2); - expect((await readFile(join(nsfMnt, FILENAME))).toString()).toEqual( - FILE_CONTENT, - ); - expect((await readFile(join(nsfMnt, FILENAME2))).toString()).toEqual( - FILE_CONTENT2, - ); - }); - - it("stop NFS share and confirms it no longers works", async () => { - await executeCode({ - command: "sudo", - args: ["umount", nsfMnt], - }); - await restartNfsServer(); - await unshareNFS({ project_id, client }); - try { - await mount(); - throw Error("bug -- mount should fail"); - } catch (err) { - expect(`${err}`).toMatch(/not permitted|denied/); - } - }); -}); diff --git a/src/packages/file-server/zfs/test/pull.test.ts b/src/packages/file-server/zfs/test/pull.test.ts deleted file mode 100644 index 06a5c4af4c..0000000000 --- a/src/packages/file-server/zfs/test/pull.test.ts +++ /dev/null @@ -1,315 +0,0 @@ -/* -DEVELOPMENT: - -This tests pull replication by setting up two separate file-servers on disk locally -and doing pulls from one to the other over ssh. This involves password-less ssh -to root on localhost, and creating multiple pools, so use with caution and don't -expect this to work unless you really know what you're doing. -Also, these tests are going to take a while. - -Efficient powerful backup isn't trivial and is very valuable, so -its' worth the wait! - -pnpm exec jest --watch pull.test.ts -*/ - -import { join } from "path"; -import { createTestPools, deleteTestPools, init, describe } from "./util"; -import { - createFilesystem, - createSnapshot, - deleteSnapshot, - deleteFilesystem, - pull, - archiveFilesystem, - dearchiveFilesystem, -} from "@cocalc/file-server/zfs"; -import { context, setContext } from "@cocalc/file-server/zfs/config"; -import { filesystemMountpoint } from "@cocalc/file-server/zfs/names"; -import { readFile, writeFile } from "fs/promises"; -import { filesystemExists, get } from "@cocalc/file-server/zfs/db"; -import { SYNCED_FIELDS } from "../pull"; - -describe("create two separate file servers, then do pulls to sync one to the other under various conditions", () => { - let one: any = null, - two: any = null; - const data1 = context.DATA + ".1"; - const data2 = context.DATA + ".2"; - const remote = "root@localhost"; - - beforeAll(async () => { - one = await createTestPools({ count: 1, size: "1G", data: data1 }); - setContext({ data: data1 }); - await init(); - two = await createTestPools({ - count: 1, - size: "1G", - data: data2, - }); - setContext({ data: data2 }); - await init(); - }); - - afterAll(async () => { - await deleteTestPools(one); - await deleteTestPools(two); - }); - - it("creates a filesystem in pool one, writes a file and takes a snapshot", async () => { - setContext({ data: data1 }); - const fs = await createFilesystem({ - project_id: "00000000-0000-0000-0000-000000000001", - }); - await writeFile(join(filesystemMountpoint(fs), "a.txt"), "hello"); - await createSnapshot(fs); - expect(await filesystemExists(fs)).toEqual(true); - }); - - it("pulls filesystem one to filesystem two, and confirms the fs and file were indeed sync'd", async () => { - setContext({ data: data2 }); - expect( - await filesystemExists({ - project_id: "00000000-0000-0000-0000-000000000001", - }), - ).toEqual(false); - - // first dryRun - const { toUpdate, toDelete } = await pull({ - remote, - data: data1, - dryRun: true, - }); - expect(toDelete.length).toBe(0); - expect(toUpdate.length).toBe(1); - expect(toUpdate[0].remoteFs.owner_id).toEqual( - "00000000-0000-0000-0000-000000000001", - ); - expect(toUpdate[0].localFs).toBe(undefined); - - // now for real - const { toUpdate: toUpdate1, toDelete: toDelete1 } = await pull({ - remote, - data: data1, - }); - - expect(toDelete1).toEqual(toDelete); - expect(toUpdate1).toEqual(toUpdate); - const fs = { project_id: "00000000-0000-0000-0000-000000000001" }; - expect(await filesystemExists(fs)).toEqual(true); - expect( - (await readFile(join(filesystemMountpoint(fs), "a.txt"))).toString(), - ).toEqual("hello"); - - // nothing if we sync again: - const { toUpdate: toUpdate2, toDelete: toDelete2 } = await pull({ - remote, - data: data1, - }); - expect(toDelete2.length).toBe(0); - expect(toUpdate2.length).toBe(0); - }); - - it("creates another file in our filesystem, creates another snapshot, syncs again, and sees that the sync worked", async () => { - setContext({ data: data1 }); - const fs = { project_id: "00000000-0000-0000-0000-000000000001" }; - await writeFile(join(filesystemMountpoint(fs), "b.txt"), "cocalc"); - await createSnapshot({ ...fs, force: true }); - const { snapshots } = get(fs); - expect(snapshots.length).toBe(2); - - setContext({ data: data2 }); - await pull({ remote, data: data1 }); - - expect( - (await readFile(join(filesystemMountpoint(fs), "b.txt"))).toString(), - ).toEqual("cocalc"); - }); - - it("archives the project, does sync, and see the other one got archived", async () => { - const fs = { project_id: "00000000-0000-0000-0000-000000000001" }; - setContext({ data: data2 }); - const project2before = get(fs); - expect(project2before.archived).toBe(false); - - setContext({ data: data1 }); - await archiveFilesystem(fs); - const project1 = get(fs); - expect(project1.archived).toBe(true); - - setContext({ data: data2 }); - await pull({ remote, data: data1 }); - const project2 = get(fs); - expect(project2.archived).toBe(true); - expect(project1.last_edited).toEqual(project2.last_edited); - }); - - it("dearchives, does sync, then sees the other gets dearchived; this just tests that sync de-archives, but works even if there are no new snapshots", async () => { - const fs = { project_id: "00000000-0000-0000-0000-000000000001" }; - setContext({ data: data1 }); - await dearchiveFilesystem(fs); - const project1 = get(fs); - expect(project1.archived).toBe(false); - - setContext({ data: data2 }); - await pull({ remote, data: data1 }); - const project2 = get(fs); - expect(project2.archived).toBe(false); - }); - - it("archives project, does sync, de-archives project, adds another snapshot, then does sync, thus testing that sync both de-archives *and* pulls latest snapshot", async () => { - const fs = { project_id: "00000000-0000-0000-0000-000000000001" }; - setContext({ data: data1 }); - expect(get(fs).archived).toBe(false); - await archiveFilesystem(fs); - expect(get(fs).archived).toBe(true); - setContext({ data: data2 }); - await pull({ remote, data: data1 }); - expect(get(fs).archived).toBe(true); - - // now dearchive - setContext({ data: data1 }); - await dearchiveFilesystem(fs); - // write content - await writeFile(join(filesystemMountpoint(fs), "d.txt"), "hello"); - // snapshot - await createSnapshot({ ...fs, force: true }); - const project1 = get(fs); - - setContext({ data: data2 }); - await pull({ remote, data: data1 }); - const project2 = get(fs); - expect(project2.snapshots).toEqual(project1.snapshots); - expect(project2.archived).toBe(false); - }); - - it("deletes project, does sync, then sees the other does NOT gets deleted without passing the deleteFilesystemCutoff option, and also with deleteFilesystemCutoff an hour ago, but does get deleted with it now", async () => { - const fs = { project_id: "00000000-0000-0000-0000-000000000001" }; - setContext({ data: data1 }); - expect(await filesystemExists(fs)).toEqual(true); - await deleteFilesystem(fs); - expect(await filesystemExists(fs)).toEqual(false); - - setContext({ data: data2 }); - expect(await filesystemExists(fs)).toEqual(true); - await pull({ remote, data: data1 }); - expect(await filesystemExists(fs)).toEqual(true); - - await pull({ - remote, - data: data1, - deleteFilesystemCutoff: new Date(Date.now() - 1000 * 60 * 60), - }); - expect(await filesystemExists(fs)).toEqual(true); - - await pull({ - remote, - data: data1, - deleteFilesystemCutoff: new Date(), - }); - expect(await filesystemExists(fs)).toEqual(false); - }); - - const v = [ - { project_id: "00000000-0000-0000-0000-000000000001", affinity: "math" }, - { - account_id: "00000000-0000-0000-0000-000000000002", - name: "cocalc", - affinity: "math", - }, - { - group_id: "00000000-0000-0000-0000-000000000003", - namespace: "test", - name: "data", - affinity: "sage", - }, - ]; - it("creates 3 filesystems in 2 different namespaces, and confirms sync works", async () => { - setContext({ data: data1 }); - for (const fs of v) { - await createFilesystem(fs); - } - // write files to fs2 and fs3, so data will get sync'd too - await writeFile(join(filesystemMountpoint(v[1]), "a.txt"), "hello"); - await writeFile(join(filesystemMountpoint(v[2]), "b.txt"), "cocalc"); - // snapshot - await createSnapshot({ ...v[1], force: true }); - await createSnapshot({ ...v[2], force: true }); - const p = v.map((x) => get(x)); - - // do the sync - setContext({ data: data2 }); - await pull({ remote, data: data1 }); - - // verify that we have everything - for (const fs of v) { - expect(await filesystemExists(fs)).toEqual(true); - } - const p2 = v.map((x) => get(x)); - for (let i = 0; i < p.length; i++) { - // everything matches (even snapshots, since no trimming happened) - for (const field of SYNCED_FIELDS) { - expect({ i, field, value: p[i][field] }).toEqual({ - i, - field, - value: p2[i][field], - }); - } - } - }); - - it("edits some files on one of the above filesystems, snapshots, sync's, goes back and deletes a snapshot, edits more files, sync's, and notices that snapshots on sync target properly match snapshots on source.", async () => { - // edits some files on one of the above filesystems, snapshots: - setContext({ data: data1 }); - await writeFile(join(filesystemMountpoint(v[1]), "a2.txt"), "hello2"); - await createSnapshot({ ...v[1], force: true }); - - // sync's - setContext({ data: data2 }); - await pull({ remote, data: data1 }); - - // delete snapshot - setContext({ data: data1 }); - const fs1 = get(v[1]); - await deleteSnapshot({ ...v[1], snapshot: fs1.snapshots[0] }); - - // do more edits and make another snapshot - await writeFile(join(filesystemMountpoint(v[1]), "a3.txt"), "hello3"); - await createSnapshot({ ...v[1], force: true }); - const snapshots1 = get(v[1]).snapshots; - - // sync - setContext({ data: data2 }); - await pull({ remote, data: data1 }); - - // snapshots do NOT initially match, since we didn't enable snapshot deleting! - let snapshots2 = get(v[1]).snapshots; - expect(snapshots1).not.toEqual(snapshots2); - - await pull({ remote, data: data1, deleteSnapshots: true }); - // now snapshots should match exactly! - snapshots2 = get(v[1]).snapshots; - expect(snapshots1).toEqual(snapshots2); - }); - - it("test directly pulling one filesystem, rather than doing a full sync", async () => { - setContext({ data: data1 }); - await writeFile(join(filesystemMountpoint(v[1]), "a3.txt"), "hello2"); - await createSnapshot({ ...v[1], force: true }); - await writeFile(join(filesystemMountpoint(v[2]), "a4.txt"), "hello"); - await createSnapshot({ ...v[2], force: true }); - const p = v.map((x) => get(x)); - - setContext({ data: data2 }); - await pull({ remote, data: data1, filesystem: v[1] }); - const p2 = v.map((x) => get(x)); - - // now filesystem 1 should match, but not filesystem 2 - expect(p[1].snapshots).toEqual(p2[1].snapshots); - expect(p[2].snapshots).not.toEqual(p2[2].snapshots); - - // finally a full sync will get filesystem 2 - await pull({ remote, data: data1 }); - const p2b = v.map((x) => get(x)); - expect(p[2].snapshots).toEqual(p2b[2].snapshots); - }); -}); diff --git a/src/packages/file-server/zfs/test/util.ts b/src/packages/file-server/zfs/test/util.ts deleted file mode 100644 index 34bacecad4..0000000000 --- a/src/packages/file-server/zfs/test/util.ts +++ /dev/null @@ -1,63 +0,0 @@ -// application/typescript text -import { context, setContext } from "@cocalc/file-server/zfs/config"; -import { mkdtemp } from "fs/promises"; -import { tmpdir } from "os"; -import { join } from "path"; -import { executeCode } from "@cocalc/backend/execute-code"; -import { initDataDir } from "@cocalc/file-server/zfs/util"; -import { resetDb } from "@cocalc/file-server/zfs/db"; - -// export "describe" from here that is a no-op unless TEST_ZFS is set - -const Describe = process.env.TEST_ZFS ? describe : describe.skip; -const describe0 = describe; -export { Describe as describe, describe0 }; - -export async function init() { - if (!context.PREFIX.includes("test")) { - throw Error("context.PREFIX must contain 'test'"); - } - await initDataDir(); - resetDb(); -} - -export async function createTestPools({ - size = "10G", - count = 1, - data, -}: { - size?: string; - count?: number; - data?: string; -}): Promise<{ tempDir: string; data?: string }> { - console.log("TODO:", { size, count }); - setContext({ data }); - if (!context.DATA.includes("test")) { - throw Error(`context.DATA=${context.DATA} must contain 'test'`); - } - // Create temp directory - const tempDir = await mkdtemp(join(tmpdir(), "test-")); - return { tempDir, data }; -} - -// Even after setting sharefnfs=off, it can be a while (a minute?) until NFS -// fully frees up the share so we can destroy the pool. This makes it instant, -// which is very useful for unit testing. -export async function restartNfsServer() { - await executeCode({ - command: "sudo", - args: ["service", "nfs-kernel-server", "restart"], - }); -} - -export async function deleteTestPools(x?: { tempDir: string; data?: string }) { - if (!x) { - return; - } - const { data } = x; - setContext({ data }); - if (!context.DATA.includes("test")) { - throw Error("context.DATA must contain 'test'"); - } - throw Error("not implemented"); -} diff --git a/src/packages/file-server/zfs/types.ts b/src/packages/file-server/zfs/types.ts deleted file mode 100644 index d4c5e44526..0000000000 --- a/src/packages/file-server/zfs/types.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { context } from "./config"; -import { isValidUUID } from "@cocalc/util/misc"; - -export const OWNER_TYPES = ["account", "project", "group"] as const; - -export const OWNER_ID_FIELDS = OWNER_TYPES.map((x) => x + "_id"); - -export type OwnerType = (typeof OWNER_TYPES)[number]; - -export interface FilesystemPrimaryKey { - // The primary key (namespace, owner_type, owner_id, name): - - namespace: string; - // we support two types of filesystems: - // - 'project': owned by one project and can be used in various context associated with a single project; - // useful to all collaborators on a project. - // - 'account': owned by a user (a person) and be used in various ways on all projects they collaborator on. - // Other than the above distinction, the filesystems are treated identically by the server. - owner_type: OwnerType; - // owner_id is either a project_id or an account_id or an group_id. - owner_id: string; - // The name of the filesystem. - name: string; -} - -// This isn't exactly a FilesystemPrimaryKey, but it is convenient to -// work with and it uniquely *defines* one (or throws an error), after -// being fed through the primaryKey function below. -export interface PrimaryKey { - namespace?: string; - owner_type?: OwnerType; - owner_id?: string; - name?: string; - account_id?: string; - project_id?: string; - group_id?: string; -} - -const zfsSegmentRegex = /^[a-zA-Z0-9 _\-.:]+$/; - -export function primaryKey({ - namespace = context.namespace, - owner_type, - owner_id, - name, - account_id, - project_id, - group_id, -}: PrimaryKey): FilesystemPrimaryKey { - if ( - (account_id ? 1 : 0) + - (project_id ? 1 : 0) + - (group_id ? 1 : 0) + - (owner_type ? 1 : 0) != - 1 - ) { - throw Error( - `exactly one of account_id, project_id, group_id, or owner_type must be specified: ${JSON.stringify({ account_id, project_id, group_id, owner_type })}`, - ); - } - if ( - (account_id ? 1 : 0) + - (project_id ? 1 : 0) + - (group_id ? 1 : 0) + - (owner_id ? 1 : 0) != - 1 - ) { - throw Error( - `exactly one of account_id, project_id, group_id, or owner_type must be specified: ${JSON.stringify({ account_id, project_id, group_id, owner_id })}`, - ); - } - if (account_id) { - owner_type = "account"; - owner_id = account_id; - } else if (project_id) { - owner_type = "project"; - owner_id = project_id; - } else if (group_id) { - owner_type = "group"; - owner_id = group_id; - } - if (!owner_type || !OWNER_TYPES.includes(owner_type)) { - throw Error( - `unknown owner type '${owner_type}' -- must be one of ${JSON.stringify(OWNER_TYPES)}`, - ); - } - if (!name) { - if (owner_type == "project" && name == null) { - // the home directory of a project. - name = "home"; - } else { - throw Error("name must be nonempty"); - } - } - if (name.length >= 64) { - // this is only so mounting is reasonable on the filesystem... and could be lifted - throw Error("name must be at most 63 characters"); - } - if (!zfsSegmentRegex.test(name)) { - throw Error( - 'name must only contain alphanumeric characters, space, *, "-", "_", "." and ":"', - ); - } - - if (!isValidUUID(owner_id) || !owner_id) { - throw Error("owner_id must be a valid uuid"); - } - - return { namespace, owner_type, owner_id, name }; -} - -export interface Filesystem extends FilesystemPrimaryKey { - // Properties of the filesystem and its current state: - - // the pool is where the filesystem happened to get allocated. This can be influenced by affinity or usage. - pool: string; - // true if project is currently archived - archived: boolean; - // array of hosts (or range using CIDR notation) that we're - // granting NFS client access to. - nfs: string[]; - // list of snapshots as ISO timestamps from oldest to newest - snapshots: string[]; - // name of the most recent snapshot that was used for sending a stream - // (for incremental backups). This specified snapshot will never be - // deleted by the snapshot trimming process, until a newer send snapshot is made. - last_send_snapshot?: string; - // name of most recent bup backup - last_bup_backup?: string; - // Last_edited = last time this project was "edited" -- various - // operations cause this to get updated. - last_edited?: Date; - // optional arbitrary affinity string - we attempt if possible to put - // projects with the same affinity in the same pool, to improve chances of dedup. - affinity?: string; - // if this is set, then some sort of error that "should" never happen, - // has happened, and manual intervention is needed. - error?: string; - // when the last error actually happened - last_error?: Date; - - // Bytes used by the main project filesystem dataset, NOT counting snapshots (zfs "USED"). - // Obviously these used_by fields are NOT always up to date. They get updated on some - // standard operations, including making snapshots, so can be pretty useful for monitoring - // for issues, etc. - used_by_dataset?: number; - // Total amount of space used by snapshots (not the filesystem) - used_by_snapshots?: number; - - // Quota for dataset usage (in bytes), so used_by_dataset <= dataset_quota. This is the refquota property in ZFS. - quota?: number; -} - -// Used for set(...), main thing being each field can be FilesystemFieldFunction, -// which makes it very easy to *safely* mutate data (assuming only one process -// is using sqlite). -type FilesystemFieldFunction = (project: Filesystem) => any; -export interface SetFilesystem extends PrimaryKey { - pool?: string | FilesystemFieldFunction; - archived?: boolean | FilesystemFieldFunction; - nfs?: string[] | FilesystemFieldFunction; - snapshots?: string[] | FilesystemFieldFunction; - last_send_snapshot?: string | FilesystemFieldFunction; - last_bup_backup?: string | FilesystemFieldFunction; - last_edited?: Date | FilesystemFieldFunction; - affinity?: null | string | FilesystemFieldFunction; - error?: null | string | FilesystemFieldFunction; - last_error?: Date | FilesystemFieldFunction; - used_by_dataset?: null | number; - used_by_snapshots?: null | number; - quota?: null | number; -} - -// what is *actually* stored in sqlite -export interface RawFilesystem { - owner_type: OwnerType; - owner_id: string; - namespace: string; - pool: string; - // 0 or 1 - archived?: number; - // nfs and snasphots are v.join(',') - nfs?: string; - snapshots?: string; - last_send_snapshot?: string; - last_bup_backup?: string; - // new Date().ISOString() - last_edited?: string; - affinity?: string; - error?: string; - last_error?: string; - used_by_dataset?: number; - used_by_snapshots?: number; - quota?: number; -} diff --git a/src/packages/file-server/zfs/util.ts b/src/packages/file-server/zfs/util.ts deleted file mode 100644 index 6f60893fd6..0000000000 --- a/src/packages/file-server/zfs/util.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { executeCode } from "@cocalc/backend/execute-code"; -import { context, DEFAULT_EXEC_TIMEOUT_MS } from "./config"; -import { fatalError } from "./db"; - -export async function exec(opts) { - try { - return await executeCode({ - ...opts, - timeout: DEFAULT_EXEC_TIMEOUT_MS / 1000, - }); - } catch (err) { - if (opts.what) { - fatalError({ - ...opts.what, - err, - desc: `${opts.desc ? opts.desc : ""} "${opts.command} ${opts.args?.join(" ") ?? ""}"`, - }); - } - throw err; - } -} - -export async function initDataDir() { - await executeCode({ command: "sudo", args: ["mkdir", "-p", context.DATA] }); - await executeCode({ - command: "sudo", - args: ["chmod", "a+rxw", context.DATA], - }); -} diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 04316cec4f..2f53a5e655 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -289,37 +289,19 @@ importers: '@cocalc/backend': specifier: workspace:* version: link:../backend - '@cocalc/conat': - specifier: workspace:* - version: link:../conat '@cocalc/file-server': specifier: workspace:* version: 'link:' '@cocalc/util': specifier: workspace:* version: link:../util + devDependencies: '@types/jest': specifier: ^29.5.14 version: 29.5.14 '@types/node': specifier: ^18.16.14 version: 18.19.118 - awaiting: - specifier: ^3.0.0 - version: 3.0.0 - better-sqlite3: - specifier: ^11.10.0 - version: 11.10.0 - lodash: - specifier: ^4.17.21 - version: 4.17.21 - devDependencies: - '@types/better-sqlite3': - specifier: ^7.6.13 - version: 7.6.13 - '@types/lodash': - specifier: ^4.14.202 - version: 4.17.20 frontend: dependencies: From d087c8e5bc053cd42226b0b2ea9d50c6da8cc368 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 01:20:59 +0000 Subject: [PATCH 28/47] add more btrfs tests --- src/packages/file-server/btrfs/subvolume.ts | 81 +++++++++++----- .../btrfs/test/filesystem-stress.test.ts | 97 +++++++++++++++++++ src/packages/file-server/btrfs/test/setup.ts | 12 ++- .../file-server/btrfs/test/subvolume.test.ts | 45 +++++++++ src/packages/file-server/btrfs/util.ts | 71 ++++++++++++++ src/packages/file-server/package.json | 3 +- src/packages/pnpm-lock.yaml | 3 + 7 files changed, 284 insertions(+), 28 deletions(-) create mode 100644 src/packages/file-server/btrfs/test/filesystem-stress.test.ts create mode 100644 src/packages/file-server/btrfs/test/subvolume.test.ts create mode 100644 src/packages/file-server/btrfs/util.ts diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 768fa87bee..ab14014642 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -4,25 +4,17 @@ A subvolume import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; -import { - exists, - listdir, - mkdirp, - sudo, -} from "./util"; +import { exists, listdir, mkdirp, sudo } from "./util"; import { join } from "path"; import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; -import { human_readable_size } from "@cocalc/util/misc"; +//import { human_readable_size } from "@cocalc/util/misc"; +import getLogger from "@cocalc/backend/logger"; export const SNAPSHOTS = ".snapshots"; const SEND_SNAPSHOT_PREFIX = "send-"; - const BUP_SNAPSHOT = "temp-bup-snapshot"; - const PAD = 4; -import getLogger from "@cocalc/backend/logger"; - const logger = getLogger("file-server:storage-btrfs:subvolume"); interface Options { @@ -88,6 +80,20 @@ export class Subvolume { return x["qgroup-show"][0]; }; + quota = async (): Promise<{ + size: number; + used: number; + }> => { + let { max_referenced: size, referenced: used } = await this.quotaInfo(); + if (size == "none") { + size = null; + } + return { + used, + size, + }; + }; + size = async (size: string | number) => { if (!size) { throw Error("size must be specified"); @@ -98,23 +104,50 @@ export class Subvolume { }); }; + du = async () => { + return await sudo({ + command: "btrfs", + args: ["filesystem", "du", "-s", this.path], + }); + }; + usage = async (): Promise<{ + // used and free in bytes + used: number; + free: number; size: number; - usage: number; - human: { size: string; usage: string }; }> => { - let { max_referenced: size, referenced: usage } = await this.quotaInfo(); - if (size == "none") { - size = null; + const { stdout } = await sudo({ + command: "btrfs", + args: ["filesystem", "usage", "-b", this.path], + }); + let used: number = -1; + let free: number = -1; + let size: number = -1; + for (const x of stdout.split("\n")) { + if (used == -1) { + const i = x.indexOf("Used:"); + if (i != -1) { + used = parseInt(x.split(":")[1].trim()); + continue; + } + } + if (free == -1) { + const i = x.indexOf("Free (statfs, df):"); + if (i != -1) { + free = parseInt(x.split(":")[1].trim()); + continue; + } + } + if (size == -1) { + const i = x.indexOf("Device size:"); + if (i != -1) { + size = parseInt(x.split(":")[1].trim()); + continue; + } + } } - return { - usage, - size, - human: { - usage: human_readable_size(usage), - size: size != null ? human_readable_size(size) : size, - }, - }; + return { used, free, size }; }; private makeSnapshotsDir = async () => { diff --git a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts new file mode 100644 index 0000000000..76642be115 --- /dev/null +++ b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts @@ -0,0 +1,97 @@ +import { before, after, fs } from "./setup"; +import { writeFile } from "fs/promises"; +import { join } from "path"; + +beforeAll(before); + +//const log = console.log; +const log = (..._args) => {}; + +describe("stress operations with subvolumes", () => { + const count1 = 10; + it(`create ${count1} subvolumes in serial`, async () => { + const t = Date.now(); + for (let i = 0; i < count1; i++) { + await fs.subvolume(`${i}`); + } + log( + `created ${Math.round((count1 / (Date.now() - t)) * 1000)} subvolumes per second serial`, + ); + }); + + it("list them and confirm", async () => { + const v = await fs.list(); + expect(v.length).toBe(count1); + }); + + let count2 = 10; + it(`create ${count2} subvolumes in parallel`, async () => { + const v: any[] = []; + const t = Date.now(); + for (let i = 0; i < count2; i++) { + v.push(fs.subvolume(`p-${i}`)); + } + await Promise.all(v); + log( + `created ${Math.round((count2 / (Date.now() - t)) * 1000)} subvolumes per second in parallel`, + ); + }); + + it("list them and confirm", async () => { + const v = await fs.list(); + expect(v.length).toBe(count1 + count2); + }); + + it("write a file to each volume", async () => { + for (const name of await fs.list()) { + const vol = await fs.subvolume(name); + await writeFile(join(vol.path, "a.txt"), "hi"); + } + }); + + it("clone the first group in serial", async () => { + const t = Date.now(); + for (let i = 0; i < count1; i++) { + await fs.cloneSubvolume(`${i}`, `clone-of-${i}`); + } + log( + `cloned ${Math.round((count1 / (Date.now() - t)) * 1000)} subvolumes per second serial`, + ); + }); + + it("clone the second group in parallel", async () => { + const t = Date.now(); + const v: any[] = []; + for (let i = 0; i < count2; i++) { + v.push(fs.cloneSubvolume(`p-${i}`, `clone-of-p-${i}`)); + } + await Promise.all(v); + log( + `cloned ${Math.round((count2 / (Date.now() - t)) * 1000)} subvolumes per second parallel`, + ); + }); + + it("delete the first batch serial", async () => { + const t = Date.now(); + for (let i = 0; i < count1; i++) { + await fs.deleteSubvolume(`${i}`); + } + log( + `deleted ${Math.round((count1 / (Date.now() - t)) * 1000)} subvolumes per second serial`, + ); + }); + + it("delete the second batch in parallel", async () => { + const v: any[] = []; + const t = Date.now(); + for (let i = 0; i < count2; i++) { + v.push(fs.deleteSubvolume(`p-${i}`)); + } + await Promise.all(v); + log( + `deleted ${Math.round((count2 / (Date.now() - t)) * 1000)} subvolumes per second in parallel`, + ); + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index 9487833385..180c327e5c 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -6,6 +6,7 @@ import process from "node:process"; import { chmod, mkdtemp, mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; +import { until } from "@cocalc/util/async-utils"; export let fs: Filesystem; let tempDir; @@ -26,8 +27,13 @@ export async function before() { } export async function after() { - try { - await fs.unmount(); - } catch {} + await until(async () => { + try { + await fs.unmount(); + return true; + } catch { + return false; + } + }); await rm(tempDir, { force: true, recursive: true }); } diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts new file mode 100644 index 0000000000..24699b5e56 --- /dev/null +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -0,0 +1,45 @@ +import { before, after, fs } from "./setup"; +import { writeFile } from "fs/promises"; +import { join } from "path"; +import { delay } from "awaiting"; +import { wait } from "@cocalc/backend/conat/test/util"; +import { randomBytes } from "crypto"; + +beforeAll(before); + +jest.setTimeout(20000); +describe("setting and getting quota of a subvolume", () => { + let vol; + it("set the quota of a subvolume to 5 M", async () => { + vol = await fs.subvolume("q"); + await vol.size("5M"); + + const { size, used } = await vol.quota(); + expect(size).toBe(5 * 1024 * 1024); + expect(used).toBe(0); + }); + + it("write a file and check usage goes up", async () => { + const buf = randomBytes(4 * 1024 * 1024); + await writeFile(join(vol.path, "buf"), buf); + await wait({ + until: async () => { + await delay(1000); + const { used } = await vol.usage(); + return used > 0; + }, + }); + const { used } = await vol.usage(); + expect(used).toBeGreaterThan(0); + }); + + it("fail to write a 50MB file (due to quota)", async () => { + const buf2 = randomBytes(50 * 1024 * 1024); + const b = join(vol.path, "buf2"); + expect(async () => { + await writeFile(b, buf2); + }).rejects.toThrow("write"); + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/btrfs/util.ts b/src/packages/file-server/btrfs/util.ts new file mode 100644 index 0000000000..bc986e968e --- /dev/null +++ b/src/packages/file-server/btrfs/util.ts @@ -0,0 +1,71 @@ +import { + type ExecuteCodeOptions, + type ExecuteCodeOutput, +} from "@cocalc/util/types/execute-code"; +import { executeCode } from "@cocalc/backend/execute-code"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("file-server:storage:util"); + +const DEFAULT_EXEC_TIMEOUT_MS = 60 * 1000; + +export async function exists(path: string) { + try { + await sudo({ command: "ls", args: [path], verbose: false }); + return true; + } catch { + return false; + } +} + +export async function mkdirp(paths: string[]) { + if (paths.length == 0) return; + await sudo({ command: "mkdir", args: ["-p", ...paths] }); +} + +export async function chmod(args: string[]) { + await sudo({ command: "chmod", args: args }); +} + +export async function sudo( + opts: ExecuteCodeOptions & { desc?: string }, +): Promise { + if (opts.verbose !== false && opts.desc) { + logger.debug("exec", opts.desc); + } + let command, args; + if (opts.bash) { + command = `sudo ${opts.command}`; + args = undefined; + } else { + command = "sudo"; + args = [opts.command, ...(opts.args ?? [])]; + } + return await executeCode({ + verbose: true, + timeout: DEFAULT_EXEC_TIMEOUT_MS / 1000, + ...opts, + command, + args, + }); +} + +export async function rm(paths: string[]) { + if (paths.length == 0) return; + await sudo({ command: "rm", args: paths }); +} + +export async function rmdir(paths: string[]) { + if (paths.length == 0) return; + await sudo({ command: "rmdir", args: paths }); +} + +export async function listdir(path: string) { + const { stdout } = await sudo({ command: "ls", args: ["-1", path] }); + return stdout.split("\n").filter((x) => x); +} + +export async function isdir(path: string) { + const { stdout } = await sudo({ command: "stat", args: ["-c", "%F", path] }); + return stdout.trim() == "directory"; +} diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index d6480a39bf..691aed12f9 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -29,7 +29,8 @@ "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/file-server": "workspace:*", - "@cocalc/util": "workspace:*" + "@cocalc/util": "workspace:*", + "awaiting": "^3.0.0" }, "devDependencies": { "@types/jest": "^29.5.14", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 2f53a5e655..cf70f203ef 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -295,6 +295,9 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util + awaiting: + specifier: ^3.0.0 + version: 3.0.0 devDependencies: '@types/jest': specifier: ^29.5.14 From 1a3019b529256c92fb6b368cafd4a63f0e0ef223 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 04:30:32 +0000 Subject: [PATCH 29/47] more btrfs unit tests --- src/packages/file-server/btrfs/subvolume.ts | 6 +- src/packages/file-server/btrfs/test/setup.ts | 2 + .../file-server/btrfs/test/subvolume.test.ts | 57 +++++++++++++++++-- 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index ab14014642..280155a109 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -219,7 +219,11 @@ export class Subvolume { const s = await this.snapshots(); if (s.length == 0) { // more than just the SNAPSHOTS directory? - return (await listdir(this.path)).length > 1; + const v = await listdir(this.path); + if (v.length == 0 || (v.length == 1 && v[0] == this.snapshotsDir)) { + return false; + } + return true; } const pathGen = await getGeneration(this.path); const snapGen = await getGeneration( diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index 180c327e5c..7c55c359eb 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -7,6 +7,8 @@ import { chmod, mkdtemp, mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; import { until } from "@cocalc/util/async-utils"; +export { sudo } from "../util"; +export { delay } from "awaiting"; export let fs: Filesystem; let tempDir; diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 24699b5e56..4c9bc41981 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -1,7 +1,6 @@ -import { before, after, fs } from "./setup"; -import { writeFile } from "fs/promises"; +import { before, after, fs, sudo } from "./setup"; +import { readFile, writeFile, unlink } from "fs/promises"; import { join } from "path"; -import { delay } from "awaiting"; import { wait } from "@cocalc/backend/conat/test/util"; import { randomBytes } from "crypto"; @@ -24,7 +23,7 @@ describe("setting and getting quota of a subvolume", () => { await writeFile(join(vol.path, "buf"), buf); await wait({ until: async () => { - await delay(1000); + await sudo({ command: "sync" }); const { used } = await vol.usage(); return used > 0; }, @@ -42,4 +41,54 @@ describe("setting and getting quota of a subvolume", () => { }); }); +describe("test snapshots", () => { + let vol; + it("creates a volume and write a file to it", async () => { + vol = await fs.subvolume("snapper"); + expect(await vol.hasUnsavedChanges()).toBe(false); + await writeFile(join(vol.path, "a.txt"), "hello"); + expect(await vol.hasUnsavedChanges()).toBe(true); + }); + + it("snapshot the volume", async () => { + expect(await vol.snapshots()).toEqual([]); + await vol.createSnapshot("snap1"); + expect(await vol.snapshots()).toEqual(["snap1"]); + expect(await vol.hasUnsavedChanges()).toBe(false); + }); + + it("create a file see that we know there are unsaved changes", async () => { + await writeFile(join(vol.path, "b.txt"), "world"); + await sudo({ command: "sync" }); + expect(await vol.hasUnsavedChanges()).toBe(true); + }); + + it("delete our file, but then read it in a snapshot", async () => { + await unlink(join(vol.path, "a.txt")); + const b = await readFile(join(vol.snapshotsDir, "snap1", "a.txt"), "utf8"); + expect(b).toEqual("hello"); + }); + + it("verifies snapshot exists", async () => { + expect(await vol.snapshotExists("snap1")).toBe(true); + expect(await vol.snapshotExists("snap2")).toBe(false); + }); + + it("lock our snapshot and confirm it prevents deletion", async () => { + await vol.lockSnapshot("snap1"); + expect(async () => { + await vol.deleteSnapshot("snap1"); + }).rejects.toThrow("locked"); + }); + + it("unlock our snapshot and delete it", async () => { + await vol.unlockSnapshot("snap1"); + await vol.deleteSnapshot("snap1"); + expect(await vol.snapshotExists("snap1")).toBe(false); + expect(await vol.snapshots()).toEqual([]); + }); + + +}); + afterAll(after); From 1dc6687bca3c7d2374c2afdd6b296e9ba803aa02 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 05:39:17 +0000 Subject: [PATCH 30/47] add snapshot stress test --- .../btrfs/test/subvolume-stress.test.ts | 41 +++++++++++++++++++ .../file-server/btrfs/test/subvolume.test.ts | 7 +++- 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 src/packages/file-server/btrfs/test/subvolume-stress.test.ts diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts new file mode 100644 index 0000000000..a96913583a --- /dev/null +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -0,0 +1,41 @@ +import { before, after, fs, sudo } from "./setup"; +import { readFile, writeFile, unlink } from "fs/promises"; +import { join } from "path"; +import { wait } from "@cocalc/backend/conat/test/util"; +import { randomBytes } from "crypto"; + +beforeAll(before); +//const log = console.log; +const log = (..._args) => {}; + +describe("stress test creating many snapshots", () => { + let vol; + it("creates a volume and write a file to it", async () => { + vol = await fs.subvolume("stress"); + }); + + const count = 25; + it(`create file and snapshot the volume ${count} times`, async () => { + const snaps: string[] = []; + const start = Date.now(); + for (let i = 0; i < count; i++) { + await writeFile(join(vol.path, `${i}.txt`), "world"); + await vol.createSnapshot(`snap${i}`); + snaps.push(`snap${i}`); + } + log( + `created ${Math.round((count / (Date.now() - start)) * 1000)} snapshots per second in serial`, + ); + snaps.sort(); + expect(await vol.snapshots()).toEqual(snaps); + }); + + it(`delete our ${count} snapshots`, async () => { + for (let i = 0; i < count; i++) { + await vol.deleteSnapshot(`snap${i}`); + } + expect(await vol.snapshots()).toEqual([]); + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 4c9bc41981..684182c9da 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -87,8 +87,11 @@ describe("test snapshots", () => { expect(await vol.snapshotExists("snap1")).toBe(false); expect(await vol.snapshots()).toEqual([]); }); - - }); +// describe("test bup backups", ()=>{ +// let vol; +// it('creates a volume') +// }) + afterAll(after); From 1afa4ce04edd3c28f88592ec458194a92b57a018 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 05:40:00 +0000 Subject: [PATCH 31/47] ts --- .../file-server/btrfs/test/subvolume-stress.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts index a96913583a..45ad9b58fd 100644 --- a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -1,8 +1,6 @@ -import { before, after, fs, sudo } from "./setup"; -import { readFile, writeFile, unlink } from "fs/promises"; +import { before, after, fs } from "./setup"; +import { writeFile } from "fs/promises"; import { join } from "path"; -import { wait } from "@cocalc/backend/conat/test/util"; -import { randomBytes } from "crypto"; beforeAll(before); //const log = console.log; From d663508d72690f32f1b9112274d4f842b8b31e0a Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 06:28:30 +0000 Subject: [PATCH 32/47] btrfs: unit testing bup integration --- src/packages/file-server/btrfs/subvolume.ts | 72 ++++++++++++++++++- .../file-server/btrfs/test/subvolume.test.ts | 53 ++++++++++++-- src/packages/file-server/btrfs/util.ts | 16 +++++ 3 files changed, 135 insertions(+), 6 deletions(-) diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 280155a109..26b1d4cb8e 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -5,7 +5,7 @@ A subvolume import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; import { exists, listdir, mkdirp, sudo } from "./util"; -import { join } from "path"; +import { join, normalize } from "path"; import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; //import { human_readable_size } from "@cocalc/util/misc"; import getLogger from "@cocalc/backend/logger"; @@ -292,6 +292,76 @@ export class Subvolume { .filter((x) => x); }; + bupRestore = async (path: string) => { + path = normalize(path); + // outdir -- path relative to subvolume + // path -- /branch/revision/path/to/dir + await sudo({ + command: "bup", + args: [ + "-d", + this.filesystem.bup, + "restore", + "-C", + this.path, //join(this.path, outdir), + join(`/${this.name}`, path), + "--quiet", + ], + }); + }; + + bupLs = async ( + path: string, + ): Promise< + { + path: string; + size: number; + timestamp: number; + isdir: boolean; + }[] + > => { + path = normalize(path); + const { stdout } = await sudo({ + command: "bup", + args: [ + "-d", + this.filesystem.bup, + "ls", + "--almost-all", + "--file-type", + "-l", + join(`/${this.name}`, path), + ], + }); + const v: { + path: string; + size: number; + timestamp: number; + isdir: boolean; + }[] = []; + for (const x of stdout.split("\n")) { + // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"] + const w = x.split(/\s+/); + if (w.length >= 6) { + let isdir, path; + if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) { + w[5] = w[5].slice(0, -1); + } + if (w[5].endsWith("/")) { + isdir = true; + path = w[5].slice(0, -1); + } else { + path = w[5]; + isdir = false; + } + const size = parseInt(w[2]); + const timestamp = new Date(w[3] + "T" + w[4]).valueOf(); + v.push({ path, size, timestamp, isdir }); + } + } + return v; + }; + bupPrune = async ({ dailies = "1w", monthlies = "4m", diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 684182c9da..68f2015870 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -1,8 +1,9 @@ import { before, after, fs, sudo } from "./setup"; -import { readFile, writeFile, unlink } from "fs/promises"; +import { mkdir, readFile, writeFile, unlink } from "fs/promises"; import { join } from "path"; import { wait } from "@cocalc/backend/conat/test/util"; import { randomBytes } from "crypto"; +import { parseBupTime } from "../util"; beforeAll(before); @@ -89,9 +90,51 @@ describe("test snapshots", () => { }); }); -// describe("test bup backups", ()=>{ -// let vol; -// it('creates a volume') -// }) +describe("test bup backups", () => { + let vol; + it("creates a volume", async () => { + vol = await fs.subvolume("bup-test"); + await writeFile(join(vol.path, "a.txt"), "hello"); + }); + + it("create a bup backup", async () => { + await vol.createBupBackup(); + }); + + it("list bup backups of this vol -- there are 2, one for the date and 'latest'", async () => { + const v = await vol.bupBackups(); + expect(v.length).toBe(2); + const t = parseBupTime(v[0]); + expect(Math.abs(t.valueOf() - Date.now())).toBeLessThan(10_000); + }); + + it("confirm a.txt is in our backup", async () => { + const x = await vol.bupLs("latest"); + expect(x).toEqual([ + { path: "a.txt", size: 5, timestamp: x[0].timestamp, isdir: false }, + ]); + }); + + it("restore a.txt from our backup", async () => { + await writeFile(join(vol.path, "a.txt"), "hello2"); + await vol.bupRestore("latest/a.txt"); + expect(await readFile(join(vol.path, "a.txt"), "utf8")).toEqual("hello"); + }); + + it("prune bup backups does nothing since we have so few", async () => { + await vol.bupPrune(); + expect((await vol.bupBackups()).length).toBe(2); + }); + + it("add a directory and back up", async () => { + await mkdir(join(vol.path, "mydir")); + await vol.createBupBackup(); + const x = await vol.bupLs("latest"); + expect(x).toEqual([ + { path: "a.txt", size: 5, timestamp: x[0].timestamp, isdir: false }, + { path: "mydir", size: 0, timestamp: x[1].timestamp, isdir: true }, + ]); + }); +}); afterAll(after); diff --git a/src/packages/file-server/btrfs/util.ts b/src/packages/file-server/btrfs/util.ts index bc986e968e..c8dc221411 100644 --- a/src/packages/file-server/btrfs/util.ts +++ b/src/packages/file-server/btrfs/util.ts @@ -69,3 +69,19 @@ export async function isdir(path: string) { const { stdout } = await sudo({ command: "stat", args: ["-c", "%F", path] }); return stdout.trim() == "directory"; } + +export function parseBupTime(s: string): Date { + const [year, month, day, time] = s.split("-"); + const hours = time.slice(0, 2); + const minutes = time.slice(2, 4); + const seconds = time.slice(4, 6); + + return new Date( + Number(year), + Number(month) - 1, // JS months are 0-based + Number(day), + Number(hours), + Number(minutes), + Number(seconds), + ); +} From 91da892dcfdca1ca4de083b19d6d97cb814f1a77 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 15:17:52 +0000 Subject: [PATCH 33/47] btrfs tests: create more loopback devices automatically so we can run btrfs unit tests in parallel --- src/packages/file-server/btrfs/test/setup.ts | 19 ++++++++++++++++++- .../file-server/btrfs/test/subvolume.test.ts | 2 +- src/packages/file-server/package.json | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index 08afd9028c..e09999b3d8 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -3,7 +3,7 @@ import { type Filesystem, } from "@cocalc/file-server/btrfs/filesystem"; import process from "node:process"; -import { chmod, mkdtemp, mkdir, rm } from "node:fs/promises"; +import { chmod, mkdtemp, mkdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; import { until } from "@cocalc/util/async-utils"; @@ -16,12 +16,29 @@ let tempDir; const TEMP_PREFIX = "cocalc-test-btrfs-"; +async function ensureMoreLoops() { + // to run tests, this is helpful + //for i in $(seq 8 63); do sudo mknod -m660 /dev/loop$i b 7 $i; sudo chown root:disk /dev/loop$i; done + for (let i = 0; i < 64; i++) { + try { + await stat(`/dev/loop${i}`); + continue; + } catch {} + await sudo({ + command: "mknod", + args: ["-m660", `/dev/loop${i}`, "b", "7", `${i}`], + }); + await sudo({ command: "chown", args: ["root:disk", `/dev/loop${i}`] }); + } +} + export async function before() { try { const command = `umount ${join(tmpdir(), TEMP_PREFIX)}*/mnt`; // attempt to unmount any mounts left from previous runs await sudo({ command, bash: true }); } catch {} + await ensureMoreLoops(); tempDir = await mkdtemp(join(tmpdir(), TEMP_PREFIX)); // Set world read/write/execute await chmod(tempDir, 0o777); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index f6e1f4bd8f..6cf2e8828b 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -152,4 +152,4 @@ describe("test bup backups", () => { }); }); -//afterAll(after); +afterAll(after); diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index 691aed12f9..58285c4bf6 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -10,7 +10,7 @@ "preinstall": "npx only-allow pnpm", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest --runInBand", + "test": "pnpm exec jest", "depcheck": "pnpx depcheck", "clean": "rm -rf node_modules dist" }, From caac6dd903793bbf61a9cbd8250bde5ec1e0fc23 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 16:59:30 +0000 Subject: [PATCH 34/47] btrfs: adding more testing and fs operations support --- src/packages/backend/get-listing.ts | 2 +- src/packages/file-server/btrfs/subvolume.ts | 135 ++++++++++++++---- .../btrfs/test/subvolume-stress.test.ts | 65 +++++++-- .../file-server/btrfs/test/subvolume.test.ts | 48 ++++--- 4 files changed, 193 insertions(+), 57 deletions(-) diff --git a/src/packages/backend/get-listing.ts b/src/packages/backend/get-listing.ts index 0854ea82b1..02bcdbc835 100644 --- a/src/packages/backend/get-listing.ts +++ b/src/packages/backend/get-listing.ts @@ -4,7 +4,7 @@ */ /* -Server directory listing through the HTTP server and Websocket API. +This is used by backends to serve directory listings to clients: {files:[..., {size:?,name:?,mtime:?,isdir:?}]} diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 786acab4ad..b882a287ac 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -4,10 +4,12 @@ A subvolume import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; -import { exists, listdir, mkdirp, sudo } from "./util"; +import { readFile, writeFile, unlink } from "node:fs/promises"; +import { exists, isdir, listdir, mkdirp, sudo } from "./util"; import { join, normalize } from "path"; import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; -//import { human_readable_size } from "@cocalc/util/misc"; +import { DirectoryListingEntry } from "@cocalc/util/types"; +import getListing from "@cocalc/backend/get-listing"; import getLogger from "@cocalc/backend/logger"; export const SNAPSHOTS = ".snapshots"; @@ -23,10 +25,11 @@ interface Options { } export class Subvolume { - private filesystem: Filesystem; public readonly name: string; - public readonly path: string; - public readonly snapshotsDir: string; + + private filesystem: Filesystem; + private readonly path: string; + private readonly snapshotsDir: string; constructor({ filesystem, name }: Options) { this.filesystem = filesystem; @@ -70,6 +73,72 @@ export class Subvolume { }); }; + // this should provide a path that is guaranteed to be + // inside this.path on the filesystem or throw error + // [ ] TODO: not sure if the code here is sufficient!! + private normalize = (path: string) => { + return join(this.path, normalize(path)); + }; + + ///////////// + // Files + ///////////// + ls = async ( + path: string, + { hidden, limit }: { hidden?: boolean; limit?: number } = {}, + ): Promise => { + path = normalize(path); + return await getListing(this.normalize(path), hidden, { + limit, + home: "/", + }); + }; + + readFile = async (path: string, encoding?: any): Promise => { + path = normalize(path); + return await readFile(this.normalize(path), encoding); + }; + + writeFile = async (path: string, data: string | Buffer) => { + path = normalize(path); + return await writeFile(this.normalize(path), data); + }; + + unlink = async (path: string) => { + await unlink(this.normalize(path)); + }; + + rsync = async ({ + src, + target, + args = ["-axH"], + timeout = 5 * 60 * 1000, + }: { + src: string; + target: string; + args?: string[]; + timeout?: number; + }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { + let srcPath = this.normalize(src); + let targetPath = this.normalize(target); + if (!srcPath.endsWith("/") && (await isdir(srcPath))) { + srcPath += "/"; + if (!targetPath.endsWith("/")) { + targetPath += "/"; + } + } + return await sudo({ + command: "rsync", + args: [...args, srcPath, targetPath], + err_on_exit: false, + timeout: timeout / 1000, + }); + }; + + ///////////// + // QUOTA + ///////////// + private quotaInfo = async () => { const { stdout } = await sudo({ verbose: false, @@ -150,6 +219,13 @@ export class Subvolume { return { used, free, size }; }; + ///////////// + // SNAPSHOTS + ///////////// + snapshotPath = (snapshot: string, ...segments) => { + return join(SNAPSHOTS, snapshot, ...segments); + }; + private makeSnapshotsDir = async () => { if (await exists(this.snapshotsDir)) { return; @@ -233,6 +309,13 @@ export class Subvolume { return snapGen < pathGen; }; + ///////////// + // BACKUPS + // There is a single global dedup'd backup archive stored in the btrfs filesystem. + // Obviously, admins should rsync this regularly to a separate location as a genuine + // backup strategy. + ///////////// + // create a new bup backup createBupBackup = async ({ // timeout used for bup index and bup save commands @@ -304,7 +387,7 @@ export class Subvolume { const i = path.indexOf("/"); // remove the commit name await sudo({ command: "rm", - args: ["-rf", join(this.path, path.slice(i + 1))], + args: ["-rf", this.normalize(path.slice(i + 1))], }); await sudo({ command: "bup", @@ -320,16 +403,7 @@ export class Subvolume { }); }; - bupLs = async ( - path: string, - ): Promise< - { - path: string; - size: number; - timestamp: number; - isdir: boolean; - }[] - > => { + bupLs = async (path: string): Promise => { path = normalize(path); const { stdout } = await sudo({ command: "bup", @@ -343,30 +417,25 @@ export class Subvolume { join(`/${this.name}`, path), ], }); - const v: { - path: string; - size: number; - timestamp: number; - isdir: boolean; - }[] = []; + const v: DirectoryListingEntry[] = []; for (const x of stdout.split("\n")) { // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"] const w = x.split(/\s+/); if (w.length >= 6) { - let isdir, path; + let isdir, name; if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) { w[5] = w[5].slice(0, -1); } if (w[5].endsWith("/")) { isdir = true; - path = w[5].slice(0, -1); + name = w[5].slice(0, -1); } else { - path = w[5]; + name = w[5]; isdir = false; } const size = parseInt(w[2]); - const timestamp = new Date(w[3] + "T" + w[4]).valueOf(); - v.push({ path, size, timestamp, isdir }); + const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000; + v.push({ name, size, mtime, isdir }); } } return v; @@ -392,6 +461,18 @@ export class Subvolume { }); }; + ///////////// + // BTRFS send/recv + // Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since: + // - much easier to check they are valid + // - decoupled from any btrfs issues + // - not tied to any specific filesystem at all + // - easier to offsite via incremntal rsync + // - much more space efficient with *global* dedup and compression + // - bup is really just git, which is very proven + // The drawback is speed. + ///////////// + // this was just a quick proof of concept -- I don't like it. Should switch to using // timestamps and a lock. // To recover these, doing recv for each in order does work. Then you have to diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts index 45ad9b58fd..a957039a29 100644 --- a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -1,39 +1,84 @@ import { before, after, fs } from "./setup"; -import { writeFile } from "fs/promises"; +import { mkdir, writeFile } from "fs/promises"; import { join } from "path"; +const DEBUG = false; +const log = DEBUG ? console.log : (..._args) => {}; + +const numSnapshots = 25; +const numFiles = 1000; + beforeAll(before); -//const log = console.log; -const log = (..._args) => {}; -describe("stress test creating many snapshots", () => { +describe(`stress test creating ${numSnapshots} snapshots`, () => { let vol; it("creates a volume and write a file to it", async () => { vol = await fs.subvolume("stress"); }); - const count = 25; - it(`create file and snapshot the volume ${count} times`, async () => { + it(`create file and snapshot the volume ${numSnapshots} times`, async () => { const snaps: string[] = []; const start = Date.now(); - for (let i = 0; i < count; i++) { + for (let i = 0; i < numSnapshots; i++) { await writeFile(join(vol.path, `${i}.txt`), "world"); await vol.createSnapshot(`snap${i}`); snaps.push(`snap${i}`); } log( - `created ${Math.round((count / (Date.now() - start)) * 1000)} snapshots per second in serial`, + `created ${Math.round((numSnapshots / (Date.now() - start)) * 1000)} snapshots per second in serial`, ); snaps.sort(); expect(await vol.snapshots()).toEqual(snaps); }); - it(`delete our ${count} snapshots`, async () => { - for (let i = 0; i < count; i++) { + it(`delete our ${numSnapshots} snapshots`, async () => { + for (let i = 0; i < numSnapshots; i++) { await vol.deleteSnapshot(`snap${i}`); } expect(await vol.snapshots()).toEqual([]); }); }); +describe(`create ${numFiles} files`, () => { + let vol; + it("creates a volume", async () => { + vol = await fs.subvolume("many-files"); + }); + + it(`creates ${numFiles} files`, async () => { + const names: string[] = []; + const start = Date.now(); + for (let i = 0; i < numFiles; i++) { + await writeFile(join(vol.path, `${i}`), "world"); + names.push(`${i}`); + } + log( + `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in serial`, + ); + const v = await vol.ls(""); + const w = v.map(({ name }) => name); + expect(w.sort()).toEqual(names.sort()); + }); + + it(`creates ${numFiles} files in parallel`, async () => { + await mkdir(join(vol.path, "p")); + const names: string[] = []; + const start = Date.now(); + const z: any[] = []; + for (let i = 0; i < numFiles; i++) { + z.push(writeFile(join(vol.path, `p/${i}`), "world")); + names.push(`${i}`); + } + await Promise.all(z); + log( + `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in parallel`, + ); + const t0 = Date.now(); + const v = await vol.ls("p"); + log("get listing of files took", Date.now() - t0, "ms"); + const w = v.map(({ name }) => name); + expect(w.sort()).toEqual(names.sort()); + }); +}); + afterAll(after); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 6cf2e8828b..b637a44e90 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -1,5 +1,5 @@ import { before, after, fs, sudo } from "./setup"; -import { mkdir, readFile, writeFile, unlink } from "fs/promises"; +import { mkdir } from "fs/promises"; import { join } from "path"; import { wait } from "@cocalc/backend/conat/test/util"; import { randomBytes } from "crypto"; @@ -18,9 +18,14 @@ describe("setting and getting quota of a subvolume", () => { expect(used).toBe(0); }); + it("get directory listing", async () => { + const v = await vol.ls(""); + expect(v).toEqual([]); + }); + it("write a file and check usage goes up", async () => { const buf = randomBytes(4 * 1024 * 1024); - await writeFile(join(vol.path, "buf"), buf); + await vol.writeFile("buf", buf); await wait({ until: async () => { await sudo({ command: "sync" }); @@ -30,13 +35,16 @@ describe("setting and getting quota of a subvolume", () => { }); const { used } = await vol.usage(); expect(used).toBeGreaterThan(0); + + const v = await vol.ls(""); + // size is potentially random, reflecting compression + expect(v).toEqual([{ name: "buf", mtime: v[0].mtime, size: v[0].size }]); }); it("fail to write a 50MB file (due to quota)", async () => { const buf2 = randomBytes(50 * 1024 * 1024); - const b = join(vol.path, "buf2"); expect(async () => { - await writeFile(b, buf2); + await vol.writeFile("buf2", buf2); }).rejects.toThrow("write"); }); }); @@ -46,7 +54,7 @@ describe("test snapshots", () => { it("creates a volume and write a file to it", async () => { vol = await fs.subvolume("snapper"); expect(await vol.hasUnsavedChanges()).toBe(false); - await writeFile(join(vol.path, "a.txt"), "hello"); + await vol.writeFile("a.txt", "hello"); expect(await vol.hasUnsavedChanges()).toBe(true); }); @@ -58,14 +66,14 @@ describe("test snapshots", () => { }); it("create a file see that we know there are unsaved changes", async () => { - await writeFile(join(vol.path, "b.txt"), "world"); + await vol.writeFile("b.txt", "world"); await sudo({ command: "sync" }); expect(await vol.hasUnsavedChanges()).toBe(true); }); it("delete our file, but then read it in a snapshot", async () => { - await unlink(join(vol.path, "a.txt")); - const b = await readFile(join(vol.snapshotsDir, "snap1", "a.txt"), "utf8"); + await vol.unlink("a.txt"); + const b = await vol.readFile(vol.snapshotPath("snap1", "a.txt"), "utf8"); expect(b).toEqual("hello"); }); @@ -93,7 +101,7 @@ describe("test bup backups", () => { let vol; it("creates a volume", async () => { vol = await fs.subvolume("bup-test"); - await writeFile(join(vol.path, "a.txt"), "hello"); + await vol.writeFile("a.txt", "hello"); }); it("create a bup backup", async () => { @@ -110,14 +118,14 @@ describe("test bup backups", () => { it("confirm a.txt is in our backup", async () => { const x = await vol.bupLs("latest"); expect(x).toEqual([ - { path: "a.txt", size: 5, timestamp: x[0].timestamp, isdir: false }, + { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, ]); }); it("restore a.txt from our backup", async () => { - await writeFile(join(vol.path, "a.txt"), "hello2"); + await vol.writeFile("a.txt", "hello2"); await vol.bupRestore("latest/a.txt"); - expect(await readFile(join(vol.path, "a.txt"), "utf8")).toEqual("hello"); + expect(await vol.readFile("a.txt", "utf8")).toEqual("hello"); }); it("prune bup backups does nothing since we have so few", async () => { @@ -127,19 +135,21 @@ describe("test bup backups", () => { it("add a directory and back up", async () => { await mkdir(join(vol.path, "mydir")); - await writeFile(join(vol.path, "mydir", "file.txt"), "hello3"); + await vol.writeFile(join("mydir", "file.txt"), "hello3"); + expect((await vol.ls("mydir"))[0].name).toBe("file.txt"); await vol.createBupBackup(); const x = await vol.bupLs("latest"); expect(x).toEqual([ - { path: "a.txt", size: 5, timestamp: x[0].timestamp, isdir: false }, - { path: "mydir", size: 0, timestamp: x[1].timestamp, isdir: true }, + { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, + { name: "mydir", size: 0, mtime: x[1].mtime, isdir: true }, ]); + expect(Math.abs(x[0].mtime * 1000 - Date.now())).toBeLessThan(60_000); }); it("change file in the directory, then restore from backup whole dir", async () => { - await writeFile(join(vol.path, "mydir", "file.txt"), "changed"); + await vol.writeFile(join("mydir", "file.txt"), "changed"); await vol.bupRestore("latest/mydir"); - expect(await readFile(join(vol.path, "mydir", "file.txt"), "utf8")).toEqual( + expect(await vol.readFile(join("mydir", "file.txt"), "utf8")).toEqual( "hello3", ); }); @@ -147,8 +157,8 @@ describe("test bup backups", () => { it("most recent snapshot has a backup before the restore", async () => { const s = await vol.snapshots(); const recent = s.slice(-1)[0]; - const p = join(vol.snapshotsDir, recent, "mydir", "file.txt"); - expect(await readFile(p, "utf8")).toEqual("changed"); + const p = vol.snapshotPath(recent, "mydir", "file.txt"); + expect(await vol.readFile(p, "utf8")).toEqual("changed"); }); }); From 4e2563aad0e33b04b6c78f4e7f2c6ce07d697e07 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 19:10:44 +0000 Subject: [PATCH 35/47] btrfs: add more filesystem support --- src/packages/file-server/btrfs/subvolume.ts | 93 ++++++++++++- .../btrfs/test/filesystem-stress.test.ts | 4 +- .../file-server/btrfs/test/filesystem.test.ts | 8 +- src/packages/file-server/btrfs/test/setup.ts | 3 +- .../file-server/btrfs/test/subvolume.test.ts | 123 ++++++++++++++++++ 5 files changed, 221 insertions(+), 10 deletions(-) diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index b882a287ac..cd4cd9f021 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -4,13 +4,33 @@ A subvolume import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; -import { readFile, writeFile, unlink } from "node:fs/promises"; +import { + appendFile, + chmod, + cp, + copyFile, + link, + readFile, + realpath, + rename, + rm, + rmdir, + mkdir, + stat, + symlink, + truncate, + writeFile, + unlink, + utimes, + watch, +} from "node:fs/promises"; import { exists, isdir, listdir, mkdirp, sudo } from "./util"; import { join, normalize } from "path"; import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; import { DirectoryListingEntry } from "@cocalc/util/types"; import getListing from "@cocalc/backend/get-listing"; import getLogger from "@cocalc/backend/logger"; +import { exists as pathExists } from "@cocalc/backend/misc/async-utils-node"; export const SNAPSHOTS = ".snapshots"; const SEND_SNAPSHOT_PREFIX = "send-"; @@ -104,10 +124,73 @@ export class Subvolume { return await writeFile(this.normalize(path), data); }; + appendFile = async (path: string, data: string | Buffer, encoding?) => { + path = normalize(path); + return await appendFile(this.normalize(path), data, encoding); + }; + unlink = async (path: string) => { await unlink(this.normalize(path)); }; + stat = async (path: string) => { + return await stat(this.normalize(path)); + }; + + exists = async (path: string) => { + return await pathExists(this.normalize(path)); + }; + + // hard link + link = async (existingPath: string, newPath: string) => { + return await link(this.normalize(existingPath), this.normalize(newPath)); + }; + + symlink = async (target: string, path: string) => { + return await symlink(this.normalize(target), this.normalize(path)); + }; + + realpath = async (path: string) => { + const x = await realpath(this.normalize(path)); + return x.slice(this.path.length + 1); + }; + + rename = async (oldPath: string, newPath: string) => { + await rename(this.normalize(oldPath), this.normalize(newPath)); + }; + + utimes = async ( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) => { + await utimes(this.normalize(path), atime, mtime); + }; + + watch = (filename: string, options?) => { + return watch(this.normalize(filename), options); + }; + + truncate = async (path: string, len?: number) => { + await truncate(this.normalize(path), len); + }; + + copyFile = async (src: string, dest: string) => { + await copyFile(this.normalize(src), this.normalize(dest)); + }; + + cp = async (src: string, dest: string, options?) => { + await cp(this.normalize(src), this.normalize(dest), options); + }; + + chmod = async (path: string, mode: string | number) => { + await chmod(this.normalize(path), mode); + }; + + mkdir = async (path: string, options?) => { + await mkdir(this.normalize(path), options); + }; + rsync = async ({ src, target, @@ -135,6 +218,14 @@ export class Subvolume { }); }; + rmdir = async (path: string, options?) => { + await rmdir(this.normalize(path), options); + }; + + rm = async (path: string, options?) => { + await rm(this.normalize(path), options); + }; + ///////////// // QUOTA ///////////// diff --git a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts index 76642be115..86cc05b8ee 100644 --- a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts @@ -1,6 +1,4 @@ import { before, after, fs } from "./setup"; -import { writeFile } from "fs/promises"; -import { join } from "path"; beforeAll(before); @@ -45,7 +43,7 @@ describe("stress operations with subvolumes", () => { it("write a file to each volume", async () => { for (const name of await fs.list()) { const vol = await fs.subvolume(name); - await writeFile(join(vol.path, "a.txt"), "hi"); + await vol.writeFile("a.txt", "hi"); } }); diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index 17d181f307..554b1bacf1 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -1,7 +1,5 @@ import { before, after, fs } from "./setup"; import { isValidUUID } from "@cocalc/util/misc"; -import { readFile, writeFile } from "fs/promises"; -import { join } from "path"; beforeAll(before); @@ -63,16 +61,16 @@ describe("operations with subvolumes", () => { it("rsync an actual file", async () => { const sagemath = await fs.subvolume("sagemath"); const cython = await fs.subvolume("cython"); - await writeFile(join(sagemath.path, "README.md"), "hi"); + await sagemath.writeFile("README.md", "hi"); await fs.rsync({ src: "sagemath", target: "cython" }); - const copy = await readFile(join(cython.path, "README.md"), "utf8"); + const copy = await cython.readFile("README.md", "utf8"); expect(copy).toEqual("hi"); }); it("clone a subvolume with contents", async () => { await fs.cloneSubvolume("cython", "pyrex"); const pyrex = await fs.subvolume("pyrex"); - const clone = await readFile(join(pyrex.path, "README.md"), "utf8"); + const clone = await pyrex.readFile("README.md", "utf8"); expect(clone).toEqual("hi"); }); }); diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index e09999b3d8..918c1e69f2 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -35,7 +35,8 @@ async function ensureMoreLoops() { export async function before() { try { const command = `umount ${join(tmpdir(), TEMP_PREFIX)}*/mnt`; - // attempt to unmount any mounts left from previous runs + // attempt to unmount any mounts left from previous runs. + // TODO: this could impact runs in parallel await sudo({ command, bash: true }); } catch {} await ensureMoreLoops(); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index b637a44e90..3e9fb47d2c 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -49,6 +49,129 @@ describe("setting and getting quota of a subvolume", () => { }); }); +describe("the filesystem operations", () => { + let vol; + + it("creates a volume and get empty listing", async () => { + vol = await fs.subvolume("fs"); + expect(await vol.ls("")).toEqual([]); + }); + + it("error listing non-existent path", async () => { + vol = await fs.subvolume("fs"); + expect(async () => { + await vol.ls("no-such-path"); + }).rejects.toThrow("ENOENT"); + }); + + it("creates a text file to it", async () => { + await vol.writeFile("a.txt", "hello"); + const ls = await vol.ls(""); + expect(ls).toEqual([{ name: "a.txt", mtime: ls[0].mtime, size: 5 }]); + }); + + it("read the file we just created as utf8", async () => { + expect(await vol.readFile("a.txt", "utf8")).toEqual("hello"); + }); + + it("read the file we just created as a binary buffer", async () => { + expect(await vol.readFile("a.txt")).toEqual(Buffer.from("hello")); + }); + + it("stat the file we just created", async () => { + const s = await vol.stat("a.txt"); + expect(s.size).toBe(5); + expect(Math.abs(s.mtimeMs - Date.now())).toBeLessThan(60_000); + }); + + let origStat; + it("snapshot filesystem and see file is in snapshot", async () => { + await vol.createSnapshot("snap"); + const s = await vol.ls(vol.snapshotPath("snap")); + expect(s).toEqual([{ name: "a.txt", mtime: s[0].mtime, size: 5 }]); + + const stat = await vol.stat("a.txt"); + origStat = stat; + expect(stat.mtimeMs / 1000).toBeCloseTo(s[0].mtime); + }); + + it("unlink (delete) our file", async () => { + await vol.unlink("a.txt"); + expect(await vol.ls("")).toEqual([]); + }); + + it("snapshot still exists", async () => { + expect(await vol.exists(vol.snapshotPath("snap", "a.txt"))); + }); + + it("copy file from snapshot and note it has the same mode as before (so much nicer than what happens with zfs)", async () => { + await vol.copyFile(vol.snapshotPath("snap", "a.txt"), "a.txt"); + const stat = await vol.stat("a.txt"); + expect(stat.mode).toEqual(origStat.mode); + }); + + it("create and copy a folder", async () => { + await vol.mkdir("my-folder"); + await vol.writeFile("my-folder/foo.txt", "foo"); + await vol.cp("my-folder", "folder2", { recursive: true }); + expect(await vol.readFile("folder2/foo.txt", "utf8")).toEqual("foo"); + }); + + it("append to a file", async () => { + await vol.writeFile("b.txt", "hell"); + await vol.appendFile("b.txt", "-o"); + expect(await vol.readFile("b.txt", "utf8")).toEqual("hell-o"); + }); + + it("make a file readonly, then change it back", async () => { + await vol.writeFile("c.txt", "hi"); + await vol.chmod("c.txt", "444"); + expect(async () => { + await vol.appendFile("c.txt", " there"); + }).rejects.toThrow("EACCES"); + await vol.chmod("c.txt", "666"); + await vol.appendFile("c.txt", " there"); + }); + + it("realpath of a symlink", async () => { + await vol.writeFile("real.txt", "i am real"); + await vol.symlink("real.txt", "link.txt"); + expect(await vol.realpath("link.txt")).toBe("real.txt"); + }); + + it("watch for changes", async () => { + await vol.writeFile("w.txt", "hi"); + const ac = new AbortController(); + const { signal } = ac; + const watcher = vol.watch("w.txt", { signal }); + vol.appendFile("w.txt", " there"); + const { value, done } = await watcher.next(); + expect(done).toBe(false); + expect(value).toEqual({ eventType: "change", filename: "w.txt" }); + ac.abort(); + + expect(async () => { + await watcher.next(); + }).rejects.toThrow("aborted"); + }); + + it("rename a file", async () => { + await vol.writeFile("old", "hi"); + await vol.rename("old", "new"); + expect(await vol.readFile("new", "utf8")).toEqual("hi"); + }); + + it("create and remove a directory", async () => { + await vol.mkdir("path"); + await vol.rmdir("path"); + }); + + it("create a directory recursively and remove", async () => { + await vol.mkdir("path/to/stuff", { recursive: true }); + await vol.rm("path", { recursive: true }); + }); +}); + describe("test snapshots", () => { let vol; it("creates a volume and write a file to it", async () => { From 0268a407df7b96719fe8677c385661f9d5caa23a Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 19:25:27 +0000 Subject: [PATCH 36/47] btrfs: refactor fs code --- .../file-server/btrfs/subvolume-fs.ts | 150 +++++++++++++++++ src/packages/file-server/btrfs/subvolume.ts | 157 +----------------- .../btrfs/test/filesystem-stress.test.ts | 2 +- .../file-server/btrfs/test/filesystem.test.ts | 6 +- .../btrfs/test/subvolume-stress.test.ts | 4 +- .../file-server/btrfs/test/subvolume.test.ts | 110 ++++++------ 6 files changed, 219 insertions(+), 210 deletions(-) create mode 100644 src/packages/file-server/btrfs/subvolume-fs.ts diff --git a/src/packages/file-server/btrfs/subvolume-fs.ts b/src/packages/file-server/btrfs/subvolume-fs.ts new file mode 100644 index 0000000000..f1f2dd3677 --- /dev/null +++ b/src/packages/file-server/btrfs/subvolume-fs.ts @@ -0,0 +1,150 @@ +import { + appendFile, + chmod, + cp, + copyFile, + link, + readFile, + realpath, + rename, + rm, + rmdir, + mkdir, + stat, + symlink, + truncate, + writeFile, + unlink, + utimes, + watch, +} from "node:fs/promises"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { type DirectoryListingEntry } from "@cocalc/util/types"; +import getListing from "@cocalc/backend/get-listing"; +import { type Subvolume } from "./subvolume"; +import { isdir, sudo } from "./util"; + +export class SubvolumeFilesystem { + constructor(private subvolume: Subvolume) {} + + private normalize = this.subvolume.normalize; + + ls = async ( + path: string, + { hidden, limit }: { hidden?: boolean; limit?: number } = {}, + ): Promise => { + return await getListing(this.normalize(path), hidden, { + limit, + home: "/", + }); + }; + + readFile = async (path: string, encoding?: any): Promise => { + return await readFile(this.normalize(path), encoding); + }; + + writeFile = async (path: string, data: string | Buffer) => { + return await writeFile(this.normalize(path), data); + }; + + appendFile = async (path: string, data: string | Buffer, encoding?) => { + return await appendFile(this.normalize(path), data, encoding); + }; + + unlink = async (path: string) => { + await unlink(this.normalize(path)); + }; + + stat = async (path: string) => { + return await stat(this.normalize(path)); + }; + + exists = async (path: string) => { + return await exists(this.normalize(path)); + }; + + // hard link + link = async (existingPath: string, newPath: string) => { + return await link(this.normalize(existingPath), this.normalize(newPath)); + }; + + symlink = async (target: string, path: string) => { + return await symlink(this.normalize(target), this.normalize(path)); + }; + + realpath = async (path: string) => { + const x = await realpath(this.normalize(path)); + return x.slice(this.subvolume.path.length + 1); + }; + + rename = async (oldPath: string, newPath: string) => { + await rename(this.normalize(oldPath), this.normalize(newPath)); + }; + + utimes = async ( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) => { + await utimes(this.normalize(path), atime, mtime); + }; + + watch = (filename: string, options?) => { + return watch(this.normalize(filename), options); + }; + + truncate = async (path: string, len?: number) => { + await truncate(this.normalize(path), len); + }; + + copyFile = async (src: string, dest: string) => { + await copyFile(this.normalize(src), this.normalize(dest)); + }; + + cp = async (src: string, dest: string, options?) => { + await cp(this.normalize(src), this.normalize(dest), options); + }; + + chmod = async (path: string, mode: string | number) => { + await chmod(this.normalize(path), mode); + }; + + mkdir = async (path: string, options?) => { + await mkdir(this.normalize(path), options); + }; + + rsync = async ({ + src, + target, + args = ["-axH"], + timeout = 5 * 60 * 1000, + }: { + src: string; + target: string; + args?: string[]; + timeout?: number; + }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { + let srcPath = this.normalize(src); + let targetPath = this.normalize(target); + if (!srcPath.endsWith("/") && (await isdir(srcPath))) { + srcPath += "/"; + if (!targetPath.endsWith("/")) { + targetPath += "/"; + } + } + return await sudo({ + command: "rsync", + args: [...args, srcPath, targetPath], + err_on_exit: false, + timeout: timeout / 1000, + }); + }; + + rmdir = async (path: string, options?) => { + await rmdir(this.normalize(path), options); + }; + + rm = async (path: string, options?) => { + await rm(this.normalize(path), options); + }; +} diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index cd4cd9f021..968cb57031 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -4,33 +4,12 @@ A subvolume import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; -import { - appendFile, - chmod, - cp, - copyFile, - link, - readFile, - realpath, - rename, - rm, - rmdir, - mkdir, - stat, - symlink, - truncate, - writeFile, - unlink, - utimes, - watch, -} from "node:fs/promises"; -import { exists, isdir, listdir, mkdirp, sudo } from "./util"; +import { exists, listdir, mkdirp, sudo } from "./util"; import { join, normalize } from "path"; import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; -import { DirectoryListingEntry } from "@cocalc/util/types"; -import getListing from "@cocalc/backend/get-listing"; +import { type DirectoryListingEntry } from "@cocalc/util/types"; import getLogger from "@cocalc/backend/logger"; -import { exists as pathExists } from "@cocalc/backend/misc/async-utils-node"; +import { SubvolumeFilesystem } from "./subvolume-fs"; export const SNAPSHOTS = ".snapshots"; const SEND_SNAPSHOT_PREFIX = "send-"; @@ -48,14 +27,16 @@ export class Subvolume { public readonly name: string; private filesystem: Filesystem; - private readonly path: string; - private readonly snapshotsDir: string; + public readonly path: string; + public readonly snapshotsDir: string; + public readonly fs: SubvolumeFilesystem; constructor({ filesystem, name }: Options) { this.filesystem = filesystem; this.name = name; this.path = join(filesystem.opts.mount, name); this.snapshotsDir = join(this.path, SNAPSHOTS); + this.fs = new SubvolumeFilesystem(this); } init = async () => { @@ -96,135 +77,13 @@ export class Subvolume { // this should provide a path that is guaranteed to be // inside this.path on the filesystem or throw error // [ ] TODO: not sure if the code here is sufficient!! - private normalize = (path: string) => { + normalize = (path: string) => { return join(this.path, normalize(path)); }; ///////////// // Files ///////////// - ls = async ( - path: string, - { hidden, limit }: { hidden?: boolean; limit?: number } = {}, - ): Promise => { - path = normalize(path); - return await getListing(this.normalize(path), hidden, { - limit, - home: "/", - }); - }; - - readFile = async (path: string, encoding?: any): Promise => { - path = normalize(path); - return await readFile(this.normalize(path), encoding); - }; - - writeFile = async (path: string, data: string | Buffer) => { - path = normalize(path); - return await writeFile(this.normalize(path), data); - }; - - appendFile = async (path: string, data: string | Buffer, encoding?) => { - path = normalize(path); - return await appendFile(this.normalize(path), data, encoding); - }; - - unlink = async (path: string) => { - await unlink(this.normalize(path)); - }; - - stat = async (path: string) => { - return await stat(this.normalize(path)); - }; - - exists = async (path: string) => { - return await pathExists(this.normalize(path)); - }; - - // hard link - link = async (existingPath: string, newPath: string) => { - return await link(this.normalize(existingPath), this.normalize(newPath)); - }; - - symlink = async (target: string, path: string) => { - return await symlink(this.normalize(target), this.normalize(path)); - }; - - realpath = async (path: string) => { - const x = await realpath(this.normalize(path)); - return x.slice(this.path.length + 1); - }; - - rename = async (oldPath: string, newPath: string) => { - await rename(this.normalize(oldPath), this.normalize(newPath)); - }; - - utimes = async ( - path: string, - atime: number | string | Date, - mtime: number | string | Date, - ) => { - await utimes(this.normalize(path), atime, mtime); - }; - - watch = (filename: string, options?) => { - return watch(this.normalize(filename), options); - }; - - truncate = async (path: string, len?: number) => { - await truncate(this.normalize(path), len); - }; - - copyFile = async (src: string, dest: string) => { - await copyFile(this.normalize(src), this.normalize(dest)); - }; - - cp = async (src: string, dest: string, options?) => { - await cp(this.normalize(src), this.normalize(dest), options); - }; - - chmod = async (path: string, mode: string | number) => { - await chmod(this.normalize(path), mode); - }; - - mkdir = async (path: string, options?) => { - await mkdir(this.normalize(path), options); - }; - - rsync = async ({ - src, - target, - args = ["-axH"], - timeout = 5 * 60 * 1000, - }: { - src: string; - target: string; - args?: string[]; - timeout?: number; - }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = this.normalize(src); - let targetPath = this.normalize(target); - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { - srcPath += "/"; - if (!targetPath.endsWith("/")) { - targetPath += "/"; - } - } - return await sudo({ - command: "rsync", - args: [...args, srcPath, targetPath], - err_on_exit: false, - timeout: timeout / 1000, - }); - }; - - rmdir = async (path: string, options?) => { - await rmdir(this.normalize(path), options); - }; - - rm = async (path: string, options?) => { - await rm(this.normalize(path), options); - }; ///////////// // QUOTA diff --git a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts index 86cc05b8ee..176cdf6a5a 100644 --- a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts @@ -43,7 +43,7 @@ describe("stress operations with subvolumes", () => { it("write a file to each volume", async () => { for (const name of await fs.list()) { const vol = await fs.subvolume(name); - await vol.writeFile("a.txt", "hi"); + await vol.fs.writeFile("a.txt", "hi"); } }); diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index 554b1bacf1..e22a69377e 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -61,16 +61,16 @@ describe("operations with subvolumes", () => { it("rsync an actual file", async () => { const sagemath = await fs.subvolume("sagemath"); const cython = await fs.subvolume("cython"); - await sagemath.writeFile("README.md", "hi"); + await sagemath.fs.writeFile("README.md", "hi"); await fs.rsync({ src: "sagemath", target: "cython" }); - const copy = await cython.readFile("README.md", "utf8"); + const copy = await cython.fs.readFile("README.md", "utf8"); expect(copy).toEqual("hi"); }); it("clone a subvolume with contents", async () => { await fs.cloneSubvolume("cython", "pyrex"); const pyrex = await fs.subvolume("pyrex"); - const clone = await pyrex.readFile("README.md", "utf8"); + const clone = await pyrex.fs.readFile("README.md", "utf8"); expect(clone).toEqual("hi"); }); }); diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts index a957039a29..851d24a315 100644 --- a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -55,7 +55,7 @@ describe(`create ${numFiles} files`, () => { log( `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in serial`, ); - const v = await vol.ls(""); + const v = await vol.fs.ls(""); const w = v.map(({ name }) => name); expect(w.sort()).toEqual(names.sort()); }); @@ -74,7 +74,7 @@ describe(`create ${numFiles} files`, () => { `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in parallel`, ); const t0 = Date.now(); - const v = await vol.ls("p"); + const v = await vol.fs.ls("p"); log("get listing of files took", Date.now() - t0, "ms"); const w = v.map(({ name }) => name); expect(w.sort()).toEqual(names.sort()); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 3e9fb47d2c..3bce6bbacc 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -19,13 +19,13 @@ describe("setting and getting quota of a subvolume", () => { }); it("get directory listing", async () => { - const v = await vol.ls(""); + const v = await vol.fs.ls(""); expect(v).toEqual([]); }); it("write a file and check usage goes up", async () => { const buf = randomBytes(4 * 1024 * 1024); - await vol.writeFile("buf", buf); + await vol.fs.writeFile("buf", buf); await wait({ until: async () => { await sudo({ command: "sync" }); @@ -36,7 +36,7 @@ describe("setting and getting quota of a subvolume", () => { const { used } = await vol.usage(); expect(used).toBeGreaterThan(0); - const v = await vol.ls(""); + const v = await vol.fs.ls(""); // size is potentially random, reflecting compression expect(v).toEqual([{ name: "buf", mtime: v[0].mtime, size: v[0].size }]); }); @@ -44,7 +44,7 @@ describe("setting and getting quota of a subvolume", () => { it("fail to write a 50MB file (due to quota)", async () => { const buf2 = randomBytes(50 * 1024 * 1024); expect(async () => { - await vol.writeFile("buf2", buf2); + await vol.fs.writeFile("buf2", buf2); }).rejects.toThrow("write"); }); }); @@ -54,32 +54,32 @@ describe("the filesystem operations", () => { it("creates a volume and get empty listing", async () => { vol = await fs.subvolume("fs"); - expect(await vol.ls("")).toEqual([]); + expect(await vol.fs.ls("")).toEqual([]); }); it("error listing non-existent path", async () => { vol = await fs.subvolume("fs"); expect(async () => { - await vol.ls("no-such-path"); + await vol.fs.ls("no-such-path"); }).rejects.toThrow("ENOENT"); }); it("creates a text file to it", async () => { - await vol.writeFile("a.txt", "hello"); - const ls = await vol.ls(""); + await vol.fs.writeFile("a.txt", "hello"); + const ls = await vol.fs.ls(""); expect(ls).toEqual([{ name: "a.txt", mtime: ls[0].mtime, size: 5 }]); }); it("read the file we just created as utf8", async () => { - expect(await vol.readFile("a.txt", "utf8")).toEqual("hello"); + expect(await vol.fs.readFile("a.txt", "utf8")).toEqual("hello"); }); it("read the file we just created as a binary buffer", async () => { - expect(await vol.readFile("a.txt")).toEqual(Buffer.from("hello")); + expect(await vol.fs.readFile("a.txt")).toEqual(Buffer.from("hello")); }); it("stat the file we just created", async () => { - const s = await vol.stat("a.txt"); + const s = await vol.fs.stat("a.txt"); expect(s.size).toBe(5); expect(Math.abs(s.mtimeMs - Date.now())).toBeLessThan(60_000); }); @@ -87,64 +87,64 @@ describe("the filesystem operations", () => { let origStat; it("snapshot filesystem and see file is in snapshot", async () => { await vol.createSnapshot("snap"); - const s = await vol.ls(vol.snapshotPath("snap")); + const s = await vol.fs.ls(vol.snapshotPath("snap")); expect(s).toEqual([{ name: "a.txt", mtime: s[0].mtime, size: 5 }]); - const stat = await vol.stat("a.txt"); + const stat = await vol.fs.stat("a.txt"); origStat = stat; expect(stat.mtimeMs / 1000).toBeCloseTo(s[0].mtime); }); it("unlink (delete) our file", async () => { - await vol.unlink("a.txt"); - expect(await vol.ls("")).toEqual([]); + await vol.fs.unlink("a.txt"); + expect(await vol.fs.ls("")).toEqual([]); }); it("snapshot still exists", async () => { - expect(await vol.exists(vol.snapshotPath("snap", "a.txt"))); + expect(await vol.fs.exists(vol.snapshotPath("snap", "a.txt"))); }); it("copy file from snapshot and note it has the same mode as before (so much nicer than what happens with zfs)", async () => { - await vol.copyFile(vol.snapshotPath("snap", "a.txt"), "a.txt"); - const stat = await vol.stat("a.txt"); + await vol.fs.copyFile(vol.snapshotPath("snap", "a.txt"), "a.txt"); + const stat = await vol.fs.stat("a.txt"); expect(stat.mode).toEqual(origStat.mode); }); it("create and copy a folder", async () => { - await vol.mkdir("my-folder"); - await vol.writeFile("my-folder/foo.txt", "foo"); - await vol.cp("my-folder", "folder2", { recursive: true }); - expect(await vol.readFile("folder2/foo.txt", "utf8")).toEqual("foo"); + await vol.fs.mkdir("my-folder"); + await vol.fs.writeFile("my-folder/foo.txt", "foo"); + await vol.fs.cp("my-folder", "folder2", { recursive: true }); + expect(await vol.fs.readFile("folder2/foo.txt", "utf8")).toEqual("foo"); }); it("append to a file", async () => { - await vol.writeFile("b.txt", "hell"); - await vol.appendFile("b.txt", "-o"); - expect(await vol.readFile("b.txt", "utf8")).toEqual("hell-o"); + await vol.fs.writeFile("b.txt", "hell"); + await vol.fs.appendFile("b.txt", "-o"); + expect(await vol.fs.readFile("b.txt", "utf8")).toEqual("hell-o"); }); it("make a file readonly, then change it back", async () => { - await vol.writeFile("c.txt", "hi"); - await vol.chmod("c.txt", "444"); + await vol.fs.writeFile("c.txt", "hi"); + await vol.fs.chmod("c.txt", "444"); expect(async () => { - await vol.appendFile("c.txt", " there"); + await vol.fs.appendFile("c.txt", " there"); }).rejects.toThrow("EACCES"); - await vol.chmod("c.txt", "666"); - await vol.appendFile("c.txt", " there"); + await vol.fs.chmod("c.txt", "666"); + await vol.fs.appendFile("c.txt", " there"); }); it("realpath of a symlink", async () => { - await vol.writeFile("real.txt", "i am real"); - await vol.symlink("real.txt", "link.txt"); - expect(await vol.realpath("link.txt")).toBe("real.txt"); + await vol.fs.writeFile("real.txt", "i am real"); + await vol.fs.symlink("real.txt", "link.txt"); + expect(await vol.fs.realpath("link.txt")).toBe("real.txt"); }); it("watch for changes", async () => { - await vol.writeFile("w.txt", "hi"); + await vol.fs.writeFile("w.txt", "hi"); const ac = new AbortController(); const { signal } = ac; - const watcher = vol.watch("w.txt", { signal }); - vol.appendFile("w.txt", " there"); + const watcher = vol.fs.watch("w.txt", { signal }); + vol.fs.appendFile("w.txt", " there"); const { value, done } = await watcher.next(); expect(done).toBe(false); expect(value).toEqual({ eventType: "change", filename: "w.txt" }); @@ -156,19 +156,19 @@ describe("the filesystem operations", () => { }); it("rename a file", async () => { - await vol.writeFile("old", "hi"); - await vol.rename("old", "new"); - expect(await vol.readFile("new", "utf8")).toEqual("hi"); + await vol.fs.writeFile("old", "hi"); + await vol.fs.rename("old", "new"); + expect(await vol.fs.readFile("new", "utf8")).toEqual("hi"); }); it("create and remove a directory", async () => { - await vol.mkdir("path"); - await vol.rmdir("path"); + await vol.fs.mkdir("path"); + await vol.fs.rmdir("path"); }); it("create a directory recursively and remove", async () => { - await vol.mkdir("path/to/stuff", { recursive: true }); - await vol.rm("path", { recursive: true }); + await vol.fs.mkdir("path/to/stuff", { recursive: true }); + await vol.fs.rm("path", { recursive: true }); }); }); @@ -177,7 +177,7 @@ describe("test snapshots", () => { it("creates a volume and write a file to it", async () => { vol = await fs.subvolume("snapper"); expect(await vol.hasUnsavedChanges()).toBe(false); - await vol.writeFile("a.txt", "hello"); + await vol.fs.writeFile("a.txt", "hello"); expect(await vol.hasUnsavedChanges()).toBe(true); }); @@ -189,14 +189,14 @@ describe("test snapshots", () => { }); it("create a file see that we know there are unsaved changes", async () => { - await vol.writeFile("b.txt", "world"); + await vol.fs.writeFile("b.txt", "world"); await sudo({ command: "sync" }); expect(await vol.hasUnsavedChanges()).toBe(true); }); it("delete our file, but then read it in a snapshot", async () => { - await vol.unlink("a.txt"); - const b = await vol.readFile(vol.snapshotPath("snap1", "a.txt"), "utf8"); + await vol.fs.unlink("a.txt"); + const b = await vol.fs.readFile(vol.snapshotPath("snap1", "a.txt"), "utf8"); expect(b).toEqual("hello"); }); @@ -224,7 +224,7 @@ describe("test bup backups", () => { let vol; it("creates a volume", async () => { vol = await fs.subvolume("bup-test"); - await vol.writeFile("a.txt", "hello"); + await vol.fs.writeFile("a.txt", "hello"); }); it("create a bup backup", async () => { @@ -246,9 +246,9 @@ describe("test bup backups", () => { }); it("restore a.txt from our backup", async () => { - await vol.writeFile("a.txt", "hello2"); + await vol.fs.writeFile("a.txt", "hello2"); await vol.bupRestore("latest/a.txt"); - expect(await vol.readFile("a.txt", "utf8")).toEqual("hello"); + expect(await vol.fs.readFile("a.txt", "utf8")).toEqual("hello"); }); it("prune bup backups does nothing since we have so few", async () => { @@ -258,8 +258,8 @@ describe("test bup backups", () => { it("add a directory and back up", async () => { await mkdir(join(vol.path, "mydir")); - await vol.writeFile(join("mydir", "file.txt"), "hello3"); - expect((await vol.ls("mydir"))[0].name).toBe("file.txt"); + await vol.fs.writeFile(join("mydir", "file.txt"), "hello3"); + expect((await vol.fs.ls("mydir"))[0].name).toBe("file.txt"); await vol.createBupBackup(); const x = await vol.bupLs("latest"); expect(x).toEqual([ @@ -270,9 +270,9 @@ describe("test bup backups", () => { }); it("change file in the directory, then restore from backup whole dir", async () => { - await vol.writeFile(join("mydir", "file.txt"), "changed"); + await vol.fs.writeFile(join("mydir", "file.txt"), "changed"); await vol.bupRestore("latest/mydir"); - expect(await vol.readFile(join("mydir", "file.txt"), "utf8")).toEqual( + expect(await vol.fs.readFile(join("mydir", "file.txt"), "utf8")).toEqual( "hello3", ); }); @@ -281,7 +281,7 @@ describe("test bup backups", () => { const s = await vol.snapshots(); const recent = s.slice(-1)[0]; const p = vol.snapshotPath(recent, "mydir", "file.txt"); - expect(await vol.readFile(p, "utf8")).toEqual("changed"); + expect(await vol.fs.readFile(p, "utf8")).toEqual("changed"); }); }); From 636d5890f4382edce558c94c5ed1a03325cae385 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 20:29:00 +0000 Subject: [PATCH 37/47] btrfs: refactor bup backup code --- .../file-server/btrfs/subvolume-bup.ts | 178 ++++++++++++++++++ src/packages/file-server/btrfs/subvolume.ts | 165 +--------------- .../btrfs/test/subvolume-stress.test.ts | 5 +- .../file-server/btrfs/test/subvolume.test.ts | 41 ++-- 4 files changed, 209 insertions(+), 180 deletions(-) create mode 100644 src/packages/file-server/btrfs/subvolume-bup.ts diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts new file mode 100644 index 0000000000..c4563fbdd9 --- /dev/null +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -0,0 +1,178 @@ +import { type DirectoryListingEntry } from "@cocalc/util/types"; +import { type Subvolume } from "./subvolume"; +import { sudo, parseBupTime } from "./util"; +import { join, normalize } from "path"; +import getLogger from "@cocalc/backend/logger"; + +const BUP_SNAPSHOT = "temp-bup-snapshot"; + +const logger = getLogger("file-server:storage-btrfs:subvolume-bup"); + +export class SubvolumeBup { + constructor(private subvolume: Subvolume) {} + + ///////////// + // BACKUPS + // There is a single global dedup'd backup archive stored in the btrfs filesystem. + // Obviously, admins should rsync this regularly to a separate location as a genuine + // backup strategy. + ///////////// + + // create a new bup backup + save = async ({ + // timeout used for bup index and bup save commands + timeout = 30 * 60 * 1000, + }: { timeout?: number } = {}) => { + if (await this.subvolume.snapshotExists(BUP_SNAPSHOT)) { + logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`); + await this.subvolume.deleteSnapshot(BUP_SNAPSHOT); + } + try { + logger.debug( + `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, + ); + await this.subvolume.createSnapshot(BUP_SNAPSHOT); + const target = this.subvolume.normalize( + this.subvolume.snapshotPath(BUP_SNAPSHOT), + ); + + logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`); + await sudo({ + command: "bup", + args: [ + "-d", + this.subvolume.filesystem.bup, + "index", + "--exclude", + join(target, ".snapshots"), + "-x", + target, + ], + timeout, + }); + + logger.debug(`createBackup: saving ${BUP_SNAPSHOT}`); + await sudo({ + command: "bup", + args: [ + "-d", + this.subvolume.filesystem.bup, + "save", + "--strip", + "-n", + this.subvolume.name, + target, + ], + timeout, + }); + } finally { + logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`); + await this.subvolume.deleteSnapshot(BUP_SNAPSHOT); + } + }; + + restore = async (path: string) => { + // path -- branch/revision/path/to/dir + if (path.startsWith("/")) { + path = path.slice(1); + } + path = normalize(path); + // ... but to avoid potential data loss, we make a snapshot before deleting it. + await this.subvolume.createSnapshot(); + const i = path.indexOf("/"); // remove the commit name + // remove the target we're about to restore + await this.subvolume.fs.rm(path.slice(i + 1), { recursive: true }); + await sudo({ + command: "bup", + args: [ + "-d", + this.subvolume.filesystem.bup, + "restore", + "-C", + this.subvolume.path, + join(`/${this.subvolume.name}`, path), + "--quiet", + ], + }); + }; + + ls = async (path: string = ""): Promise => { + if (!path) { + const { stdout } = await sudo({ + command: "bup", + args: ["-d", this.subvolume.filesystem.bup, "ls", this.subvolume.name], + }); + const v: DirectoryListingEntry[] = []; + let newest = 0; + for (const x of stdout.trim().split("\n")) { + const name = x.split(" ").slice(-1)[0]; + if (name == "latest") { + continue; + } + const mtime = parseBupTime(name).valueOf() / 1000; + newest = Math.max(mtime, newest); + v.push({ name, isdir: true, mtime }); + } + if (v.length > 0) { + v.push({ name: "latest", isdir: true, mtime: newest }); + } + return v; + } + + path = normalize(path); + const { stdout } = await sudo({ + command: "bup", + args: [ + "-d", + this.subvolume.filesystem.bup, + "ls", + "--almost-all", + "--file-type", + "-l", + join(`/${this.subvolume.name}`, path), + ], + }); + const v: DirectoryListingEntry[] = []; + for (const x of stdout.split("\n")) { + // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"] + const w = x.split(/\s+/); + if (w.length >= 6) { + let isdir, name; + if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) { + w[5] = w[5].slice(0, -1); + } + if (w[5].endsWith("/")) { + isdir = true; + name = w[5].slice(0, -1); + } else { + name = w[5]; + isdir = false; + } + const size = parseInt(w[2]); + const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000; + v.push({ name, size, mtime, isdir }); + } + } + return v; + }; + + prune = async ({ + dailies = "1w", + monthlies = "4m", + all = "3d", + }: { dailies?: string; monthlies?: string; all?: string } = {}) => { + await sudo({ + command: "bup", + args: [ + "-d", + this.subvolume.filesystem.bup, + "prune-older", + `--keep-dailies-for=${dailies}`, + `--keep-monthlies-for=${monthlies}`, + `--keep-all-for=${all}`, + "--unsafe", + this.subvolume.name, + ], + }); + }; +} diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 968cb57031..206fdc45e5 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -7,13 +7,12 @@ import refCache from "@cocalc/util/refcache"; import { exists, listdir, mkdirp, sudo } from "./util"; import { join, normalize } from "path"; import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; -import { type DirectoryListingEntry } from "@cocalc/util/types"; -import getLogger from "@cocalc/backend/logger"; import { SubvolumeFilesystem } from "./subvolume-fs"; +import { SubvolumeBup } from "./subvolume-bup"; +import getLogger from "@cocalc/backend/logger"; export const SNAPSHOTS = ".snapshots"; const SEND_SNAPSHOT_PREFIX = "send-"; -const BUP_SNAPSHOT = "temp-bup-snapshot"; const PAD = 4; const logger = getLogger("file-server:storage-btrfs:subvolume"); @@ -26,10 +25,11 @@ interface Options { export class Subvolume { public readonly name: string; - private filesystem: Filesystem; + public readonly filesystem: Filesystem; public readonly path: string; public readonly snapshotsDir: string; public readonly fs: SubvolumeFilesystem; + public readonly bup: SubvolumeBup; constructor({ filesystem, name }: Options) { this.filesystem = filesystem; @@ -37,6 +37,7 @@ export class Subvolume { this.path = join(filesystem.opts.mount, name); this.snapshotsDir = join(this.path, SNAPSHOTS); this.fs = new SubvolumeFilesystem(this); + this.bup = new SubvolumeBup(this); } init = async () => { @@ -81,10 +82,6 @@ export class Subvolume { return join(this.path, normalize(path)); }; - ///////////// - // Files - ///////////// - ///////////// // QUOTA ///////////// @@ -259,158 +256,6 @@ export class Subvolume { return snapGen < pathGen; }; - ///////////// - // BACKUPS - // There is a single global dedup'd backup archive stored in the btrfs filesystem. - // Obviously, admins should rsync this regularly to a separate location as a genuine - // backup strategy. - ///////////// - - // create a new bup backup - createBupBackup = async ({ - // timeout used for bup index and bup save commands - timeout = 30 * 60 * 1000, - }: { timeout?: number } = {}) => { - if (await this.snapshotExists(BUP_SNAPSHOT)) { - logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`); - await this.deleteSnapshot(BUP_SNAPSHOT); - } - try { - logger.debug( - `createBupBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, - ); - await this.createSnapshot(BUP_SNAPSHOT); - const target = join(this.snapshotsDir, BUP_SNAPSHOT); - logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`); - await sudo({ - command: "bup", - args: [ - "-d", - this.filesystem.bup, - "index", - "--exclude", - join(target, ".snapshots"), - "-x", - target, - ], - timeout, - }); - logger.debug(`createBupBackup: saving ${BUP_SNAPSHOT}`); - await sudo({ - command: "bup", - args: [ - "-d", - this.filesystem.bup, - "save", - "--strip", - "-n", - this.name, - target, - ], - timeout, - }); - } finally { - logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`); - await this.deleteSnapshot(BUP_SNAPSHOT); - } - }; - - bupBackups = async (): Promise => { - const { stdout } = await sudo({ - command: "bup", - args: ["-d", this.filesystem.bup, "ls", this.name], - }); - return stdout - .split("\n") - .map((x) => x.split(" ").slice(-1)[0]) - .filter((x) => x); - }; - - bupRestore = async (path: string) => { - // path -- branch/revision/path/to/dir - if (path.startsWith("/")) { - path = path.slice(1); - } - path = normalize(path); - // ... but to avoid potential data loss, we make a snapshot before deleting it. - await this.createSnapshot(); - const i = path.indexOf("/"); // remove the commit name - await sudo({ - command: "rm", - args: ["-rf", this.normalize(path.slice(i + 1))], - }); - await sudo({ - command: "bup", - args: [ - "-d", - this.filesystem.bup, - "restore", - "-C", - this.path, - join(`/${this.name}`, path), - "--quiet", - ], - }); - }; - - bupLs = async (path: string): Promise => { - path = normalize(path); - const { stdout } = await sudo({ - command: "bup", - args: [ - "-d", - this.filesystem.bup, - "ls", - "--almost-all", - "--file-type", - "-l", - join(`/${this.name}`, path), - ], - }); - const v: DirectoryListingEntry[] = []; - for (const x of stdout.split("\n")) { - // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"] - const w = x.split(/\s+/); - if (w.length >= 6) { - let isdir, name; - if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) { - w[5] = w[5].slice(0, -1); - } - if (w[5].endsWith("/")) { - isdir = true; - name = w[5].slice(0, -1); - } else { - name = w[5]; - isdir = false; - } - const size = parseInt(w[2]); - const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000; - v.push({ name, size, mtime, isdir }); - } - } - return v; - }; - - bupPrune = async ({ - dailies = "1w", - monthlies = "4m", - all = "3d", - }: { dailies?: string; monthlies?: string; all?: string } = {}) => { - await sudo({ - command: "bup", - args: [ - "-d", - this.filesystem.bup, - "prune-older", - `--keep-dailies-for=${dailies}`, - `--keep-monthlies-for=${monthlies}`, - `--keep-all-for=${all}`, - "--unsafe", - this.name, - ], - }); - }; - ///////////// // BTRFS send/recv // Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since: diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts index 851d24a315..a9f0c5e33c 100644 --- a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -1,6 +1,7 @@ import { before, after, fs } from "./setup"; import { mkdir, writeFile } from "fs/promises"; import { join } from "path"; +import { type Subvolume } from "../subvolume"; const DEBUG = false; const log = DEBUG ? console.log : (..._args) => {}; @@ -11,7 +12,7 @@ const numFiles = 1000; beforeAll(before); describe(`stress test creating ${numSnapshots} snapshots`, () => { - let vol; + let vol: Subvolume; it("creates a volume and write a file to it", async () => { vol = await fs.subvolume("stress"); }); @@ -40,7 +41,7 @@ describe(`stress test creating ${numSnapshots} snapshots`, () => { }); describe(`create ${numFiles} files`, () => { - let vol; + let vol: Subvolume; it("creates a volume", async () => { vol = await fs.subvolume("many-files"); }); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 3bce6bbacc..ecfdecc24a 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -3,12 +3,12 @@ import { mkdir } from "fs/promises"; import { join } from "path"; import { wait } from "@cocalc/backend/conat/test/util"; import { randomBytes } from "crypto"; -import { parseBupTime } from "../util"; +import { type Subvolume } from "../subvolume"; beforeAll(before); describe("setting and getting quota of a subvolume", () => { - let vol; + let vol: Subvolume; it("set the quota of a subvolume to 5 M", async () => { vol = await fs.subvolume("q"); await vol.size("5M"); @@ -50,7 +50,7 @@ describe("setting and getting quota of a subvolume", () => { }); describe("the filesystem operations", () => { - let vol; + let vol: Subvolume; it("creates a volume and get empty listing", async () => { vol = await fs.subvolume("fs"); @@ -92,7 +92,7 @@ describe("the filesystem operations", () => { const stat = await vol.fs.stat("a.txt"); origStat = stat; - expect(stat.mtimeMs / 1000).toBeCloseTo(s[0].mtime); + expect(stat.mtimeMs / 1000).toBeCloseTo(s[0].mtime ?? 0); }); it("unlink (delete) our file", async () => { @@ -145,12 +145,14 @@ describe("the filesystem operations", () => { const { signal } = ac; const watcher = vol.fs.watch("w.txt", { signal }); vol.fs.appendFile("w.txt", " there"); + // @ts-ignore const { value, done } = await watcher.next(); expect(done).toBe(false); expect(value).toEqual({ eventType: "change", filename: "w.txt" }); ac.abort(); expect(async () => { + // @ts-ignore await watcher.next(); }).rejects.toThrow("aborted"); }); @@ -173,7 +175,8 @@ describe("the filesystem operations", () => { }); describe("test snapshots", () => { - let vol; + let vol: Subvolume; + it("creates a volume and write a file to it", async () => { vol = await fs.subvolume("snapper"); expect(await vol.hasUnsavedChanges()).toBe(false); @@ -220,26 +223,26 @@ describe("test snapshots", () => { }); }); -describe("test bup backups", () => { - let vol; +describe.only("test bup backups", () => { + let vol: Subvolume; it("creates a volume", async () => { vol = await fs.subvolume("bup-test"); await vol.fs.writeFile("a.txt", "hello"); }); it("create a bup backup", async () => { - await vol.createBupBackup(); + await vol.bup.save(); }); it("list bup backups of this vol -- there are 2, one for the date and 'latest'", async () => { - const v = await vol.bupBackups(); + const v = await vol.bup.ls(); expect(v.length).toBe(2); - const t = parseBupTime(v[0]); + const t = (v[0].mtime ?? 0) * 1000; expect(Math.abs(t.valueOf() - Date.now())).toBeLessThan(10_000); }); it("confirm a.txt is in our backup", async () => { - const x = await vol.bupLs("latest"); + const x = await vol.bup.ls("latest"); expect(x).toEqual([ { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, ]); @@ -247,31 +250,33 @@ describe("test bup backups", () => { it("restore a.txt from our backup", async () => { await vol.fs.writeFile("a.txt", "hello2"); - await vol.bupRestore("latest/a.txt"); + await vol.bup.restore("latest/a.txt"); expect(await vol.fs.readFile("a.txt", "utf8")).toEqual("hello"); }); it("prune bup backups does nothing since we have so few", async () => { - await vol.bupPrune(); - expect((await vol.bupBackups()).length).toBe(2); + await vol.bup.prune(); + expect((await vol.bup.ls()).length).toBe(2); }); it("add a directory and back up", async () => { await mkdir(join(vol.path, "mydir")); await vol.fs.writeFile(join("mydir", "file.txt"), "hello3"); expect((await vol.fs.ls("mydir"))[0].name).toBe("file.txt"); - await vol.createBupBackup(); - const x = await vol.bupLs("latest"); + await vol.bup.save(); + const x = await vol.bup.ls("latest"); expect(x).toEqual([ { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, { name: "mydir", size: 0, mtime: x[1].mtime, isdir: true }, ]); - expect(Math.abs(x[0].mtime * 1000 - Date.now())).toBeLessThan(60_000); + expect(Math.abs((x[0].mtime ?? 0) * 1000 - Date.now())).toBeLessThan( + 60_000, + ); }); it("change file in the directory, then restore from backup whole dir", async () => { await vol.fs.writeFile(join("mydir", "file.txt"), "changed"); - await vol.bupRestore("latest/mydir"); + await vol.bup.restore("latest/mydir"); expect(await vol.fs.readFile(join("mydir", "file.txt"), "utf8")).toEqual( "hello3", ); From 5e5e772bef07adcc83b03deed636194149f9469d Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 21:14:28 +0000 Subject: [PATCH 38/47] btrfs: refactor snapshot code --- src/packages/file-server/btrfs/filesystem.ts | 14 +- src/packages/file-server/btrfs/snapshots.ts | 28 ++-- .../file-server/btrfs/subvolume-bup.ts | 12 +- .../file-server/btrfs/subvolume-snapshot.ts | 111 ++++++++++++++++ src/packages/file-server/btrfs/subvolume.ts | 120 ++---------------- .../file-server/btrfs/test/filesystem.test.ts | 2 +- .../btrfs/test/subvolume-stress.test.ts | 10 +- .../file-server/btrfs/test/subvolume.test.ts | 49 +++---- src/packages/file-server/btrfs/util.ts | 6 + 9 files changed, 189 insertions(+), 163 deletions(-) create mode 100644 src/packages/file-server/btrfs/subvolume-snapshot.ts diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 79e1f19785..39416e3c87 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -13,7 +13,8 @@ a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({devic import refCache from "@cocalc/util/refcache"; import { exists, isdir, listdir, mkdirp, rmdir, sudo } from "./util"; -import { subvolume, SNAPSHOTS, type Subvolume } from "./subvolume"; +import { subvolume, type Subvolume } from "./subvolume"; +import { SNAPSHOTS } from "./subvolume-snapshot"; import { join, normalize } from "path"; // default size of btrfs filesystem if creating an image file. @@ -201,10 +202,13 @@ export class Filesystem { command: "mv", args: [join(this.opts.mount, source, name), join(this.opts.mount, name)], }); - const snapshots = await listdir(join(this.opts.mount, name, SNAPSHOTS)); - await rmdir( - snapshots.map((x) => join(this.opts.mount, name, SNAPSHOTS, x)), - ); + const snapdir = join(this.opts.mount, name, SNAPSHOTS); + if (await exists(snapdir)) { + const snapshots = await listdir(snapdir); + await rmdir( + snapshots.map((x) => join(this.opts.mount, name, SNAPSHOTS, x)), + ); + } const src = await this.subvolume(source); const vol = await this.subvolume(name); const { size } = await src.usage(); diff --git a/src/packages/file-server/btrfs/snapshots.ts b/src/packages/file-server/btrfs/snapshots.ts index f0dd6dff7a..10bc81d81a 100644 --- a/src/packages/file-server/btrfs/snapshots.ts +++ b/src/packages/file-server/btrfs/snapshots.ts @@ -1,4 +1,4 @@ -import { type Subvolume } from "./subvolume"; +import { type SubvolumeSnapshot } from "./subvolume-snapshot"; import getLogger from "@cocalc/backend/logger"; const logger = getLogger("file-server:storage-btrfs:snapshots"); @@ -30,17 +30,17 @@ export interface SnapshotCounts { } export async function updateRollingSnapshots({ - subvolume, + snapshot, counts, }: { - subvolume: Subvolume; + snapshot: SubvolumeSnapshot; counts?: Partial; }) { counts = { ...DEFAULT_SNAPSHOT_COUNTS, ...counts }; - const changed = await subvolume.hasUnsavedChanges(); + const changed = await snapshot.hasUnsavedChanges(); logger.debug("updateRollingSnapshots", { - name: subvolume.name, + name: snapshot.subvolume.name, counts, changed, }); @@ -50,9 +50,9 @@ export async function updateRollingSnapshots({ } // get exactly the iso timestamp snapshot names: - const snapshots = (await subvolume.snapshots()).filter((x) => - DATE_REGEXP.test(x), - ); + const snapshots = (await snapshot.ls()) + .map((x) => x.name) + .filter((name) => DATE_REGEXP.test(name)); snapshots.sort(); if (snapshots.length > 0) { const age = Date.now() - new Date(snapshots.slice(-1)[0]).valueOf(); @@ -61,7 +61,7 @@ export async function updateRollingSnapshots({ if (age < SNAPSHOT_INTERVALS_MS[key]) { // no need to snapshot since there is already a sufficiently recent snapshot logger.debug("updateRollingSnapshots: no need to snapshot", { - name: subvolume.name, + name: snapshot.subvolume.name, }); return; } @@ -72,14 +72,14 @@ export async function updateRollingSnapshots({ } // make a new snapshot - const snapshot = new Date().toISOString(); - await subvolume.createSnapshot(snapshot); + const name = new Date().toISOString(); + await snapshot.create(name); // delete extra snapshots - snapshots.push(snapshot); + snapshots.push(name); const toDelete = snapshotsToDelete({ counts, snapshots }); - for (const snapshot of toDelete) { + for (const expired of toDelete) { try { - await subvolume.deleteSnapshot(snapshot); + await snapshot.delete(expired); } catch { // some snapshots can't be deleted, e.g., they were used for the last send. } diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index c4563fbdd9..2380298090 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -23,17 +23,17 @@ export class SubvolumeBup { // timeout used for bup index and bup save commands timeout = 30 * 60 * 1000, }: { timeout?: number } = {}) => { - if (await this.subvolume.snapshotExists(BUP_SNAPSHOT)) { + if (await this.subvolume.snapshot.exists(BUP_SNAPSHOT)) { logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`); - await this.subvolume.deleteSnapshot(BUP_SNAPSHOT); + await this.subvolume.snapshot.delete(BUP_SNAPSHOT); } try { logger.debug( `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, ); - await this.subvolume.createSnapshot(BUP_SNAPSHOT); + await this.subvolume.snapshot.create(BUP_SNAPSHOT); const target = this.subvolume.normalize( - this.subvolume.snapshotPath(BUP_SNAPSHOT), + this.subvolume.snapshot.path(BUP_SNAPSHOT), ); logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`); @@ -67,7 +67,7 @@ export class SubvolumeBup { }); } finally { logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`); - await this.subvolume.deleteSnapshot(BUP_SNAPSHOT); + await this.subvolume.snapshot.delete(BUP_SNAPSHOT); } }; @@ -78,7 +78,7 @@ export class SubvolumeBup { } path = normalize(path); // ... but to avoid potential data loss, we make a snapshot before deleting it. - await this.subvolume.createSnapshot(); + await this.subvolume.snapshot.create(); const i = path.indexOf("/"); // remove the commit name // remove the target we're about to restore await this.subvolume.fs.rm(path.slice(i + 1), { recursive: true }); diff --git a/src/packages/file-server/btrfs/subvolume-snapshot.ts b/src/packages/file-server/btrfs/subvolume-snapshot.ts new file mode 100644 index 0000000000..4fb74a50ac --- /dev/null +++ b/src/packages/file-server/btrfs/subvolume-snapshot.ts @@ -0,0 +1,111 @@ +import { type Subvolume } from "./subvolume"; +import { btrfs } from "./util"; +import getLogger from "@cocalc/backend/logger"; +import { join } from "path"; +import { type DirectoryListingEntry } from "@cocalc/util/types"; +import { SnapshotCounts, updateRollingSnapshots } from "./snapshots"; + +export const SNAPSHOTS = ".snapshots"; +const logger = getLogger("file-server:storage-btrfs:subvolume-snapshot"); + +export class SubvolumeSnapshot { + public readonly snapshotsDir: string; + + constructor(public subvolume: Subvolume) { + this.snapshotsDir = join(this.subvolume.path, SNAPSHOTS); + } + + path = (snapshot?: string, ...segments) => { + if (!snapshot) { + return SNAPSHOTS; + } + return join(SNAPSHOTS, snapshot, ...segments); + }; + + private makeSnapshotsDir = async () => { + if (await this.subvolume.fs.exists(SNAPSHOTS)) { + return; + } + await this.subvolume.fs.mkdir(SNAPSHOTS); + await this.subvolume.fs.chmod(SNAPSHOTS, "0555"); + }; + + create = async (name?: string) => { + if (name?.startsWith(".")) { + throw Error("snapshot name must not start with '.'"); + } + name ??= new Date().toISOString(); + logger.debug("create", { name, subvolume: this.subvolume.name }); + await this.makeSnapshotsDir(); + await btrfs({ + args: [ + "subvolume", + "snapshot", + "-r", + this.subvolume.path, + join(this.snapshotsDir, name), + ], + }); + }; + + ls = async (): Promise => { + await this.makeSnapshotsDir(); + return await this.subvolume.fs.ls(SNAPSHOTS, { hidden: false }); + }; + + lock = async (name: string) => { + if (await this.subvolume.fs.exists(this.path(name))) { + this.subvolume.fs.writeFile(this.path(`.${name}.lock`), ""); + } else { + throw Error(`snapshot ${name} does not exist`); + } + }; + + unlock = async (name: string) => { + await this.subvolume.fs.rm(this.path(`.${name}.lock`)); + }; + + exists = async (name: string) => { + return await this.subvolume.fs.exists(this.path(name)); + }; + + delete = async (name) => { + if (await this.subvolume.fs.exists(this.path(`.${name}.lock`))) { + throw Error(`snapshot ${name} is locked`); + } + await btrfs({ + args: ["subvolume", "delete", join(this.snapshotsDir, name)], + }); + }; + + // update the rolling snapshots schedule + update = async (counts?: Partial) => { + return await updateRollingSnapshots({ snapshot: this, counts }); + }; + + // has newly written changes since last snapshot + hasUnsavedChanges = async (): Promise => { + const s = await this.ls(); + if (s.length == 0) { + // more than just the SNAPSHOTS directory? + const v = await this.subvolume.fs.ls("", { hidden: true }); + if (v.length == 0 || (v.length == 1 && v[0].name == SNAPSHOTS)) { + return false; + } + return true; + } + const pathGen = await getGeneration(this.subvolume.path); + const snapGen = await getGeneration( + join(this.snapshotsDir, s[s.length - 1].name), + ); + return snapGen < pathGen; + }; +} + +async function getGeneration(path: string): Promise { + const { stdout } = await btrfs({ + args: ["subvolume", "show", path], + verbose: false, + }); + return parseInt(stdout.split("Generation:")[1].split("\n")[0].trim()); +} diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 206fdc45e5..ea2a72b777 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -6,12 +6,11 @@ import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; import { exists, listdir, mkdirp, sudo } from "./util"; import { join, normalize } from "path"; -import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots"; import { SubvolumeFilesystem } from "./subvolume-fs"; import { SubvolumeBup } from "./subvolume-bup"; +import { SubvolumeSnapshot } from "./subvolume-snapshot"; import getLogger from "@cocalc/backend/logger"; -export const SNAPSHOTS = ".snapshots"; const SEND_SNAPSHOT_PREFIX = "send-"; const PAD = 4; @@ -27,26 +26,26 @@ export class Subvolume { public readonly filesystem: Filesystem; public readonly path: string; - public readonly snapshotsDir: string; public readonly fs: SubvolumeFilesystem; public readonly bup: SubvolumeBup; + public readonly snapshot: SubvolumeSnapshot; constructor({ filesystem, name }: Options) { this.filesystem = filesystem; this.name = name; this.path = join(filesystem.opts.mount, name); - this.snapshotsDir = join(this.path, SNAPSHOTS); this.fs = new SubvolumeFilesystem(this); this.bup = new SubvolumeBup(this); + this.snapshot = new SubvolumeSnapshot(this); } init = async () => { if (!(await exists(this.path))) { + logger.debug(`creating ${this.name} at ${this.path}`); await sudo({ command: "btrfs", args: ["subvolume", "create", this.path], }); - await this.makeSnapshotsDir(); await this.chown(this.path); await this.size( this.filesystem.opts.defaultSize ?? DEFAULT_SUBVOLUME_SIZE, @@ -166,96 +165,6 @@ export class Subvolume { return { used, free, size }; }; - ///////////// - // SNAPSHOTS - ///////////// - snapshotPath = (snapshot: string, ...segments) => { - return join(SNAPSHOTS, snapshot, ...segments); - }; - - private makeSnapshotsDir = async () => { - if (await exists(this.snapshotsDir)) { - return; - } - await mkdirp([this.snapshotsDir]); - await this.chown(this.snapshotsDir); - await sudo({ command: "chmod", args: ["a-w", this.snapshotsDir] }); - }; - - createSnapshot = async (name?: string) => { - name ??= new Date().toISOString(); - logger.debug("createSnapshot", { name, subvolume: this.name }); - await this.makeSnapshotsDir(); - await sudo({ - command: "btrfs", - args: [ - "subvolume", - "snapshot", - "-r", - this.path, - join(this.snapshotsDir, name), - ], - }); - }; - - snapshots = async (): Promise => { - return (await listdir(this.snapshotsDir)).sort(); - }; - - lockSnapshot = async (name) => { - if (await exists(join(this.snapshotsDir, name))) { - await sudo({ - command: "touch", - args: [join(this.snapshotsDir, `.${name}.lock`)], - }); - } else { - throw Error(`snapshot ${name} does not exist`); - } - }; - - unlockSnapshot = async (name) => { - await sudo({ - command: "rm", - args: ["-f", join(this.snapshotsDir, `.${name}.lock`)], - }); - }; - - snapshotExists = async (name: string) => { - return await exists(join(this.snapshotsDir, name)); - }; - - deleteSnapshot = async (name) => { - if (await exists(join(this.snapshotsDir, `.${name}.lock`))) { - throw Error(`snapshot ${name} is locked`); - } - await sudo({ - command: "btrfs", - args: ["subvolume", "delete", join(this.snapshotsDir, name)], - }); - }; - - updateRollingSnapshots = async (counts?: Partial) => { - return await updateRollingSnapshots({ subvolume: this, counts }); - }; - - // has newly written changes since last snapshot - hasUnsavedChanges = async (): Promise => { - const s = await this.snapshots(); - if (s.length == 0) { - // more than just the SNAPSHOTS directory? - const v = await listdir(this.path); - if (v.length == 0 || (v.length == 1 && v[0] == this.snapshotsDir)) { - return false; - } - return true; - } - const pathGen = await getGeneration(this.path); - const snapGen = await getGeneration( - join(this.snapshotsDir, s[s.length - 1]), - ); - return snapGen < pathGen; - }; - ///////////// // BTRFS send/recv // Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since: @@ -278,7 +187,7 @@ export class Subvolume { const streams = new Set( await listdir(join(this.filesystem.streams, this.name)), ); - const allSnapshots = await this.snapshots(); + const allSnapshots = (await this.snapshot.ls()).map((x) => x.name); const snapshots = allSnapshots.filter( (x) => x.startsWith(SEND_SNAPSHOT_PREFIX) && streams.has(x), ); @@ -298,22 +207,22 @@ export class Subvolume { } const send = `${SEND_SNAPSHOT_PREFIX}${seq}`; if (allSnapshots.includes(send)) { - await this.deleteSnapshot(send); + await this.snapshot.delete(send); } - await this.createSnapshot(send); + await this.snapshot.create(send); await sudo({ command: "btrfs", args: [ "send", "--compressed-data", - join(this.snapshotsDir, send), - ...(last ? ["-p", join(this.snapshotsDir, parent)] : []), + join(this.snapshot.path(), send), + ...(last ? ["-p", this.snapshot.path(parent)] : []), "-f", join(this.filesystem.streams, this.name, send), ], }); if (parent) { - await this.deleteSnapshot(parent); + await this.snapshot.delete(parent); } }; @@ -330,15 +239,6 @@ export class Subvolume { // }; } -async function getGeneration(path: string): Promise { - const { stdout } = await sudo({ - command: "btrfs", - args: ["subvolume", "show", path], - verbose: false, - }); - return parseInt(stdout.split("Generation:")[1].split("\n")[0].trim()); -} - const cache = refCache({ name: "btrfs-subvolumes", createObject: async (options: Options) => { diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index e22a69377e..364b74ba0e 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -30,7 +30,7 @@ describe("operations with subvolumes", () => { const vol = await fs.subvolume("cocalc"); expect(vol.name).toBe("cocalc"); // it has no snapshots - expect(await vol.snapshots()).toEqual([]); + expect(await vol.snapshot.ls()).toEqual([]); }); it("our subvolume is in the list", async () => { diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts index a9f0c5e33c..0937b7e7e5 100644 --- a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -22,21 +22,23 @@ describe(`stress test creating ${numSnapshots} snapshots`, () => { const start = Date.now(); for (let i = 0; i < numSnapshots; i++) { await writeFile(join(vol.path, `${i}.txt`), "world"); - await vol.createSnapshot(`snap${i}`); + await vol.snapshot.create(`snap${i}`); snaps.push(`snap${i}`); } log( `created ${Math.round((numSnapshots / (Date.now() - start)) * 1000)} snapshots per second in serial`, ); snaps.sort(); - expect(await vol.snapshots()).toEqual(snaps); + expect((await vol.snapshot.ls()).map(({ name }) => name).sort()).toEqual( + snaps.sort(), + ); }); it(`delete our ${numSnapshots} snapshots`, async () => { for (let i = 0; i < numSnapshots; i++) { - await vol.deleteSnapshot(`snap${i}`); + await vol.snapshot.delete(`snap${i}`); } - expect(await vol.snapshots()).toEqual([]); + expect(await vol.snapshot.ls()).toEqual([]); }); }); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index ecfdecc24a..e68d837db7 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -86,8 +86,8 @@ describe("the filesystem operations", () => { let origStat; it("snapshot filesystem and see file is in snapshot", async () => { - await vol.createSnapshot("snap"); - const s = await vol.fs.ls(vol.snapshotPath("snap")); + await vol.snapshot.create("snap"); + const s = await vol.fs.ls(vol.snapshot.path("snap")); expect(s).toEqual([{ name: "a.txt", mtime: s[0].mtime, size: 5 }]); const stat = await vol.fs.stat("a.txt"); @@ -101,11 +101,11 @@ describe("the filesystem operations", () => { }); it("snapshot still exists", async () => { - expect(await vol.fs.exists(vol.snapshotPath("snap", "a.txt"))); + expect(await vol.fs.exists(vol.snapshot.path("snap", "a.txt"))); }); it("copy file from snapshot and note it has the same mode as before (so much nicer than what happens with zfs)", async () => { - await vol.fs.copyFile(vol.snapshotPath("snap", "a.txt"), "a.txt"); + await vol.fs.copyFile(vol.snapshot.path("snap", "a.txt"), "a.txt"); const stat = await vol.fs.stat("a.txt"); expect(stat.mode).toEqual(origStat.mode); }); @@ -179,47 +179,50 @@ describe("test snapshots", () => { it("creates a volume and write a file to it", async () => { vol = await fs.subvolume("snapper"); - expect(await vol.hasUnsavedChanges()).toBe(false); + expect(await vol.snapshot.hasUnsavedChanges()).toBe(false); await vol.fs.writeFile("a.txt", "hello"); - expect(await vol.hasUnsavedChanges()).toBe(true); + expect(await vol.snapshot.hasUnsavedChanges()).toBe(true); }); it("snapshot the volume", async () => { - expect(await vol.snapshots()).toEqual([]); - await vol.createSnapshot("snap1"); - expect(await vol.snapshots()).toEqual(["snap1"]); - expect(await vol.hasUnsavedChanges()).toBe(false); + expect(await vol.snapshot.ls()).toEqual([]); + await vol.snapshot.create("snap1"); + expect((await vol.snapshot.ls()).map((x) => x.name)).toEqual(["snap1"]); + expect(await vol.snapshot.hasUnsavedChanges()).toBe(false); }); it("create a file see that we know there are unsaved changes", async () => { await vol.fs.writeFile("b.txt", "world"); await sudo({ command: "sync" }); - expect(await vol.hasUnsavedChanges()).toBe(true); + expect(await vol.snapshot.hasUnsavedChanges()).toBe(true); }); it("delete our file, but then read it in a snapshot", async () => { await vol.fs.unlink("a.txt"); - const b = await vol.fs.readFile(vol.snapshotPath("snap1", "a.txt"), "utf8"); + const b = await vol.fs.readFile( + vol.snapshot.path("snap1", "a.txt"), + "utf8", + ); expect(b).toEqual("hello"); }); it("verifies snapshot exists", async () => { - expect(await vol.snapshotExists("snap1")).toBe(true); - expect(await vol.snapshotExists("snap2")).toBe(false); + expect(await vol.snapshot.exists("snap1")).toBe(true); + expect(await vol.snapshot.exists("snap2")).toBe(false); }); it("lock our snapshot and confirm it prevents deletion", async () => { - await vol.lockSnapshot("snap1"); + await vol.snapshot.lock("snap1"); expect(async () => { - await vol.deleteSnapshot("snap1"); + await vol.snapshot.delete("snap1"); }).rejects.toThrow("locked"); }); it("unlock our snapshot and delete it", async () => { - await vol.unlockSnapshot("snap1"); - await vol.deleteSnapshot("snap1"); - expect(await vol.snapshotExists("snap1")).toBe(false); - expect(await vol.snapshots()).toEqual([]); + await vol.snapshot.unlock("snap1"); + await vol.snapshot.delete("snap1"); + expect(await vol.snapshot.exists("snap1")).toBe(false); + expect(await vol.snapshot.ls()).toEqual([]); }); }); @@ -283,9 +286,9 @@ describe.only("test bup backups", () => { }); it("most recent snapshot has a backup before the restore", async () => { - const s = await vol.snapshots(); - const recent = s.slice(-1)[0]; - const p = vol.snapshotPath(recent, "mydir", "file.txt"); + const s = await vol.snapshot.ls(); + const recent = s.slice(-1)[0].name; + const p = vol.snapshot.path(recent, "mydir", "file.txt"); expect(await vol.fs.readFile(p, "utf8")).toEqual("changed"); }); }); diff --git a/src/packages/file-server/btrfs/util.ts b/src/packages/file-server/btrfs/util.ts index c8dc221411..981e4d96b0 100644 --- a/src/packages/file-server/btrfs/util.ts +++ b/src/packages/file-server/btrfs/util.ts @@ -50,6 +50,12 @@ export async function sudo( }); } +export async function btrfs( + opts: Partial, +) { + return await sudo({ ...opts, command: "btrfs" }); +} + export async function rm(paths: string[]) { if (paths.length == 0) return; await sudo({ command: "rm", args: paths }); From 8458477b4cfd53305cf1970f8444aebc1c9d3d39 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 21:19:17 +0000 Subject: [PATCH 39/47] btrfs: snapshot --> snapshots --- src/packages/file-server/btrfs/filesystem.ts | 2 +- src/packages/file-server/btrfs/snapshots.ts | 28 ++++++------ .../file-server/btrfs/subvolume-bup.ts | 12 ++--- ...ume-snapshot.ts => subvolume-snapshots.ts} | 6 +-- src/packages/file-server/btrfs/subvolume.ts | 18 ++++---- .../file-server/btrfs/test/filesystem.test.ts | 2 +- .../btrfs/test/subvolume-stress.test.ts | 8 ++-- .../file-server/btrfs/test/subvolume.test.ts | 44 +++++++++---------- 8 files changed, 60 insertions(+), 60 deletions(-) rename src/packages/file-server/btrfs/{subvolume-snapshot.ts => subvolume-snapshots.ts} (96%) diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 39416e3c87..599297c4be 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -14,7 +14,7 @@ a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({devic import refCache from "@cocalc/util/refcache"; import { exists, isdir, listdir, mkdirp, rmdir, sudo } from "./util"; import { subvolume, type Subvolume } from "./subvolume"; -import { SNAPSHOTS } from "./subvolume-snapshot"; +import { SNAPSHOTS } from "./subvolume-snapshots"; import { join, normalize } from "path"; // default size of btrfs filesystem if creating an image file. diff --git a/src/packages/file-server/btrfs/snapshots.ts b/src/packages/file-server/btrfs/snapshots.ts index 10bc81d81a..eb1fbfb1a7 100644 --- a/src/packages/file-server/btrfs/snapshots.ts +++ b/src/packages/file-server/btrfs/snapshots.ts @@ -1,4 +1,4 @@ -import { type SubvolumeSnapshot } from "./subvolume-snapshot"; +import { type SubvolumeSnapshots } from "./subvolume-snapshots"; import getLogger from "@cocalc/backend/logger"; const logger = getLogger("file-server:storage-btrfs:snapshots"); @@ -30,17 +30,17 @@ export interface SnapshotCounts { } export async function updateRollingSnapshots({ - snapshot, + snapshots, counts, }: { - snapshot: SubvolumeSnapshot; + snapshots: SubvolumeSnapshots; counts?: Partial; }) { counts = { ...DEFAULT_SNAPSHOT_COUNTS, ...counts }; - const changed = await snapshot.hasUnsavedChanges(); + const changed = await snapshots.hasUnsavedChanges(); logger.debug("updateRollingSnapshots", { - name: snapshot.subvolume.name, + name: snapshots.subvolume.name, counts, changed, }); @@ -50,18 +50,18 @@ export async function updateRollingSnapshots({ } // get exactly the iso timestamp snapshot names: - const snapshots = (await snapshot.ls()) + const snapshotNames = (await snapshots.ls()) .map((x) => x.name) .filter((name) => DATE_REGEXP.test(name)); - snapshots.sort(); - if (snapshots.length > 0) { - const age = Date.now() - new Date(snapshots.slice(-1)[0]).valueOf(); + snapshotNames.sort(); + if (snapshotNames.length > 0) { + const age = Date.now() - new Date(snapshotNames.slice(-1)[0]).valueOf(); for (const key in SNAPSHOT_INTERVALS_MS) { if (counts[key]) { if (age < SNAPSHOT_INTERVALS_MS[key]) { // no need to snapshot since there is already a sufficiently recent snapshot logger.debug("updateRollingSnapshots: no need to snapshot", { - name: snapshot.subvolume.name, + name: snapshots.subvolume.name, }); return; } @@ -73,13 +73,13 @@ export async function updateRollingSnapshots({ // make a new snapshot const name = new Date().toISOString(); - await snapshot.create(name); + await snapshots.create(name); // delete extra snapshots - snapshots.push(name); - const toDelete = snapshotsToDelete({ counts, snapshots }); + snapshotNames.push(name); + const toDelete = snapshotsToDelete({ counts, snapshots: snapshotNames }); for (const expired of toDelete) { try { - await snapshot.delete(expired); + await snapshots.delete(expired); } catch { // some snapshots can't be deleted, e.g., they were used for the last send. } diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index 2380298090..814e0f2f1c 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -23,17 +23,17 @@ export class SubvolumeBup { // timeout used for bup index and bup save commands timeout = 30 * 60 * 1000, }: { timeout?: number } = {}) => { - if (await this.subvolume.snapshot.exists(BUP_SNAPSHOT)) { + if (await this.subvolume.snapshots.exists(BUP_SNAPSHOT)) { logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`); - await this.subvolume.snapshot.delete(BUP_SNAPSHOT); + await this.subvolume.snapshots.delete(BUP_SNAPSHOT); } try { logger.debug( `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, ); - await this.subvolume.snapshot.create(BUP_SNAPSHOT); + await this.subvolume.snapshots.create(BUP_SNAPSHOT); const target = this.subvolume.normalize( - this.subvolume.snapshot.path(BUP_SNAPSHOT), + this.subvolume.snapshots.path(BUP_SNAPSHOT), ); logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`); @@ -67,7 +67,7 @@ export class SubvolumeBup { }); } finally { logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`); - await this.subvolume.snapshot.delete(BUP_SNAPSHOT); + await this.subvolume.snapshots.delete(BUP_SNAPSHOT); } }; @@ -78,7 +78,7 @@ export class SubvolumeBup { } path = normalize(path); // ... but to avoid potential data loss, we make a snapshot before deleting it. - await this.subvolume.snapshot.create(); + await this.subvolume.snapshots.create(); const i = path.indexOf("/"); // remove the commit name // remove the target we're about to restore await this.subvolume.fs.rm(path.slice(i + 1), { recursive: true }); diff --git a/src/packages/file-server/btrfs/subvolume-snapshot.ts b/src/packages/file-server/btrfs/subvolume-snapshots.ts similarity index 96% rename from src/packages/file-server/btrfs/subvolume-snapshot.ts rename to src/packages/file-server/btrfs/subvolume-snapshots.ts index 4fb74a50ac..59732d9e79 100644 --- a/src/packages/file-server/btrfs/subvolume-snapshot.ts +++ b/src/packages/file-server/btrfs/subvolume-snapshots.ts @@ -6,9 +6,9 @@ import { type DirectoryListingEntry } from "@cocalc/util/types"; import { SnapshotCounts, updateRollingSnapshots } from "./snapshots"; export const SNAPSHOTS = ".snapshots"; -const logger = getLogger("file-server:storage-btrfs:subvolume-snapshot"); +const logger = getLogger("file-server:storage-btrfs:subvolume-snapshots"); -export class SubvolumeSnapshot { +export class SubvolumeSnapshots { public readonly snapshotsDir: string; constructor(public subvolume: Subvolume) { @@ -80,7 +80,7 @@ export class SubvolumeSnapshot { // update the rolling snapshots schedule update = async (counts?: Partial) => { - return await updateRollingSnapshots({ snapshot: this, counts }); + return await updateRollingSnapshots({ snapshots: this, counts }); }; // has newly written changes since last snapshot diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index ea2a72b777..7b79a163d3 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -8,7 +8,7 @@ import { exists, listdir, mkdirp, sudo } from "./util"; import { join, normalize } from "path"; import { SubvolumeFilesystem } from "./subvolume-fs"; import { SubvolumeBup } from "./subvolume-bup"; -import { SubvolumeSnapshot } from "./subvolume-snapshot"; +import { SubvolumeSnapshots } from "./subvolume-snapshots"; import getLogger from "@cocalc/backend/logger"; const SEND_SNAPSHOT_PREFIX = "send-"; @@ -28,7 +28,7 @@ export class Subvolume { public readonly path: string; public readonly fs: SubvolumeFilesystem; public readonly bup: SubvolumeBup; - public readonly snapshot: SubvolumeSnapshot; + public readonly snapshots: SubvolumeSnapshots; constructor({ filesystem, name }: Options) { this.filesystem = filesystem; @@ -36,7 +36,7 @@ export class Subvolume { this.path = join(filesystem.opts.mount, name); this.fs = new SubvolumeFilesystem(this); this.bup = new SubvolumeBup(this); - this.snapshot = new SubvolumeSnapshot(this); + this.snapshots = new SubvolumeSnapshots(this); } init = async () => { @@ -187,7 +187,7 @@ export class Subvolume { const streams = new Set( await listdir(join(this.filesystem.streams, this.name)), ); - const allSnapshots = (await this.snapshot.ls()).map((x) => x.name); + const allSnapshots = (await this.snapshots.ls()).map((x) => x.name); const snapshots = allSnapshots.filter( (x) => x.startsWith(SEND_SNAPSHOT_PREFIX) && streams.has(x), ); @@ -207,22 +207,22 @@ export class Subvolume { } const send = `${SEND_SNAPSHOT_PREFIX}${seq}`; if (allSnapshots.includes(send)) { - await this.snapshot.delete(send); + await this.snapshots.delete(send); } - await this.snapshot.create(send); + await this.snapshots.create(send); await sudo({ command: "btrfs", args: [ "send", "--compressed-data", - join(this.snapshot.path(), send), - ...(last ? ["-p", this.snapshot.path(parent)] : []), + join(this.snapshots.path(), send), + ...(last ? ["-p", this.snapshots.path(parent)] : []), "-f", join(this.filesystem.streams, this.name, send), ], }); if (parent) { - await this.snapshot.delete(parent); + await this.snapshots.delete(parent); } }; diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index 364b74ba0e..eaea74cfcf 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -30,7 +30,7 @@ describe("operations with subvolumes", () => { const vol = await fs.subvolume("cocalc"); expect(vol.name).toBe("cocalc"); // it has no snapshots - expect(await vol.snapshot.ls()).toEqual([]); + expect(await vol.snapshots.ls()).toEqual([]); }); it("our subvolume is in the list", async () => { diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts index 0937b7e7e5..9f8567a7c1 100644 --- a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -22,23 +22,23 @@ describe(`stress test creating ${numSnapshots} snapshots`, () => { const start = Date.now(); for (let i = 0; i < numSnapshots; i++) { await writeFile(join(vol.path, `${i}.txt`), "world"); - await vol.snapshot.create(`snap${i}`); + await vol.snapshots.create(`snap${i}`); snaps.push(`snap${i}`); } log( `created ${Math.round((numSnapshots / (Date.now() - start)) * 1000)} snapshots per second in serial`, ); snaps.sort(); - expect((await vol.snapshot.ls()).map(({ name }) => name).sort()).toEqual( + expect((await vol.snapshots.ls()).map(({ name }) => name).sort()).toEqual( snaps.sort(), ); }); it(`delete our ${numSnapshots} snapshots`, async () => { for (let i = 0; i < numSnapshots; i++) { - await vol.snapshot.delete(`snap${i}`); + await vol.snapshots.delete(`snap${i}`); } - expect(await vol.snapshot.ls()).toEqual([]); + expect(await vol.snapshots.ls()).toEqual([]); }); }); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index e68d837db7..f5d8ed87c8 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -86,8 +86,8 @@ describe("the filesystem operations", () => { let origStat; it("snapshot filesystem and see file is in snapshot", async () => { - await vol.snapshot.create("snap"); - const s = await vol.fs.ls(vol.snapshot.path("snap")); + await vol.snapshots.create("snap"); + const s = await vol.fs.ls(vol.snapshots.path("snap")); expect(s).toEqual([{ name: "a.txt", mtime: s[0].mtime, size: 5 }]); const stat = await vol.fs.stat("a.txt"); @@ -101,11 +101,11 @@ describe("the filesystem operations", () => { }); it("snapshot still exists", async () => { - expect(await vol.fs.exists(vol.snapshot.path("snap", "a.txt"))); + expect(await vol.fs.exists(vol.snapshots.path("snap", "a.txt"))); }); it("copy file from snapshot and note it has the same mode as before (so much nicer than what happens with zfs)", async () => { - await vol.fs.copyFile(vol.snapshot.path("snap", "a.txt"), "a.txt"); + await vol.fs.copyFile(vol.snapshots.path("snap", "a.txt"), "a.txt"); const stat = await vol.fs.stat("a.txt"); expect(stat.mode).toEqual(origStat.mode); }); @@ -179,50 +179,50 @@ describe("test snapshots", () => { it("creates a volume and write a file to it", async () => { vol = await fs.subvolume("snapper"); - expect(await vol.snapshot.hasUnsavedChanges()).toBe(false); + expect(await vol.snapshots.hasUnsavedChanges()).toBe(false); await vol.fs.writeFile("a.txt", "hello"); - expect(await vol.snapshot.hasUnsavedChanges()).toBe(true); + expect(await vol.snapshots.hasUnsavedChanges()).toBe(true); }); it("snapshot the volume", async () => { - expect(await vol.snapshot.ls()).toEqual([]); - await vol.snapshot.create("snap1"); - expect((await vol.snapshot.ls()).map((x) => x.name)).toEqual(["snap1"]); - expect(await vol.snapshot.hasUnsavedChanges()).toBe(false); + expect(await vol.snapshots.ls()).toEqual([]); + await vol.snapshots.create("snap1"); + expect((await vol.snapshots.ls()).map((x) => x.name)).toEqual(["snap1"]); + expect(await vol.snapshots.hasUnsavedChanges()).toBe(false); }); it("create a file see that we know there are unsaved changes", async () => { await vol.fs.writeFile("b.txt", "world"); await sudo({ command: "sync" }); - expect(await vol.snapshot.hasUnsavedChanges()).toBe(true); + expect(await vol.snapshots.hasUnsavedChanges()).toBe(true); }); it("delete our file, but then read it in a snapshot", async () => { await vol.fs.unlink("a.txt"); const b = await vol.fs.readFile( - vol.snapshot.path("snap1", "a.txt"), + vol.snapshots.path("snap1", "a.txt"), "utf8", ); expect(b).toEqual("hello"); }); it("verifies snapshot exists", async () => { - expect(await vol.snapshot.exists("snap1")).toBe(true); - expect(await vol.snapshot.exists("snap2")).toBe(false); + expect(await vol.snapshots.exists("snap1")).toBe(true); + expect(await vol.snapshots.exists("snap2")).toBe(false); }); it("lock our snapshot and confirm it prevents deletion", async () => { - await vol.snapshot.lock("snap1"); + await vol.snapshots.lock("snap1"); expect(async () => { - await vol.snapshot.delete("snap1"); + await vol.snapshots.delete("snap1"); }).rejects.toThrow("locked"); }); it("unlock our snapshot and delete it", async () => { - await vol.snapshot.unlock("snap1"); - await vol.snapshot.delete("snap1"); - expect(await vol.snapshot.exists("snap1")).toBe(false); - expect(await vol.snapshot.ls()).toEqual([]); + await vol.snapshots.unlock("snap1"); + await vol.snapshots.delete("snap1"); + expect(await vol.snapshots.exists("snap1")).toBe(false); + expect(await vol.snapshots.ls()).toEqual([]); }); }); @@ -286,9 +286,9 @@ describe.only("test bup backups", () => { }); it("most recent snapshot has a backup before the restore", async () => { - const s = await vol.snapshot.ls(); + const s = await vol.snapshots.ls(); const recent = s.slice(-1)[0].name; - const p = vol.snapshot.path(recent, "mydir", "file.txt"); + const p = vol.snapshots.path(recent, "mydir", "file.txt"); expect(await vol.fs.readFile(p, "utf8")).toEqual("changed"); }); }); From 2fe725ad06894934cb07ca38720b1fdbe1eec6bc Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 22:01:23 +0000 Subject: [PATCH 40/47] btrfs: refactor quota --- src/packages/file-server/btrfs/filesystem.ts | 4 +- .../file-server/btrfs/subvolume-bup.ts | 28 ++- .../file-server/btrfs/subvolume-quota.ts | 86 +++++++++ src/packages/file-server/btrfs/subvolume.ts | 167 +----------------- .../file-server/btrfs/test/subvolume.test.ts | 8 +- 5 files changed, 118 insertions(+), 175 deletions(-) create mode 100644 src/packages/file-server/btrfs/subvolume-quota.ts diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 599297c4be..3bbaea1837 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -211,9 +211,9 @@ export class Filesystem { } const src = await this.subvolume(source); const vol = await this.subvolume(name); - const { size } = await src.usage(); + const { size } = await src.quota.get(); if (size) { - await vol.size(size); + await vol.quota.set(size); } return vol; }; diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index 814e0f2f1c..69e02f2b70 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -1,3 +1,24 @@ +/* + +BUP Architecture: + +There is a single global dedup'd backup archive stored in the btrfs filesystem. +Obviously, admins should rsync this regularly to a separate location as a genuine +backup strategy. + +NOTE: we use bup instead of btrfs send/recv ! + +Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since: + - much easier to check they are valid + - decoupled from any btrfs issues + - not tied to any specific filesystem at all + - easier to offsite via incremental rsync + - much more space efficient with *global* dedup and compression + - bup is really just git, which is much more proven than even btrfs + +The drawback is speed, but that can be managed. +*/ + import { type DirectoryListingEntry } from "@cocalc/util/types"; import { type Subvolume } from "./subvolume"; import { sudo, parseBupTime } from "./util"; @@ -11,13 +32,6 @@ const logger = getLogger("file-server:storage-btrfs:subvolume-bup"); export class SubvolumeBup { constructor(private subvolume: Subvolume) {} - ///////////// - // BACKUPS - // There is a single global dedup'd backup archive stored in the btrfs filesystem. - // Obviously, admins should rsync this regularly to a separate location as a genuine - // backup strategy. - ///////////// - // create a new bup backup save = async ({ // timeout used for bup index and bup save commands diff --git a/src/packages/file-server/btrfs/subvolume-quota.ts b/src/packages/file-server/btrfs/subvolume-quota.ts new file mode 100644 index 0000000000..ac6261f5ab --- /dev/null +++ b/src/packages/file-server/btrfs/subvolume-quota.ts @@ -0,0 +1,86 @@ +import { type Subvolume } from "./subvolume"; +import { btrfs } from "./util"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("file-server:storage-btrfs:subvolume-quota"); + +export class SubvolumeQuota { + constructor(public subvolume: Subvolume) {} + + private qgroup = async () => { + const { stdout } = await btrfs({ + verbose: false, + args: ["--format=json", "qgroup", "show", "-reF", this.subvolume.path], + }); + const x = JSON.parse(stdout); + return x["qgroup-show"][0]; + }; + + get = async (): Promise<{ + size: number; + used: number; + }> => { + let { max_referenced: size, referenced: used } = await this.qgroup(); + if (size == "none") { + size = null; + } + return { + used, + size, + }; + }; + + set = async (size: string | number) => { + if (!size) { + throw Error("size must be specified"); + } + logger.debug("setQuota ", this.subvolume.path, size); + await btrfs({ + args: ["qgroup", "limit", `${size}`, this.subvolume.path], + }); + }; + + du = async () => { + return await btrfs({ + args: ["filesystem", "du", "-s", this.subvolume.path], + }); + }; + + usage = async (): Promise<{ + // used and free in bytes + used: number; + free: number; + size: number; + }> => { + const { stdout } = await btrfs({ + args: ["filesystem", "usage", "-b", this.subvolume.path], + }); + let used: number = -1; + let free: number = -1; + let size: number = -1; + for (const x of stdout.split("\n")) { + if (used == -1) { + const i = x.indexOf("Used:"); + if (i != -1) { + used = parseInt(x.split(":")[1].trim()); + continue; + } + } + if (free == -1) { + const i = x.indexOf("Free (statfs, df):"); + if (i != -1) { + free = parseInt(x.split(":")[1].trim()); + continue; + } + } + if (size == -1) { + const i = x.indexOf("Device size:"); + if (i != -1) { + size = parseInt(x.split(":")[1].trim()); + continue; + } + } + } + return { used, free, size }; + }; +} diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 7b79a163d3..5706416ad1 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -4,16 +4,14 @@ A subvolume import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; -import { exists, listdir, mkdirp, sudo } from "./util"; +import { exists, sudo } from "./util"; import { join, normalize } from "path"; import { SubvolumeFilesystem } from "./subvolume-fs"; import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; +import { SubvolumeQuota } from "./subvolume-quota"; import getLogger from "@cocalc/backend/logger"; -const SEND_SNAPSHOT_PREFIX = "send-"; -const PAD = 4; - const logger = getLogger("file-server:storage-btrfs:subvolume"); interface Options { @@ -29,6 +27,7 @@ export class Subvolume { public readonly fs: SubvolumeFilesystem; public readonly bup: SubvolumeBup; public readonly snapshots: SubvolumeSnapshots; + public readonly quota: SubvolumeQuota; constructor({ filesystem, name }: Options) { this.filesystem = filesystem; @@ -37,6 +36,7 @@ export class Subvolume { this.fs = new SubvolumeFilesystem(this); this.bup = new SubvolumeBup(this); this.snapshots = new SubvolumeSnapshots(this); + this.quota = new SubvolumeQuota(this); } init = async () => { @@ -47,7 +47,7 @@ export class Subvolume { args: ["subvolume", "create", this.path], }); await this.chown(this.path); - await this.size( + await this.quota.set( this.filesystem.opts.defaultSize ?? DEFAULT_SUBVOLUME_SIZE, ); } @@ -80,163 +80,6 @@ export class Subvolume { normalize = (path: string) => { return join(this.path, normalize(path)); }; - - ///////////// - // QUOTA - ///////////// - - private quotaInfo = async () => { - const { stdout } = await sudo({ - verbose: false, - command: "btrfs", - args: ["--format=json", "qgroup", "show", "-reF", this.path], - }); - const x = JSON.parse(stdout); - return x["qgroup-show"][0]; - }; - - quota = async (): Promise<{ - size: number; - used: number; - }> => { - let { max_referenced: size, referenced: used } = await this.quotaInfo(); - if (size == "none") { - size = null; - } - return { - used, - size, - }; - }; - - size = async (size: string | number) => { - if (!size) { - throw Error("size must be specified"); - } - await sudo({ - command: "btrfs", - args: ["qgroup", "limit", `${size}`, this.path], - }); - }; - - du = async () => { - return await sudo({ - command: "btrfs", - args: ["filesystem", "du", "-s", this.path], - }); - }; - - usage = async (): Promise<{ - // used and free in bytes - used: number; - free: number; - size: number; - }> => { - const { stdout } = await sudo({ - command: "btrfs", - args: ["filesystem", "usage", "-b", this.path], - }); - let used: number = -1; - let free: number = -1; - let size: number = -1; - for (const x of stdout.split("\n")) { - if (used == -1) { - const i = x.indexOf("Used:"); - if (i != -1) { - used = parseInt(x.split(":")[1].trim()); - continue; - } - } - if (free == -1) { - const i = x.indexOf("Free (statfs, df):"); - if (i != -1) { - free = parseInt(x.split(":")[1].trim()); - continue; - } - } - if (size == -1) { - const i = x.indexOf("Device size:"); - if (i != -1) { - size = parseInt(x.split(":")[1].trim()); - continue; - } - } - } - return { used, free, size }; - }; - - ///////////// - // BTRFS send/recv - // Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since: - // - much easier to check they are valid - // - decoupled from any btrfs issues - // - not tied to any specific filesystem at all - // - easier to offsite via incremntal rsync - // - much more space efficient with *global* dedup and compression - // - bup is really just git, which is very proven - // The drawback is speed. - ///////////// - - // this was just a quick proof of concept -- I don't like it. Should switch to using - // timestamps and a lock. - // To recover these, doing recv for each in order does work. Then you have to - // snapshot all of the results to move them. It's awkward, but efficient - // and works fine. - send = async () => { - await mkdirp([join(this.filesystem.streams, this.name)]); - const streams = new Set( - await listdir(join(this.filesystem.streams, this.name)), - ); - const allSnapshots = (await this.snapshots.ls()).map((x) => x.name); - const snapshots = allSnapshots.filter( - (x) => x.startsWith(SEND_SNAPSHOT_PREFIX) && streams.has(x), - ); - const nums = snapshots.map((x) => - parseInt(x.slice(SEND_SNAPSHOT_PREFIX.length)), - ); - nums.sort(); - const last = nums.slice(-1)[0]; - let seq, parent; - if (last) { - seq = `${last + 1}`.padStart(PAD, "0"); - const l = `${last}`.padStart(PAD, "0"); - parent = `${SEND_SNAPSHOT_PREFIX}${l}`; - } else { - seq = "1".padStart(PAD, "0"); - parent = ""; - } - const send = `${SEND_SNAPSHOT_PREFIX}${seq}`; - if (allSnapshots.includes(send)) { - await this.snapshots.delete(send); - } - await this.snapshots.create(send); - await sudo({ - command: "btrfs", - args: [ - "send", - "--compressed-data", - join(this.snapshots.path(), send), - ...(last ? ["-p", this.snapshots.path(parent)] : []), - "-f", - join(this.filesystem.streams, this.name, send), - ], - }); - if (parent) { - await this.snapshots.delete(parent); - } - }; - - // recv = async (target: string) => { - // const streamsDir = join(this.filesystem.streams, this.name); - // const streams = await listdir(streamsDir); - // streams.sort(); - // for (const stream of streams) { - // await sudo({ - // command: "btrfs", - // args: ["recv", "-f", join(streamsDir, stream)], - // }); - // } - // }; } const cache = refCache({ diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index f5d8ed87c8..a3326d8958 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -11,9 +11,9 @@ describe("setting and getting quota of a subvolume", () => { let vol: Subvolume; it("set the quota of a subvolume to 5 M", async () => { vol = await fs.subvolume("q"); - await vol.size("5M"); + await vol.quota.set("5M"); - const { size, used } = await vol.quota(); + const { size, used } = await vol.quota.get(); expect(size).toBe(5 * 1024 * 1024); expect(used).toBe(0); }); @@ -29,11 +29,11 @@ describe("setting and getting quota of a subvolume", () => { await wait({ until: async () => { await sudo({ command: "sync" }); - const { used } = await vol.usage(); + const { used } = await vol.quota.usage(); return used > 0; }, }); - const { used } = await vol.usage(); + const { used } = await vol.quota.usage(); expect(used).toBeGreaterThan(0); const v = await vol.fs.ls(""); From f22c75f798718d9929db7affe4f80698647c7e73 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 22:50:21 +0000 Subject: [PATCH 41/47] btrfs: subvolumes refactor --- src/packages/file-server/btrfs/filesystem.ts | 148 ++++-------------- src/packages/file-server/btrfs/snapshots.ts | 2 +- .../file-server/btrfs/subvolume-bup.ts | 2 +- .../file-server/btrfs/subvolume-quota.ts | 2 +- .../file-server/btrfs/subvolume-snapshots.ts | 2 +- src/packages/file-server/btrfs/subvolume.ts | 12 +- src/packages/file-server/btrfs/subvolumes.ts | 122 +++++++++++++++ .../btrfs/test/filesystem-stress.test.ts | 20 +-- .../file-server/btrfs/test/filesystem.test.ts | 42 +++-- src/packages/file-server/btrfs/test/setup.ts | 2 - .../btrfs/test/subvolume-stress.test.ts | 4 +- .../file-server/btrfs/test/subvolume.test.ts | 10 +- 12 files changed, 202 insertions(+), 166 deletions(-) create mode 100644 src/packages/file-server/btrfs/subvolumes.ts diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 3bbaea1837..16dd035304 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -1,5 +1,5 @@ /* -A BTRFS Filesystem +BTRFS Filesystem DEVELOPMENT: @@ -7,15 +7,14 @@ Start node, then: DEBUG="cocalc:*file-server*" DEBUG_CONSOLE=yes node -a = require('@cocalc/file-server/storage-btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', formatIfNeeded:true, mount:'/mnt/btrfs', uid:293597964}) +a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', formatIfNeeded:true, mount:'/mnt/btrfs', uid:293597964}) */ import refCache from "@cocalc/util/refcache"; -import { exists, isdir, listdir, mkdirp, rmdir, sudo } from "./util"; -import { subvolume, type Subvolume } from "./subvolume"; -import { SNAPSHOTS } from "./subvolume-snapshots"; -import { join, normalize } from "path"; +import { exists, mkdirp, btrfs, sudo } from "./util"; +import { join } from "path"; +import { Subvolumes } from "./subvolumes"; // default size of btrfs filesystem if creating an image file. const DEFAULT_FILESYSTEM_SIZE = "10G"; @@ -25,8 +24,6 @@ export const DEFAULT_SUBVOLUME_SIZE = "1G"; const MOUNT_ERROR = "wrong fs type, bad option, bad superblock"; -const RESERVED = new Set(["bup", "recv", "streams", SNAPSHOTS]); - export interface Options { // the underlying block device. // If this is a file (or filename) ending in .img, then it's a sparse file mounted as a loopback device. @@ -40,9 +37,6 @@ export interface Options { // where the btrfs filesystem is mounted mount: string; - // all subvolumes will have this owner - uid?: number; - // default size of newly created subvolumes defaultSize?: string | number; defaultFilesystemSize?: string | number; @@ -52,6 +46,7 @@ export class Filesystem { public readonly opts: Options; public readonly bup: string; public readonly streams: string; + public readonly subvolumes: Subvolumes; constructor(opts: Options) { opts = { @@ -62,6 +57,7 @@ export class Filesystem { this.opts = opts; this.bup = join(this.opts.mount, "bup"); this.streams = join(this.opts.mount, "streams"); + this.subvolumes = new Subvolumes(this); } init = async () => { @@ -71,8 +67,7 @@ export class Filesystem { await this.initDevice(); await this.mountFilesystem(); await sudo({ command: "chmod", args: ["a+rx", this.opts.mount] }); - await sudo({ - command: "btrfs", + await btrfs({ args: ["quota", "enable", "--simple", this.opts.mount], }); await sudo({ @@ -95,8 +90,7 @@ export class Filesystem { }; info = async (): Promise<{ [field: string]: string }> => { - const { stdout } = await sudo({ - command: "btrfs", + const { stdout } = await btrfs({ args: ["subvolume", "show", this.opts.mount], }); const obj: { [field: string]: string } = {}; @@ -108,7 +102,6 @@ export class Filesystem { return obj; }; - // private mountFilesystem = async () => { try { await this.info(); @@ -148,11 +141,25 @@ export class Filesystem { "btrfs", this.opts.mount, ); - return await sudo({ - command: "mount", - args, + { + const { stderr, exit_code } = await sudo({ + command: "mount", + args, + err_on_exit: false, + }); + if (exit_code) { + return { stderr, exit_code }; + } + } + const { stderr, exit_code } = await sudo({ + command: "chown", + args: [ + `${process.getuid?.() ?? 0}:${process.getgid?.() ?? 0}`, + this.opts.mount, + ], err_on_exit: false, }); + return { stderr, exit_code }; }; unmount = async () => { @@ -167,108 +174,7 @@ export class Filesystem { await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); }; - close = () => { - // nothing, yet - }; - - subvolume = async (name: string): Promise => { - if (RESERVED.has(name)) { - throw Error(`${name} is reserved`); - } - return await subvolume({ filesystem: this, name }); - }; - - // create a subvolume by cloning an existing one. - cloneSubvolume = async (source: string, name: string) => { - if (RESERVED.has(name)) { - throw Error(`${name} is reserved`); - } - if (!(await exists(join(this.opts.mount, source)))) { - throw Error(`subvolume ${source} does not exist`); - } - if (await exists(join(this.opts.mount, name))) { - throw Error(`subvolume ${name} already exists`); - } - await sudo({ - command: "btrfs", - args: [ - "subvolume", - "snapshot", - join(this.opts.mount, source), - join(this.opts.mount, source, name), - ], - }); - await sudo({ - command: "mv", - args: [join(this.opts.mount, source, name), join(this.opts.mount, name)], - }); - const snapdir = join(this.opts.mount, name, SNAPSHOTS); - if (await exists(snapdir)) { - const snapshots = await listdir(snapdir); - await rmdir( - snapshots.map((x) => join(this.opts.mount, name, SNAPSHOTS, x)), - ); - } - const src = await this.subvolume(source); - const vol = await this.subvolume(name); - const { size } = await src.quota.get(); - if (size) { - await vol.quota.set(size); - } - return vol; - }; - - deleteSubvolume = async (name: string) => { - await sudo({ - command: "btrfs", - args: ["subvolume", "delete", join(this.opts.mount, name)], - }); - }; - - list = async (): Promise => { - const { stdout } = await sudo({ - command: "btrfs", - args: ["subvolume", "list", this.opts.mount], - }); - return stdout - .split("\n") - .map((x) => x.split(" ").slice(-1)[0]) - .filter((x) => x) - .sort(); - }; - - rsync = async ({ - src, - target, - args = ["-axH"], - timeout = 5 * 60 * 1000, - }: { - src: string; - target: string; - args?: string[]; - timeout?: number; - }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = normalize(join(this.opts.mount, src)); - if (!srcPath.startsWith(this.opts.mount)) { - throw Error("suspicious source"); - } - let targetPath = normalize(join(this.opts.mount, target)); - if (!targetPath.startsWith(this.opts.mount)) { - throw Error("suspicious target"); - } - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { - srcPath += "/"; - if (!targetPath.endsWith("/")) { - targetPath += "/"; - } - } - return await sudo({ - command: "rsync", - args: [...args, srcPath, targetPath], - err_on_exit: false, - timeout: timeout / 1000, - }); - }; + close = () => {}; } function isImageFile(name: string) { diff --git a/src/packages/file-server/btrfs/snapshots.ts b/src/packages/file-server/btrfs/snapshots.ts index eb1fbfb1a7..daf8e09212 100644 --- a/src/packages/file-server/btrfs/snapshots.ts +++ b/src/packages/file-server/btrfs/snapshots.ts @@ -1,7 +1,7 @@ import { type SubvolumeSnapshots } from "./subvolume-snapshots"; import getLogger from "@cocalc/backend/logger"; -const logger = getLogger("file-server:storage-btrfs:snapshots"); +const logger = getLogger("file-server:btrfs:snapshots"); const DATE_REGEXP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index 69e02f2b70..21cbbff364 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -27,7 +27,7 @@ import getLogger from "@cocalc/backend/logger"; const BUP_SNAPSHOT = "temp-bup-snapshot"; -const logger = getLogger("file-server:storage-btrfs:subvolume-bup"); +const logger = getLogger("file-server:btrfs:subvolume-bup"); export class SubvolumeBup { constructor(private subvolume: Subvolume) {} diff --git a/src/packages/file-server/btrfs/subvolume-quota.ts b/src/packages/file-server/btrfs/subvolume-quota.ts index ac6261f5ab..b4288cfc57 100644 --- a/src/packages/file-server/btrfs/subvolume-quota.ts +++ b/src/packages/file-server/btrfs/subvolume-quota.ts @@ -2,7 +2,7 @@ import { type Subvolume } from "./subvolume"; import { btrfs } from "./util"; import getLogger from "@cocalc/backend/logger"; -const logger = getLogger("file-server:storage-btrfs:subvolume-quota"); +const logger = getLogger("file-server:btrfs:subvolume-quota"); export class SubvolumeQuota { constructor(public subvolume: Subvolume) {} diff --git a/src/packages/file-server/btrfs/subvolume-snapshots.ts b/src/packages/file-server/btrfs/subvolume-snapshots.ts index 59732d9e79..a230621522 100644 --- a/src/packages/file-server/btrfs/subvolume-snapshots.ts +++ b/src/packages/file-server/btrfs/subvolume-snapshots.ts @@ -6,7 +6,7 @@ import { type DirectoryListingEntry } from "@cocalc/util/types"; import { SnapshotCounts, updateRollingSnapshots } from "./snapshots"; export const SNAPSHOTS = ".snapshots"; -const logger = getLogger("file-server:storage-btrfs:subvolume-snapshots"); +const logger = getLogger("file-server:btrfs:subvolume-snapshots"); export class SubvolumeSnapshots { public readonly snapshotsDir: string; diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 5706416ad1..ba2011d16b 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -12,7 +12,7 @@ import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; import getLogger from "@cocalc/backend/logger"; -const logger = getLogger("file-server:storage-btrfs:subvolume"); +const logger = getLogger("file-server:btrfs:subvolume"); interface Options { filesystem: Filesystem; @@ -62,15 +62,16 @@ export class Subvolume { delete this.path; // @ts-ignore delete this.snapshotsDir; + for (const sub of ["fs", "bup", "snapshots", "quota"]) { + this[sub].close?.(); + delete this[sub]; + } }; private chown = async (path: string) => { - if (!this.filesystem.opts.uid) { - return; - } await sudo({ command: "chown", - args: [`${this.filesystem.opts.uid}:${this.filesystem.opts.uid}`, path], + args: [`${process.getuid?.() ?? 0}:${process.getgid?.() ?? 0}`, path], }); }; @@ -84,6 +85,7 @@ export class Subvolume { const cache = refCache({ name: "btrfs-subvolumes", + createKey: ({ name }) => name, createObject: async (options: Options) => { const subvolume = new Subvolume(options); await subvolume.init(); diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts new file mode 100644 index 0000000000..debfd30e6f --- /dev/null +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -0,0 +1,122 @@ +import { type Filesystem } from "./filesystem"; +import { subvolume, type Subvolume } from "./subvolume"; +import getLogger from "@cocalc/backend/logger"; +import { SNAPSHOTS } from "./subvolume-snapshots"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { join, normalize } from "path"; +import { btrfs } from "./util"; +import { rename, readdir, rm, stat } from "fs/promises"; +import { executeCode } from "@cocalc/backend/execute-code"; + +const RESERVED = new Set(["bup", "recv", "streams", SNAPSHOTS]); + +const logger = getLogger("file-server:btrfs:subvolumes"); + +export class Subvolumes { + constructor(public filesystem: Filesystem) {} + + get = async (name: string): Promise => { + if (RESERVED.has(name)) { + throw Error(`${name} is reserved`); + } + return await subvolume({ filesystem: this.filesystem, name }); + }; + + // create a subvolume by cloning an existing one. + clone = async (source: string, dest: string) => { + logger.debug("clone ", { source, dest }); + if (RESERVED.has(dest)) { + throw Error(`${dest} is reserved`); + } + if (!(await exists(join(this.filesystem.opts.mount, source)))) { + throw Error(`subvolume ${source} does not exist`); + } + if (await exists(join(this.filesystem.opts.mount, dest))) { + throw Error(`subvolume ${dest} already exists`); + } + await btrfs({ + args: [ + "subvolume", + "snapshot", + join(this.filesystem.opts.mount, source), + join(this.filesystem.opts.mount, source, dest), + ], + }); + await rename( + join(this.filesystem.opts.mount, source, dest), + join(this.filesystem.opts.mount, dest), + ); + const snapdir = join(this.filesystem.opts.mount, dest, SNAPSHOTS); + if (await exists(snapdir)) { + const snapshots = await readdir(snapdir); + const f = async (x) => { + await rm(join(this.filesystem.opts.mount, dest, SNAPSHOTS, x), { + recursive: true, + force: true, + }); + }; + await Promise.all(snapshots.map(f)); + } + const src = await this.get(source); + const dst = await this.get(dest); + const { size } = await src.quota.get(); + if (size) { + await dst.quota.set(size); + } + return dst; + }; + + delete = async (name: string) => { + await btrfs({ + args: ["subvolume", "delete", join(this.filesystem.opts.mount, name)], + }); + }; + + list = async (): Promise => { + const { stdout } = await btrfs({ + args: ["subvolume", "list", this.filesystem.opts.mount], + }); + return stdout + .split("\n") + .map((x) => x.split(" ").slice(-1)[0]) + .filter((x) => x) + .sort(); + }; + + rsync = async ({ + src, + target, + args = ["-axH"], + timeout = 5 * 60 * 1000, + }: { + src: string; + target: string; + args?: string[]; + timeout?: number; + }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { + let srcPath = normalize(join(this.filesystem.opts.mount, src)); + if (!srcPath.startsWith(this.filesystem.opts.mount)) { + throw Error("suspicious source"); + } + let targetPath = normalize(join(this.filesystem.opts.mount, target)); + if (!targetPath.startsWith(this.filesystem.opts.mount)) { + throw Error("suspicious target"); + } + if (!srcPath.endsWith("/") && (await isdir(srcPath))) { + srcPath += "/"; + if (!targetPath.endsWith("/")) { + targetPath += "/"; + } + } + return await executeCode({ + command: "rsync", + args: [...args, srcPath, targetPath], + err_on_exit: false, + timeout: timeout / 1000, + }); + }; +} + +async function isdir(path: string) { + return (await stat(path)).isDirectory(); +} diff --git a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts index 176cdf6a5a..5c3455da53 100644 --- a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts @@ -10,7 +10,7 @@ describe("stress operations with subvolumes", () => { it(`create ${count1} subvolumes in serial`, async () => { const t = Date.now(); for (let i = 0; i < count1; i++) { - await fs.subvolume(`${i}`); + await fs.subvolumes.get(`${i}`); } log( `created ${Math.round((count1 / (Date.now() - t)) * 1000)} subvolumes per second serial`, @@ -18,7 +18,7 @@ describe("stress operations with subvolumes", () => { }); it("list them and confirm", async () => { - const v = await fs.list(); + const v = await fs.subvolumes.list(); expect(v.length).toBe(count1); }); @@ -27,7 +27,7 @@ describe("stress operations with subvolumes", () => { const v: any[] = []; const t = Date.now(); for (let i = 0; i < count2; i++) { - v.push(fs.subvolume(`p-${i}`)); + v.push(fs.subvolumes.get(`p-${i}`)); } await Promise.all(v); log( @@ -36,13 +36,13 @@ describe("stress operations with subvolumes", () => { }); it("list them and confirm", async () => { - const v = await fs.list(); + const v = await fs.subvolumes.list(); expect(v.length).toBe(count1 + count2); }); it("write a file to each volume", async () => { - for (const name of await fs.list()) { - const vol = await fs.subvolume(name); + for (const name of await fs.subvolumes.list()) { + const vol = await fs.subvolumes.get(name); await vol.fs.writeFile("a.txt", "hi"); } }); @@ -50,7 +50,7 @@ describe("stress operations with subvolumes", () => { it("clone the first group in serial", async () => { const t = Date.now(); for (let i = 0; i < count1; i++) { - await fs.cloneSubvolume(`${i}`, `clone-of-${i}`); + await fs.subvolumes.clone(`${i}`, `clone-of-${i}`); } log( `cloned ${Math.round((count1 / (Date.now() - t)) * 1000)} subvolumes per second serial`, @@ -61,7 +61,7 @@ describe("stress operations with subvolumes", () => { const t = Date.now(); const v: any[] = []; for (let i = 0; i < count2; i++) { - v.push(fs.cloneSubvolume(`p-${i}`, `clone-of-p-${i}`)); + v.push(fs.subvolumes.clone(`p-${i}`, `clone-of-p-${i}`)); } await Promise.all(v); log( @@ -72,7 +72,7 @@ describe("stress operations with subvolumes", () => { it("delete the first batch serial", async () => { const t = Date.now(); for (let i = 0; i < count1; i++) { - await fs.deleteSubvolume(`${i}`); + await fs.subvolumes.delete(`${i}`); } log( `deleted ${Math.round((count1 / (Date.now() - t)) * 1000)} subvolumes per second serial`, @@ -83,7 +83,7 @@ describe("stress operations with subvolumes", () => { const v: any[] = []; const t = Date.now(); for (let i = 0; i < count2; i++) { - v.push(fs.deleteSubvolume(`p-${i}`)); + v.push(fs.subvolumes.delete(`p-${i}`)); } await Promise.all(v); log( diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index eaea74cfcf..172e2fb6ba 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -15,61 +15,69 @@ describe("some basic tests", () => { }); it("lists the subvolumes (there are none)", async () => { - expect(await fs.list()).toEqual([]); + expect(await fs.subvolumes.list()).toEqual([]); }); }); describe("operations with subvolumes", () => { it("can't use a reserved subvolume name", async () => { expect(async () => { - await fs.subvolume("bup"); + await fs.subvolumes.get("bup"); }).rejects.toThrow("is reserved"); }); it("creates a subvolume", async () => { - const vol = await fs.subvolume("cocalc"); + const vol = await fs.subvolumes.get("cocalc"); expect(vol.name).toBe("cocalc"); // it has no snapshots expect(await vol.snapshots.ls()).toEqual([]); }); it("our subvolume is in the list", async () => { - expect(await fs.list()).toEqual(["cocalc"]); + expect(await fs.subvolumes.list()).toEqual(["cocalc"]); }); it("create another two subvolumes", async () => { - await fs.subvolume("sagemath"); - await fs.subvolume("a-math"); + await fs.subvolumes.get("sagemath"); + await fs.subvolumes.get("a-math"); // list is sorted: - expect(await fs.list()).toEqual(["a-math", "cocalc", "sagemath"]); + expect(await fs.subvolumes.list()).toEqual([ + "a-math", + "cocalc", + "sagemath", + ]); }); it("delete a subvolume", async () => { - await fs.deleteSubvolume("a-math"); - expect(await fs.list()).toEqual(["cocalc", "sagemath"]); + await fs.subvolumes.delete("a-math"); + expect(await fs.subvolumes.list()).toEqual(["cocalc", "sagemath"]); }); it("clone a subvolume", async () => { - await fs.cloneSubvolume("sagemath", "cython"); - expect(await fs.list()).toEqual(["cocalc", "cython", "sagemath"]); + await fs.subvolumes.clone("sagemath", "cython"); + expect(await fs.subvolumes.list()).toEqual([ + "cocalc", + "cython", + "sagemath", + ]); }); it("rsync from one volume to another", async () => { - await fs.rsync({ src: "sagemath", target: "cython" }); + await fs.subvolumes.rsync({ src: "sagemath", target: "cython" }); }); it("rsync an actual file", async () => { - const sagemath = await fs.subvolume("sagemath"); - const cython = await fs.subvolume("cython"); + const sagemath = await fs.subvolumes.get("sagemath"); + const cython = await fs.subvolumes.get("cython"); await sagemath.fs.writeFile("README.md", "hi"); - await fs.rsync({ src: "sagemath", target: "cython" }); + await fs.subvolumes.rsync({ src: "sagemath", target: "cython" }); const copy = await cython.fs.readFile("README.md", "utf8"); expect(copy).toEqual("hi"); }); it("clone a subvolume with contents", async () => { - await fs.cloneSubvolume("cython", "pyrex"); - const pyrex = await fs.subvolume("pyrex"); + await fs.subvolumes.clone("cython", "pyrex"); + const pyrex = await fs.subvolumes.get("pyrex"); const clone = await pyrex.fs.readFile("README.md", "utf8"); expect(clone).toEqual("hi"); }); diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index 918c1e69f2..0904f2a0b5 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -2,7 +2,6 @@ import { filesystem, type Filesystem, } from "@cocalc/file-server/btrfs/filesystem"; -import process from "node:process"; import { chmod, mkdtemp, mkdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; @@ -50,7 +49,6 @@ export async function before() { device: join(tempDir, "btrfs.img"), formatIfNeeded: true, mount: join(tempDir, "mnt"), - uid: process.getuid?.(), }); } diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts index 9f8567a7c1..29bc048e69 100644 --- a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -14,7 +14,7 @@ beforeAll(before); describe(`stress test creating ${numSnapshots} snapshots`, () => { let vol: Subvolume; it("creates a volume and write a file to it", async () => { - vol = await fs.subvolume("stress"); + vol = await fs.subvolumes.get("stress"); }); it(`create file and snapshot the volume ${numSnapshots} times`, async () => { @@ -45,7 +45,7 @@ describe(`stress test creating ${numSnapshots} snapshots`, () => { describe(`create ${numFiles} files`, () => { let vol: Subvolume; it("creates a volume", async () => { - vol = await fs.subvolume("many-files"); + vol = await fs.subvolumes.get("many-files"); }); it(`creates ${numFiles} files`, async () => { diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index a3326d8958..c7c16a40bf 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -10,7 +10,7 @@ beforeAll(before); describe("setting and getting quota of a subvolume", () => { let vol: Subvolume; it("set the quota of a subvolume to 5 M", async () => { - vol = await fs.subvolume("q"); + vol = await fs.subvolumes.get("q"); await vol.quota.set("5M"); const { size, used } = await vol.quota.get(); @@ -53,12 +53,12 @@ describe("the filesystem operations", () => { let vol: Subvolume; it("creates a volume and get empty listing", async () => { - vol = await fs.subvolume("fs"); + vol = await fs.subvolumes.get("fs"); expect(await vol.fs.ls("")).toEqual([]); }); it("error listing non-existent path", async () => { - vol = await fs.subvolume("fs"); + vol = await fs.subvolumes.get("fs"); expect(async () => { await vol.fs.ls("no-such-path"); }).rejects.toThrow("ENOENT"); @@ -178,7 +178,7 @@ describe("test snapshots", () => { let vol: Subvolume; it("creates a volume and write a file to it", async () => { - vol = await fs.subvolume("snapper"); + vol = await fs.subvolumes.get("snapper"); expect(await vol.snapshots.hasUnsavedChanges()).toBe(false); await vol.fs.writeFile("a.txt", "hello"); expect(await vol.snapshots.hasUnsavedChanges()).toBe(true); @@ -229,7 +229,7 @@ describe("test snapshots", () => { describe.only("test bup backups", () => { let vol: Subvolume; it("creates a volume", async () => { - vol = await fs.subvolume("bup-test"); + vol = await fs.subvolumes.get("bup-test"); await vol.fs.writeFile("a.txt", "hello"); }); From 46ad996d4d10361178d65cfb97a9e8c37aee3c9d Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 23:00:08 +0000 Subject: [PATCH 42/47] btrfs cleanup code --- src/packages/file-server/btrfs/filesystem.ts | 45 +++++++++++--------- src/packages/file-server/btrfs/subvolumes.ts | 2 +- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 16dd035304..b52b4f477b 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -12,9 +12,12 @@ a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({device:'/tmp/ */ import refCache from "@cocalc/util/refcache"; -import { exists, mkdirp, btrfs, sudo } from "./util"; +import { mkdirp, btrfs, sudo } from "./util"; import { join } from "path"; import { Subvolumes } from "./subvolumes"; +import { mkdir } from "fs/promises"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { executeCode } from "@cocalc/backend/execute-code"; // default size of btrfs filesystem if creating an image file. const DEFAULT_FILESYSTEM_SIZE = "10G"; @@ -45,7 +48,6 @@ export interface Options { export class Filesystem { public readonly opts: Options; public readonly bup: string; - public readonly streams: string; public readonly subvolumes: Subvolumes; constructor(opts: Options) { @@ -56,26 +58,30 @@ export class Filesystem { }; this.opts = opts; this.bup = join(this.opts.mount, "bup"); - this.streams = join(this.opts.mount, "streams"); this.subvolumes = new Subvolumes(this); } init = async () => { - await mkdirp( - [this.opts.mount, this.streams, this.bup].filter((x) => x) as string[], - ); + await mkdirp([this.opts.mount]); await this.initDevice(); await this.mountFilesystem(); await sudo({ command: "chmod", args: ["a+rx", this.opts.mount] }); await btrfs({ args: ["quota", "enable", "--simple", this.opts.mount], }); + await this.initBup(); + }; + + unmount = async () => { await sudo({ - bash: true, - command: `BUP_DIR=${this.bup} bup init`, + command: "umount", + args: [this.opts.mount], + err_on_exit: true, }); }; + close = () => {}; + private initDevice = async () => { if (!isImageFile(this.opts.device)) { // raw block device -- nothing to do @@ -125,6 +131,10 @@ export class Filesystem { } }; + private formatDevice = async () => { + await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); + }; + private _mountFilesystem = async () => { const args: string[] = isImageFile(this.opts.device) ? ["-o", "loop"] : []; args.push( @@ -162,19 +172,16 @@ export class Filesystem { return { stderr, exit_code }; }; - unmount = async () => { - await sudo({ - command: "umount", - args: [this.opts.mount], - err_on_exit: true, + private initBup = async () => { + if (!(await exists(this.bup))) { + await mkdir(this.bup); + } + await executeCode({ + command: "bup", + args: ["init"], + env: { BUP_DIR: this.bup }, }); }; - - private formatDevice = async () => { - await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); - }; - - close = () => {}; } function isImageFile(name: string) { diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts index debfd30e6f..98489a66c6 100644 --- a/src/packages/file-server/btrfs/subvolumes.ts +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -8,7 +8,7 @@ import { btrfs } from "./util"; import { rename, readdir, rm, stat } from "fs/promises"; import { executeCode } from "@cocalc/backend/execute-code"; -const RESERVED = new Set(["bup", "recv", "streams", SNAPSHOTS]); +const RESERVED = new Set(["bup", SNAPSHOTS]); const logger = getLogger("file-server:btrfs:subvolumes"); From 832e1248655bec40d44966bf0ada28e8a9226a84 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 23:14:30 +0000 Subject: [PATCH 43/47] btrfs: cloning with subvolumes (fix permission issues) --- .../file-server/btrfs/subvolume-snapshots.ts | 2 +- src/packages/file-server/btrfs/subvolumes.ts | 15 ++++++--------- .../file-server/btrfs/test/filesystem.test.ts | 19 +++++++++++++++++++ .../file-server/btrfs/test/subvolume.test.ts | 4 ++-- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/packages/file-server/btrfs/subvolume-snapshots.ts b/src/packages/file-server/btrfs/subvolume-snapshots.ts index a230621522..ffe71fe6fc 100644 --- a/src/packages/file-server/btrfs/subvolume-snapshots.ts +++ b/src/packages/file-server/btrfs/subvolume-snapshots.ts @@ -27,7 +27,7 @@ export class SubvolumeSnapshots { return; } await this.subvolume.fs.mkdir(SNAPSHOTS); - await this.subvolume.fs.chmod(SNAPSHOTS, "0555"); + await this.subvolume.fs.chmod(SNAPSHOTS, "0550"); }; create = async (name?: string) => { diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts index 98489a66c6..29725f6043 100644 --- a/src/packages/file-server/btrfs/subvolumes.ts +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -5,7 +5,7 @@ import { SNAPSHOTS } from "./subvolume-snapshots"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { join, normalize } from "path"; import { btrfs } from "./util"; -import { rename, readdir, rm, stat } from "fs/promises"; +import { chmod, rename, readdir, rm, stat } from "fs/promises"; import { executeCode } from "@cocalc/backend/execute-code"; const RESERVED = new Set(["bup", SNAPSHOTS]); @@ -48,14 +48,11 @@ export class Subvolumes { ); const snapdir = join(this.filesystem.opts.mount, dest, SNAPSHOTS); if (await exists(snapdir)) { - const snapshots = await readdir(snapdir); - const f = async (x) => { - await rm(join(this.filesystem.opts.mount, dest, SNAPSHOTS, x), { - recursive: true, - force: true, - }); - }; - await Promise.all(snapshots.map(f)); + await chmod(snapdir, "0700"); + await rm(snapdir, { + recursive: true, + force: true, + }); } const src = await this.get(source); const dst = await this.get(dest); diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index 172e2fb6ba..673980af89 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -83,4 +83,23 @@ describe("operations with subvolumes", () => { }); }); +describe("clone of a subvolume with snapshots should have no snapshots", () => { + it("creates a subvolume, a file, and a snapshot", async () => { + const x = await fs.subvolumes.get("my-volume"); + await x.fs.writeFile("abc.txt", "hi"); + await x.snapshots.create("my-snap"); + }); + + it("clones my-volume", async () => { + await fs.subvolumes.clone("my-volume", "my-clone"); + }); + + it("clone has no snapshots", async () => { + const clone = await fs.subvolumes.get("my-clone"); + expect(await clone.fs.readFile("abc.txt", "utf8")).toEqual("hi"); + expect(await clone.snapshots.ls()).toEqual([]); + await clone.snapshots.create("my-clone-snap"); + }); +}); + afterAll(after); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index c7c16a40bf..e586cc656b 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -125,11 +125,11 @@ describe("the filesystem operations", () => { it("make a file readonly, then change it back", async () => { await vol.fs.writeFile("c.txt", "hi"); - await vol.fs.chmod("c.txt", "444"); + await vol.fs.chmod("c.txt", "440"); expect(async () => { await vol.fs.appendFile("c.txt", " there"); }).rejects.toThrow("EACCES"); - await vol.fs.chmod("c.txt", "666"); + await vol.fs.chmod("c.txt", "660"); await vol.fs.appendFile("c.txt", " there"); }); From f05f5118d4bad2f3ce3ad4c46830c79800f0c3f6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 15 Jul 2025 23:22:27 +0000 Subject: [PATCH 44/47] btrfs: cleanup util --- src/packages/file-server/btrfs/filesystem.ts | 1 - src/packages/file-server/btrfs/subvolume.ts | 4 ++- src/packages/file-server/btrfs/subvolumes.ts | 8 ++--- .../btrfs/test/filesystem-stress.test.ts | 4 +-- src/packages/file-server/btrfs/util.ts | 32 ++----------------- 5 files changed, 9 insertions(+), 40 deletions(-) diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index b52b4f477b..927fb23548 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -65,7 +65,6 @@ export class Filesystem { await mkdirp([this.opts.mount]); await this.initDevice(); await this.mountFilesystem(); - await sudo({ command: "chmod", args: ["a+rx", this.opts.mount] }); await btrfs({ args: ["quota", "enable", "--simple", this.opts.mount], }); diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index ba2011d16b..3f77abd9c4 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -4,12 +4,14 @@ A subvolume import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; -import { exists, sudo } from "./util"; +import { sudo } from "./util"; import { join, normalize } from "path"; import { SubvolumeFilesystem } from "./subvolume-fs"; import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; + import getLogger from "@cocalc/backend/logger"; const logger = getLogger("file-server:btrfs:subvolume"); diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts index 29725f6043..0f8da1468f 100644 --- a/src/packages/file-server/btrfs/subvolumes.ts +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -4,8 +4,8 @@ import getLogger from "@cocalc/backend/logger"; import { SNAPSHOTS } from "./subvolume-snapshots"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { join, normalize } from "path"; -import { btrfs } from "./util"; -import { chmod, rename, readdir, rm, stat } from "fs/promises"; +import { btrfs, isdir } from "./util"; +import { chmod, rename, rm } from "node:fs/promises"; import { executeCode } from "@cocalc/backend/execute-code"; const RESERVED = new Set(["bup", SNAPSHOTS]); @@ -113,7 +113,3 @@ export class Subvolumes { }); }; } - -async function isdir(path: string) { - return (await stat(path)).isDirectory(); -} diff --git a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts index 5c3455da53..a2e1de9531 100644 --- a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts @@ -2,8 +2,8 @@ import { before, after, fs } from "./setup"; beforeAll(before); -//const log = console.log; -const log = (..._args) => {}; +const DEBUG = false; +const log = DEBUG ? console.log : (..._args) => {}; describe("stress operations with subvolumes", () => { const count1 = 10; diff --git a/src/packages/file-server/btrfs/util.ts b/src/packages/file-server/btrfs/util.ts index 981e4d96b0..b6408f8440 100644 --- a/src/packages/file-server/btrfs/util.ts +++ b/src/packages/file-server/btrfs/util.ts @@ -4,29 +4,17 @@ import { } from "@cocalc/util/types/execute-code"; import { executeCode } from "@cocalc/backend/execute-code"; import getLogger from "@cocalc/backend/logger"; +import { stat } from "node:fs/promises"; const logger = getLogger("file-server:storage:util"); const DEFAULT_EXEC_TIMEOUT_MS = 60 * 1000; -export async function exists(path: string) { - try { - await sudo({ command: "ls", args: [path], verbose: false }); - return true; - } catch { - return false; - } -} - export async function mkdirp(paths: string[]) { if (paths.length == 0) return; await sudo({ command: "mkdir", args: ["-p", ...paths] }); } -export async function chmod(args: string[]) { - await sudo({ command: "chmod", args: args }); -} - export async function sudo( opts: ExecuteCodeOptions & { desc?: string }, ): Promise { @@ -56,24 +44,8 @@ export async function btrfs( return await sudo({ ...opts, command: "btrfs" }); } -export async function rm(paths: string[]) { - if (paths.length == 0) return; - await sudo({ command: "rm", args: paths }); -} - -export async function rmdir(paths: string[]) { - if (paths.length == 0) return; - await sudo({ command: "rmdir", args: paths }); -} - -export async function listdir(path: string) { - const { stdout } = await sudo({ command: "ls", args: ["-1", path] }); - return stdout.split("\n").filter((x) => x); -} - export async function isdir(path: string) { - const { stdout } = await sudo({ command: "stat", args: ["-c", "%F", path] }); - return stdout.trim() == "directory"; + return (await stat(path)).isDirectory(); } export function parseBupTime(s: string): Date { From a03d97e781b836350af590c4ae6a94096b69e7d2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 16 Jul 2025 14:34:35 +0000 Subject: [PATCH 45/47] btrfs testing: fix initializing many block devices at once --- src/packages/file-server/btrfs/test/setup.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index 0904f2a0b5..c9736c8b79 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -23,10 +23,13 @@ async function ensureMoreLoops() { await stat(`/dev/loop${i}`); continue; } catch {} - await sudo({ - command: "mknod", - args: ["-m660", `/dev/loop${i}`, "b", "7", `${i}`], - }); + try { + // also try/catch this because ensureMoreLoops happens in parallel many times at once... + await sudo({ + command: "mknod", + args: ["-m660", `/dev/loop${i}`, "b", "7", `${i}`], + }); + } catch {} await sudo({ command: "chown", args: ["root:disk", `/dev/loop${i}`] }); } } From d4e03b1a98129f263943d14c5dfa535487f3dcd6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 16 Jul 2025 15:33:55 +0000 Subject: [PATCH 46/47] delete the support code for using the socketio cluster module - confusing - not needed for anything anymore --- src/package.json | 3 +- src/packages/pnpm-lock.yaml | 28 +--- src/packages/server/conat/persist/index.ts | 35 +++++ src/packages/server/conat/socketio/cluster.ts | 137 ------------------ .../server/conat/socketio/start-cluster.ts | 58 -------- src/packages/server/package.json | 5 +- 6 files changed, 40 insertions(+), 226 deletions(-) create mode 100644 src/packages/server/conat/persist/index.ts delete mode 100644 src/packages/server/conat/socketio/cluster.ts delete mode 100644 src/packages/server/conat/socketio/start-cluster.ts diff --git a/src/package.json b/src/package.json index ef75b66aa8..d0f0701277 100644 --- a/src/package.json +++ b/src/package.json @@ -18,11 +18,10 @@ "version-check": "pip3 install typing_extensions mypy || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test", - "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter --retries=1", + "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter,file-server --retries=1", "depcheck": "cd packages && pnpm run -r --parallel depcheck", "prettier-all": "cd packages/", "local-ci": "./scripts/ci.sh", - "conat-server": "cd packages/server && pnpm conat-server", "conat-connections": "cd packages/backend && pnpm conat-connections", "conat-watch": "cd packages/backend && pnpm conat-watch", "conat-inventory": "cd packages/backend && pnpm conat-inventory" diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 5ba4513c11..1fe4499fe0 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1240,12 +1240,6 @@ importers: '@sendgrid/mail': specifier: ^8.1.4 version: 8.1.5 - '@socket.io/cluster-adapter': - specifier: ^0.2.2 - version: 0.2.2(socket.io-adapter@2.5.5) - '@socket.io/sticky': - specifier: ^1.0.4 - version: 1.0.4 '@zxcvbn-ts/core': specifier: ^3.0.4 version: 3.0.4 @@ -3953,18 +3947,9 @@ packages: '@sinonjs/fake-timers@13.0.5': resolution: {integrity: sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==} - '@socket.io/cluster-adapter@0.2.2': - resolution: {integrity: sha512-/tNcY6qQx0BOgjl4mFk3YxX6pjaPdEyeWhP88Ea9gTlISY4SfA7t8VxbryeAs5/9QgXzChlvSN/i37Gog3kWag==} - engines: {node: '>=10.0.0'} - peerDependencies: - socket.io-adapter: ^2.4.0 - '@socket.io/component-emitter@3.1.2': resolution: {integrity: sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==} - '@socket.io/sticky@1.0.4': - resolution: {integrity: sha512-VuauT5CJLvzYtKIgouFSQ8rUaygseR+zRutnwh6ZA2QYcXx+8g52EoJ8V2SLxfo+Tfs3ELUDy08oEXxlWNrxaw==} - '@stripe/react-stripe-js@3.7.0': resolution: {integrity: sha512-PYls/2S9l0FF+2n0wHaEJsEU8x7CmBagiH7zYOsxbBlLIHEsqUIQ4MlIAbV9Zg6xwT8jlYdlRIyBTHmO3yM7kQ==} peerDependencies: @@ -14153,17 +14138,8 @@ snapshots: dependencies: '@sinonjs/commons': 3.0.1 - '@socket.io/cluster-adapter@0.2.2(socket.io-adapter@2.5.5)': - dependencies: - debug: 4.3.7 - socket.io-adapter: 2.5.5 - transitivePeerDependencies: - - supports-color - '@socket.io/component-emitter@3.1.2': {} - '@socket.io/sticky@1.0.4': {} - '@stripe/react-stripe-js@3.7.0(@stripe/stripe-js@5.10.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': dependencies: '@stripe/stripe-js': 5.10.0 @@ -15345,7 +15321,7 @@ snapshots: axios@1.10.0: dependencies: - follow-redirects: 1.15.9(debug@4.4.1) + follow-redirects: 1.15.9 form-data: 4.0.3 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -17481,6 +17457,8 @@ snapshots: dependencies: dtype: 2.0.0 + follow-redirects@1.15.9: {} + follow-redirects@1.15.9(debug@4.4.1): optionalDependencies: debug: 4.4.1 diff --git a/src/packages/server/conat/persist/index.ts b/src/packages/server/conat/persist/index.ts new file mode 100644 index 0000000000..a17ba419ed --- /dev/null +++ b/src/packages/server/conat/persist/index.ts @@ -0,0 +1,35 @@ +import { loadConatConfiguration } from "./configuration"; +import { initPersistServer } from "@cocalc/backend/conat/persist"; +import { conatPersistCount } from "@cocalc/backend/data"; + +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("server:conat:persist"); + +export async function initConatPersist() { + logger.debug("initPersistServer: sqlite3 stream persistence", { + conatPersistCount, + }); + if (!conatPersistCount || conatPersistCount <= 1) { + // only 1, so no need to use separate processes + await loadConatConfiguration(); + initPersistServer(); + return; + } + + // more than 1 so no possible value to multiple servers if we don't + // use separate processes + createPersistCluster(); +} + +async function createPersistCluster() { + logger.debug( + "initPersistServer: creating cluster with", + conatPersistCount, + "nodes", + ); + await loadConatConfiguration(); + for (let i = 0; i < conatPersistCount; i++) { + initPersistServer(); + } +} diff --git a/src/packages/server/conat/socketio/cluster.ts b/src/packages/server/conat/socketio/cluster.ts deleted file mode 100644 index 7ada080fd6..0000000000 --- a/src/packages/server/conat/socketio/cluster.ts +++ /dev/null @@ -1,137 +0,0 @@ -/* - -To start this: - - pnpm conat-server - -Run this to be able to use all the cores, since nodejs is (mostly) single threaded. -*/ - -import { init as createConatServer } from "@cocalc/conat/core/server"; -import cluster from "node:cluster"; -import * as http from "http"; -import { availableParallelism } from "os"; -import { - setupMaster as setupPrimarySticky, - setupWorker, -} from "@socket.io/sticky"; -import { createAdapter, setupPrimary } from "@socket.io/cluster-adapter"; -import { getUser, isAllowed } from "./auth"; -import { secureRandomString } from "@cocalc/backend/misc"; -import basePath from "@cocalc/backend/base-path"; -import port from "@cocalc/backend/port"; -import { - conatSocketioCount, - conatClusterPort, - conatClusterHealthPort, -} from "@cocalc/backend/data"; -import { loadConatConfiguration } from "../configuration"; -import { join } from "path"; - -// ensure conat logging, credentials, etc. is setup -import "@cocalc/backend/conat"; - -console.log(`* CONAT Core Pub/Sub Server on port ${port} *`); - -async function primary() { - console.log(`Socketio Server Primary pid=${process.pid} is running`); - - await loadConatConfiguration(); - - const httpServer = http.createServer(); - setupPrimarySticky(httpServer, { - loadBalancingMethod: "least-connection", - }); - - setupPrimary(); - cluster.setupPrimary({ serialization: "advanced" }); - httpServer.listen(getPort()); - - if (conatClusterHealthPort) { - console.log( - `starting /health socketio server on port ${conatClusterHealthPort}`, - ); - const healthServer = http.createServer(); - healthServer.listen(conatClusterHealthPort); - healthServer.on("request", (req, res) => { - // unhealthy if >3 deaths in 1 min - handleHealth(req, res, recentDeaths.length <= 3, "Too many worker exits"); - }); - } - - const numWorkers = conatSocketioCount - ? conatSocketioCount - : availableParallelism(); - const systemAccountPassword = await secureRandomString(32); - for (let i = 0; i < numWorkers; i++) { - cluster.fork({ SYSTEM_ACCOUNT_PASSWORD: systemAccountPassword }); - } - console.log({ numWorkers, port, basePath }); - - const recentDeaths: number[] = []; - cluster.on("exit", (worker) => { - if (conatClusterHealthPort) { - recentDeaths.push(Date.now()); - // Remove entries older than X seconds (e.g. 60s) - while (recentDeaths.length && recentDeaths[0] < Date.now() - 60_000) { - recentDeaths.shift(); - } - } - - console.log(`Worker ${worker.process.pid} died, so making a new one`); - cluster.fork(); - }); -} - -async function worker() { - console.log("BASE_PATH=", process.env.BASE_PATH); - await loadConatConfiguration(); - - const path = join(basePath, "conat"); - console.log(`Socketio Worker pid=${process.pid} started with path=${path}`); - - const httpServer = http.createServer(); - const id = `${cluster.worker?.id ?? ""}`; - const systemAccountPassword = process.env.SYSTEM_ACCOUNT_PASSWORD; - delete process.env.SYSTEM_ACCOUNT_PASSWORD; - - const conatServer = createConatServer({ - path, - httpServer, - id, - getUser, - isAllowed, - systemAccountPassword, - // port -- server needs to know implicitly to make a clients - port: getPort(), - }); - conatServer.io.adapter(createAdapter()); - setupWorker(conatServer.io); -} - -function getPort() { - return conatClusterPort ? conatClusterPort : port; -} - -if (cluster.isPrimary) { - primary(); -} else { - worker(); -} - -function handleHealth( - req: http.IncomingMessage, - res: http.ServerResponse, - status: boolean, - msg?: string, -) { - if (req.method === "GET") { - if (status) { - res.statusCode = 200; - res.end("healthy"); - } else { - res.statusCode = 500; - res.end(msg || "Unhealthy"); - } - } -} diff --git a/src/packages/server/conat/socketio/start-cluster.ts b/src/packages/server/conat/socketio/start-cluster.ts deleted file mode 100644 index da6da241f8..0000000000 --- a/src/packages/server/conat/socketio/start-cluster.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { spawn, ChildProcess } from "node:child_process"; -import { join } from "path"; -import { conatSocketioCount, conatClusterPort } from "@cocalc/backend/data"; -import basePath from "@cocalc/backend/base-path"; - -const servers: { close: Function }[] = []; - -export default function startCluster({ - port = conatClusterPort, - numWorkers = conatSocketioCount, -}: { port?: number; numWorkers?: number } = {}) { - const child: ChildProcess = spawn( - process.argv[0], - [join(__dirname, "cluster.js")], - { - stdio: "inherit", - detached: false, - cwd: __dirname, - env: { - ...process.env, - PORT: `${port}`, - CONAT_SOCKETIO_COUNT: `${numWorkers}`, - BASE_PATH: basePath, - }, - }, - ); - - let closed = false; - const close = () => { - if (closed) return; - closed = true; - if (!child?.pid) return; - try { - process.kill(child.pid, "SIGKILL"); - } catch { - // already dead or not found - } - }; - - const server = { - close, - }; - servers.push(server); - return server; -} - -process.once("exit", () => { - for (const { close } of servers) { - try { - close(); - } catch {} - } -}); -["SIGINT", "SIGTERM", "SIGQUIT"].forEach((sig) => { - process.once(sig, () => { - process.exit(); - }); -}); diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 10173819ce..fd46d02793 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -37,8 +37,7 @@ "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput ", "test": "TZ=UTC jest --forceExit --runInBand", "depcheck": "pnpx depcheck", - "prepublishOnly": "test", - "conat-server": "node ./dist/conat/socketio/cluster.js" + "prepublishOnly": "test" }, "author": "SageMath, Inc.", "license": "SEE LICENSE.md", @@ -68,8 +67,6 @@ "@passport-next/passport-oauth2": "^2.1.4", "@sendgrid/client": "^8.1.4", "@sendgrid/mail": "^8.1.4", - "@socket.io/cluster-adapter": "^0.2.2", - "@socket.io/sticky": "^1.0.4", "@zxcvbn-ts/core": "^3.0.4", "@zxcvbn-ts/language-common": "^3.0.4", "@zxcvbn-ts/language-en": "^3.0.2", From 8a826081ef72884096b6ba2a171e97b5a0d528bd Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 16 Jul 2025 16:11:45 +0000 Subject: [PATCH 47/47] use separate process for each persist server; also include pid in logger output --- src/packages/backend/logger.ts | 2 +- src/packages/server/conat/index.ts | 14 ++---------- src/packages/server/conat/persist/index.ts | 7 +++--- .../conat/persist/start-persist-node.ts | 16 ++++++++++++++ .../conat/socketio/start-cluster-node.ts | 22 ++++++++++++++++++- 5 files changed, 44 insertions(+), 17 deletions(-) create mode 100644 src/packages/server/conat/persist/start-persist-node.ts diff --git a/src/packages/backend/logger.ts b/src/packages/backend/logger.ts index 3b89197d97..6d1dcea6a3 100644 --- a/src/packages/backend/logger.ts +++ b/src/packages/backend/logger.ts @@ -128,7 +128,7 @@ function initTransports() { // Similar as in debug source code, except I stuck a timestamp // at the beginning, which I like... except also aware of // non-printf formatting. - const line = `${new Date().toISOString()}: ${myFormat(...args)}\n`; + const line = `${new Date().toISOString()} (${process.pid}):${myFormat(...args)}\n`; if (transports.console) { // the console transport: diff --git a/src/packages/server/conat/index.ts b/src/packages/server/conat/index.ts index d78fab9d5f..56e08e45b5 100644 --- a/src/packages/server/conat/index.ts +++ b/src/packages/server/conat/index.ts @@ -4,8 +4,8 @@ import { init as initChangefeedServer } from "@cocalc/database/conat/changefeed- import { init as initLLM } from "./llm"; import { loadConatConfiguration } from "./configuration"; import { createTimeService } from "@cocalc/conat/service/time"; -import { initPersistServer } from "@cocalc/backend/conat/persist"; -import { conatPersistCount, conatApiCount } from "@cocalc/backend/data"; +export { initConatPersist } from "./persist"; +import { conatApiCount } from "@cocalc/backend/data"; export { loadConatConfiguration }; @@ -19,16 +19,6 @@ export async function initConatChangefeedServer() { initChangefeedServer(); } -export async function initConatPersist() { - logger.debug("initPersistServer: sqlite3 stream persistence", { - conatPersistCount, - }); - await loadConatConfiguration(); - for (let i = 0; i < conatPersistCount; i++) { - initPersistServer(); - } -} - export async function initConatApi() { logger.debug("initConatApi: the central api services", { conatApiCount }); await loadConatConfiguration(); diff --git a/src/packages/server/conat/persist/index.ts b/src/packages/server/conat/persist/index.ts index a17ba419ed..2a31892ea9 100644 --- a/src/packages/server/conat/persist/index.ts +++ b/src/packages/server/conat/persist/index.ts @@ -1,6 +1,7 @@ -import { loadConatConfiguration } from "./configuration"; +import { loadConatConfiguration } from "../configuration"; import { initPersistServer } from "@cocalc/backend/conat/persist"; import { conatPersistCount } from "@cocalc/backend/data"; +import { createForkedPersistServer } from "./start-server"; import getLogger from "@cocalc/backend/logger"; @@ -28,8 +29,8 @@ async function createPersistCluster() { conatPersistCount, "nodes", ); - await loadConatConfiguration(); for (let i = 0; i < conatPersistCount; i++) { - initPersistServer(); + logger.debug("initPersistServer: starting node ", i); + createForkedPersistServer(); } } diff --git a/src/packages/server/conat/persist/start-persist-node.ts b/src/packages/server/conat/persist/start-persist-node.ts new file mode 100644 index 0000000000..57e27939f2 --- /dev/null +++ b/src/packages/server/conat/persist/start-persist-node.ts @@ -0,0 +1,16 @@ +import { loadConatConfiguration } from "../configuration"; +import { initPersistServer } from "@cocalc/backend/conat/persist"; +import { getLogger } from "@cocalc/backend/logger"; +import { addErrorListeners } from "@cocalc/server/metrics/error-listener"; + +const logger = getLogger("server:conat:persist:start-persist-node"); + +async function main() { + logger.debug("starting forked persist node in process", process.pid); + console.log("starting forked persist node in process", process.pid); + addErrorListeners(); + await loadConatConfiguration(); + await initPersistServer(); +} + +main(); diff --git a/src/packages/server/conat/socketio/start-cluster-node.ts b/src/packages/server/conat/socketio/start-cluster-node.ts index 92823a2fb7..54433fa497 100644 --- a/src/packages/server/conat/socketio/start-cluster-node.ts +++ b/src/packages/server/conat/socketio/start-cluster-node.ts @@ -1,3 +1,23 @@ +/* +The code here is run when conatSocketioCount > 1 (i.e., env var CONAT_SOCKETIO_COUNT). +This does NOT use the socketio cluster adapter or the nodejs cluster module. +Every worker process this spawns runs independently after it starts and there is +no single node coordinating communications like with the socketio cluster adapter, +and traffic across the cluster is minimal. This will thus *scale* much better, +though this is also just using normal TCP networking for communication instead of +IPC (like socketios cluster adapter). Also, the traffic between nodes is precisely +what is needed for Conat, so it's really solving a differnet problem than socketio's +cluster adapter, and under the hood what this actually does is much more sophisticated, +with each node maintaining and serving sqlite backed streams of data about their state. + +This code exists mainly for testing and potentially also for scaling cocalc to +more traffic when running on a single machine without Kubernetes. + +One cpu support several hundred simultaneous active connections -- if you want to +have 500 active users all using projects (that also means additional connections) -- +you will definitely need more than one node. +*/ + import "@cocalc/backend/conat/persist"; import { init, type Options } from "@cocalc/conat/core/server"; import { getUser, isAllowed } from "./auth"; @@ -6,7 +26,7 @@ import { loadConatConfiguration } from "../configuration"; import { getLogger } from "@cocalc/backend/logger"; async function main() { - console.log("main"); + console.log("conat server: starting a cluster node"); addErrorListeners(); const configDone = loadConatConfiguration();