Skip to content

Commit

Permalink
fix(xsnap)!: upgrade XS to fix memory leak
Browse files Browse the repository at this point in the history
We upgrade the XS submodule to the latest version:
Moddable-OpenSource/moddable@10cc52e

This fixes a major memory leak: 64 bytes per Map `delete()`, 32 per Set
`delete()`. We believe this should: closes #3839

Unfortunately Map/Set deletion is now O(N) not O(1).

This version of XS also fixes a bug that might be the cause of #3877 "cannot
read (corrupted?) snapshot", but we're not sure.

Note that this breaks snapshot compatibility (snapshots created before this
version cannot be loaded by code after this version). It might also
break metering compatibility, but the chances seem low enough that we decided
to leave the metering version alone.

closes #3889
  • Loading branch information
warner committed Sep 30, 2021
1 parent bdc626b commit 47bfa42
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 13 deletions.
45 changes: 33 additions & 12 deletions packages/SwingSet/test/test-xsnap-store.js
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ const relativeSize = (fn, fullSize) =>
const snapSize = {
raw: 417,
SESboot: 858,
compression: 0.1,
};

/**
Expand Down Expand Up @@ -69,7 +68,7 @@ async function bootSESWorker(name, handleCommand) {
return bootWorker(name, handleCommand, bootScript);
}

test(`create XS Machine, snapshot (${snapSize.raw} Kb), compress to ${snapSize.compression}x`, async t => {
test(`create XS Machine, snapshot (${snapSize.raw} Kb), compress to smaller`, async t => {
const vat = await bootWorker('xs1', async m => m, '1 + 1');
t.teardown(() => vat.close());

Expand All @@ -84,9 +83,8 @@ test(`create XS Machine, snapshot (${snapSize.raw} Kb), compress to ${snapSize.c
});

const zfile = path.resolve(pool.name, `${h}.gz`);
t.is(
relativeSize(zfile, snapSize.raw),
snapSize.compression,
t.true(
relativeSize(zfile, snapSize.raw) < 0.5,
'compressed snapshots are smaller',
);
});
Expand All @@ -107,9 +105,8 @@ test('SES bootstrap, save, compress', async t => {
});

const zfile = path.resolve(pool.name, `${h}.gz`);
t.is(
relativeSize(zfile, snapSize.SESboot),
0.2,
t.true(
relativeSize(zfile, snapSize.SESboot) < 0.5,
'compressed snapshots are smaller',
);
});
Expand Down Expand Up @@ -140,8 +137,9 @@ test('create SES worker, save, restore, resume', async t => {
* sensitive to any changes in bundle-ses-boot.umd.js;
* that is: any changes to the SES shim or to the
* xsnap-worker supervisor.
* They are also sensitive to the XS code itself.
*/
test('XS + SES snapshots are deterministic', async t => {
test('XS + SES snapshots are long-term deterministic', async t => {
const pool = tmp.dirSync({ unsafeCleanup: true });
t.teardown(() => pool.removeCallback());
t.log({ pool: pool.name });
Expand All @@ -155,7 +153,7 @@ test('XS + SES snapshots are deterministic', async t => {

t.is(
h1,
'817ce29f1f0f460a0066ec257b9941803b3f9fea4ad87a8a7a76f458d9f8a65b',
'9b90f329ae31e65bfb9b3f8436ceb581e160b3ec9984e9ac1dcef23ae78338fb',
'initial snapshot',
);

Expand All @@ -167,15 +165,38 @@ test('XS + SES snapshots are deterministic', async t => {
const h2 = await store.save(vat.snapshot);
t.is(
h2,
'f62c6fce5accbbfbb5f08bb25f9414e9ba611359af1fb3a889d6171e25e58d6a',
'b3bd291a9b42abb6acbe488a4da0c0eacee417a9f0ca94b88f7fbe4191bc43a0',
'after SES boot',
);

await vat.evaluate('globalThis.x = harden({a: 1})');
const h3 = await store.save(vat.snapshot);
t.is(
h3,
'911f4ea5fa42b5245d7024f269c59bdf382dc0521146ecc03e017136023f9435',
'834f6333c6c51aec41cb9aa8e8665c04a868c7c57ff0a356a0b72d82eeca578b',
'after use of harden()',
);
});

async function makeTestSnapshot(t) {
const pool = tmp.dirSync({ unsafeCleanup: true });
t.teardown(() => pool.removeCallback());
// t.log({ pool: pool.name });
await fs.promises.mkdir(pool.name, { recursive: true });
const store = makeSnapStore(pool.name, makeSnapStoreIO());
const vat = await bootWorker('xs1', async m => m, '1 + 1');
const bootScript = await ld.asset(
'@agoric/xsnap/dist/bundle-ses-boot.umd.js',
);
await vat.evaluate(bootScript);
await vat.evaluate('globalThis.x = harden({a: 1})');
const hash = await store.save(vat.snapshot);
await vat.close();
return hash;
}

test('XS + SES snapshots are short-term deterministic', async t => {
const h1 = await makeTestSnapshot(t);
const h2 = await makeTestSnapshot(t);
t.is(h1, h2);
});
2 changes: 1 addition & 1 deletion packages/xsnap/moddable
Submodule moddable updated 73 files
+1 −1 build/devices/esp/targets/moddable_display_1/host/provider.js
+1 −1 build/devices/esp/targets/moddable_display_3/host/provider.js
+1 −1 build/devices/esp/targets/moddable_one/host/provider.js
+1 −1 build/devices/esp/targets/moddable_three/host/provider.js
+1 −1 build/devices/esp/targets/nodemcu/host/provider.js
+93 −37 build/devices/esp32/setup/piu.js
+1 −1 build/devices/esp32/targets/esp32_thing/host/provider.js
+1 −1 build/devices/esp32/targets/esp32_thing_plus/host/provider.js
+1 −1 build/devices/esp32/targets/kaluga/host/provider.js
+1 −1 build/devices/esp32/targets/lilygo_ttgo/host/provider.js
+1 −1 build/devices/esp32/targets/m5atom_echo/host/provider.js
+1 −1 build/devices/esp32/targets/m5atom_lite/host/provider.js
+1 −1 build/devices/esp32/targets/m5atom_matrix/host/provider.js
+1 −1 build/devices/esp32/targets/m5stack/host/provider.js
+1 −1 build/devices/esp32/targets/m5stack/manifest.json
+1 −1 build/devices/esp32/targets/m5stack_core2/host/provider.js
+1 −1 build/devices/esp32/targets/m5stack_fire/host/provider.js
+1 −1 build/devices/esp32/targets/m5stick_c/host/provider.js
+1 −1 build/devices/esp32/targets/moddable_display_2/host/provider.js
+1 −1 build/devices/esp32/targets/moddable_two/host/provider.js
+197 −0 build/devices/esp32/targets/moddable_two_io/host/provider.js
+53 −0 build/devices/esp32/targets/moddable_two_io/manifest.json
+15 −0 build/devices/esp32/targets/moddable_two_io/setup-target.js
+1 −1 build/devices/esp32/targets/nodemcu/host/provider.js
+1 −1 build/devices/esp32/targets/saola_wroom/host/provider.js
+1 −1 build/devices/esp32/targets/saola_wrover/host/provider.js
+1 −1 build/devices/esp32/targets/wrover_kit/host/provider.js
+32 −0 build/devices/esp32/xsProj-esp32/versionCheck.py
+32 −0 build/devices/esp32/xsProj-esp32s2/versionCheck.py
+32 −0 build/devices/esp32/xsProj-esp32s3/versionCheck.py
+1 −1 build/makefiles/wasm/tools.mk
+47 −36 documentation/devices/esp32.md
+7 −120 examples/drivers/BMP180/main.js
+11 −0 examples/drivers/BMP180/manifest.json
+1 −1 examples/io/digital/blink/main.js
+2 −2 examples/io/digital/button/main.js
+78 −36 examples/io/i2c/ft6206/ft6206.js
+13 −12 examples/io/i2c/ft6206/main.js
+2 −2 examples/io/i2c/ft6206/manifest.json
+1 −1 examples/io/pwm/led/main.js
+4 −15 examples/io/spi/ili9341/main.js
+1 −1 modules/commodetto/commodettoPocoBlit.c
+54 −12 modules/commodetto/gif/pocogif.c
+3 −3 modules/commodetto/gif/pocogif.js
+124 −0 modules/drivers/bmp180/bmp180.js
+11 −0 modules/drivers/bmp180/manifest.json
+0 −1 modules/drivers/sensors/ccs811/ccs811.js
+11 −5 modules/files/flash/esp/flash.c
+10 −6 modules/files/flash/esp32/flash.c
+1 −1 modules/io/host/esp/provider.js
+1 −1 modules/io/host/esp32/provider.js
+11 −10 modules/io/i2c/esp/_i2c.c
+10 −16 modules/io/i2c/esp32/_i2c.c
+8 −7 modules/io/i2c/smbus.js
+2 −2 modules/io/pwm/esp32/_pwm.c
+12 −2 modules/io/serial/esp32/serial.c
+2 −2 modules/network/ble/btutils.js
+12 −11 modules/network/dns/moddnsparser.c
+1 −1 modules/network/mdns/mdns.js
+14 −15 modules/network/websocket/websocket.js
+11 −18 tools/mcconfig/make.esp32.mk
+12 −5 tools/mcconfig/nmake.esp32.mk
+1 −1 tools/mcmanifest.js
+0 −7 xs/platforms/xsPlatform.h
+2 −1 xs/sources/xsAll.h
+1 −1 xs/sources/xsCommon.h
+6 −0 xs/sources/xsDataView.c
+51 −37 xs/sources/xsMapSet.c
+8 −9 xs/sources/xsRegExp.c
+1 −1 xs/sources/xsRun.c
+76 −38 xs/sources/xsSnapshot.c
+4 −2 xs/sources/xsString.c
+8 −5 xs/tools/xst.c

0 comments on commit 47bfa42

Please sign in to comment.