fix(SwingSet): VOM tracks Presence vrefs in virtualized data

If userspace puts a Presence into the `state` of a virtual object, or
somewhere inside the value stored in vref-keyed `makeWeakStore()` entry, it
gets serialized and stored as a vref, which doesn't (and should not) keep the
Presence object alive. Allowing this Presence to leave RAM, remembering only
the vref on disk, is a non-trivial part of the memory savings we obtain by
using virtualized data.

However, just because there is currently no Presence (for a given vref)
does *not* mean that the vat cannot reach the vref. Liveslots will observe
the Presence being collected (when the finalizer runs), but if the vref is
still stored somewhere in virtualized data, liveslots must not emit a
`syscall.dropImport` for it.

This changes the virtual object manager to keep track of Presences used in
virtualized data, and remember their vref in a Set. When liveslots' wants to
`dropImport` a vref that no longer has a Presence, it will ask the VOM first.
With this Set, the VOM can inhibit the `dropImport` call until later.

At this stage, we simply add the vref to a Set and never remove it. This is
safe but conservative. In the future, we'll need some form of refcounting to
detect when the vref is no longer mentioned anywhere in virtualized data. At
that point, the VOM will need to inform liveslots (or some sort of
"reachability manager") that the VOM no longer needs the vref kept alive. The
`syscall.dropImport` can be sent when neither the VOM nor a Presence is
causing the vref to remain reachable.

closes #3133
refs #3106

packages/SwingSet/src/kernel/virtualObjectManager.js

@@ -196,6 +196,32 @@ export function makeVirtualObjectManager(
    */
   const kindTable = new Map();
 
+  /**
+   * Set of all import vrefs which are reachable from our virtualized data.
+   * These were Presences at one point. We add to this set whenever we store
+   * a Presence into the state of a virtual object, or the value of a
+   * makeWeakStore() instance. We currently never remove anything from the
+   * set, but eventually we'll use refcounts within virtual data to figure
+   * out when the vref becomes unreachable, allowing the vat do send a
+   * dropImport into the kernel and release the object.
+   *
+   */
+  /** @type {Set<string>} of vrefs */
+  const reachableVrefs = new Set();
+
+  // We track imports, to preserve their vrefs against syscall.dropImport
+  // when the Presence goes away.
+  function addReachablePresence(vref) {
+    const { type, allocatedByVat } = parseVatSlot(vref);
+    if (type === 'object' && !allocatedByVat) {
+      reachableVrefs.add(vref);
+    }
+  }
+
+  function isVrefReachable(vref) {
+    return reachableVrefs.has(vref);
+  }
+
   /**
    * Set of all Remotables which are reachable by our virtualized data, e.g.
    * `makeWeakStore().set(key, remotable)` or `virtualObject.state.foo =
@@ -305,6 +331,7 @@ export function makeVirtualObjectManager(
             X`${q(keyName)} already registered: ${key}`,
           );
           const data = m.serialize(value);
+          data.slots.map(addReachablePresence);
           data.slots.map(addReachableRemotable);
           syscall.vatstoreSet(vkey, JSON.stringify(data));
         } else {
@@ -328,6 +355,7 @@ export function makeVirtualObjectManager(
         if (vkey) {
           assert(syscall.vatstoreGet(vkey), X`${q(keyName)} not found: ${key}`);
           const data = m.serialize(harden(value));
+          data.slots.map(addReachablePresence);
           data.slots.map(addReachableRemotable);
           syscall.vatstoreSet(vkey, JSON.stringify(data));
         } else {
@@ -566,6 +594,7 @@ export function makeVirtualObjectManager(
             },
             set: value => {
               const serializedValue = m.serialize(value);
+              serializedValue.slots.map(addReachablePresence);
               serializedValue.slots.map(addReachableRemotable);
               ensureState();
               innerSelf.rawData[prop] = serializedValue;
@@ -625,6 +654,7 @@ export function makeVirtualObjectManager(
       for (const prop of Object.getOwnPropertyNames(initialData)) {
         try {
           const data = m.serialize(initialData[prop]);
+          data.slots.map(addReachablePresence);
           data.slots.map(addReachableRemotable);
           rawData[prop] = data;
         } catch (e) {
@@ -649,6 +679,7 @@ export function makeVirtualObjectManager(
     makeKind,
     VirtualObjectAwareWeakMap,
     VirtualObjectAwareWeakSet,
+    isVrefReachable,
     flushCache: cache.flush,
     makeVirtualObjectRepresentative,
   });

packages/SwingSet/test/virtualObjects/test-reachable-vrefs.js

@@ -0,0 +1,88 @@
+import { test } from '../../tools/prepare-test-env-ava';
+
+// eslint-disable-next-line import/order
+import { Far, Remotable } from '@agoric/marshal';
+
+import { makeVatSlot } from '../../src/parseVatSlots';
+import { makeFakeVirtualObjectManager } from '../../tools/fakeVirtualObjectManager';
+
+// empty object, used as makeWeakStore() key
+function makeKeyInstance(_state) {
+  return {
+    init() {},
+    self: Far('key'),
+  };
+}
+
+function makeHolderInstance(state) {
+  return {
+    init(held) {
+      state.held = held;
+    },
+    self: Far('holder', {
+      setHeld(held) {
+        state.held = held;
+      },
+      getHeld() {
+        return state.held;
+      },
+    }),
+  };
+}
+
+test('VOM tracks reachable vrefs', async t => {
+  const vomOptions = { cacheSize: 3 };
+  const vom = makeFakeVirtualObjectManager(vomOptions);
+  const { makeWeakStore, makeKind } = vom;
+  const weakStore = makeWeakStore();
+  const keyMaker = makeKind(makeKeyInstance);
+  const holderMaker = makeKind(makeHolderInstance);
+
+  let count = 1001;
+  function makePresence() {
+    // Both Remotable() and the Far() convenience wrapper mark things as
+    // pass-by-reference. They are used when creating an (imported) Presence,
+    // not just an (exported) "Remotable".
+    const pres = Remotable(`Alleged: presence-${count}`, undefined, {});
+    const vref = makeVatSlot('object', false, count);
+    vom.registerEntry(vref, pres);
+    count += 1;
+    return [vref, pres];
+  }
+
+  const [vref1, obj1] = makePresence();
+  const key1 = keyMaker();
+  t.falsy(vom.isVrefReachable(vref1));
+  weakStore.init(key1, obj1);
+  t.truthy(vom.isVrefReachable(vref1));
+
+  const [vref2, obj2] = makePresence();
+  const key2 = keyMaker();
+  weakStore.init(key2, 'not yet');
+  t.falsy(vom.isVrefReachable(vref2));
+  weakStore.set(key2, obj2);
+  t.truthy(vom.isVrefReachable(vref2));
+
+  // storing Presences as the value for a non-virtual key just holds on to
+  // the Presence directly, and does not track the vref
+
+  const [vref3, obj3] = makePresence();
+  const key3 = {};
+  weakStore.init(key3, obj3);
+  weakStore.set(key3, obj3);
+  t.falsy(vom.isVrefReachable(vref3));
+
+  // now check that Presences are tracked when in the state of a virtual
+  // object
+  const [vref4, obj4] = makePresence();
+  t.falsy(vom.isVrefReachable(vref4));
+  // eslint-disable-next-line no-unused-vars
+  const holder4 = holderMaker(obj4);
+  t.truthy(vom.isVrefReachable(vref4));
+
+  const [vref5, obj5] = makePresence();
+  const holder5 = holderMaker('not yet');
+  t.falsy(vom.isVrefReachable(vref5));
+  holder5.setHeld(obj5);
+  t.truthy(vom.isVrefReachable(vref5));
+});

packages/SwingSet/tools/fakeVirtualObjectManager.js

@@ -106,6 +106,7 @@ export function makeFakeVirtualObjectManager(options = {}) {
     makeKind,
     VirtualObjectAwareWeakMap,
     VirtualObjectAwareWeakSet,
+    isVrefReachable,
     flushCache,
   } = makeVirtualObjectManager(
     fakeSyscall,
@@ -122,6 +123,7 @@ export function makeFakeVirtualObjectManager(options = {}) {
     makeWeakStore,
     VirtualObjectAwareWeakMap,
     VirtualObjectAwareWeakSet,
+    isVrefReachable,
   };
 
   const debugTools = {