Skip to content

Commit

Permalink
fix(lambda-tiler): do not keep failed tiffs in memory (#3331)
Browse files Browse the repository at this point in the history
### Motivation

when requests fail, sometimes they can be retried later and succeed,
right now we are caching the error state.

### Modifications

On error remove the cached tiff/tar from the cache

### Verification

added unit tests.
  • Loading branch information
blacha authored Aug 26, 2024
1 parent b9fe33f commit 89e72ea
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 3 deletions.
22 changes: 19 additions & 3 deletions packages/lambda-tiler/src/util/__test__/cache.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,31 @@ import { fsa, FsMemory } from '@chunkd/fs';
import { SourceCache } from '../source.cache.js';

describe('CoSourceCache', () => {
it('should not exit if a promise rejection happens', async () => {
it('should not exit if a promise rejection happens for tiff', async () => {
const cache = new SourceCache(5);

const mem = new FsMemory();
const tiffLoc = new URL('memory://foo/bar.tiff');
await mem.write(tiffLoc, Buffer.from('ABC123'));
fsa.register('memory://', mem);

await cache.getCog(tiffLoc).catch(() => null);
assert.equal(cache.cache.currentSize, 1);
let failCount = 0;
await cache.getCog(tiffLoc).catch(() => failCount++);
assert.equal(cache.cache.currentSize, 0);
assert.equal(failCount, 1);
});

it('should not exit if a promise rejection happens for tar', async () => {
const cache = new SourceCache(5);

const mem = new FsMemory();
const tiffLoc = new URL('memory://foo/bar.tar');
await mem.write(tiffLoc, Buffer.from('ABC123'));
fsa.register('memory://', mem);

let failCount = 0;
await cache.getCotar(tiffLoc).catch(() => failCount++);
assert.equal(cache.cache.currentSize, 0);
assert.equal(failCount, 1);
});
});
2 changes: 2 additions & 0 deletions packages/lambda-tiler/src/util/source.cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ export class SourceCache {
}
const value = Tiff.create(fsa.source(location));
this.cache.set(location.href, { type: 'cog', value, size: 1 });
value.catch(() => this.cache.delete(location.href));
return value;
}

Expand All @@ -43,6 +44,7 @@ export class SourceCache {
}
const value = Cotar.fromTar(fsa.source(location));
this.cache.set(location.href, { type: 'cotar', value, size: 1 });
value.catch(() => this.cache.delete(location.href));
return value;
}
}
Expand Down
5 changes: 5 additions & 0 deletions packages/lambda-tiler/src/util/swapping.lru.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ export class SwappingLru<T extends { size: number }> {
this.clears++;
}

delete(id: string): void {
this.cacheA.delete(id);
this.cacheB.delete(id);
}

set(id: string, tiff: T): void {
this.cacheA.set(id, tiff);
this.check();
Expand Down

0 comments on commit 89e72ea

Please sign in to comment.