Skip to content

Commit

Permalink
refactor: add promisified wrapper for MBTiles (#1321)
Browse files Browse the repository at this point in the history 
Signed-off-by: Aarni Koskela <akx@iki.fi>
  • Loading branch information
@akx
akx authored Aug 28, 2024
1 parent 00d9189 commit 664afe6
Show file tree
Hide file tree
Showing 4 changed files with 149 additions and 142 deletions.
99 changes: 46 additions & 53 deletions src/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@ import path from 'path';
import { fileURLToPath } from 'url';
import axios from 'axios';
import { server } from './server.js';
import MBTiles from '@mapbox/mbtiles';
import { isValidHttpUrl } from './utils.js';
import { openPMtiles, getPMtilesInfo } from './pmtiles_adapter.js';
import { program } from 'commander';
import { existsP } from './promises.js';
import { openMbTilesWrapper } from './mbtiles_wrapper.js';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
Expand All @@ -23,8 +25,6 @@ if (args.length >= 3 && args[2][0] !== '-') {
args.splice(2, 0, '--mbtiles');
}

import { program } from 'commander';
import { existsP } from './promises.js';
program
.description('tileserver-gl startup options')
.usage('tileserver-gl [mbtiles] [options]')
Expand Down Expand Up @@ -184,62 +184,55 @@ const startWithInputFile = async (inputFile) => {
);
process.exit(1);
}
const instance = new MBTiles(inputFile + '?mode=ro', (err) => {
if (err) {
console.log('ERROR: Unable to open MBTiles.');
console.log(`Make sure ${path.basename(inputFile)} is valid MBTiles.`);
process.exit(1);
}

instance.getInfo(async (err, info) => {
if (err || !info) {
console.log('ERROR: Metadata missing in the MBTiles.');
console.log(
`Make sure ${path.basename(inputFile)} is valid MBTiles.`,
);
process.exit(1);
}
const bounds = info.bounds;
let info;
try {
const mbw = await openMbTilesWrapper(inputFile);
info = await mbw.getInfo();
if (!info) throw new Error('Metadata missing in the MBTiles.');
} catch (err) {
console.log('ERROR: Unable to open MBTiles or read metadata:', err);
console.log(`Make sure ${path.basename(inputFile)} is valid MBTiles.`);
process.exit(1);
}
const bounds = info.bounds;

if (
info.format === 'pbf' &&
info.name.toLowerCase().indexOf('openmaptiles') > -1
) {
config['data'][`v3`] = {
mbtiles: path.basename(inputFile),
};
if (
info.format === 'pbf' &&
info.name.toLowerCase().indexOf('openmaptiles') > -1
) {
config['data'][`v3`] = {
mbtiles: path.basename(inputFile),
};

const styles = await fsp.readdir(path.resolve(styleDir, 'styles'));
for (const styleName of styles) {
const styleFileRel = styleName + '/style.json';
const styleFile = path.resolve(styleDir, 'styles', styleFileRel);
if (await existsP(styleFile)) {
config['styles'][styleName] = {
style: styleFileRel,
tilejson: {
bounds,
},
};
}
}
} else {
console.log(
`WARN: MBTiles not in "openmaptiles" format. Serving raw data only...`,
);
config['data'][(info.id || 'mbtiles').replace(/[?/:]/g, '_')] = {
mbtiles: path.basename(inputFile),
const styles = await fsp.readdir(path.resolve(styleDir, 'styles'));
for (const styleName of styles) {
const styleFileRel = styleName + '/style.json';
const styleFile = path.resolve(styleDir, 'styles', styleFileRel);
if (await existsP(styleFile)) {
config['styles'][styleName] = {

Check warning on line 212 in src/main.js

View workflow job for this annotation

GitHub Actions / ESLint

src/main.js#L212 

Generic Object Injection Sink (security/detect-object-injection)
style: styleFileRel,
tilejson: {
bounds,
},
};
}
}
} else {
console.log(
`WARN: MBTiles not in "openmaptiles" format. Serving raw data only...`,
);
config['data'][(info.id || 'mbtiles').replace(/[?/:]/g, '_')] = {
mbtiles: path.basename(inputFile),
};
}

if (opts.verbose) {
console.log(JSON.stringify(config, undefined, 2));
} else {
console.log('Run with --verbose to see the config file here.');
}
if (opts.verbose) {
console.log(JSON.stringify(config, undefined, 2));
} else {
console.log('Run with --verbose to see the config file here.');
}

return startServer(null, config);
});
});
return startServer(null, config);
}
};

Expand Down
46 changes: 46 additions & 0 deletions src/mbtiles_wrapper.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import MBTiles from '@mapbox/mbtiles';
import util from 'node:util';

/**
* Promise-ful wrapper around the MBTiles class.
*/
class MBTilesWrapper {
constructor(mbtiles) {
this._mbtiles = mbtiles;
this._getInfoP = util.promisify(mbtiles.getInfo.bind(mbtiles));
}

/**
* Get the underlying MBTiles object.
* @returns {MBTiles}

Check warning on line 15 in src/mbtiles_wrapper.js

View workflow job for this annotation

GitHub Actions / ESLint

src/mbtiles_wrapper.js#L15 

Missing JSDoc @returns description (jsdoc/require-returns-description)
*/
getMbTiles() {
return this._mbtiles;
}

/**
* Get the MBTiles metadata object.
* @returns {Promise<object>}

Check warning on line 23 in src/mbtiles_wrapper.js

View workflow job for this annotation

GitHub Actions / ESLint

src/mbtiles_wrapper.js#L23 

Missing JSDoc @returns description (jsdoc/require-returns-description)
*/
getInfo() {
return this._getInfoP();
}
}

/**
* Open the given MBTiles file and return a promise that resolves with a
* MBTilesWrapper instance.
* @param inputFile Input file

Check warning on line 33 in src/mbtiles_wrapper.js

View workflow job for this annotation

GitHub Actions / ESLint

src/mbtiles_wrapper.js#L33 

Missing JSDoc @param "inputFile" type (jsdoc/require-param-type)
* @returns {Promise<MBTilesWrapper>}

Check warning on line 34 in src/mbtiles_wrapper.js

View workflow job for this annotation

GitHub Actions / ESLint

src/mbtiles_wrapper.js#L34 

Missing JSDoc @returns description (jsdoc/require-returns-description)
*/
export function openMbTilesWrapper(inputFile) {
return new Promise((resolve, reject) => {
const mbtiles = new MBTiles(inputFile + '?mode=ro', (err) => {
if (err) {
reject(err);
return;
}
resolve(new MBTilesWrapper(mbtiles));
});
});
}
46 changes: 16 additions & 30 deletions src/serve_data.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import path from 'path';

import clone from 'clone';
import express from 'express';
import MBTiles from '@mapbox/mbtiles';
import Pbf from 'pbf';
import { VectorTile } from '@mapbox/vector-tile';

Expand All @@ -16,6 +15,7 @@ import {
openPMtiles,
} from './pmtiles_adapter.js';
import { gunzipP, gzipP } from './promises.js';
import { openMbTilesWrapper } from './mbtiles_wrapper.js';

export const serve_data = {
init: (options, repo) => {
Expand Down Expand Up @@ -242,39 +242,25 @@ export const serve_data = {
}
} else if (inputType === 'mbtiles') {
sourceType = 'mbtiles';
const sourceInfoPromise = new Promise((resolve, reject) => {
source = new MBTiles(inputFile + '?mode=ro', (err) => {
if (err) {
reject(err);
return;
}
source.getInfo((err, info) => {
if (err) {
reject(err);
return;
}
tileJSON['name'] = id;
tileJSON['format'] = 'pbf';

Object.assign(tileJSON, info);
const mbw = await openMbTilesWrapper(inputFile);
const info = await mbw.getInfo();
source = mbw.getMbTiles();
tileJSON['name'] = id;
tileJSON['format'] = 'pbf';

tileJSON['tilejson'] = '2.0.0';
delete tileJSON['filesize'];
delete tileJSON['mtime'];
delete tileJSON['scheme'];
Object.assign(tileJSON, info);

Object.assign(tileJSON, params.tilejson || {});
fixTileJSONCenter(tileJSON);
tileJSON['tilejson'] = '2.0.0';
delete tileJSON['filesize'];
delete tileJSON['mtime'];
delete tileJSON['scheme'];

if (options.dataDecoratorFunc) {
tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON);
}
resolve();
});
});
});
Object.assign(tileJSON, params.tilejson || {});
fixTileJSONCenter(tileJSON);

await sourceInfoPromise;
if (options.dataDecoratorFunc) {
tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON);
}
}

repo[id] = {

Check warning on line 266 in src/serve_data.js

View workflow job for this annotation

GitHub Actions / ESLint

src/serve_data.js#L266 

Generic Object Injection Sink (security/detect-object-injection)
Expand Down
100 changes: 41 additions & 59 deletions src/serve_rendered.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import express from 'express';
import sanitize from 'sanitize-filename';
import SphericalMercator from '@mapbox/sphericalmercator';
import mlgl from '@maplibre/maplibre-gl-native';
import MBTiles from '@mapbox/mbtiles';
import polyline from '@mapbox/polyline';
import proj4 from 'proj4';
import axios from 'axios';
Expand All @@ -43,6 +42,7 @@ import {
import { renderOverlay, renderWatermark, renderAttribution } from './render.js';
import fsp from 'node:fs/promises';
import { gunzipP } from './promises.js';
import { openMbTilesWrapper } from './mbtiles_wrapper.js';

const FLOAT_PATTERN = '[+-]?(?:\\d+|\\d+.?\\d+)';
const PATH_PATTERN =
Expand Down Expand Up @@ -1131,7 +1131,6 @@ export const serve_rendered = {
};
repo[id] = repoobj;

const queue = [];
for (const name of Object.keys(styleJSON.sources)) {
let sourceType;
let source = styleJSON.sources[name];
Expand Down Expand Up @@ -1205,69 +1204,52 @@ export const serve_rendered = {
}
}
} else {
queue.push(
new Promise(async (resolve, reject) => {
inputFile = path.resolve(options.paths.mbtiles, inputFile);
const inputFileStats = await fsp.stat(inputFile);
if (!inputFileStats.isFile() || inputFileStats.size === 0) {
throw Error(`Not valid MBTiles file: "${inputFile}"`);
}
map.sources[name] = new MBTiles(inputFile + '?mode=ro', (err) => {
map.sources[name].getInfo((err, info) => {
if (err) {
console.error(err);
return;
}
map.sourceTypes[name] = 'mbtiles';

if (!repoobj.dataProjWGStoInternalWGS && info.proj4) {
// how to do this for multiple sources with different proj4 defs?
const to3857 = proj4('EPSG:3857');
const toDataProj = proj4(info.proj4);
repoobj.dataProjWGStoInternalWGS = (xy) =>
to3857.inverse(toDataProj.forward(xy));
}
const inputFileStats = await fsp.stat(inputFile);
if (!inputFileStats.isFile() || inputFileStats.size === 0) {
throw Error(`Not valid MBTiles file: "${inputFile}"`);
}
const mbw = await openMbTilesWrapper(inputFile);
const info = await mbw.getInfo();
map.sources[name] = mbw.getMbTiles();
map.sourceTypes[name] = 'mbtiles';

const type = source.type;
Object.assign(source, info);
source.type = type;
source.tiles = [
// meta url which will be detected when requested
`mbtiles://${name}/{z}/{x}/{y}.${info.format || 'pbf'}`,
];
delete source.scheme;

if (options.dataDecoratorFunc) {
source = options.dataDecoratorFunc(
name,
'tilejson',
source,
);
}
if (!repoobj.dataProjWGStoInternalWGS && info.proj4) {
// how to do this for multiple sources with different proj4 defs?
const to3857 = proj4('EPSG:3857');
const toDataProj = proj4(info.proj4);
repoobj.dataProjWGStoInternalWGS = (xy) =>
to3857.inverse(toDataProj.forward(xy));
}

if (
!attributionOverride &&
source.attribution &&
source.attribution.length > 0
) {
if (!tileJSON.attribution.includes(source.attribution)) {
if (tileJSON.attribution.length > 0) {
tileJSON.attribution += ' | ';
}
tileJSON.attribution += source.attribution;
}
}
resolve();
});
});
}),
);
const type = source.type;
Object.assign(source, info);
source.type = type;
source.tiles = [
// meta url which will be detected when requested
`mbtiles://${name}/{z}/{x}/{y}.${info.format || 'pbf'}`,
];
delete source.scheme;

if (options.dataDecoratorFunc) {
source = options.dataDecoratorFunc(name, 'tilejson', source);
}

if (
!attributionOverride &&
source.attribution &&
source.attribution.length > 0
) {
if (!tileJSON.attribution.includes(source.attribution)) {
if (tileJSON.attribution.length > 0) {
tileJSON.attribution += ' | ';
}
tileJSON.attribution += source.attribution;
}
}
}
}
}

await Promise.all(queue);

// standard and @2x tiles are much more usual -> default to larger pools
const minPoolSizes = options.minRendererPoolSizes || [8, 4, 2];
const maxPoolSizes = options.maxRendererPoolSizes || [16, 8, 4];
Expand Down

0 comments on commit 664afe6

Please sign in to comment.