Skip to content
This repository was archived by the owner on Aug 4, 2023. It is now read-only.

update dyno version; add support for before script #24

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Default ignored files
/.idea/workspace.xml
.idea
node_modules
7 changes: 6 additions & 1 deletion bin/migrate.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ function usage() {
console.error(' - live [false]: if not specified, the migration script will not receive a database reference');
console.error(' - dyno [false]: if not specified, it is assumed that the objects are formatted using standard DynamoDB syntax. Pass the `--dyno` flag to the migrator if your input JSON objects are in a format suitable for direct usage in dyno (https://github.com/mapbox/dyno)');
console.error(' - rate [false]: log information about the rate at which migration is running. Will interfere with a migration script\'s logs');
console.log('To run using npm:');
console.log('npm run dynamodb-migrate <method> <database> <script> -- <options>');
console.log('eg: npm run dynamodb-migrate scan local/teams my_script.js -- --live')
}

if (args.help) {
Expand Down Expand Up @@ -58,5 +61,7 @@ var options = {
};

migration(options, function(err) {
if (err) throw err;
if (err) {
throw err;
}
});
23 changes: 12 additions & 11 deletions index.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
var Parser = require('./lib/parser');
var Migrator = require('./lib/migrator');
var Dyno = require('dyno');
var Dyno = require('@mapbox/dyno');
var split = require('split');
var Readable = require('stream').Readable;
var util = require('util');
Expand All @@ -26,29 +26,27 @@ module.exports = function(options, callback) {
if (region === 'local') {
params.accessKeyId = 'fake';
params.secretAccessKey = 'fake';
params.endpoint = 'http://localhost:4567';
params.endpoint = 'http://localhost:8000';
}

var dyno = Dyno(params);

var parser = Parser(method === 'scan', plainJSON);
var migrator = Migrator(migrate, dyno, concurrency, live);

var scanner = (function() {
var scanner = (function () {
if (method === 'scan') return dyno.scanStream();
if (method === 'stream') return process.stdin.pipe(split());
if (method instanceof Readable) return method.pipe(split());
})();

if (rateLogging) {
scanner.scans = 0;
scanner.on('dbrequest', function() {
scanner.on('dbrequest', function () {
scanner.scans++;
});

var starttime = Date.now();

setInterval(function() {
setInterval(function () {
var msg = util.format(
'\r\033[KScanner scans: %s, read depth: %s, Parser write depth: %s, read depth: %s | Migrator depth: %s, active: %s, %s/s',
scanner.scans,
Expand All @@ -63,13 +61,16 @@ module.exports = function(options, callback) {
}, 50).unref();
}

scanner
.pipe(parser)
let successCallback = function() {
scanner
.pipe(parser)
.on('error', callback)
.pipe(migrator)
.pipe(migrator)
.on('error', callback)
.on('finish', function() {
.on('finish', function () {
if (migrate.finish) migrate.finish(live ? dyno : null, callback);
else callback();
});
}
migrate.before(dyno, successCallback, callback);
};
2 changes: 1 addition & 1 deletion lib/parser.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
var stream = require('stream');
var Dyno = require('dyno');
var Dyno = require('@mapbox/dyno');

module.exports = function(objectMode, plainJSON) {
var parser = new stream.Transform();
Expand Down
Loading