forked from jo/couchdb-bulk
-
Notifications
You must be signed in to change notification settings - Fork 1
/
cli.js
executable file
·86 lines (67 loc) · 2.36 KB
/
cli.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
#!/usr/bin/env node
const fs = require('fs')
const split = require('split')
const { program } = require('commander')
const { version } = require('./package.json')
program
.arguments('<url> [file]')
.option('-l, --batch-length <number>', 'set the number of documents to be sent in bulk to CouchDB per batch (default: 1000)')
.option('-s, --sleep <milliseconds>', 'defines the amount of time (in milliseconds) to wait once a batch was sent before sending a new one (default: 0)')
.version(version)
program.addHelpText('after', `
Notes:
The [file] argument is optional, if its missing (or if its '-'),
input is expected to be piped via stdin. The tool
is intended to be used in a command chain like
cat docs.ndjson | couchdb-bulk
couchdb-bulk2 expects input to be newline-delimited JSON.
See http://jsonlines.org for more info on this format.
Each line should be a single doc:
{ "_id": "one" }
This newline-delimited JSON format can easily be obtained from a JSON document
containing an array of docs using a tool such as jq https://stedolan.github.io/jq/
cat view_reponse.json | jq -c '.docs[]'
`)
program.parse(process.argv)
const [ url, file ] = program.args
// Lowest end of the recommended range
// See https://docs.couchdb.org/en/stable/maintenance/performance.html#network
let docsPerBulk = 1000
const { batchLength, sleep } = program.opts()
if (batchLength) {
docsPerBulk = parseInt(batchLength)
}
const logErrorAndExit = (errMessage) => {
console.error(errMessage)
process.exit(1)
}
if (!url) logErrorAndExit('missing url')
if (!url.startsWith('http')) logErrorAndExit(`invalid url: ${url}`)
const inStream = (!process.stdin.isTTY || file === '-' || !file)
? process.stdin
: fs.createReadStream(file)
const bulkPost = require('./bulk_post')(url)
const wait = ms => new Promise(resolve => setTimeout(resolve, ms))
let batch = []
inStream
.pipe(split())
.on('data', async function (line) {
// weed empty lines
if (line === '') return
batch.push(line)
if (batch.length >= docsPerBulk) {
try {
this.pause()
const batchToPost = batch
batch = []
await bulkPost(batchToPost)
if (sleep) await wait(sleep)
this.resume()
} catch (err) {
console.error('bulk error', err)
process.exit(1)
}
}
})
.on('close', () => bulkPost(batch))
.on('error', console.error)