feat: add it-ndjson

packages/it-foreach/README.md

@@ -1,6 +1,6 @@
 # it-foreach
 
-[![Build status](https://github.com/achingbrain/it/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/achingbrain/it/actions/workflows/test.yml) [![Coverage Status](https://coveralls.io/repos/github/achingbrain/it/badge.svg?branch=master)](https://coveralls.io/github/achingbrain/it?branch=master) [![Dependencies Status](https://david-dm.org/achingbrain/it/status.svg?path=packages/it-all)](https://david-dm.org/achingbrain/it?path=packages/it-all)
+[![Build status](https://github.com/achingbrain/it/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/achingbrain/it/actions/workflows/test.yml) [![Coverage Status](https://coveralls.io/repos/github/achingbrain/it/badge.svg?branch=master)](https://coveralls.io/github/achingbrain/it?branch=master) [![Dependencies Status](https://david-dm.org/achingbrain/it/status.svg?path=packages/it-foreach)](https://david-dm.org/achingbrain/it?path=packages/it-foreach)
 
 > Invokes the passed function for each item in an iterable
 

packages/it-ndjson/.npmignore

@@ -0,0 +1,8 @@
+node_modules
+*.log
+.DS_Store
+Thumbs.db
+.vscode
+.nyc_output
+coverage
+bin

packages/it-ndjson/README.md

@@ -0,0 +1,25 @@
+# it-ndjson
+
+[![Build status](https://github.com/achingbrain/it/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/achingbrain/it/actions/workflows/test.yml) [![Coverage Status](https://coveralls.io/repos/github/achingbrain/it/badge.svg?branch=master)](https://coveralls.io/github/achingbrain/it?branch=master) [![Dependencies Status](https://david-dm.org/achingbrain/it/status.svg?path=packages/it-ndjson)](https://david-dm.org/achingbrain/it?path=packages/it-ndjson)
+
+> Parse iterators as ndjson and transform iterators to ndjson
+
+## Install
+
+```sh
+$ npm install --save it-ndjson
+```
+
+## Usage
+
+```javascript
+const ndjson = require('it-ndjson')
+const all = require('it-all')
+
+// This can also be an iterator, async iterator, generator, etc
+const values = [0, 1, 2, 3, 4]
+
+const arr = await all(ndjson.stringify(values))
+
+console.info(arr) // '0\n', '1\n', '2\n', '3\n', '4\n'
+```

packages/it-ndjson/index.js

@@ -0,0 +1,7 @@
+const parse = require('./parse')
+const stringify = require('./stringify')
+
+module.exports = {
+  parse,
+  stringify
+}

packages/it-ndjson/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "it-ndjson",
+  "version": "0.0.0",
+  "description": "Parse iterators as ndjson and transform iterators to ndjson",
+  "main": "index.js",
+  "repository": "github:achingbrain/it",
+  "homepage": "https://github.com/achingbrain/it/tree/master/packages/it-ndjson#readme",
+  "bugs": "https://github.com/achingbrain/it/issues",
+  "scripts": {
+    "test": "ava",
+    "lint": "standard",
+    "coverage": "nyc --reporter html --reporter lcov ava",
+    "clean": "rm -rf .nyc_output coverage dist",
+    "check": "tsc --noEmit",
+    "build": "npm run build:types",
+    "build:types": "tsc --emitDeclarationOnly --declarationDir dist",
+    "prepublishOnly": "npm run build"
+  },
+  "author": "Alex Potsides <alex@achingbrain.net>",
+  "license": "ISC",
+  "devDependencies": {
+    "ava": "^3.12.1",
+    "buffer": "^6.0.3",
+    "nyc": "^15.1.0",
+    "standard": "^16.0.3",
+    "typescript": "^4.0.2"
+  },
+  "types": "dist/src/index.d.ts"
+}

packages/it-ndjson/parse.js

@@ -0,0 +1,30 @@
+/**
+ * @param {AsyncIterable<Uint8Array> | Iterable<Uint8Array>} source
+ */
+async function * parse (source) {
+  const matcher = /\r?\n/
+  const decoder = new TextDecoder('utf8')
+  let buffer = ''
+
+  for await (let chunk of source) {
+    if (typeof chunk === 'string') {
+      chunk = new TextEncoder().encode(chunk)
+    }
+
+    buffer += decoder.decode(chunk, { stream: true })
+    const parts = buffer.split(matcher)
+    buffer = parts.pop() || ''
+
+    for (let i = 0; i < parts.length; i++) {
+      yield JSON.parse(parts[i])
+    }
+  }
+
+  buffer += decoder.decode()
+
+  if (buffer) {
+    yield JSON.parse(buffer)
+  }
+}
+
+module.exports = parse

packages/it-ndjson/stringify.js

@@ -0,0 +1,11 @@
+
+/**
+ * @param {AsyncIterable<any> | Iterable<any>} source
+ */
+async function * stringify (source) {
+  for await (const obj of source) {
+    yield JSON.stringify(obj) + '\n'
+  }
+}
+
+module.exports = stringify

packages/it-ndjson/test.js

@@ -0,0 +1,108 @@
+const test = require('ava')
+const ndjson = require('.')
+const { Buffer } = require('buffer')
+
+function toAsyncIterator (array) {
+  return (async function * () {
+    for (let i = 0; i < array.length; i++) {
+      yield new Promise(resolve => setTimeout(() => resolve(array[i])))
+    }
+  })()
+}
+
+function toUint8Array (str) {
+  const arr = new Uint8Array(str.length)
+  for (let i = 0; i < str.length; i++) {
+    arr[i] = str.charCodeAt(i)
+  }
+  return arr
+}
+
+test('should split 1 item from 1 chunk', async t => {
+  const source = toAsyncIterator(['{ "id": 1 }\n'])
+  const results = []
+
+  for await (const value of ndjson.parse(source)) {
+    results.push(value)
+  }
+
+  t.deepEqual(results, [{ id: 1 }])
+})
+
+test('should split 1 item from 2 chunks', async t => {
+  const source = toAsyncIterator(['{ "id', '": 1 }\n'])
+  const results = []
+
+  for await (const value of ndjson.parse(source)) {
+    results.push(value)
+  }
+
+  t.deepEqual(results, [{ id: 1 }])
+})
+
+test('should split 2 items from 2 chunks', async t => {
+  const source = toAsyncIterator(['{ "id": 1 }\n', '{ "id": 2 }'])
+  const results = []
+
+  for await (const value of ndjson.parse(source)) {
+    results.push(value)
+  }
+
+  t.deepEqual(results, [{ id: 1 }, { id: 2 }])
+})
+
+test('should split 2 items from 1 chunk', async t => {
+  const source = toAsyncIterator(['{ "id": 1 }\n{ "id": 2 }'])
+  const results = []
+
+  for await (const value of ndjson.parse(source)) {
+    results.push(value)
+  }
+
+  t.deepEqual(results, [{ id: 1 }, { id: 2 }])
+})
+
+test('should split 3 items from 2 chunks', async t => {
+  const source = toAsyncIterator(['{ "id": 1 }\n{ "i', 'd": 2 }', '\n{"id":3}'])
+  const results = []
+
+  for await (const value of ndjson.parse(source)) {
+    results.push(value)
+  }
+
+  t.deepEqual(results, [{ id: 1 }, { id: 2 }, { id: 3 }])
+})
+
+test('should split from Buffers', async t => {
+  const source = toAsyncIterator([Buffer.from('{ "id": 1 }\n{ "i'), Buffer.from('d": 2 }'), Buffer.from('\n{"id":3}')])
+  const results = []
+
+  for await (const value of ndjson.parse(source)) {
+    results.push(value)
+  }
+
+  t.deepEqual(results, [{ id: 1 }, { id: 2 }, { id: 3 }])
+})
+
+test('should split from Uint8Arrays', async t => {
+  const source = toAsyncIterator([toUint8Array('{ "id": 1 }\n{ "i'), toUint8Array('d": 2 }'), toUint8Array('\n{"id":3}')])
+  const results = []
+
+  for await (const value of ndjson.parse(source)) {
+    results.push(value)
+  }
+
+  t.deepEqual(results, [{ id: 1 }, { id: 2 }, { id: 3 }])
+})
+
+test('should round trip', async t => {
+  const input = '{"id":1}\n{"id":2}\n{"id":3}\n'
+  const source = toAsyncIterator([input])
+  const results = []
+
+  for await (const value of ndjson.stringify(ndjson.parse(source))) {
+    results.push(value)
+  }
+
+  t.is(results.join(''), input)
+})

packages/it-ndjson/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../../typescript.json",
+  "include": [
+    "."
+  ]
+}