dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-18 05:36:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

stream header flag + export from mongo

Browse Source
This commit is contained in:
Jan Prochazka
2021-04-08 17:49:57 +02:00
parent a5d37eb528
commit 20fccf51d9
9 changed files with 83 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
packages/api/src/shell/jsonLinesReader.js
View File

@@ -3,9 +3,8 @@ const stream = require('stream');
const byline = require('byline');
class ParseStream extends stream.Transform {
constructor({ header, limitRows }) {
constructor({ limitRows }) {
super({ objectMode: true });
this.header = header;
this.wasHeader = false;
this.limitRows = limitRows;
this.rowsWritten = 0;
@@ -13,7 +12,14 @@ class ParseStream extends stream.Transform {
_transform(chunk, encoding, done) {
const obj = JSON.parse(chunk);
if (!this.wasHeader) {
if (!this.header) this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
if (
!obj.__isStreamHeader &&
// TODO remove isArray test
!Array.isArray(obj.columns)
) {
this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
}
this.wasHeader = true;
}
if (!this.limitRows || this.rowsWritten < this.limitRows) {
@@ -24,12 +30,12 @@ class ParseStream extends stream.Transform {
}
}
async function jsonLinesReader({ fileName, encoding = 'utf-8', header = true, limitRows = undefined }) {
async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
console.log(`Reading file ${fileName}`);
const fileStream = fs.createReadStream(fileName, encoding);
const liner = byline(fileStream);
const parser = new ParseStream({ header, limitRows });
const parser = new ParseStream({ limitRows });
liner.pipe(parser);
return parser;
}