dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-19 08:26:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

JSON object import

Browse Source
This commit is contained in:
Jan Prochazka
2024-09-17 12:52:53 +02:00
parent 503e09ddd1
commit fd8a28831e
2 changed files with 54 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
packages/api/src/shell/jsonReader.js
View File

@@ -6,12 +6,15 @@ const logger = getLogger('jsonReader');
const { parser } = require('stream-json');
const { pick } = require('stream-json/filters/Pick');
const { streamArray } = require('stream-json/streamers/StreamArray');
const { streamObject } = require('stream-json/streamers/StreamObject');
class ParseStream extends stream.Transform {
constructor({ limitRows }) {
constructor({ limitRows, jsonStyle, keyField }) {
super({ objectMode: true });
this.wasHeader = false;
this.limitRows = limitRows;
this.jsonStyle = jsonStyle;
this.keyField = keyField;
this.rowsWritten = 0;
}
_transform(chunk, encoding, done) {
@@ -24,14 +27,22 @@ class ParseStream extends stream.Transform {
this.wasHeader = true;
}
if (!this.limitRows || this.rowsWritten < this.limitRows) {
this.push(chunk.value);
if (this.jsonStyle === 'object') {
this.push({
...chunk.value,
[this.keyField]: chunk.key,
});
} else {
this.push(chunk.value);
}
this.rowsWritten += 1;
}
done();
}
}
async function jsonReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
async function jsonReader({ fileName, jsonStyle, keyField = '_key', encoding = 'utf-8', limitRows = undefined }) {
logger.info(`Reading file ${fileName}`);
const fileStream = fs.createReadStream(
@@ -42,14 +53,17 @@ async function jsonReader({ fileName, encoding = 'utf-8', limitRows = undefined
const parseJsonStream = parser();
fileStream.pipe(parseJsonStream);
const { streamArray } = require('stream-json/streamers/StreamArray');
const streamArrayStream = streamArray();
const parseStream = new ParseStream({ limitRows, jsonStyle, keyField });
parseJsonStream.pipe(streamArrayStream);
const parseStream = new ParseStream({ limitRows });
streamArrayStream.pipe(parseStream);
if (jsonStyle === 'object') {
const tramsformer = streamObject();
parseJsonStream.pipe(tramsformer);
tramsformer.pipe(parseStream);
} else {
const tramsformer = streamArray();
parseJsonStream.pipe(tramsformer);
tramsformer.pipe(parseStream);
}
return parseStream;
}