dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-19 06:06:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

download fileat first in imports

Browse Source
This commit is contained in:
SPRINX0\prochazka
2024-09-17 15:06:54 +02:00
parent 260b2e4b12
commit fc6a43b4fe
6 changed files with 16 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
packages/api/src/shell/jsonLinesReader.js
View File

@@ -2,6 +2,7 @@ const fs = require('fs');
const stream = require('stream');
const byline = require('byline');
const { getLogger } = require('dbgate-tools');
const download = require('./download');
const logger = getLogger('jsonLinesReader');
class ParseStream extends stream.Transform {
@@ -35,8 +36,10 @@ class ParseStream extends stream.Transform {
async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
logger.info(`Reading file ${fileName}`);
const downloadedFile = await download(fileName);
const fileStream = fs.createReadStream(
fileName,
downloadedFile,
// @ts-ignore
encoding
);

7
packages/api/src/shell/jsonReader.js
View File

@@ -6,16 +6,18 @@ const { parser } = require('stream-json');
const { pick } = require('stream-json/filters/Pick');
const { streamArray } = require('stream-json/streamers/StreamArray');
const { streamObject } = require('stream-json/streamers/StreamObject');
const download = require('./download');
const logger = getLogger('jsonReader');
class ParseStream extends stream.Transform {
constructor({ limitRows, jsonStyle, keyField }) {
super({ objectMode: true });
this.wasHeader = false;
this.limitRows = limitRows;
this.jsonStyle = jsonStyle;
this.keyField = keyField;
this.keyField = keyField || '_key';
this.rowsWritten = 0;
}
_transform(chunk, encoding, done) {
@@ -53,8 +55,9 @@ async function jsonReader({
}) {
logger.info(`Reading file ${fileName}`);
const downloadedFile = await download(fileName);
const fileStream = fs.createReadStream(
fileName,
downloadedFile,
// @ts-ignore
encoding
);

9
packages/api/src/shell/jsonWriter.js
View File

@@ -11,12 +11,11 @@ class StringifyStream extends stream.Transform {
this.wasHeader = false;
this.wasRecord = false;
this.jsonStyle = jsonStyle;
this.keyField = keyField;
this.keyField = keyField || '_key';
this.rootField = rootField;
}
_transform(chunk, encoding, done) {
let skip = false;
const keyField = this.keyField || '_key';
if (!this.wasHeader) {
skip = chunk.__isStreamHeader;
@@ -43,8 +42,8 @@ class StringifyStream extends stream.Transform {
this.wasRecord = true;
if (this.jsonStyle === 'object') {
const key = chunk[keyField] ?? chunk[Object.keys(chunk)[0]];
this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [keyField]))}`);
const key = chunk[this.keyField] ?? chunk[Object.keys(chunk)[0]];
this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [this.keyField]))}`);
} else {
this.push(JSON.stringify(chunk));
}
@@ -86,7 +85,7 @@ class StringifyStream extends stream.Transform {
}
}
async function jsonWriter({ fileName, jsonStyle, keyField, rootField, encoding = 'utf-8' }) {
async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, encoding = 'utf-8' }) {
logger.info(`Writing file ${fileName}`);
const stringify = new StringifyStream({ jsonStyle, keyField, rootField });
const fileStream = fs.createWriteStream(fileName, encoding);