download fileat first in imports

This commit is contained in:
SPRINX0\prochazka
2024-09-17 15:06:54 +02:00
parent 260b2e4b12
commit fc6a43b4fe
6 changed files with 16 additions and 11 deletions

View File

@@ -6,16 +6,18 @@ const { parser } = require('stream-json');
const { pick } = require('stream-json/filters/Pick');
const { streamArray } = require('stream-json/streamers/StreamArray');
const { streamObject } = require('stream-json/streamers/StreamObject');
const download = require('./download');
const logger = getLogger('jsonReader');
class ParseStream extends stream.Transform {
constructor({ limitRows, jsonStyle, keyField }) {
super({ objectMode: true });
this.wasHeader = false;
this.limitRows = limitRows;
this.jsonStyle = jsonStyle;
this.keyField = keyField;
this.keyField = keyField || '_key';
this.rowsWritten = 0;
}
_transform(chunk, encoding, done) {
@@ -53,8 +55,9 @@ async function jsonReader({
}) {
logger.info(`Reading file ${fileName}`);
const downloadedFile = await download(fileName);
const fileStream = fs.createReadStream(
fileName,
downloadedFile,
// @ts-ignore
encoding
);