diff --git a/packages/api/src/shell/jsonLinesReader.js b/packages/api/src/shell/jsonLinesReader.js index b226e487e..db36d966f 100644 --- a/packages/api/src/shell/jsonLinesReader.js +++ b/packages/api/src/shell/jsonLinesReader.js @@ -2,6 +2,7 @@ const fs = require('fs'); const stream = require('stream'); const byline = require('byline'); const { getLogger } = require('dbgate-tools'); +const download = require('./download'); const logger = getLogger('jsonLinesReader'); class ParseStream extends stream.Transform { @@ -35,8 +36,10 @@ class ParseStream extends stream.Transform { async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) { logger.info(`Reading file ${fileName}`); + const downloadedFile = await download(fileName); + const fileStream = fs.createReadStream( - fileName, + downloadedFile, // @ts-ignore encoding ); diff --git a/packages/api/src/shell/jsonReader.js b/packages/api/src/shell/jsonReader.js index 1347595eb..48423f3ca 100644 --- a/packages/api/src/shell/jsonReader.js +++ b/packages/api/src/shell/jsonReader.js @@ -6,16 +6,18 @@ const { parser } = require('stream-json'); const { pick } = require('stream-json/filters/Pick'); const { streamArray } = require('stream-json/streamers/StreamArray'); const { streamObject } = require('stream-json/streamers/StreamObject'); +const download = require('./download'); const logger = getLogger('jsonReader'); + class ParseStream extends stream.Transform { constructor({ limitRows, jsonStyle, keyField }) { super({ objectMode: true }); this.wasHeader = false; this.limitRows = limitRows; this.jsonStyle = jsonStyle; - this.keyField = keyField; + this.keyField = keyField || '_key'; this.rowsWritten = 0; } _transform(chunk, encoding, done) { @@ -53,8 +55,9 @@ async function jsonReader({ }) { logger.info(`Reading file ${fileName}`); + const downloadedFile = await download(fileName); const fileStream = fs.createReadStream( - fileName, + downloadedFile, // @ts-ignore encoding ); diff --git a/packages/api/src/shell/jsonWriter.js b/packages/api/src/shell/jsonWriter.js index 3dcb4fc63..a0ce19f24 100644 --- a/packages/api/src/shell/jsonWriter.js +++ b/packages/api/src/shell/jsonWriter.js @@ -11,12 +11,11 @@ class StringifyStream extends stream.Transform { this.wasHeader = false; this.wasRecord = false; this.jsonStyle = jsonStyle; - this.keyField = keyField; + this.keyField = keyField || '_key'; this.rootField = rootField; } _transform(chunk, encoding, done) { let skip = false; - const keyField = this.keyField || '_key'; if (!this.wasHeader) { skip = chunk.__isStreamHeader; @@ -43,8 +42,8 @@ class StringifyStream extends stream.Transform { this.wasRecord = true; if (this.jsonStyle === 'object') { - const key = chunk[keyField] ?? chunk[Object.keys(chunk)[0]]; - this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [keyField]))}`); + const key = chunk[this.keyField] ?? chunk[Object.keys(chunk)[0]]; + this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [this.keyField]))}`); } else { this.push(JSON.stringify(chunk)); } @@ -86,7 +85,7 @@ class StringifyStream extends stream.Transform { } } -async function jsonWriter({ fileName, jsonStyle, keyField, rootField, encoding = 'utf-8' }) { +async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, encoding = 'utf-8' }) { logger.info(`Writing file ${fileName}`); const stringify = new StringifyStream({ jsonStyle, keyField, rootField }); const fileStream = fs.createWriteStream(fileName, encoding); diff --git a/packages/web/src/impexp/FilesInput.svelte b/packages/web/src/impexp/FilesInput.svelte index 81a4e4c60..37e4d719b 100644 --- a/packages/web/src/impexp/FilesInput.svelte +++ b/packages/web/src/impexp/FilesInput.svelte @@ -1,6 +1,6 @@ - + Download imported file from web