diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index ff0a610c9..b8ceda796 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -4,6 +4,7 @@ on: branches: - master - develop + - 'feature/**' jobs: test-runner: diff --git a/integration-tests/__tests__/import-formats.spec.js b/integration-tests/__tests__/import-formats.spec.js new file mode 100644 index 000000000..21dc38e21 --- /dev/null +++ b/integration-tests/__tests__/import-formats.spec.js @@ -0,0 +1,151 @@ +const dbgateApi = require('dbgate-api/src/shell'); +// const jsonLinesWriter = require('dbgate-api/src/shell/jsonLinesWriter'); +const tmp = require('tmp'); +// const dbgatePluginCsv = require('dbgate-plugin-csv/src/backend'); +const fs = require('fs'); +const requirePlugin = require('dbgate-api/src/shell/requirePlugin'); + +const CSV_DATA = `Issue Number; Title; Github URL; Labels; State; Created At; Updated At; Reporter; Assignee +801; "Does it 'burst' the database on startup or first lUI load ? "; https://github.com/dbgate/dbgate/issues/801; ""; open; 05/23/2024; 05/23/2024; rgarrigue; +799; "BUG: latest AppImage crashes on opening in Fedora 39"; https://github.com/dbgate/dbgate/issues/799; ""; open; 05/21/2024; 05/24/2024; BenGraham-Git; +798; "MongoDB write operations fail"; https://github.com/dbgate/dbgate/issues/798; "bug,solved"; open; 05/21/2024; 05/24/2024; mahmed0715; +797; "BUG: Unable to open SQL files"; https://github.com/dbgate/dbgate/issues/797; "bug"; open; 05/20/2024; 05/21/2024; cesarValdivia; +795; "BUG: MS SQL Server connection error (KEY_USAGE_BIT_INCORRECT)"; https://github.com/dbgate/dbgate/issues/795; ""; open; 05/20/2024; 05/20/2024; keskinonur; +794; "GLIBC_2.29' not found and i have 2.31"; https://github.com/dbgate/dbgate/issues/794; ""; closed; 05/20/2024; 05/21/2024; MFdanGM; +793; "BUG: PostgresSQL doesn't show tables when connected"; https://github.com/dbgate/dbgate/issues/793; ""; open; 05/20/2024; 05/22/2024; stomper013; +792; "FEAT: Wayland support"; https://github.com/dbgate/dbgate/issues/792; ""; closed; 05/19/2024; 05/21/2024; VosaXalo; +`; + +async function getReaderRows(reader) { + const jsonLinesFileName = tmp.tmpNameSync(); + + const writer = await dbgateApi.jsonLinesWriter({ + fileName: jsonLinesFileName, + }); + await dbgateApi.copyStream(reader, writer); + + const jsonData = fs.readFileSync(jsonLinesFileName, 'utf-8'); + const rows = jsonData + .split('\n') + .filter(x => x.trim() !== '') + .map(x => JSON.parse(x)); + + return rows; +} + +test('csv import test', async () => { + const dbgatePluginCsv = requirePlugin('dbgate-plugin-csv'); + + const csvFileName = tmp.tmpNameSync(); + + fs.writeFileSync(csvFileName, CSV_DATA); + + const reader = await dbgatePluginCsv.shellApi.reader({ + fileName: csvFileName, + }); + + const rows = await getReaderRows(reader); + + expect(rows[0].columns).toEqual([ + { columnName: 'Issue Number' }, + { columnName: 'Title' }, + { columnName: 'Github URL' }, + { columnName: 'Labels' }, + { columnName: 'State' }, + { columnName: 'Created At' }, + { columnName: 'Updated At' }, + { columnName: 'Reporter' }, + { columnName: 'Assignee' }, + ]); + expect(rows.length).toEqual(9); + expect(rows[1]).toEqual({ + 'Issue Number': '801', + Title: "Does it 'burst' the database on startup or first lUI load ? ", + 'Github URL': 'https://github.com/dbgate/dbgate/issues/801', + Labels: '', + State: 'open', + 'Created At': '05/23/2024', + 'Updated At': '05/23/2024', + Reporter: 'rgarrigue', + Assignee: '', + }); +}); + +test('JSON array import test', async () => { + const jsonFileName = tmp.tmpNameSync(); + + fs.writeFileSync( + jsonFileName, + JSON.stringify([ + { id: 1, val: 'v1' }, + { id: 2, val: 'v2' }, + ]) + ); + + const reader = await dbgateApi.jsonReader({ + fileName: jsonFileName, + }); + + const rows = await getReaderRows(reader); + + expect(rows.length).toEqual(2); + expect(rows).toEqual([ + { id: 1, val: 'v1' }, + { id: 2, val: 'v2' }, + ]); +}); + +test('JSON object import test', async () => { + const jsonFileName = tmp.tmpNameSync(); + + fs.writeFileSync( + jsonFileName, + JSON.stringify({ + k1: { id: 1, val: 'v1' }, + k2: { id: 2, val: 'v2' }, + }) + ); + + const reader = await dbgateApi.jsonReader({ + fileName: jsonFileName, + jsonStyle: 'object', + keyField: 'mykey', + }); + + const rows = await getReaderRows(reader); + + expect(rows.length).toEqual(2); + expect(rows).toEqual([ + { mykey: 'k1', id: 1, val: 'v1' }, + { mykey: 'k2', id: 2, val: 'v2' }, + ]); +}); + +test('JSON filtered object import test', async () => { + const jsonFileName = tmp.tmpNameSync(); + + fs.writeFileSync( + jsonFileName, + JSON.stringify({ + filtered: { + k1: { id: 1, val: 'v1' }, + k2: { id: 2, val: 'v2' }, + }, + }) + ); + + const reader = await dbgateApi.jsonReader({ + fileName: jsonFileName, + jsonStyle: 'object', + keyField: 'mykey', + rootField: 'filtered', + }); + + const rows = await getReaderRows(reader); + + expect(rows.length).toEqual(2); + expect(rows).toEqual([ + { mykey: 'k1', id: 1, val: 'v1' }, + { mykey: 'k2', id: 2, val: 'v2' }, + ]); +}); diff --git a/integration-tests/package.json b/integration-tests/package.json index dcb1e67e4..6a0dea08c 100644 --- a/integration-tests/package.json +++ b/integration-tests/package.json @@ -22,6 +22,7 @@ "devDependencies": { "cross-env": "^7.0.3", "jest": "^27.0.1", - "pino-pretty": "^11.2.2" + "pino-pretty": "^11.2.2", + "tmp": "^0.2.3" } } diff --git a/integration-tests/setupTests.js b/integration-tests/setupTests.js index b52c4066e..e0f763787 100644 --- a/integration-tests/setupTests.js +++ b/integration-tests/setupTests.js @@ -1,4 +1,10 @@ +global.DBGATE_PACKAGES = { + 'dbgate-tools': require('dbgate-tools'), + 'dbgate-sqltree': require('dbgate-sqltree'), +}; + const { prettyFactory } = require('pino-pretty'); +const tmp = require('tmp'); const pretty = prettyFactory({ colorize: true, @@ -20,3 +26,5 @@ global.console = { process.stdout.write(messages.join(' ') + '\n'); }, }; + +tmp.setGracefulCleanup(); diff --git a/integration-tests/tools.js b/integration-tests/tools.js index 846c27659..4ab8c20d4 100644 --- a/integration-tests/tools.js +++ b/integration-tests/tools.js @@ -1,8 +1,3 @@ -global.DBGATE_PACKAGES = { - 'dbgate-tools': require('dbgate-tools'), - 'dbgate-sqltree': require('dbgate-sqltree'), -}; - const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver'); const crypto = require('crypto'); diff --git a/package.json b/package.json index ec6297e8c..ac6acbc09 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "build:api": "yarn workspace dbgate-api build", "build:web:docker": "yarn workspace dbgate-web build", "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend", + "build:plugins:backend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:backend", "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch", "storage-json": "dbmodel model-to-json storage-db packages/api/src/storageModel.js --commonjs", "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist", diff --git a/packages/api/package.json b/packages/api/package.json index 4812aefd0..c2813e3e8 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -57,6 +57,7 @@ "rimraf": "^3.0.0", "simple-encryptor": "^4.0.0", "ssh2": "^1.11.0", + "stream-json": "^1.8.0", "tar": "^6.0.5" }, "scripts": { diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index 0cf04a978..448ba5d3f 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -18,11 +18,14 @@ function readFirstLine(file) { } if (reader.hasNextLine()) { reader.nextLine((err, line) => { - if (err) reject(err); - resolve(line); + if (err) { + reader.close(() => reject(err)); // Ensure reader is closed on error + return; + } + reader.close(() => resolve(line)); // Ensure reader is closed after reading }); } else { - resolve(null); + reader.close(() => resolve(null)); // Properly close if no lines are present } }); }); diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index 253bb053a..1a2bef628 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -6,7 +6,7 @@ const copyStream = require('./copyStream'); const fakeObjectReader = require('./fakeObjectReader'); const consoleObjectWriter = require('./consoleObjectWriter'); const jsonLinesWriter = require('./jsonLinesWriter'); -const jsonArrayWriter = require('./jsonArrayWriter'); +const jsonWriter = require('./jsonWriter'); const jsonLinesReader = require('./jsonLinesReader'); const sqlDataWriter = require('./sqlDataWriter'); const jslDataReader = require('./jslDataReader'); @@ -29,6 +29,7 @@ const modifyJsonLinesReader = require('./modifyJsonLinesReader'); const dataDuplicator = require('./dataDuplicator'); const dbModelToJson = require('./dbModelToJson'); const jsonToDbModel = require('./jsonToDbModel'); +const jsonReader = require('./jsonReader'); const dbgateApi = { queryReader, @@ -37,8 +38,9 @@ const dbgateApi = { tableReader, copyStream, jsonLinesWriter, - jsonArrayWriter, jsonLinesReader, + jsonReader, + jsonWriter, sqlDataWriter, fakeObjectReader, consoleObjectWriter, diff --git a/packages/api/src/shell/jsonArrayWriter.js b/packages/api/src/shell/jsonArrayWriter.js deleted file mode 100644 index ed18b86f6..000000000 --- a/packages/api/src/shell/jsonArrayWriter.js +++ /dev/null @@ -1,52 +0,0 @@ -const { getLogger } = require('dbgate-tools'); -const fs = require('fs'); -const stream = require('stream'); - -const logger = getLogger('jsonArrayWriter'); - -class StringifyStream extends stream.Transform { - constructor() { - super({ objectMode: true }); - this.wasHeader = false; - this.wasRecord = false; - } - _transform(chunk, encoding, done) { - let skip = false; - - if (!this.wasHeader) { - skip = chunk.__isStreamHeader; - this.wasHeader = true; - } - if (!skip) { - if (!this.wasRecord) { - this.push('[\n'); - } else { - this.push(',\n'); - } - this.wasRecord = true; - - this.push(JSON.stringify(chunk)); - } - done(); - } - - _flush(done) { - if (!this.wasRecord) { - this.push('[]\n'); - } else { - this.push('\n]\n'); - } - done(); - } -} - -async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) { - logger.info(`Writing file ${fileName}`); - const stringify = new StringifyStream(); - const fileStream = fs.createWriteStream(fileName, encoding); - stringify.pipe(fileStream); - stringify['finisher'] = fileStream; - return stringify; -} - -module.exports = jsonArrayWriter; diff --git a/packages/api/src/shell/jsonLinesReader.js b/packages/api/src/shell/jsonLinesReader.js index b226e487e..db36d966f 100644 --- a/packages/api/src/shell/jsonLinesReader.js +++ b/packages/api/src/shell/jsonLinesReader.js @@ -2,6 +2,7 @@ const fs = require('fs'); const stream = require('stream'); const byline = require('byline'); const { getLogger } = require('dbgate-tools'); +const download = require('./download'); const logger = getLogger('jsonLinesReader'); class ParseStream extends stream.Transform { @@ -35,8 +36,10 @@ class ParseStream extends stream.Transform { async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) { logger.info(`Reading file ${fileName}`); + const downloadedFile = await download(fileName); + const fileStream = fs.createReadStream( - fileName, + downloadedFile, // @ts-ignore encoding ); diff --git a/packages/api/src/shell/jsonReader.js b/packages/api/src/shell/jsonReader.js new file mode 100644 index 000000000..48423f3ca --- /dev/null +++ b/packages/api/src/shell/jsonReader.js @@ -0,0 +1,84 @@ +const fs = require('fs'); +const stream = require('stream'); +const byline = require('byline'); +const { getLogger } = require('dbgate-tools'); +const { parser } = require('stream-json'); +const { pick } = require('stream-json/filters/Pick'); +const { streamArray } = require('stream-json/streamers/StreamArray'); +const { streamObject } = require('stream-json/streamers/StreamObject'); +const download = require('./download'); + +const logger = getLogger('jsonReader'); + + +class ParseStream extends stream.Transform { + constructor({ limitRows, jsonStyle, keyField }) { + super({ objectMode: true }); + this.wasHeader = false; + this.limitRows = limitRows; + this.jsonStyle = jsonStyle; + this.keyField = keyField || '_key'; + this.rowsWritten = 0; + } + _transform(chunk, encoding, done) { + if (!this.wasHeader) { + this.push({ + __isStreamHeader: true, + __isDynamicStructure: true, + }); + + this.wasHeader = true; + } + if (!this.limitRows || this.rowsWritten < this.limitRows) { + if (this.jsonStyle === 'object') { + this.push({ + ...chunk.value, + [this.keyField]: chunk.key, + }); + } else { + this.push(chunk.value); + } + + this.rowsWritten += 1; + } + done(); + } +} + +async function jsonReader({ + fileName, + jsonStyle, + keyField = '_key', + rootField = null, + encoding = 'utf-8', + limitRows = undefined, +}) { + logger.info(`Reading file ${fileName}`); + + const downloadedFile = await download(fileName); + const fileStream = fs.createReadStream( + downloadedFile, + // @ts-ignore + encoding + ); + const parseJsonStream = parser(); + fileStream.pipe(parseJsonStream); + + const parseStream = new ParseStream({ limitRows, jsonStyle, keyField }); + + const tramsformer = jsonStyle === 'object' ? streamObject() : streamArray(); + + if (rootField) { + const filterStream = pick({ filter: rootField }); + parseJsonStream.pipe(filterStream); + filterStream.pipe(tramsformer); + } else { + parseJsonStream.pipe(tramsformer); + } + + tramsformer.pipe(parseStream); + + return parseStream; +} + +module.exports = jsonReader; diff --git a/packages/api/src/shell/jsonWriter.js b/packages/api/src/shell/jsonWriter.js new file mode 100644 index 000000000..a0ce19f24 --- /dev/null +++ b/packages/api/src/shell/jsonWriter.js @@ -0,0 +1,97 @@ +const { getLogger } = require('dbgate-tools'); +const fs = require('fs'); +const stream = require('stream'); +const _ = require('lodash'); + +const logger = getLogger('jsonArrayWriter'); + +class StringifyStream extends stream.Transform { + constructor({ jsonStyle, keyField, rootField }) { + super({ objectMode: true }); + this.wasHeader = false; + this.wasRecord = false; + this.jsonStyle = jsonStyle; + this.keyField = keyField || '_key'; + this.rootField = rootField; + } + _transform(chunk, encoding, done) { + let skip = false; + + if (!this.wasHeader) { + skip = chunk.__isStreamHeader; + this.wasHeader = true; + } + if (!skip) { + if (!this.wasRecord) { + if (this.rootField) { + if (this.jsonStyle === 'object') { + this.push(`{"${this.rootField}": {\n`); + } else { + this.push(`{"${this.rootField}": [\n`); + } + } else { + if (this.jsonStyle === 'object') { + this.push('{\n'); + } else { + this.push('[\n'); + } + } + } else { + this.push(',\n'); + } + this.wasRecord = true; + + if (this.jsonStyle === 'object') { + const key = chunk[this.keyField] ?? chunk[Object.keys(chunk)[0]]; + this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [this.keyField]))}`); + } else { + this.push(JSON.stringify(chunk)); + } + } + done(); + } + + _flush(done) { + if (!this.wasRecord) { + if (this.rootField) { + if (this.jsonStyle === 'object') { + this.push(`{"${this.rootField}": {}}\n`); + } else { + this.push(`{"${this.rootField}": []}\n`); + } + } else { + if (this.jsonStyle === 'object') { + this.push('{}\n'); + } else { + this.push('[]\n'); + } + } + } else { + if (this.rootField) { + if (this.jsonStyle === 'object') { + this.push('\n}}\n'); + } else { + this.push('\n]}\n'); + } + } else { + if (this.jsonStyle === 'object') { + this.push('\n}\n'); + } else { + this.push('\n]\n'); + } + } + } + done(); + } +} + +async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, encoding = 'utf-8' }) { + logger.info(`Writing file ${fileName}`); + const stringify = new StringifyStream({ jsonStyle, keyField, rootField }); + const fileStream = fs.createWriteStream(fileName, encoding); + stringify.pipe(fileStream); + stringify['finisher'] = fileStream; + return stringify; +} + +module.exports = jsonWriter; diff --git a/packages/api/src/shell/modifyJsonLinesReader.js b/packages/api/src/shell/modifyJsonLinesReader.js index 7aa007369..dec137ba0 100644 --- a/packages/api/src/shell/modifyJsonLinesReader.js +++ b/packages/api/src/shell/modifyJsonLinesReader.js @@ -66,7 +66,7 @@ class ParseStream extends stream.Transform { ...obj, ...update.fields, }, - (v, k) => v.$$undefined$$ + (v, k) => v?.$$undefined$$ ); } } diff --git a/packages/web/src/appobj/ArchiveFileAppObject.svelte b/packages/web/src/appobj/ArchiveFileAppObject.svelte index 20457d54f..97874510d 100644 --- a/packages/web/src/appobj/ArchiveFileAppObject.svelte +++ b/packages/web/src/appobj/ArchiveFileAppObject.svelte @@ -65,18 +65,18 @@
diff --git a/packages/web/src/impexp/SourceTargetConfig.svelte b/packages/web/src/impexp/SourceTargetConfig.svelte index 241c02ecb..d25e55128 100644 --- a/packages/web/src/impexp/SourceTargetConfig.svelte +++ b/packages/web/src/impexp/SourceTargetConfig.svelte @@ -196,6 +196,7 @@ width: 20vw; margin-left: var(--dim-large-form-margin); margin-bottom: var(--dim-large-form-margin); + border: 1px solid var(--theme-border); } .label { diff --git a/packages/web/src/impexp/createImpExpScript.ts b/packages/web/src/impexp/createImpExpScript.ts index 5ea6c3b4d..cb001994f 100644 --- a/packages/web/src/impexp/createImpExpScript.ts +++ b/packages/web/src/impexp/createImpExpScript.ts @@ -192,7 +192,7 @@ export function normalizeExportColumnMap(colmap) { return null; } -export default async function createImpExpScript(extensions, values, addEditorInfo = true, forceScript = false) { +export default async function createImpExpScript(extensions, values, forceScript = false) { const config = getCurrentConfig(); const script = config.allowShellScripting || forceScript @@ -233,10 +233,6 @@ export default async function createImpExpScript(extensions, values, addEditorIn script.copyStream(sourceVar, targetVar, colmapVar); script.endLine(); } - if (addEditorInfo) { - script.comment('@ImportExportConfigurator'); - script.comment(JSON.stringify(values)); - } return script.getScript(values.schedule); } diff --git a/packages/web/src/modals/ChangeDownloadUrlModal.svelte b/packages/web/src/modals/ChangeDownloadUrlModal.svelte index 86ffb41ec..980f0aaf1 100644 --- a/packages/web/src/modals/ChangeDownloadUrlModal.svelte +++ b/packages/web/src/modals/ChangeDownloadUrlModal.svelte @@ -8,6 +8,7 @@ import { closeCurrentModal } from './modalTools'; export let onConfirm; + export let url; const handleSubmit = e => { onConfirm(e.detail.url); @@ -15,7 +16,7 @@ }; - + Download imported file from web diff --git a/packages/web/src/modals/ImportExportModal.svelte b/packages/web/src/modals/ImportExportModal.svelte deleted file mode 100644 index 1d1c6dc78..000000000 --- a/packages/web/src/modals/ImportExportModal.svelte +++ /dev/null @@ -1,203 +0,0 @@ - - - - - - Import/Export - {#if busy} - - {/if} - - - -
- - - {#if busy} - - {/if} -
- - - - - - - - - - - - - - - - - - - - - -
- {#if busy} - Stop - {:else} - Run - {/if} - Generate script - - Close -
- - - - - diff --git a/packages/web/src/plugins/fileformats.ts b/packages/web/src/plugins/fileformats.ts index a65aef9df..94b49411e 100644 --- a/packages/web/src/plugins/fileformats.ts +++ b/packages/web/src/plugins/fileformats.ts @@ -13,7 +13,33 @@ const jsonFormat = { storageType: 'json', extension: 'json', name: 'JSON', - writerFunc: 'jsonArrayWriter', + readerFunc: 'jsonReader', + writerFunc: 'jsonWriter', + + args: [ + { + type: 'select', + name: 'jsonStyle', + label: 'JSON style', + options: [ + { name: 'Array', value: '' }, + { name: 'Object', value: 'object' }, + ], + apiName: 'jsonStyle', + }, + { + type: 'text', + name: 'keyField', + label: 'Key field (only for "Object" style)', + apiName: 'keyField', + }, + { + type: 'text', + name: 'rootField', + label: 'Root field', + apiName: 'rootField', + }, + ], }; const sqlFormat = { @@ -38,7 +64,7 @@ const jsonQuickExport = { label: 'JSON', extension: 'json', createWriter: fileName => ({ - functionName: 'jsonArrayWriter', + functionName: 'jsonWriter', props: { fileName, }, diff --git a/packages/web/src/tabs/ImportExportTab.svelte b/packages/web/src/tabs/ImportExportTab.svelte index 2e065d33f..b7d0a664c 100644 --- a/packages/web/src/tabs/ImportExportTab.svelte +++ b/packages/web/src/tabs/ImportExportTab.svelte @@ -1,22 +1,39 @@ + + - - -
- + + + +
+ - {#if busy} - - {/if} -
+ {#if busy} + + {/if} +
- - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + {#if busy} + Stop + {:else} + Run + {/if} + Generate script + + + diff --git a/packages/web/src/tabs/ShellTab.svelte b/packages/web/src/tabs/ShellTab.svelte index 05a4eea74..6a8100067 100644 --- a/packages/web/src/tabs/ShellTab.svelte +++ b/packages/web/src/tabs/ShellTab.svelte @@ -23,15 +23,6 @@ onClick: () => getCurrentEditor().copyNodeScript(), }); - registerCommand({ - id: 'shell.openWizard', - category: 'Shell', - name: 'Open wizard', - // testEnabled: () => getCurrentEditor()?.openWizardEnabled(), - onClick: () => getCurrentEditor().openWizard(), - }); - - const configRegex = /\s*\/\/\s*@ImportExportConfigurator\s*\n\s*\/\/\s*(\{[^\n]+\})\n/; const requireRegex = /\s*(\/\/\s*@require\s+[^\n]+)\n/g; const initRegex = /([^\n]+\/\/\s*@init)/g; @@ -47,8 +38,6 @@ import { registerFileCommands } from '../commands/stdCommands'; import VerticalSplitter from '../elements/VerticalSplitter.svelte'; - import ImportExportModal from '../modals/ImportExportModal.svelte'; - import { showModal } from '../modals/modalTools'; import AceEditor from '../query/AceEditor.svelte'; import RunnerOutputPane from '../query/RunnerOutputPane.svelte'; import useEditorData from '../query/useEditorData'; @@ -60,7 +49,7 @@ import { showSnackbarError } from '../utility/snackbar'; import useEffect from '../utility/useEffect'; import useTimerLabel from '../utility/useTimerLabel'; - + export let tabid; const tabVisible: any = getContext('tabVisible'); @@ -149,19 +138,6 @@ copyTextToClipboard(resp); } - // export function openWizardEnabled() { - // return ($editorValue || '').match(configRegex); - // } - - export function openWizard() { - const jsonTextMatch = ($editorValue || '').match(configRegex); - if (jsonTextMatch) { - showModal(ImportExportModal, { initialValues: JSON.parse(jsonTextMatch[1]) }); - } else { - showSnackbarError('No wizard info found'); - } - } - function getActiveScript() { const selectedText = domEditor.getEditor().getSelectedText(); const editorText = $editorValue; @@ -208,7 +184,6 @@ return [ { command: 'shell.execute' }, { command: 'shell.kill' }, - { command: 'shell.openWizard' }, { divider: true }, { command: 'shell.toggleComment' }, { divider: true }, diff --git a/packages/web/src/utility/exportFileTools.ts b/packages/web/src/utility/exportFileTools.ts index 39ce5d676..4704aefed 100644 --- a/packages/web/src/utility/exportFileTools.ts +++ b/packages/web/src/utility/exportFileTools.ts @@ -42,8 +42,8 @@ export async function exportSqlDump(outputFile, connection, databaseName, pureFi onOpenResult: pureFileName && !getElectron() ? () => { - downloadFromApi(`uploads/get?file=${pureFileName}`, 'file.sql'); - } + downloadFromApi(`uploads/get?file=${pureFileName}`, 'file.sql'); + } : null, openResultLabel: 'Download SQL file', }); @@ -226,17 +226,18 @@ export async function downloadFromApi(route: string, donloadName: string) { method: 'GET', headers: resolveApiHeaders(), }) - .then((res) => res.blob()) - .then((blob) => { + .then(res => res.blob()) + .then(blob => { const objUrl = URL.createObjectURL(blob); - const a = document.createElement("a"); + const a = document.createElement('a'); document.body.appendChild(a); a.download = donloadName; a.href = objUrl; a.click(); a.remove(); setTimeout(() => { - URL.revokeObjectURL(objUrl) - }) - }) + URL.revokeObjectURL(objUrl); + }); + }); } + diff --git a/packages/web/src/utility/importExportTools.ts b/packages/web/src/utility/importExportTools.ts new file mode 100644 index 000000000..bed2b20db --- /dev/null +++ b/packages/web/src/utility/importExportTools.ts @@ -0,0 +1,15 @@ +import openNewTab from './openNewTab'; + +export function openImportExportTab(editorProps, additionalProps = {}) { + openNewTab( + { + tabComponent: 'ImportExportTab', + title: 'Import/Export', + icon: 'img export', + props: additionalProps, + }, + { + editor: editorProps, + } + ); +} diff --git a/packages/web/src/utility/openElectronFile.ts b/packages/web/src/utility/openElectronFile.ts index a7f1e4fef..c4e0b09d0 100644 --- a/packages/web/src/utility/openElectronFile.ts +++ b/packages/web/src/utility/openElectronFile.ts @@ -1,17 +1,17 @@ import { showModal } from '../modals/modalTools'; import { get } from 'svelte/store'; import newQuery from '../query/newQuery'; -import ImportExportModal from '../modals/ImportExportModal.svelte'; import getElectron from './getElectron'; import { currentDatabase, extensions, getCurrentDatabase } from '../stores'; import { getUploadListener } from './uploadFiles'; -import {getConnectionLabel, getDatabaseFileLabel } from 'dbgate-tools'; +import { getConnectionLabel, getDatabaseFileLabel } from 'dbgate-tools'; import { apiCall } from './api'; import openNewTab from './openNewTab'; import { openJsonDocument } from '../tabs/JsonTab.svelte'; import { SAVED_FILE_HANDLERS } from '../appobj/SavedFileAppObject.svelte'; import _ from 'lodash'; import ErrorMessageModal from '../modals/ErrorMessageModal.svelte'; +import { openImportExportTab } from './importExportTools'; export function canOpenByElectron(file, extensions) { if (!file) return false; @@ -178,17 +178,30 @@ export function openElectronFileCore(filePath, extensions) { shortName: parsed.name, }); } else { - showModal(ImportExportModal, { - openedFile: { - filePath, - storageType: format.storageType, - shortName: parsed.name, - }, - importToCurrentTarget: true, - initialValues: { + openImportExportTab( + { sourceStorageType: format.storageType, }, - }); + { + openedFile: { + filePath, + storageType: format.storageType, + shortName: parsed.name, + }, + } + ); + + // showModal(ImportExportModal, { + // openedFile: { + // filePath, + // storageType: format.storageType, + // shortName: parsed.name, + // }, + // importToCurrentTarget: true, + // initialValues: { + // sourceStorageType: format.storageType, + // }, + // }); } } } diff --git a/packages/web/src/utility/uploadFiles.ts b/packages/web/src/utility/uploadFiles.ts index 88d6a8d89..ddbcfa503 100644 --- a/packages/web/src/utility/uploadFiles.ts +++ b/packages/web/src/utility/uploadFiles.ts @@ -5,9 +5,9 @@ import getElectron from './getElectron'; import resolveApi, { resolveApiHeaders } from './resolveApi'; import { findFileFormat } from '../plugins/fileformats'; import { showModal } from '../modals/modalTools'; -import ImportExportModal from '../modals/ImportExportModal.svelte'; import ErrorMessageModal from '../modals/ErrorMessageModal.svelte'; import openNewTab from './openNewTab'; +import { openImportExportTab } from './importExportTools'; let uploadListener; @@ -79,13 +79,23 @@ export default function uploadFiles(files) { uploadListener(fileData); } else { if (findFileFormat(ext, fileData.storageType)) { - showModal(ImportExportModal, { - uploadedFile: fileData, - importToCurrentTarget: true, - initialValues: { + openImportExportTab( + { sourceStorageType: fileData.storageType, }, - }); + { + uploadedFile: fileData, + importToCurrentTarget: true, + } + ); + + // showModal(ImportExportModal, { + // uploadedFile: fileData, + // importToCurrentTarget: true, + // initialValues: { + // sourceStorageType: fileData.storageType, + // }, + // }); } } diff --git a/packages/web/src/widgets/SavedFilesList.svelte b/packages/web/src/widgets/SavedFilesList.svelte index 9d7989039..44d8da920 100644 --- a/packages/web/src/widgets/SavedFilesList.svelte +++ b/packages/web/src/widgets/SavedFilesList.svelte @@ -20,6 +20,7 @@ const queryFiles = useFiles({ folder: 'query' }); const sqliteFiles = useFiles({ folder: 'sqlite' }); const diagramFiles = useFiles({ folder: 'diagrams' }); + const jobFiles = useFiles({ folder: 'jobs' }); const perspectiveFiles = useFiles({ folder: 'perspectives' }); $: files = [ @@ -31,11 +32,12 @@ ...($sqliteFiles || []), ...($diagramFiles || []), ...($perspectiveFiles || []), + ...($jobFiles || []), ]; function handleRefreshFiles() { apiCall('files/refresh', { - folders: ['sql', 'shell', 'markdown', 'charts', 'query', 'sqlite', 'diagrams', 'perspectives'], + folders: ['sql', 'shell', 'markdown', 'charts', 'query', 'sqlite', 'diagrams', 'perspectives', 'jobs'], }); } diff --git a/plugins/dbgate-plugin-csv/package.json b/plugins/dbgate-plugin-csv/package.json index 08fefed64..72bc027a3 100644 --- a/plugins/dbgate-plugin-csv/package.json +++ b/plugins/dbgate-plugin-csv/package.json @@ -32,10 +32,11 @@ "prepublishOnly": "yarn build" }, "devDependencies": { - "csv": "^5.3.2", + "csv": "^6.3.10", "dbgate-plugin-tools": "^1.0.7", + "line-reader": "^0.4.0", "lodash": "^4.17.21", "webpack": "^5.91.0", "webpack-cli": "^5.1.4" } -} \ No newline at end of file +} diff --git a/plugins/dbgate-plugin-csv/src/backend/reader.js b/plugins/dbgate-plugin-csv/src/backend/reader.js index 6c07518f5..53d0ad335 100644 --- a/plugins/dbgate-plugin-csv/src/backend/reader.js +++ b/plugins/dbgate-plugin-csv/src/backend/reader.js @@ -2,8 +2,32 @@ const zipObject = require('lodash/zipObject'); const csv = require('csv'); const fs = require('fs'); const stream = require('stream'); +const lineReader = require('line-reader'); let dbgateApi; + +function readFirstLine(file) { + return new Promise((resolve, reject) => { + lineReader.open(file, (err, reader) => { + if (err) { + reject(err); + return; + } + if (reader.hasNextLine()) { + reader.nextLine((err, line) => { + if (err) { + reader.close(() => reject(err)); // Ensure reader is closed on error + return; + } + reader.close(() => resolve(line)); // Ensure reader is closed after reading + }); + } else { + reader.close(() => resolve(null)); // Properly close if no lines are present + } + }); + }); +} + class CsvPrepareStream extends stream.Transform { constructor({ header }) { super({ objectMode: true }); @@ -46,13 +70,29 @@ class CsvPrepareStream extends stream.Transform { async function reader({ fileName, encoding = 'utf-8', header = true, delimiter, limitRows = undefined }) { console.log(`Reading file ${fileName}`); + const downloadedFile = await dbgateApi.download(fileName); + + if (!delimiter) { + // auto detect delimiter + // read first line from downloadedFile + const firstLine = await readFirstLine(downloadedFile); + if (firstLine) { + const delimiterCounts = { + ',': firstLine.replace(/[^,]/g, '').length, + ';': firstLine.replace(/[^;]/g, '').length, + '|': firstLine.replace(/[^|]/g, '').length, + }; + + delimiter = Object.keys(delimiterCounts).reduce((a, b) => (delimiterCounts[a] > delimiterCounts[b] ? a : b), ','); + } + } const csvStream = csv.parse({ // @ts-ignore delimiter, skip_lines_with_error: true, to_line: limitRows ? limitRows + 1 : undefined, + ltrim: true, }); - const downloadedFile = await dbgateApi.download(fileName); const fileStream = fs.createReadStream(downloadedFile, encoding); const csvPrepare = new CsvPrepareStream({ header }); fileStream.pipe(csvStream); diff --git a/plugins/dbgate-plugin-csv/src/frontend/index.js b/plugins/dbgate-plugin-csv/src/frontend/index.js index 54ae04114..ef43d8ded 100644 --- a/plugins/dbgate-plugin-csv/src/frontend/index.js +++ b/plugins/dbgate-plugin-csv/src/frontend/index.js @@ -17,6 +17,7 @@ const fileFormat = { name: 'delimiter', label: 'Delimiter', options: [ + { name: 'Auto-detect', value: '' }, { name: 'Comma (,)', value: ',' }, { name: 'Semicolon (;)', value: ';' }, { name: 'Tab', value: '\t' }, diff --git a/yarn.lock b/yarn.lock index 97bc9f2ef..66c1a2057 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3116,30 +3116,30 @@ cssstyle@^2.3.0: dependencies: cssom "~0.3.6" -csv-generate@^3.4.3: - version "3.4.3" - resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-3.4.3.tgz#bc42d943b45aea52afa896874291da4b9108ffff" - integrity sha512-w/T+rqR0vwvHqWs/1ZyMDWtHHSJaN06klRqJXBEpDJaM/+dZkso0OKh1VcuuYvK3XM53KysVNq8Ko/epCK8wOw== +csv-generate@^4.4.1: + version "4.4.1" + resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-4.4.1.tgz#729781ace8d1b92f6bfb407d1ab9548728c55681" + integrity sha512-O/einO0v4zPmXaOV+sYqGa02VkST4GP5GLpWBNHEouIU7pF3kpGf3D0kCCvX82ydIY4EKkOK+R8b1BYsRXravg== -csv-parse@^4.16.3: - version "4.16.3" - resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.16.3.tgz#7ca624d517212ebc520a36873c3478fa66efbaf7" - integrity sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg== +csv-parse@^5.5.6: + version "5.5.6" + resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.5.6.tgz#0d726d58a60416361358eec291a9f93abe0b6b1a" + integrity sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A== -csv-stringify@^5.6.5: - version "5.6.5" - resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-5.6.5.tgz#c6d74badda4b49a79bf4e72f91cce1e33b94de00" - integrity sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A== +csv-stringify@^6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.5.1.tgz#a31837dd35e34787e3c248159c982a21af964d94" + integrity sha512-+9lpZfwpLntpTIEpFbwQyWuW/hmI/eHuJZD1XzeZpfZTqkf1fyvBbBLXTJJMsBuuS11uTShMqPwzx4A6ffXgRQ== -csv@^5.3.2: - version "5.5.3" - resolved "https://registry.yarnpkg.com/csv/-/csv-5.5.3.tgz#cd26c1e45eae00ce6a9b7b27dcb94955ec95207d" - integrity sha512-QTaY0XjjhTQOdguARF0lGKm5/mEq9PD9/VhZZegHDIBq2tQwgNpHc3dneD4mGo2iJs+fTKv5Bp0fZ+BRuY3Z0g== +csv@^6.3.10: + version "6.3.10" + resolved "https://registry.yarnpkg.com/csv/-/csv-6.3.10.tgz#960a3a9cef08573ecca2d80ddb71152aca383088" + integrity sha512-5NYZG4AN2ZUthmNxIudgBEdMPUnbQHu9V4QTzBPqQzUP3KQsFiJo+8HQ0+oVxj1PomIT1/f67VI1QH/hsrZLKA== dependencies: - csv-generate "^3.4.3" - csv-parse "^4.16.3" - csv-stringify "^5.6.5" - stream-transform "^2.1.3" + csv-generate "^4.4.1" + csv-parse "^5.5.6" + csv-stringify "^6.5.1" + stream-transform "^3.3.2" dashdash@^1.12.0: version "1.14.1" @@ -7456,11 +7456,6 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mixme@^0.5.1: - version "0.5.10" - resolved "https://registry.yarnpkg.com/mixme/-/mixme-0.5.10.tgz#d653b2984b75d9018828f1ea333e51717ead5f51" - integrity sha512-5H76ANWinB1H3twpJ6JY8uvAtpmFvHNArpilJAjXRKXSDDLPIMoZArw5SH0q9z+lLs8IrMw7Q2VWpWimFKFT1Q== - mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" @@ -9750,12 +9745,22 @@ stoppable@^1.1.0: resolved "https://registry.yarnpkg.com/stoppable/-/stoppable-1.1.0.tgz#32da568e83ea488b08e4d7ea2c3bcc9d75015d5b" integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw== -stream-transform@^2.1.3: - version "2.1.3" - resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-2.1.3.tgz#a1c3ecd72ddbf500aa8d342b0b9df38f5aa598e3" - integrity sha512-9GHUiM5hMiCi6Y03jD2ARC1ettBXkQBoQAe7nJsPknnI0ow10aXjTnew8QtYQmLjzn974BnmWEAJgCY6ZP1DeQ== +stream-chain@^2.2.5: + version "2.2.5" + resolved "https://registry.yarnpkg.com/stream-chain/-/stream-chain-2.2.5.tgz#b30967e8f14ee033c5b9a19bbe8a2cba90ba0d09" + integrity sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA== + +stream-json@^1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/stream-json/-/stream-json-1.8.0.tgz#53f486b2e3b4496c506131f8d7260ba42def151c" + integrity sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw== dependencies: - mixme "^0.5.1" + stream-chain "^2.2.5" + +stream-transform@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-3.3.2.tgz#398c67b2f3b6ed5d04ceadde9e412bda8416c8ab" + integrity sha512-v64PUnPy9Qw94NGuaEMo+9RHQe4jTBYf+NkTtqkCgeuiNo8NlL0LtLR7fkKWNVFtp3RhIm5Dlxkgm5uz7TDimQ== streamsearch@^1.1.0: version "1.1.0" @@ -10182,6 +10187,11 @@ tmp@^0.0.33: dependencies: os-tmpdir "~1.0.2" +tmp@^0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.3.tgz#eb783cc22bc1e8bebd0671476d46ea4eb32a79ae" + integrity sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w== + tmpl@1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc"