From 3a75ad61f3c5cabf83111efc75e7f9a3cc54d481 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 14 Mar 2025 10:45:30 +0100 Subject: [PATCH] SYNC: Merge branch 'feature/backup-restore' --- .../src/controllers/databaseConnections.js | 126 +++++++++++++++++ packages/api/src/controllers/runners.js | 76 +++++++++- packages/api/src/controllers/uploads.js | 49 ++++++- packages/api/src/main.js | 3 +- packages/api/src/shell/dumpDatabase.js | 47 ------- packages/api/src/shell/index.js | 2 - packages/tools/src/ScriptWriter.ts | 14 -- packages/tools/src/getConnectionLabel.ts | 8 ++ packages/types/engines.d.ts | 44 +++++- .../web/src/appobj/ConnectionAppObject.svelte | 22 +-- .../web/src/appobj/DatabaseAppObject.svelte | 37 +++-- .../src/appobj/DatabaseObjectAppObject.svelte | 8 +- .../web/src/buttons/InlineButtonLabel.svelte | 74 ++++++++++ .../src/elements/HorizontalSplitter.svelte | 3 + packages/web/src/elements/TabControl.svelte | 2 + packages/web/src/forms/FormArgument.svelte | 14 +- packages/web/src/icons/FontIcon.svelte | 3 + .../src/modals/ExportDatabaseDumpModal.svelte | 117 ---------------- .../src/modals/ImportDatabaseDumpModal.svelte | 130 ------------------ .../web/src/modals/SqlGeneratorModal.svelte | 1 + .../web/src/query/RunnerOutputFiles.svelte | 66 +++++---- .../web/src/settings/SettingsModal.svelte | 26 ++++ packages/web/src/utility/exportFileTools.ts | 20 --- packages/web/src/utility/openWebFile.ts | 25 ++++ packages/web/src/utility/uploadFiles.ts | 6 + .../web/src/widgets/SavedFilesList.svelte | 58 +++++++- .../src/widgets/WidgetsInnerContainer.svelte | 17 ++- plugins/dbgate-plugin-mysql/package.json | 1 - .../src/backend/drivers.js | 10 -- .../src/frontend/drivers.js | 119 +++++++++++++++- .../src/frontend/drivers.js | 125 +++++++++++++++++ 31 files changed, 842 insertions(+), 411 deletions(-) delete mode 100644 packages/api/src/shell/dumpDatabase.js create mode 100644 packages/web/src/buttons/InlineButtonLabel.svelte delete mode 100644 packages/web/src/modals/ExportDatabaseDumpModal.svelte delete mode 100644 packages/web/src/modals/ImportDatabaseDumpModal.svelte create mode 100644 packages/web/src/utility/openWebFile.ts diff --git a/packages/api/src/controllers/databaseConnections.js b/packages/api/src/controllers/databaseConnections.js index 7e811a7fd..5af26851a 100644 --- a/packages/api/src/controllers/databaseConnections.js +++ b/packages/api/src/controllers/databaseConnections.js @@ -1,4 +1,5 @@ const connections = require('./connections'); +const runners = require('./runners'); const archive = require('./archive'); const socket = require('../utility/socket'); const { fork } = require('child_process'); @@ -613,4 +614,129 @@ module.exports = { return res; }, + + async getNativeOpCommandArgs( + command, + { conid, database, outputFile, inputFile, options, selectedTables, skippedTables, argsFormat } + ) { + const connection = await connections.getCore({ conid }); + const driver = requireEngineDriver(connection); + + const settingsValue = await config.getSettings(); + + const externalTools = {}; + for (const pair of Object.entries(settingsValue || {})) { + const [name, value] = pair; + if (name.startsWith('externalTools.')) { + externalTools[name.substring('externalTools.'.length)] = value; + } + } + + return { + ...(command == 'backup' + ? driver.backupDatabaseCommand( + connection, + { outputFile, database, options, selectedTables, skippedTables, argsFormat }, + // @ts-ignore + externalTools + ) + : driver.restoreDatabaseCommand( + connection, + { inputFile, database, options, argsFormat }, + // @ts-ignore + externalTools + )), + transformMessage: driver.transformNativeCommandMessage + ? message => driver.transformNativeCommandMessage(message, command) + : null, + }; + }, + + commandArgsToCommandLine(commandArgs) { + const { command, args, stdinFilePath } = commandArgs; + let res = `${command} ${args.join(' ')}`; + if (stdinFilePath) { + res += ` < ${stdinFilePath}`; + } + return res; + }, + + nativeBackup_meta: true, + async nativeBackup({ conid, database, outputFile, runid, options, selectedTables, skippedTables }) { + const commandArgs = await this.getNativeOpCommandArgs('backup', { + conid, + database, + inputFile: undefined, + outputFile, + options, + selectedTables, + skippedTables, + argsFormat: 'spawn', + }); + + return runners.nativeRunCore(runid, { + ...commandArgs, + onFinished: () => { + socket.emitChanged(`files-changed`, { folder: 'sql' }); + socket.emitChanged(`all-files-changed`); + }, + }); + }, + + nativeBackupCommand_meta: true, + async nativeBackupCommand({ conid, database, outputFile, options, selectedTables, skippedTables }) { + const commandArgs = await this.getNativeOpCommandArgs('backup', { + conid, + database, + outputFile, + inputFile: undefined, + options, + selectedTables, + skippedTables, + argsFormat: 'shell', + }); + + return { + ...commandArgs, + transformMessage: null, + commandLine: this.commandArgsToCommandLine(commandArgs), + }; + }, + + nativeRestore_meta: true, + async nativeRestore({ conid, database, inputFile, runid }) { + const commandArgs = await this.getNativeOpCommandArgs('restore', { + conid, + database, + inputFile, + outputFile: undefined, + options: undefined, + argsFormat: 'spawn', + }); + + return runners.nativeRunCore(runid, { + ...commandArgs, + onFinished: () => { + this.syncModel({ conid, database, isFullRefresh: true }); + }, + }); + }, + + nativeRestoreCommand_meta: true, + async nativeRestoreCommand({ conid, database, inputFile }) { + const commandArgs = await this.getNativeOpCommandArgs('restore', { + conid, + database, + inputFile, + outputFile: undefined, + options: undefined, + argsFormat: 'shell', + }); + + return { + ...commandArgs, + transformMessage: null, + commandLine: this.commandArgsToCommandLine(commandArgs), + }; + }, }; diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js index 86967cf90..eabb2f1c2 100644 --- a/packages/api/src/controllers/runners.js +++ b/packages/api/src/controllers/runners.js @@ -4,7 +4,7 @@ const path = require('path'); const fs = require('fs-extra'); const byline = require('byline'); const socket = require('../utility/socket'); -const { fork } = require('child_process'); +const { fork, spawn } = require('child_process'); const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories'); const { extractShellApiPlugins, @@ -13,6 +13,8 @@ const { getLogger, safeJsonParse, pinoLogRecordToMessageRecord, + extractErrorMessage, + extractErrorLogData, } = require('dbgate-tools'); const { handleProcessCommunication } = require('../utility/processComm'); const processArgs = require('../utility/processArgs'); @@ -80,6 +82,7 @@ module.exports = { } : { message, + severity: 'info', time: new Date(), }; @@ -173,7 +176,7 @@ module.exports = { // console.log('... ERROR subprocess', error); this.rejectRequest(runid, { message: error && (error.message || error.toString()) }); console.error('... ERROR subprocess', error); - this.dispatchMessage({ + this.dispatchMessage(runid, { severity: 'error', message: error.toString(), }); @@ -192,6 +195,75 @@ module.exports = { return _.pick(newOpened, ['runid']); }, + nativeRunCore(runid, commandArgs) { + const { command, args, env, transformMessage, stdinFilePath, onFinished } = commandArgs; + const pipeDispatcher = severity => data => { + let messageObject = { + message: data.toString().trim(), + severity, + }; + if (transformMessage) { + messageObject = transformMessage(messageObject); + } + + if (messageObject) { + return this.dispatchMessage(runid, messageObject); + } + }; + + const subprocess = spawn(command, args, { env: { ...process.env, ...env } }); + + byline(subprocess.stdout).on('data', pipeDispatcher('info')); + byline(subprocess.stderr).on('data', pipeDispatcher('error')); + + subprocess.on('exit', code => { + console.log('... EXITED', code); + logger.info({ code, pid: subprocess.pid }, 'Exited process'); + this.dispatchMessage(runid, `Finished external process with code ${code}`); + socket.emit(`runner-done-${runid}`, code); + if (onFinished) { + onFinished(); + } + }); + subprocess.on('spawn', () => { + this.dispatchMessage(runid, `Started external process ${command}`); + }); + subprocess.on('error', error => { + console.log('... ERROR subprocess', error); + this.dispatchMessage(runid, { + severity: 'error', + message: error.toString(), + }); + if (error['code'] == 'ENOENT') { + this.dispatchMessage(runid, { + severity: 'error', + message: `Command ${command} not found, please install it and configure its location in DbGate settings, Settings/External tools, if ${command} is not in system PATH`, + }); + } + socket.emit(`runner-done-${runid}`); + }); + + if (stdinFilePath) { + const inputStream = fs.createReadStream(stdinFilePath); + inputStream.pipe(subprocess.stdin); + + subprocess.stdin.on('error', err => { + this.dispatchMessage(runid, { + severity: 'error', + message: extractErrorMessage(err), + }); + logger.error(extractErrorLogData(err), 'Caught error on stdin'); + }); + } + + const newOpened = { + runid, + subprocess, + }; + this.opened.push(newOpened); + return _.pick(newOpened, ['runid']); + }, + start_meta: true, async start({ script }) { const runid = crypto.randomUUID(); diff --git a/packages/api/src/controllers/uploads.js b/packages/api/src/controllers/uploads.js index 5fc67a854..b406a7599 100644 --- a/packages/api/src/controllers/uploads.js +++ b/packages/api/src/controllers/uploads.js @@ -1,6 +1,6 @@ const crypto = require('crypto'); const path = require('path'); -const { uploadsdir, getLogsFilePath } = require('../utility/directories'); +const { uploadsdir, getLogsFilePath, filesdir } = require('../utility/directories'); const { getLogger, extractErrorLogData } = require('dbgate-tools'); const logger = getLogger('uploads'); const axios = require('axios'); @@ -13,6 +13,7 @@ const serverConnections = require('./serverConnections'); const config = require('./config'); const gistSecret = require('../gistSecret'); const currentVersion = require('../currentVersion'); +const socket = require('../utility/socket'); module.exports = { upload_meta: { @@ -38,6 +39,52 @@ module.exports = { }); }, + uploadDataFile_meta: { + method: 'post', + raw: true, + }, + uploadDataFile(req, res) { + const { data } = req.files || {}; + + if (!data) { + res.json(null); + return; + } + + if (data.name.toLowerCase().endsWith('.sql')) { + logger.info(`Uploading SQL file ${data.name}, size=${data.size}`); + data.mv(path.join(filesdir(), 'sql', data.name), () => { + res.json({ + name: data.name, + folder: 'sql', + }); + + socket.emitChanged(`files-changed`, { folder: 'sql' }); + socket.emitChanged(`all-files-changed`); + }); + return; + } + + res.json(null); + }, + + saveDataFile_meta: true, + async saveDataFile({ filePath }) { + if (filePath.toLowerCase().endsWith('.sql')) { + logger.info(`Saving SQL file ${filePath}`); + await fs.copyFile(filePath, path.join(filesdir(), 'sql', path.basename(filePath))); + + socket.emitChanged(`files-changed`, { folder: 'sql' }); + socket.emitChanged(`all-files-changed`); + return { + name: path.basename(filePath), + folder: 'sql', + }; + } + + return null; + }, + get_meta: { method: 'get', raw: true, diff --git a/packages/api/src/main.js b/packages/api/src/main.js index 546bf0ed2..b3b51671c 100644 --- a/packages/api/src/main.js +++ b/packages/api/src/main.js @@ -30,7 +30,7 @@ const queryHistory = require('./controllers/queryHistory'); const onFinished = require('on-finished'); const processArgs = require('./utility/processArgs'); -const { rundir } = require('./utility/directories'); +const { rundir, filesdir } = require('./utility/directories'); const platformInfo = require('./utility/platformInfo'); const getExpressPath = require('./utility/getExpressPath'); const _ = require('lodash'); @@ -133,6 +133,7 @@ function start() { // } app.use(getExpressPath('/runners/data'), express.static(rundir())); + app.use(getExpressPath('/files/data'), express.static(filesdir())); if (platformInfo.isDocker) { const port = process.env.PORT || 3000; diff --git a/packages/api/src/shell/dumpDatabase.js b/packages/api/src/shell/dumpDatabase.js deleted file mode 100644 index 8d8a016a5..000000000 --- a/packages/api/src/shell/dumpDatabase.js +++ /dev/null @@ -1,47 +0,0 @@ -const requireEngineDriver = require('../utility/requireEngineDriver'); -const { connectUtility } = require('../utility/connectUtility'); -const { getLogger } = require('dbgate-tools'); - -const logger = getLogger('dumpDb'); - -function doDump(dumper) { - return new Promise((resolve, reject) => { - dumper.once('end', () => { - resolve(true); - }); - dumper.once('error', err => { - reject(err); - }); - dumper.run(); - }); -} - -async function dumpDatabase({ - connection = undefined, - systemConnection = undefined, - driver = undefined, - outputFile, - databaseName, - schemaName, -}) { - logger.info(`Dumping database`); - - if (!driver) driver = requireEngineDriver(connection); - - const dbhan = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true })); - - try { - const dumper = await driver.createBackupDumper(dbhan, { - outputFile, - databaseName, - schemaName, - }); - await doDump(dumper); - } finally { - if (!systemConnection) { - await driver.close(dbhan); - } - } -} - -module.exports = dumpDatabase; diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index 1648301d7..b1bd1f083 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -21,7 +21,6 @@ const executeQuery = require('./executeQuery'); const loadFile = require('./loadFile'); const deployDb = require('./deployDb'); const initializeApiEnvironment = require('./initializeApiEnvironment'); -const dumpDatabase = require('./dumpDatabase'); const importDatabase = require('./importDatabase'); const loadDatabase = require('./loadDatabase'); const generateModelSql = require('./generateModelSql'); @@ -61,7 +60,6 @@ const dbgateApi = { loadFile, deployDb, initializeApiEnvironment, - dumpDatabase, importDatabase, loadDatabase, generateModelSql, diff --git a/packages/tools/src/ScriptWriter.ts b/packages/tools/src/ScriptWriter.ts index 23afee0f4..26d622ac6 100644 --- a/packages/tools/src/ScriptWriter.ts +++ b/packages/tools/src/ScriptWriter.ts @@ -50,10 +50,6 @@ export class ScriptWriter { this._put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar}, ${opts});`); } - dumpDatabase(options) { - this._put(`await dbgateApi.dumpDatabase(${JSON.stringify(options)});`); - } - importDatabase(options) { this._put(`await dbgateApi.importDatabase(${JSON.stringify(options)});`); } @@ -135,13 +131,6 @@ export class ScriptWriterJson { }); } - dumpDatabase(options) { - this.commands.push({ - type: 'dumpDatabase', - options, - }); - } - importDatabase(options) { this.commands.push({ type: 'importDatabase', @@ -193,9 +182,6 @@ export function jsonScriptToJavascript(json) { case 'comment': script.comment(cmd.text); break; - case 'dumpDatabase': - script.dumpDatabase(cmd.options); - break; case 'importDatabase': script.importDatabase(cmd.options); break; diff --git a/packages/tools/src/getConnectionLabel.ts b/packages/tools/src/getConnectionLabel.ts index 6da1100b9..48d90cf06 100644 --- a/packages/tools/src/getConnectionLabel.ts +++ b/packages/tools/src/getConnectionLabel.ts @@ -43,3 +43,11 @@ export function getConnectionLabel(connection, { allowExplicitDatabase = true, s return res; } + +export function getEngineLabel(connection) { + const match = (connection?.engine || '').match(/^([^@]*)@/); + if (match) { + return match[1]; + } + return connection?.engine; +} diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 048aa2f98..31ff89760 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -147,6 +147,29 @@ export interface DatabaseHandle { export type StreamResult = stream.Readable | (stream.Readable | stream.Writable)[]; +export interface CommandLineDefinition { + command: string; + args: string[]; + env?: { [key: string]: string }; + stdinFilePath?: string; +} + +interface BackupRestoreSettingsBase { + database: string; + options?: { [key: string]: string }; + argsFormat: 'shell' | 'spawn'; +} + +export interface BackupDatabaseSettings extends BackupRestoreSettingsBase { + outputFile: string; + selectedTables?: { pureName: string; schemaName?: string }[]; + skippedTables?: { pureName: string; schemaName?: string }[]; +} + +export interface RestoreDatabaseSettings extends BackupRestoreSettingsBase { + inputFile: string; +} + export interface EngineDriver extends FilterBehaviourProvider { engine: string; title: string; @@ -157,7 +180,8 @@ export interface EngineDriver extends FilterBehaviourProvider { supportedKeyTypes: SupportedDbKeyType[]; dataEditorTypesBehaviour: DataEditorTypesBehaviour; supportsDatabaseUrl?: boolean; - supportsDatabaseDump?: boolean; + supportsDatabaseBackup?: boolean; + supportsDatabaseRestore?: boolean; supportsServerSummary?: boolean; supportsDatabaseProfiler?: boolean; requiresDefaultSortCriteria?: boolean; @@ -261,6 +285,24 @@ export interface EngineDriver extends FilterBehaviourProvider { // simple data type adapter adaptDataType(dataType: string): string; listSchemas(dbhan: DatabaseHandle): SchemaInfo[]; + backupDatabaseCommand( + connection: any, + settings: BackupDatabaseSettings, + externalTools: { [tool: string]: string } + ): CommandLineDefinition; + restoreDatabaseCommand( + connection: any, + settings: RestoreDatabaseSettings, + externalTools: { [tool: string]: string } + ): CommandLineDefinition; + transformNativeCommandMessage( + message: { + message: string; + severity: 'info' | 'error'; + }, + command: 'backup' | 'restore' + ): { message: string; severity: 'info' | 'error' | 'debug' } | null; + getNativeOperationFormArgs(operation: 'backup' | 'restore'): any[]; analyserClass?: any; dumperClass?: any; diff --git a/packages/web/src/appobj/ConnectionAppObject.svelte b/packages/web/src/appobj/ConnectionAppObject.svelte index 8918026a7..43a9a4663 100644 --- a/packages/web/src/appobj/ConnectionAppObject.svelte +++ b/packages/web/src/appobj/ConnectionAppObject.svelte @@ -132,7 +132,6 @@ import { getDatabaseList, useUsedApps } from '../utility/metadataLoaders'; import { getLocalStorage } from '../utility/storageCache'; import { apiCall, removeVolatileMapping } from '../utility/api'; - import ImportDatabaseDumpModal from '../modals/ImportDatabaseDumpModal.svelte'; import { closeMultipleTabs } from '../tabpanel/TabsPanel.svelte'; import AboutModal from '../modals/AboutModal.svelte'; import { tick } from 'svelte'; @@ -141,6 +140,7 @@ import { switchCurrentDatabase } from '../utility/common'; import { getConnectionClickActionSetting } from '../settings/settingsTools'; import { _t } from '../translations'; + import { isProApp } from '../utility/proTools'; export let data; export let passProps; @@ -231,9 +231,14 @@ }); }; - const handleSqlRestore = () => { - showModal(ImportDatabaseDumpModal, { - connection: data, + const handleRestoreDatabase = () => { + openNewTab({ + title: 'Restore #', + icon: 'img db-restore', + tabComponent: 'RestoreDatabaseTab', + props: { + conid: data._id, + }, }); }; @@ -364,11 +369,10 @@ ), ], - driver?.databaseEngineTypes?.includes('sql') && - !data.isReadOnly && { - onClick: handleSqlRestore, - text: _t('connection.sqlRestore', { defaultMessage: 'Restore/import SQL dump' }), - }, + driver?.supportsDatabaseRestore && + isProApp() && + hasPermission(`dbops/sql-dump/import`) && + !data.isReadOnly && { onClick: handleRestoreDatabase, text: 'Restore database backup' }, ]; }; diff --git a/packages/web/src/appobj/DatabaseAppObject.svelte b/packages/web/src/appobj/DatabaseAppObject.svelte index 277f3d2a3..37820aa95 100644 --- a/packages/web/src/appobj/DatabaseAppObject.svelte +++ b/packages/web/src/appobj/DatabaseAppObject.svelte @@ -126,16 +126,27 @@ }); }; - const handleSqlDump = () => { - showModal(ExportDatabaseDumpModal, { - connection: { ...connection, database: name }, + const handleBackupDatabase = () => { + openNewTab({ + title: 'Backup #', + icon: 'img db-backup', + tabComponent: 'BackupDatabaseTab', + props: { + conid: connection._id, + database: name, + }, }); - // exportSqlDump(connection, name); }; - const handleSqlRestore = () => { - showModal(ImportDatabaseDumpModal, { - connection: { ...connection, database: name }, + const handleRestoreDatabase = () => { + openNewTab({ + title: 'Restore #', + icon: 'img db-restore', + tabComponent: 'RestoreDatabaseTab', + props: { + conid: connection._id, + database: name, + }, }); }; @@ -376,11 +387,13 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify( !connection.isReadOnly && hasPermission(`dbops/import`) && { onClick: handleImport, text: 'Import' }, isSqlOrDoc && hasPermission(`dbops/export`) && { onClick: handleExport, text: 'Export' }, - driver?.databaseEngineTypes?.includes('sql') && + driver?.supportsDatabaseRestore && + isProApp() && hasPermission(`dbops/sql-dump/import`) && - !connection.isReadOnly && { onClick: handleSqlRestore, text: 'Restore/import SQL dump' }, - driver?.supportsDatabaseDump && - hasPermission(`dbops/sql-dump/export`) && { onClick: handleSqlDump, text: 'Backup/export SQL dump' }, + !connection.isReadOnly && { onClick: handleRestoreDatabase, text: 'Restore database backup' }, + driver?.supportsDatabaseBackup && + isProApp() && + hasPermission(`dbops/sql-dump/export`) && { onClick: handleBackupDatabase, text: 'Create database backup' }, isSqlOrDoc && !connection.isReadOnly && !connection.singleDatabase && @@ -491,8 +504,6 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify( import ConfirmSqlModal, { runOperationOnDatabase, saveScriptToDatabase } from '../modals/ConfirmSqlModal.svelte'; import { filterAppsForDatabase } from '../utility/appTools'; import newQuery from '../query/newQuery'; - import ImportDatabaseDumpModal from '../modals/ImportDatabaseDumpModal.svelte'; - import ExportDatabaseDumpModal from '../modals/ExportDatabaseDumpModal.svelte'; import ConfirmModal from '../modals/ConfirmModal.svelte'; import { closeMultipleTabs } from '../tabpanel/TabsPanel.svelte'; import NewCollectionModal from '../modals/NewCollectionModal.svelte'; diff --git a/packages/web/src/appobj/DatabaseObjectAppObject.svelte b/packages/web/src/appobj/DatabaseObjectAppObject.svelte index 405c07c0b..41d61a460 100644 --- a/packages/web/src/appobj/DatabaseObjectAppObject.svelte +++ b/packages/web/src/appobj/DatabaseObjectAppObject.svelte @@ -1086,8 +1086,12 @@ icon={databaseObjectIcons[data.objectTypeField]} menu={createMenu} showPinnedInsteadOfUnpin={passProps?.showPinnedInsteadOfUnpin} - onPin={isPinned ? null : () => pinnedTables.update(list => [...list, data])} - onUnpin={isPinned ? () => pinnedTables.update(list => list.filter(x => !testEqual(x, data))) : null} + onPin={passProps?.ingorePin ? null : isPinned ? null : () => pinnedTables.update(list => [...list, data])} + onUnpin={passProps?.ingorePin + ? null + : isPinned + ? () => pinnedTables.update(list => list.filter(x => !testEqual(x, data))) + : null} extInfo={getExtInfo(data)} isChoosed={matchDatabaseObjectAppObject($selectedDatabaseObjectAppObject, data)} on:click={() => handleObjectClick(data, 'leftClick')} diff --git a/packages/web/src/buttons/InlineButtonLabel.svelte b/packages/web/src/buttons/InlineButtonLabel.svelte new file mode 100644 index 000000000..4fc7942c8 --- /dev/null +++ b/packages/web/src/buttons/InlineButtonLabel.svelte @@ -0,0 +1,74 @@ + + + + + diff --git a/packages/web/src/elements/HorizontalSplitter.svelte b/packages/web/src/elements/HorizontalSplitter.svelte index be1a6dcd4..f107d5e91 100644 --- a/packages/web/src/elements/HorizontalSplitter.svelte +++ b/packages/web/src/elements/HorizontalSplitter.svelte @@ -33,10 +33,13 @@ let collapsed2 = false; export let size = 0; + export let onChangeSize = null; let clientWidth; let customRatio = null; $: size = computeSplitterSize(initialValue, clientWidth, customRatio, initialSizeRight); + + $: if (onChangeSize) onChangeSize(size);
diff --git a/packages/web/src/elements/TabControl.svelte b/packages/web/src/elements/TabControl.svelte index b95afae33..d6abeafd3 100644 --- a/packages/web/src/elements/TabControl.svelte +++ b/packages/web/src/elements/TabControl.svelte @@ -52,6 +52,8 @@ {:else if tab.slot == 5} {:else if tab.slot == 6} {:else if tab.slot == 7} + {:else if tab.slot == 8} + {:else if tab.slot == 9} {/if} {/if}
diff --git a/packages/web/src/forms/FormArgument.svelte b/packages/web/src/forms/FormArgument.svelte index a40fcfced..9d5c2cae0 100644 --- a/packages/web/src/forms/FormArgument.svelte +++ b/packages/web/src/forms/FormArgument.svelte @@ -13,7 +13,7 @@ $: name = `${namePrefix}${arg.name}`; - const { setFieldValue } = getFormContext(); + const { setFieldValue, values } = getFormContext(); {#if arg.type == 'text'} @@ -23,7 +23,7 @@ defaultValue={arg.default} focused={arg.focused} placeholder={arg.placeholder} - disabled={arg.disabled} + disabled={arg.disabledFn ? arg.disabledFn($values) : arg.disabled} /> {:else if arg.type == 'stringlist'} @@ -35,9 +35,15 @@ defaultValue={arg.default} focused={arg.focused} placeholder={arg.placeholder} + disabled={arg.disabledFn ? arg.disabledFn($values) : arg.disabled} /> {:else if arg.type == 'checkbox'} - + {:else if arg.type == 'select'} _.isString(opt) ? { label: opt, value: opt } : { label: opt.name, value: opt.value } )} + disabled={arg.disabledFn ? arg.disabledFn($values) : arg.disabled} /> {:else if arg.type == 'dropdowntext'} setFieldValue(name, _.isString(opt) ? opt : opt.value), })); }} + disabled={arg.disabledFn ? arg.disabledFn($values) : arg.disabled} /> {/if} diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index df808a488..74acfea14 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -306,6 +306,9 @@ 'img tip': 'mdi mdi-lightbulb-on color-icon-yellow', 'img filter-active': 'mdi mdi-filter-cog color-icon-blue', + + 'img db-backup': 'mdi mdi-database-export color-icon-yellow', + 'img db-restore': 'mdi mdi-database-import color-icon-red', }; diff --git a/packages/web/src/modals/ExportDatabaseDumpModal.svelte b/packages/web/src/modals/ExportDatabaseDumpModal.svelte deleted file mode 100644 index d3a6b42fe..000000000 --- a/packages/web/src/modals/ExportDatabaseDumpModal.svelte +++ /dev/null @@ -1,117 +0,0 @@ - - - - - Export database dump - -
- Source: - {getConnectionLabel(connection)} - {#if connection.database} - ({connection.database}) - {/if} -
- -
Target: {outputLabel}
-
- {#if electron} - - {:else} - - {/if} - -
- - - handleSubmit(e.detail)} disabled={!outputFile} /> - - - - diff --git a/packages/web/src/modals/ImportDatabaseDumpModal.svelte b/packages/web/src/modals/ImportDatabaseDumpModal.svelte deleted file mode 100644 index 828e3b551..000000000 --- a/packages/web/src/modals/ImportDatabaseDumpModal.svelte +++ /dev/null @@ -1,130 +0,0 @@ - - - - - Import database dump - -
Source: {inputLabel}
- -
- {#if electron} - - {:else} - - {/if} - - - -
- -
- Target: - {getConnectionLabel(connection)} - {#if connection.database} - ({connection.database}) - {/if} -
- - - handleSubmit(e.detail)} - disabled={!inputFile} - data-testid="ImportDatabaseDumpModal_runImport" - /> - - - - diff --git a/packages/web/src/modals/SqlGeneratorModal.svelte b/packages/web/src/modals/SqlGeneratorModal.svelte index ef4dbfa4c..4305a1b79 100644 --- a/packages/web/src/modals/SqlGeneratorModal.svelte +++ b/packages/web/src/modals/SqlGeneratorModal.svelte @@ -160,6 +160,7 @@ filter={objectsFilter} disableContextMenu {checkedObjectsStore} + passProps={{ ingorePin: true }} /> diff --git a/packages/web/src/query/RunnerOutputFiles.svelte b/packages/web/src/query/RunnerOutputFiles.svelte index a7d43ded4..28a32e9a8 100644 --- a/packages/web/src/query/RunnerOutputFiles.svelte +++ b/packages/web/src/query/RunnerOutputFiles.svelte @@ -7,6 +7,7 @@ import getElectron from '../utility/getElectron'; import { downloadFromApi } from '../utility/exportFileTools'; import useEffect from '../utility/useEffect'; + import Link from '../elements/Link.svelte'; export let runnerId; export let executeNumber; @@ -63,41 +64,38 @@ }, ]} > - { - downloadFromApi(`runners/data/${runnerId}/${row.name}`, row.name); - }} - > - download - + + { + downloadFromApi(`runners/data/${runnerId}/${row.name}`, row.name); + }} + > + download + + - { - const file = await electron.showSaveDialog({}); - if (file) { - const fs = window.require('fs'); - fs.copyFile(row.path, file, () => {}); - } - }} - > - save - + + { + const file = await electron.showSaveDialog({}); + if (file) { + const fs = window.require('fs'); + fs.copyFile(row.path, file, () => {}); + } + }} + > + save + + - { - electron.showItemInFolder(row.path); - }} - > - show - + + { + electron.showItemInFolder(row.path); + }} + > + show + + {/if} diff --git a/packages/web/src/settings/SettingsModal.svelte b/packages/web/src/settings/SettingsModal.svelte index 78bb3b7b0..9a307b4b3 100644 --- a/packages/web/src/settings/SettingsModal.svelte +++ b/packages/web/src/settings/SettingsModal.svelte @@ -104,6 +104,7 @@ ORDER BY { label: 'Themes', slot: 3 }, { label: 'Default Actions', slot: 4 }, { label: 'Behaviour', slot: 5 }, + { label: 'External tools', slot: 8 }, { label: 'Other', slot: 6 }, ]} > @@ -480,6 +481,31 @@ ORDER BY {/if} + + +
External tools
+ + + + + + diff --git a/packages/web/src/utility/exportFileTools.ts b/packages/web/src/utility/exportFileTools.ts index e80fed076..07e9e7714 100644 --- a/packages/web/src/utility/exportFileTools.ts +++ b/packages/web/src/utility/exportFileTools.ts @@ -182,26 +182,6 @@ export async function exportQuickExportFile(dataName, reader, format: QuickExpor } } -// export async function exportSqlDump(connection, databaseName) { -// await saveExportedFile( -// [{ name: 'SQL files', extensions: ['sql'] }], -// `${databaseName}.sql`, -// 'sql', -// `${databaseName}-dump`, -// filePath => { -// const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson(); - -// script.dumpDatabase({ -// connection, -// databaseName, -// outputFile: filePath, -// }); - -// return script.getScript(); -// } -// ); -// } - export async function saveFileToDisk( filePathFunc, options: any = { formatLabel: 'HTML page', formatExtension: 'html' } diff --git a/packages/web/src/utility/openWebFile.ts b/packages/web/src/utility/openWebFile.ts new file mode 100644 index 000000000..0175d198b --- /dev/null +++ b/packages/web/src/utility/openWebFile.ts @@ -0,0 +1,25 @@ +import newQuery from '../query/newQuery'; +import _ from 'lodash'; + +export function canOpenByWeb(file, extensions) { + if (!file) return false; + const nameLower = file.toLowerCase(); + if (nameLower.endsWith('.sql')) return true; + return false; +} + +export async function openWebFileCore(file, extensions) { + const nameLower = file.path.toLowerCase(); + + if (nameLower.endsWith('.sql')) { + const reader = new FileReader(); + + reader.onload = function (e) { + newQuery({ + initialData: e.target.result, + }); + }; + + reader.readAsText(file); + } +} diff --git a/packages/web/src/utility/uploadFiles.ts b/packages/web/src/utility/uploadFiles.ts index ddbcfa503..d260a1758 100644 --- a/packages/web/src/utility/uploadFiles.ts +++ b/packages/web/src/utility/uploadFiles.ts @@ -8,6 +8,7 @@ import { showModal } from '../modals/modalTools'; import ErrorMessageModal from '../modals/ErrorMessageModal.svelte'; import openNewTab from './openNewTab'; import { openImportExportTab } from './importExportTools'; +import { canOpenByWeb, openWebFileCore } from './openWebFile'; let uploadListener; @@ -28,6 +29,11 @@ export default function uploadFiles(files) { return; } + if (!electron && canOpenByWeb(file.path, ext)) { + openWebFileCore(file, ext); + return; + } + const maxSize = 32 * 1024 * 1024; if (parseInt(file.size, 10) >= maxSize) { showModal(ErrorMessageModal, { diff --git a/packages/web/src/widgets/SavedFilesList.svelte b/packages/web/src/widgets/SavedFilesList.svelte index b487d03e0..4a4c402f1 100644 --- a/packages/web/src/widgets/SavedFilesList.svelte +++ b/packages/web/src/widgets/SavedFilesList.svelte @@ -10,6 +10,9 @@ import { apiCall } from '../utility/api'; import { useFiles } from '../utility/metadataLoaders'; import WidgetsInnerContainer from './WidgetsInnerContainer.svelte'; + import getElectron from '../utility/getElectron'; + import InlineButtonLabel from '../buttons/InlineButtonLabel.svelte'; + import resolveApi, { resolveApiHeaders } from '../utility/resolveApi'; let filter = ''; @@ -24,6 +27,8 @@ const perspectiveFiles = useFiles({ folder: 'perspectives' }); const modelTransformFiles = useFiles({ folder: 'modtrans' }); + const electron = getElectron(); + $: files = [ ...($sqlFiles || []), ...($shellFiles || []), @@ -58,16 +63,67 @@ if (folder == 'modtrans') return 'Model transforms'; return _.startCase(folder); } + + async function handleUploadedFile(e) { + const files = [...e.target.files]; + + for (const file of files) { + const formData = new FormData(); + formData.append('name', file.name); + formData.append('data', file); + + const fetchOptions = { + method: 'POST', + body: formData, + headers: resolveApiHeaders(), + }; + + const apiBase = resolveApi(); + const resp = await fetch(`${apiBase}/uploads/upload-data-file`, fetchOptions); + const fileData = await resp.json(); + } + } + + async function handleOpenElectronFile() { + const filePaths = await electron.showOpenDialog({ + filters: [ + { + name: `All supported files`, + extensions: ['sql'], + }, + { name: `SQL files`, extensions: ['sql'] }, + ], + properties: ['showHiddenFiles', 'openFile'], + }); + const filePath = filePaths && filePaths[0]; + await apiCall('uploads/save-data-file', { filePath }); + } - + {#if electron} + + + + {:else} + {}} + title="Add file" + data-testid="SavedFileList_buttonAddFile" + htmlFor="uploadSavedFileButton" + > + + + {/if} + + + dataFolderTitle(data.folder)} {filter} /> diff --git a/packages/web/src/widgets/WidgetsInnerContainer.svelte b/packages/web/src/widgets/WidgetsInnerContainer.svelte index c727b45b2..5682b1d8e 100644 --- a/packages/web/src/widgets/WidgetsInnerContainer.svelte +++ b/packages/web/src/widgets/WidgetsInnerContainer.svelte @@ -2,23 +2,36 @@ let domDiv; export let hideContent = false; + export let fixedWidth = 0; export function scrollTop() { domDiv.scrollTop = 0; } -
+
+ +
diff --git a/plugins/dbgate-plugin-mysql/package.json b/plugins/dbgate-plugin-mysql/package.json index dea19457f..5c89e86f7 100644 --- a/plugins/dbgate-plugin-mysql/package.json +++ b/plugins/dbgate-plugin-mysql/package.json @@ -36,7 +36,6 @@ "webpack-cli": "^5.1.4" }, "dependencies": { - "antares-mysql-dumper": "^0.0.1", "dbgate-query-splitter": "^4.11.3", "dbgate-tools": "^6.0.0-alpha.1", "lodash": "^4.17.21", diff --git a/plugins/dbgate-plugin-mysql/src/backend/drivers.js b/plugins/dbgate-plugin-mysql/src/backend/drivers.js index 93c52bfa8..c59721964 100644 --- a/plugins/dbgate-plugin-mysql/src/backend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/backend/drivers.js @@ -5,7 +5,6 @@ const Analyser = require('./Analyser'); const mysql2 = require('mysql2'); const { getLogger, createBulkInsertStreamBase, makeUniqueColumnNames, extractErrorLogData } = global.DBGATE_PACKAGES['dbgate-tools']; -const { MySqlDumper } = require('antares-mysql-dumper'); const logger = getLogger('mysqlDriver'); @@ -203,15 +202,6 @@ const drivers = driverBases.map(driverBase => ({ // @ts-ignore return createBulkInsertStreamBase(this, stream, dbhan, name, options); }, - async createBackupDumper(dbhan, options) { - const { outputFile, databaseName, schemaName } = options; - const res = new MySqlDumper({ - connection: dbhan.client, - schema: databaseName || schemaName, - outputFile, - }); - return res; - }, getAuthTypes() { const res = [ { diff --git a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js index d8559ed4f..391e8b8b7 100644 --- a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js @@ -178,7 +178,8 @@ const mysqlDriverBase = { : mysqlSplitterOptions, readOnlySessions: true, - supportsDatabaseDump: true, + supportsDatabaseBackup: true, + supportsDatabaseRestore: true, authTypeLabel: 'Connection mode', defaultAuthTypeName: 'hostPort', defaultSocketPath: '/var/run/mysqld/mysqld.sock', @@ -199,6 +200,122 @@ const mysqlDriverBase = { }, ]; }, + getCliConnectionArgs(connection, externalTools) { + const args = [`--user=${connection.user}`, `--password=${connection.password}`, `--host=${connection.server}`]; + if (connection.port) { + args.push(`--port=${connection.port}`); + } + if (externalTools.mysqlPlugins) { + args.push(`--plugin-dir=${externalTools.mysqlPlugins}`); + } + if (connection.server == 'localhost') { + args.push(`--protocol=tcp`); + } + return args; + }, + backupDatabaseCommand(connection, settings, externalTools) { + const { outputFile, database, skippedTables, options } = settings; + const command = externalTools.mysqldump || 'mysqldump'; + const args = this.getCliConnectionArgs(connection, externalTools); + args.push(`--result-file=${outputFile}`); + args.push('--verbose'); + for (const table of skippedTables) { + args.push(`--ignore-table=${database}.${table.pureName}`); + } + if (options.noData) { + args.push('--no-data'); + } + if (options.noStructure) { + args.push('--no-create-info'); + } + if (options.includeEvents !== false && !options.noStructure) { + args.push('--events'); + } + if (options.includeRoutines !== false && !options.noStructure) { + args.push('--routines'); + } + if (options.includeTriggers !== false && !options.noStructure) { + args.push('--triggers'); + } + if (options.force) { + args.push('--force'); + } + args.push(database); + return { command, args }; + }, + restoreDatabaseCommand(connection, settings, externalTools) { + const { inputFile, database } = settings; + const command = externalTools.mysql || 'mysql'; + const args = this.getCliConnectionArgs(connection, externalTools); + if (database) { + args.push(database); + } + return { command, args, stdinFilePath: inputFile }; + }, + transformNativeCommandMessage(message) { + if (message.message?.startsWith('--')) { + if (message.message.startsWith('-- Retrieving table structure for table')) { + return { + ...message, + severity: 'info', + message: message.message.replace('-- Retrieving table structure for table', 'Processing table'), + }; + } else { + return { + ...message, + severity: 'debug', + message: message.message.replace('-- ', ''), + }; + } + } + return message; + }, + getNativeOperationFormArgs(operation) { + if (operation == 'backup') { + return [ + { + type: 'checkbox', + label: 'No data (dump only structure)', + name: 'noData', + default: false, + }, + { + type: 'checkbox', + label: 'No structure (dump only data)', + name: 'noStructure', + default: false, + }, + { + type: 'checkbox', + label: 'Force (ignore all errors)', + name: 'force', + default: false, + }, + { + type: 'checkbox', + label: 'Backup events', + name: 'includeEvents', + default: true, + disabledFn: values => values.noStructure, + }, + { + type: 'checkbox', + label: 'Backup routines', + name: 'includeRoutines', + default: true, + disabledFn: values => values.noStructure, + }, + { + type: 'checkbox', + label: 'Backup triggers', + name: 'includeTriggers', + default: true, + disabledFn: values => values.noStructure, + }, + ]; + } + return null; + }, }; /** @type {import('dbgate-types').EngineDriver} */ diff --git a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js index 5ee498357..fdba28f3a 100644 --- a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js @@ -218,10 +218,135 @@ EXECUTE FUNCTION function_name();`, defaultAuthTypeName: 'hostPort', defaultSocketPath: '/var/run/postgresql', + supportsDatabaseBackup: true, + supportsDatabaseRestore: true, + adaptDataType(dataType) { if (dataType?.toLowerCase() == 'datetime') return 'timestamp'; return dataType; }, + + getCliConnectionArgs(connection) { + const args = [`--username=${connection.user}`, `--host=${connection.server}`]; + if (connection.port) { + args.push(`--port=${connection.port}`); + } + return args; + }, + + getNativeOperationFormArgs(operation) { + if (operation == 'backup') { + return [ + { + type: 'checkbox', + label: 'Dump only data (without structure)', + name: 'dataOnly', + default: false, + }, + { + type: 'checkbox', + label: 'Dump schema only (no data)', + name: 'schemaOnly', + default: false, + }, + { + type: 'checkbox', + label: 'Use SQL insert instead of COPY for rows', + name: 'insert', + default: false, + }, + { + type: 'checkbox', + label: 'Prevent dumping of access privileges (grant/revoke)', + name: 'noPrivileges', + default: false, + }, + { + type: 'checkbox', + label: 'Do not output commands to set ownership of objects ', + name: 'noOwner', + default: false, + }, + ]; + } + return null; + }, + + backupDatabaseCommand(connection, settings, externalTools) { + const { outputFile, database, selectedTables, skippedTables, options, argsFormat } = settings; + const command = externalTools.pg_dump || 'pg_dump'; + const args = this.getCliConnectionArgs(connection, externalTools); + args.push(`--file=${outputFile}`); + args.push('--verbose'); + args.push(database); + + if (options.dataOnly) { + args.push(`--data-only`); + } + if (options.schemaOnly) { + args.push(`--schema-only`); + } + if (options.insert) { + args.push(`--insert`); + } + if (options.noPrivileges) { + args.push(`--no-privileges`); + } + if (options.noOwner) { + args.push(`--no-owner`); + } + if (skippedTables.length > 0) { + for (const table of selectedTables) { + args.push( + argsFormat == 'spawn' + ? `--table="${table.schemaName}"."${table.pureName}"` + : `--table='"${table.schemaName}"."${table.pureName}"'` + ); + } + } + + return { + command, + args, + env: { PGPASSWORD: connection.password }, + }; + }, + restoreDatabaseCommand(connection, settings, externalTools) { + const { inputFile, database } = settings; + const command = externalTools.psql || 'psql'; + const args = this.getCliConnectionArgs(connection, externalTools); + args.push(`--dbname=${database}`); + // args.push('--verbose'); + args.push(`--file=${inputFile}`); + return { + command, + args, + env: { PGPASSWORD: connection.password }, + }; + }, + transformNativeCommandMessage(message) { + if (message.message.startsWith('INSERT ') || message.message == 'SET') { + return null; + } + if (message.message.startsWith('pg_dump: processing data for table')) { + return { + ...message, + severity: 'info', + message: message.message.replace('pg_dump: processing data for table', 'Processing table'), + }; + } else if (message.message.toLowerCase().includes('error:')) { + return { + ...message, + severity: 'error', + }; + } else { + return { + ...message, + severity: 'debug', + }; + } + return message; + }, }; /** @type {import('dbgate-types').EngineDriver} */