SYNC: Merge branch 'feature/backup-restore'

This commit is contained in:
Jan Prochazka
2025-03-14 10:45:30 +01:00
committed by Diflow
parent ba9e124527
commit 3a75ad61f3
31 changed files with 842 additions and 411 deletions

View File

@@ -1,4 +1,5 @@
const connections = require('./connections');
const runners = require('./runners');
const archive = require('./archive');
const socket = require('../utility/socket');
const { fork } = require('child_process');
@@ -613,4 +614,129 @@ module.exports = {
return res;
},
async getNativeOpCommandArgs(
command,
{ conid, database, outputFile, inputFile, options, selectedTables, skippedTables, argsFormat }
) {
const connection = await connections.getCore({ conid });
const driver = requireEngineDriver(connection);
const settingsValue = await config.getSettings();
const externalTools = {};
for (const pair of Object.entries(settingsValue || {})) {
const [name, value] = pair;
if (name.startsWith('externalTools.')) {
externalTools[name.substring('externalTools.'.length)] = value;
}
}
return {
...(command == 'backup'
? driver.backupDatabaseCommand(
connection,
{ outputFile, database, options, selectedTables, skippedTables, argsFormat },
// @ts-ignore
externalTools
)
: driver.restoreDatabaseCommand(
connection,
{ inputFile, database, options, argsFormat },
// @ts-ignore
externalTools
)),
transformMessage: driver.transformNativeCommandMessage
? message => driver.transformNativeCommandMessage(message, command)
: null,
};
},
commandArgsToCommandLine(commandArgs) {
const { command, args, stdinFilePath } = commandArgs;
let res = `${command} ${args.join(' ')}`;
if (stdinFilePath) {
res += ` < ${stdinFilePath}`;
}
return res;
},
nativeBackup_meta: true,
async nativeBackup({ conid, database, outputFile, runid, options, selectedTables, skippedTables }) {
const commandArgs = await this.getNativeOpCommandArgs('backup', {
conid,
database,
inputFile: undefined,
outputFile,
options,
selectedTables,
skippedTables,
argsFormat: 'spawn',
});
return runners.nativeRunCore(runid, {
...commandArgs,
onFinished: () => {
socket.emitChanged(`files-changed`, { folder: 'sql' });
socket.emitChanged(`all-files-changed`);
},
});
},
nativeBackupCommand_meta: true,
async nativeBackupCommand({ conid, database, outputFile, options, selectedTables, skippedTables }) {
const commandArgs = await this.getNativeOpCommandArgs('backup', {
conid,
database,
outputFile,
inputFile: undefined,
options,
selectedTables,
skippedTables,
argsFormat: 'shell',
});
return {
...commandArgs,
transformMessage: null,
commandLine: this.commandArgsToCommandLine(commandArgs),
};
},
nativeRestore_meta: true,
async nativeRestore({ conid, database, inputFile, runid }) {
const commandArgs = await this.getNativeOpCommandArgs('restore', {
conid,
database,
inputFile,
outputFile: undefined,
options: undefined,
argsFormat: 'spawn',
});
return runners.nativeRunCore(runid, {
...commandArgs,
onFinished: () => {
this.syncModel({ conid, database, isFullRefresh: true });
},
});
},
nativeRestoreCommand_meta: true,
async nativeRestoreCommand({ conid, database, inputFile }) {
const commandArgs = await this.getNativeOpCommandArgs('restore', {
conid,
database,
inputFile,
outputFile: undefined,
options: undefined,
argsFormat: 'shell',
});
return {
...commandArgs,
transformMessage: null,
commandLine: this.commandArgsToCommandLine(commandArgs),
};
},
};

View File

@@ -4,7 +4,7 @@ const path = require('path');
const fs = require('fs-extra');
const byline = require('byline');
const socket = require('../utility/socket');
const { fork } = require('child_process');
const { fork, spawn } = require('child_process');
const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
const {
extractShellApiPlugins,
@@ -13,6 +13,8 @@ const {
getLogger,
safeJsonParse,
pinoLogRecordToMessageRecord,
extractErrorMessage,
extractErrorLogData,
} = require('dbgate-tools');
const { handleProcessCommunication } = require('../utility/processComm');
const processArgs = require('../utility/processArgs');
@@ -80,6 +82,7 @@ module.exports = {
}
: {
message,
severity: 'info',
time: new Date(),
};
@@ -173,7 +176,7 @@ module.exports = {
// console.log('... ERROR subprocess', error);
this.rejectRequest(runid, { message: error && (error.message || error.toString()) });
console.error('... ERROR subprocess', error);
this.dispatchMessage({
this.dispatchMessage(runid, {
severity: 'error',
message: error.toString(),
});
@@ -192,6 +195,75 @@ module.exports = {
return _.pick(newOpened, ['runid']);
},
nativeRunCore(runid, commandArgs) {
const { command, args, env, transformMessage, stdinFilePath, onFinished } = commandArgs;
const pipeDispatcher = severity => data => {
let messageObject = {
message: data.toString().trim(),
severity,
};
if (transformMessage) {
messageObject = transformMessage(messageObject);
}
if (messageObject) {
return this.dispatchMessage(runid, messageObject);
}
};
const subprocess = spawn(command, args, { env: { ...process.env, ...env } });
byline(subprocess.stdout).on('data', pipeDispatcher('info'));
byline(subprocess.stderr).on('data', pipeDispatcher('error'));
subprocess.on('exit', code => {
console.log('... EXITED', code);
logger.info({ code, pid: subprocess.pid }, 'Exited process');
this.dispatchMessage(runid, `Finished external process with code ${code}`);
socket.emit(`runner-done-${runid}`, code);
if (onFinished) {
onFinished();
}
});
subprocess.on('spawn', () => {
this.dispatchMessage(runid, `Started external process ${command}`);
});
subprocess.on('error', error => {
console.log('... ERROR subprocess', error);
this.dispatchMessage(runid, {
severity: 'error',
message: error.toString(),
});
if (error['code'] == 'ENOENT') {
this.dispatchMessage(runid, {
severity: 'error',
message: `Command ${command} not found, please install it and configure its location in DbGate settings, Settings/External tools, if ${command} is not in system PATH`,
});
}
socket.emit(`runner-done-${runid}`);
});
if (stdinFilePath) {
const inputStream = fs.createReadStream(stdinFilePath);
inputStream.pipe(subprocess.stdin);
subprocess.stdin.on('error', err => {
this.dispatchMessage(runid, {
severity: 'error',
message: extractErrorMessage(err),
});
logger.error(extractErrorLogData(err), 'Caught error on stdin');
});
}
const newOpened = {
runid,
subprocess,
};
this.opened.push(newOpened);
return _.pick(newOpened, ['runid']);
},
start_meta: true,
async start({ script }) {
const runid = crypto.randomUUID();

View File

@@ -1,6 +1,6 @@
const crypto = require('crypto');
const path = require('path');
const { uploadsdir, getLogsFilePath } = require('../utility/directories');
const { uploadsdir, getLogsFilePath, filesdir } = require('../utility/directories');
const { getLogger, extractErrorLogData } = require('dbgate-tools');
const logger = getLogger('uploads');
const axios = require('axios');
@@ -13,6 +13,7 @@ const serverConnections = require('./serverConnections');
const config = require('./config');
const gistSecret = require('../gistSecret');
const currentVersion = require('../currentVersion');
const socket = require('../utility/socket');
module.exports = {
upload_meta: {
@@ -38,6 +39,52 @@ module.exports = {
});
},
uploadDataFile_meta: {
method: 'post',
raw: true,
},
uploadDataFile(req, res) {
const { data } = req.files || {};
if (!data) {
res.json(null);
return;
}
if (data.name.toLowerCase().endsWith('.sql')) {
logger.info(`Uploading SQL file ${data.name}, size=${data.size}`);
data.mv(path.join(filesdir(), 'sql', data.name), () => {
res.json({
name: data.name,
folder: 'sql',
});
socket.emitChanged(`files-changed`, { folder: 'sql' });
socket.emitChanged(`all-files-changed`);
});
return;
}
res.json(null);
},
saveDataFile_meta: true,
async saveDataFile({ filePath }) {
if (filePath.toLowerCase().endsWith('.sql')) {
logger.info(`Saving SQL file ${filePath}`);
await fs.copyFile(filePath, path.join(filesdir(), 'sql', path.basename(filePath)));
socket.emitChanged(`files-changed`, { folder: 'sql' });
socket.emitChanged(`all-files-changed`);
return {
name: path.basename(filePath),
folder: 'sql',
};
}
return null;
},
get_meta: {
method: 'get',
raw: true,

View File

@@ -30,7 +30,7 @@ const queryHistory = require('./controllers/queryHistory');
const onFinished = require('on-finished');
const processArgs = require('./utility/processArgs');
const { rundir } = require('./utility/directories');
const { rundir, filesdir } = require('./utility/directories');
const platformInfo = require('./utility/platformInfo');
const getExpressPath = require('./utility/getExpressPath');
const _ = require('lodash');
@@ -133,6 +133,7 @@ function start() {
// }
app.use(getExpressPath('/runners/data'), express.static(rundir()));
app.use(getExpressPath('/files/data'), express.static(filesdir()));
if (platformInfo.isDocker) {
const port = process.env.PORT || 3000;

View File

@@ -1,47 +0,0 @@
const requireEngineDriver = require('../utility/requireEngineDriver');
const { connectUtility } = require('../utility/connectUtility');
const { getLogger } = require('dbgate-tools');
const logger = getLogger('dumpDb');
function doDump(dumper) {
return new Promise((resolve, reject) => {
dumper.once('end', () => {
resolve(true);
});
dumper.once('error', err => {
reject(err);
});
dumper.run();
});
}
async function dumpDatabase({
connection = undefined,
systemConnection = undefined,
driver = undefined,
outputFile,
databaseName,
schemaName,
}) {
logger.info(`Dumping database`);
if (!driver) driver = requireEngineDriver(connection);
const dbhan = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
try {
const dumper = await driver.createBackupDumper(dbhan, {
outputFile,
databaseName,
schemaName,
});
await doDump(dumper);
} finally {
if (!systemConnection) {
await driver.close(dbhan);
}
}
}
module.exports = dumpDatabase;

View File

@@ -21,7 +21,6 @@ const executeQuery = require('./executeQuery');
const loadFile = require('./loadFile');
const deployDb = require('./deployDb');
const initializeApiEnvironment = require('./initializeApiEnvironment');
const dumpDatabase = require('./dumpDatabase');
const importDatabase = require('./importDatabase');
const loadDatabase = require('./loadDatabase');
const generateModelSql = require('./generateModelSql');
@@ -61,7 +60,6 @@ const dbgateApi = {
loadFile,
deployDb,
initializeApiEnvironment,
dumpDatabase,
importDatabase,
loadDatabase,
generateModelSql,

View File

@@ -50,10 +50,6 @@ export class ScriptWriter {
this._put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar}, ${opts});`);
}
dumpDatabase(options) {
this._put(`await dbgateApi.dumpDatabase(${JSON.stringify(options)});`);
}
importDatabase(options) {
this._put(`await dbgateApi.importDatabase(${JSON.stringify(options)});`);
}
@@ -135,13 +131,6 @@ export class ScriptWriterJson {
});
}
dumpDatabase(options) {
this.commands.push({
type: 'dumpDatabase',
options,
});
}
importDatabase(options) {
this.commands.push({
type: 'importDatabase',
@@ -193,9 +182,6 @@ export function jsonScriptToJavascript(json) {
case 'comment':
script.comment(cmd.text);
break;
case 'dumpDatabase':
script.dumpDatabase(cmd.options);
break;
case 'importDatabase':
script.importDatabase(cmd.options);
break;

View File

@@ -43,3 +43,11 @@ export function getConnectionLabel(connection, { allowExplicitDatabase = true, s
return res;
}
export function getEngineLabel(connection) {
const match = (connection?.engine || '').match(/^([^@]*)@/);
if (match) {
return match[1];
}
return connection?.engine;
}

View File

@@ -147,6 +147,29 @@ export interface DatabaseHandle<TClient = any> {
export type StreamResult = stream.Readable | (stream.Readable | stream.Writable)[];
export interface CommandLineDefinition {
command: string;
args: string[];
env?: { [key: string]: string };
stdinFilePath?: string;
}
interface BackupRestoreSettingsBase {
database: string;
options?: { [key: string]: string };
argsFormat: 'shell' | 'spawn';
}
export interface BackupDatabaseSettings extends BackupRestoreSettingsBase {
outputFile: string;
selectedTables?: { pureName: string; schemaName?: string }[];
skippedTables?: { pureName: string; schemaName?: string }[];
}
export interface RestoreDatabaseSettings extends BackupRestoreSettingsBase {
inputFile: string;
}
export interface EngineDriver<TClient = any> extends FilterBehaviourProvider {
engine: string;
title: string;
@@ -157,7 +180,8 @@ export interface EngineDriver<TClient = any> extends FilterBehaviourProvider {
supportedKeyTypes: SupportedDbKeyType[];
dataEditorTypesBehaviour: DataEditorTypesBehaviour;
supportsDatabaseUrl?: boolean;
supportsDatabaseDump?: boolean;
supportsDatabaseBackup?: boolean;
supportsDatabaseRestore?: boolean;
supportsServerSummary?: boolean;
supportsDatabaseProfiler?: boolean;
requiresDefaultSortCriteria?: boolean;
@@ -261,6 +285,24 @@ export interface EngineDriver<TClient = any> extends FilterBehaviourProvider {
// simple data type adapter
adaptDataType(dataType: string): string;
listSchemas(dbhan: DatabaseHandle<TClient>): SchemaInfo[];
backupDatabaseCommand(
connection: any,
settings: BackupDatabaseSettings,
externalTools: { [tool: string]: string }
): CommandLineDefinition;
restoreDatabaseCommand(
connection: any,
settings: RestoreDatabaseSettings,
externalTools: { [tool: string]: string }
): CommandLineDefinition;
transformNativeCommandMessage(
message: {
message: string;
severity: 'info' | 'error';
},
command: 'backup' | 'restore'
): { message: string; severity: 'info' | 'error' | 'debug' } | null;
getNativeOperationFormArgs(operation: 'backup' | 'restore'): any[];
analyserClass?: any;
dumperClass?: any;

View File

@@ -132,7 +132,6 @@
import { getDatabaseList, useUsedApps } from '../utility/metadataLoaders';
import { getLocalStorage } from '../utility/storageCache';
import { apiCall, removeVolatileMapping } from '../utility/api';
import ImportDatabaseDumpModal from '../modals/ImportDatabaseDumpModal.svelte';
import { closeMultipleTabs } from '../tabpanel/TabsPanel.svelte';
import AboutModal from '../modals/AboutModal.svelte';
import { tick } from 'svelte';
@@ -141,6 +140,7 @@
import { switchCurrentDatabase } from '../utility/common';
import { getConnectionClickActionSetting } from '../settings/settingsTools';
import { _t } from '../translations';
import { isProApp } from '../utility/proTools';
export let data;
export let passProps;
@@ -231,9 +231,14 @@
});
};
const handleSqlRestore = () => {
showModal(ImportDatabaseDumpModal, {
connection: data,
const handleRestoreDatabase = () => {
openNewTab({
title: 'Restore #',
icon: 'img db-restore',
tabComponent: 'RestoreDatabaseTab',
props: {
conid: data._id,
},
});
};
@@ -364,11 +369,10 @@
),
],
driver?.databaseEngineTypes?.includes('sql') &&
!data.isReadOnly && {
onClick: handleSqlRestore,
text: _t('connection.sqlRestore', { defaultMessage: 'Restore/import SQL dump' }),
},
driver?.supportsDatabaseRestore &&
isProApp() &&
hasPermission(`dbops/sql-dump/import`) &&
!data.isReadOnly && { onClick: handleRestoreDatabase, text: 'Restore database backup' },
];
};

View File

@@ -126,16 +126,27 @@
});
};
const handleSqlDump = () => {
showModal(ExportDatabaseDumpModal, {
connection: { ...connection, database: name },
const handleBackupDatabase = () => {
openNewTab({
title: 'Backup #',
icon: 'img db-backup',
tabComponent: 'BackupDatabaseTab',
props: {
conid: connection._id,
database: name,
},
});
// exportSqlDump(connection, name);
};
const handleSqlRestore = () => {
showModal(ImportDatabaseDumpModal, {
connection: { ...connection, database: name },
const handleRestoreDatabase = () => {
openNewTab({
title: 'Restore #',
icon: 'img db-restore',
tabComponent: 'RestoreDatabaseTab',
props: {
conid: connection._id,
database: name,
},
});
};
@@ -376,11 +387,13 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify(
!connection.isReadOnly &&
hasPermission(`dbops/import`) && { onClick: handleImport, text: 'Import' },
isSqlOrDoc && hasPermission(`dbops/export`) && { onClick: handleExport, text: 'Export' },
driver?.databaseEngineTypes?.includes('sql') &&
driver?.supportsDatabaseRestore &&
isProApp() &&
hasPermission(`dbops/sql-dump/import`) &&
!connection.isReadOnly && { onClick: handleSqlRestore, text: 'Restore/import SQL dump' },
driver?.supportsDatabaseDump &&
hasPermission(`dbops/sql-dump/export`) && { onClick: handleSqlDump, text: 'Backup/export SQL dump' },
!connection.isReadOnly && { onClick: handleRestoreDatabase, text: 'Restore database backup' },
driver?.supportsDatabaseBackup &&
isProApp() &&
hasPermission(`dbops/sql-dump/export`) && { onClick: handleBackupDatabase, text: 'Create database backup' },
isSqlOrDoc &&
!connection.isReadOnly &&
!connection.singleDatabase &&
@@ -491,8 +504,6 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify(
import ConfirmSqlModal, { runOperationOnDatabase, saveScriptToDatabase } from '../modals/ConfirmSqlModal.svelte';
import { filterAppsForDatabase } from '../utility/appTools';
import newQuery from '../query/newQuery';
import ImportDatabaseDumpModal from '../modals/ImportDatabaseDumpModal.svelte';
import ExportDatabaseDumpModal from '../modals/ExportDatabaseDumpModal.svelte';
import ConfirmModal from '../modals/ConfirmModal.svelte';
import { closeMultipleTabs } from '../tabpanel/TabsPanel.svelte';
import NewCollectionModal from '../modals/NewCollectionModal.svelte';

View File

@@ -1086,8 +1086,12 @@
icon={databaseObjectIcons[data.objectTypeField]}
menu={createMenu}
showPinnedInsteadOfUnpin={passProps?.showPinnedInsteadOfUnpin}
onPin={isPinned ? null : () => pinnedTables.update(list => [...list, data])}
onUnpin={isPinned ? () => pinnedTables.update(list => list.filter(x => !testEqual(x, data))) : null}
onPin={passProps?.ingorePin ? null : isPinned ? null : () => pinnedTables.update(list => [...list, data])}
onUnpin={passProps?.ingorePin
? null
: isPinned
? () => pinnedTables.update(list => list.filter(x => !testEqual(x, data)))
: null}
extInfo={getExtInfo(data)}
isChoosed={matchDatabaseObjectAppObject($selectedDatabaseObjectAppObject, data)}
on:click={() => handleObjectClick(data, 'leftClick')}

View File

@@ -0,0 +1,74 @@
<script lang="ts">
export let disabled = false;
export let square = false;
export let narrow = false;
export let title = null;
let domButton;
export function getBoundingClientRect() {
return domButton.getBoundingClientRect();
}
</script>
<label
class="outer buttonLike"
{title}
class:disabled
class:square
class:narrow
on:click
bind:this={domButton}
{...$$restProps}
>
<div class="inner">
<slot />
</div>
</label>
<style>
.outer {
--bg-1: var(--theme-bg-1);
--bg-2: var(--theme-bg-3);
background: linear-gradient(to bottom, var(--bg-1) 5%, var(--bg-2) 100%);
background-color: var(--bg-1);
border: 1px solid var(--bg-2);
display: inline-block;
cursor: pointer;
vertical-align: middle;
color: var(--theme-font-1);
font-size: 12px;
padding: 3px;
margin: 0;
text-decoration: none;
display: flex;
}
.narrow {
padding: 3px 1px;
}
.outer.disabled {
color: var(--theme-font-3);
}
.outer:hover:not(.disabled) {
border: 1px solid var(--theme-font-1);
}
.outer:active:not(.disabled) {
background: linear-gradient(to bottom, var(--bg-2) 5%, var(--bg-1) 100%);
background-color: var(--bg-2);
}
.inner {
margin: auto;
flex: 1;
text-align: center;
}
.square {
width: 18px;
}
</style>

View File

@@ -33,10 +33,13 @@
let collapsed2 = false;
export let size = 0;
export let onChangeSize = null;
let clientWidth;
let customRatio = null;
$: size = computeSplitterSize(initialValue, clientWidth, customRatio, initialSizeRight);
$: if (onChangeSize) onChangeSize(size);
</script>
<div class="container" bind:clientWidth>

View File

@@ -52,6 +52,8 @@
{:else if tab.slot == 5}<slot name="5" />
{:else if tab.slot == 6}<slot name="6" />
{:else if tab.slot == 7}<slot name="7" />
{:else if tab.slot == 8}<slot name="8" />
{:else if tab.slot == 9}<slot name="9" />
{/if}
{/if}
</div>

View File

@@ -13,7 +13,7 @@
$: name = `${namePrefix}${arg.name}`;
const { setFieldValue } = getFormContext();
const { setFieldValue, values } = getFormContext();
</script>
{#if arg.type == 'text'}
@@ -23,7 +23,7 @@
defaultValue={arg.default}
focused={arg.focused}
placeholder={arg.placeholder}
disabled={arg.disabled}
disabled={arg.disabledFn ? arg.disabledFn($values) : arg.disabled}
/>
{:else if arg.type == 'stringlist'}
<FormStringList label={arg.label} addButtonLabel={arg.addButtonLabel} {name} placeholder={arg.placeholder} />
@@ -35,9 +35,15 @@
defaultValue={arg.default}
focused={arg.focused}
placeholder={arg.placeholder}
disabled={arg.disabledFn ? arg.disabledFn($values) : arg.disabled}
/>
{:else if arg.type == 'checkbox'}
<FormCheckboxField label={arg.label} {name} defaultValue={arg.default} />
<FormCheckboxField
label={arg.label}
{name}
defaultValue={arg.default}
disabled={arg.disabledFn ? arg.disabledFn($values) : arg.disabled}
/>
{:else if arg.type == 'select'}
<FormSelectField
label={arg.label}
@@ -47,6 +53,7 @@
options={arg.options.map(opt =>
_.isString(opt) ? { label: opt, value: opt } : { label: opt.name, value: opt.value }
)}
disabled={arg.disabledFn ? arg.disabledFn($values) : arg.disabled}
/>
{:else if arg.type == 'dropdowntext'}
<FormDropDownTextField
@@ -59,5 +66,6 @@
onClick: () => setFieldValue(name, _.isString(opt) ? opt : opt.value),
}));
}}
disabled={arg.disabledFn ? arg.disabledFn($values) : arg.disabled}
/>
{/if}

View File

@@ -306,6 +306,9 @@
'img tip': 'mdi mdi-lightbulb-on color-icon-yellow',
'img filter-active': 'mdi mdi-filter-cog color-icon-blue',
'img db-backup': 'mdi mdi-database-export color-icon-yellow',
'img db-restore': 'mdi mdi-database-import color-icon-red',
};
</script>

View File

@@ -1,117 +0,0 @@
<script lang="ts">
import { onMount, tick } from 'svelte';
import { format as dateFormat } from 'date-fns';
import FormStyledButton from '../buttons/FormStyledButton.svelte';
import UploadButton from '../buttons/UploadButton.svelte';
import FormProvider from '../forms/FormProvider.svelte';
import FormSubmit from '../forms/FormSubmit.svelte';
import { exportSqlDump, importSqlDump } from '../utility/exportFileTools';
import getElectron from '../utility/getElectron';
import { setUploadListener } from '../utility/uploadFiles';
import ChangeDownloadUrlModal from './ChangeDownloadUrlModal.svelte';
import ModalBase from './ModalBase.svelte';
import { closeCurrentModal, showModal } from './modalTools';
import InputTextModal from './InputTextModal.svelte';
import { apiCall } from '../utility/api';
import { getConnectionLabel } from 'dbgate-tools';
export let connection;
let outputLabel;
let outputFile;
let pureFileName = null;
function getDefaultFileName() {
return `${connection.database}-${dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss')}.sql`;
}
onMount(async () => {
const file = getDefaultFileName();
setFilesFolderResult(file);
});
const setFilesFolderResult = async file => {
const resp = await apiCall('files/get-file-real-path', { folder: 'sql', file });
if (!resp) return;
outputLabel = `SQL Files folder: ${file}`;
outputFile = resp;
pureFileName = null;
};
const handleSubmit = async values => {
const { value } = values;
closeCurrentModal();
exportSqlDump(outputFile, connection, connection.database, pureFileName);
};
const electron = getElectron();
const handleFilesFolder = () => {
showModal(InputTextModal, {
value: getDefaultFileName(),
label: 'New file name',
header: 'Backup/dump database',
onConfirm: async file => {
await tick();
setFilesFolderResult(file);
},
});
};
const handleBrowse = async () => {
const electron = getElectron();
const file = await electron.showSaveDialog({
properties: ['showOverwriteConfirmation'],
filters: [
{ name: 'SQL Files', extensions: ['sql'] },
{ name: 'All Files', extensions: ['*'] },
],
defaultPath: outputFile,
});
if (file) {
const path = window.require('path');
outputFile = file;
outputLabel = path.parse(outputFile).name;
pureFileName = null;
}
};
const handleDownload = async () => {
const resp = await apiCall('files/generate-uploads-file', { extension: 'sql' });
outputFile = resp.filePath;
outputLabel = 'Download';
pureFileName = resp.fileName;
};
</script>
<FormProvider>
<ModalBase {...$$restProps}>
<svelte:fragment slot="header">Export database dump</svelte:fragment>
<div class="m-3">
<strong>Source:</strong>
{getConnectionLabel(connection)}
{#if connection.database}
({connection.database})
{/if}
</div>
<div class="ml-3 mr-3 mt-3"><strong>Target:</strong> {outputLabel}</div>
<div class="flex ml-3">
{#if electron}
<FormStyledButton type="button" value="Browse" on:click={handleBrowse} />
{:else}
<FormStyledButton type="button" value="Set download" on:click={handleDownload} />
{/if}
<FormStyledButton type="button" value="Files folder" on:click={handleFilesFolder} />
</div>
<svelte:fragment slot="footer">
<FormSubmit value="Run export" on:click={e => handleSubmit(e.detail)} disabled={!outputFile} />
<FormStyledButton type="button" value="Cancel" on:click={closeCurrentModal} />
</svelte:fragment>
</ModalBase>
</FormProvider>

View File

@@ -1,130 +0,0 @@
<script lang="ts">
import { onMount, tick } from 'svelte';
import FormStyledButton from '../buttons/FormStyledButton.svelte';
import UploadButton from '../buttons/UploadButton.svelte';
import FormProvider from '../forms/FormProvider.svelte';
import FormSubmit from '../forms/FormSubmit.svelte';
import { currentDropDownMenu } from '../stores';
import { apiCall } from '../utility/api';
import { importSqlDump } from '../utility/exportFileTools';
import getElectron from '../utility/getElectron';
import { setUploadListener } from '../utility/uploadFiles';
import ChangeDownloadUrlModal from './ChangeDownloadUrlModal.svelte';
import ModalBase from './ModalBase.svelte';
import { closeCurrentModal, showModal } from './modalTools';
import { getConnectionLabel } from 'dbgate-tools';
export let connection;
let inputLabel = '(not selected)';
let inputFile = null;
let domButton;
const handleSubmit = async values => {
const { value } = values;
closeCurrentModal();
importSqlDump(inputFile, connection);
};
const electron = getElectron();
const handleUpload = file => {
inputLabel = `uploaded: ${file.shortName}`;
inputFile = file.filePath;
};
onMount(() => {
setUploadListener(handleUpload);
return () => {
setUploadListener(null);
};
});
const handleAddUrl = () => {
showModal(ChangeDownloadUrlModal, {
onConfirm: async url => {
await tick();
inputLabel = url;
inputFile = url;
},
});
};
const handleBrowse = async () => {
const electron = getElectron();
const files = await electron.showOpenDialog({
properties: ['openFile'],
filters: [
{ name: 'SQL Files', extensions: ['sql'] },
{ name: 'All Files', extensions: ['*'] },
],
});
if (files && files[0]) {
const path = window.require('path');
inputFile = files[0];
inputLabel = path.parse(inputFile).name;
}
};
async function handleFilesClick() {
const rect = domButton.getBoundingClientRect();
const left = rect.left;
const top = rect.bottom;
const files = await apiCall('files/list', { folder: 'sql' });
const menu = files.map(({ file }) => ({
label: file,
onClick: async () => {
inputFile = await apiCall('files/get-file-real-path', { folder: 'sql', file });
if (inputFile) {
inputLabel = file;
}
},
}));
currentDropDownMenu.set({ left, top, items: menu });
}
</script>
<FormProvider>
<ModalBase {...$$restProps}>
<svelte:fragment slot="header">Import database dump</svelte:fragment>
<div class="ml-3 mr-3 mt-3"><strong>Source:</strong> {inputLabel}</div>
<div class="flex ml-3 mr-3 mb-3">
{#if electron}
<FormStyledButton type="button" value="Browse" on:click={handleBrowse} />
{:else}
<UploadButton />
{/if}
<FormStyledButton value="Web URL" on:click={handleAddUrl} />
<FormStyledButton value="From files" on:click={handleFilesClick} bind:this={domButton} />
</div>
<div class="m-3">
<strong>Target:</strong>
{getConnectionLabel(connection)}
{#if connection.database}
({connection.database})
{/if}
</div>
<svelte:fragment slot="footer">
<FormSubmit
value="Run import"
on:click={e => handleSubmit(e.detail)}
disabled={!inputFile}
data-testid="ImportDatabaseDumpModal_runImport"
/>
<FormStyledButton
type="button"
value="Cancel"
on:click={closeCurrentModal}
data-testid="ImportDatabaseDumpModal_cancel"
/>
</svelte:fragment>
</ModalBase>
</FormProvider>

View File

@@ -160,6 +160,7 @@
filter={objectsFilter}
disableContextMenu
{checkedObjectsStore}
passProps={{ ingorePin: true }}
/>
</WidgetsInnerContainer>
</div>

View File

@@ -7,6 +7,7 @@
import getElectron from '../utility/getElectron';
import { downloadFromApi } from '../utility/exportFileTools';
import useEffect from '../utility/useEffect';
import Link from '../elements/Link.svelte';
export let runnerId;
export let executeNumber;
@@ -63,41 +64,38 @@
},
]}
>
<a
slot="0"
let:row
href="#"
on:click={() => {
downloadFromApi(`runners/data/${runnerId}/${row.name}`, row.name);
}}
>
download
</a>
<svelte:fragment slot="0" let:row>
<Link
onClick={() => {
downloadFromApi(`runners/data/${runnerId}/${row.name}`, row.name);
}}
>
download
</Link>
</svelte:fragment>
<a
slot="1"
let:row
href="#"
on:click={async () => {
const file = await electron.showSaveDialog({});
if (file) {
const fs = window.require('fs');
fs.copyFile(row.path, file, () => {});
}
}}
>
save
</a>
<svelte:fragment slot="1" let:row>
<Link
onClick={async () => {
const file = await electron.showSaveDialog({});
if (file) {
const fs = window.require('fs');
fs.copyFile(row.path, file, () => {});
}
}}
>
save
</Link>
</svelte:fragment>
<a
slot="2"
let:row
href="#"
on:click={() => {
electron.showItemInFolder(row.path);
}}
>
show
</a>
<svelte:fragment slot="2" let:row>
<Link
onClick={() => {
electron.showItemInFolder(row.path);
}}
>
show
</Link>
</svelte:fragment>
</TableControl>
{/if}

View File

@@ -104,6 +104,7 @@ ORDER BY
{ label: 'Themes', slot: 3 },
{ label: 'Default Actions', slot: 4 },
{ label: 'Behaviour', slot: 5 },
{ label: 'External tools', slot: 8 },
{ label: 'Other', slot: 6 },
]}
>
@@ -480,6 +481,31 @@ ORDER BY
</div>
{/if}
</svelte:fragment>
<svelte:fragment slot="8">
<div class="heading">External tools</div>
<FormTextField
name="externalTools.mysqldump"
label="mysqldump (backup MySQL database)"
defaultValue="mysqldump"
/>
<FormTextField name="externalTools.mysql" label="mysql (restore MySQL database)" defaultValue="mysql" />
<FormTextField
name="externalTools.mysqlPlugins"
label="Folder with mysql plugins (for example for authentication). Set only in case of problems"
defaultValue=""
/>
<FormTextField
name="externalTools.pg_dump"
label="pg_dump (backup PostgreSQL database)"
defaultValue="pg_dump"
/>
<FormTextField
name="externalTools.psql"
label="psql (restore PostgreSQL database)"
defaultValue="psql"
/>
</svelte:fragment>
</TabControl>
</FormValues>

View File

@@ -182,26 +182,6 @@ export async function exportQuickExportFile(dataName, reader, format: QuickExpor
}
}
// export async function exportSqlDump(connection, databaseName) {
// await saveExportedFile(
// [{ name: 'SQL files', extensions: ['sql'] }],
// `${databaseName}.sql`,
// 'sql',
// `${databaseName}-dump`,
// filePath => {
// const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson();
// script.dumpDatabase({
// connection,
// databaseName,
// outputFile: filePath,
// });
// return script.getScript();
// }
// );
// }
export async function saveFileToDisk(
filePathFunc,
options: any = { formatLabel: 'HTML page', formatExtension: 'html' }

View File

@@ -0,0 +1,25 @@
import newQuery from '../query/newQuery';
import _ from 'lodash';
export function canOpenByWeb(file, extensions) {
if (!file) return false;
const nameLower = file.toLowerCase();
if (nameLower.endsWith('.sql')) return true;
return false;
}
export async function openWebFileCore(file, extensions) {
const nameLower = file.path.toLowerCase();
if (nameLower.endsWith('.sql')) {
const reader = new FileReader();
reader.onload = function (e) {
newQuery({
initialData: e.target.result,
});
};
reader.readAsText(file);
}
}

View File

@@ -8,6 +8,7 @@ import { showModal } from '../modals/modalTools';
import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
import openNewTab from './openNewTab';
import { openImportExportTab } from './importExportTools';
import { canOpenByWeb, openWebFileCore } from './openWebFile';
let uploadListener;
@@ -28,6 +29,11 @@ export default function uploadFiles(files) {
return;
}
if (!electron && canOpenByWeb(file.path, ext)) {
openWebFileCore(file, ext);
return;
}
const maxSize = 32 * 1024 * 1024;
if (parseInt(file.size, 10) >= maxSize) {
showModal(ErrorMessageModal, {

View File

@@ -10,6 +10,9 @@
import { apiCall } from '../utility/api';
import { useFiles } from '../utility/metadataLoaders';
import WidgetsInnerContainer from './WidgetsInnerContainer.svelte';
import getElectron from '../utility/getElectron';
import InlineButtonLabel from '../buttons/InlineButtonLabel.svelte';
import resolveApi, { resolveApiHeaders } from '../utility/resolveApi';
let filter = '';
@@ -24,6 +27,8 @@
const perspectiveFiles = useFiles({ folder: 'perspectives' });
const modelTransformFiles = useFiles({ folder: 'modtrans' });
const electron = getElectron();
$: files = [
...($sqlFiles || []),
...($shellFiles || []),
@@ -58,16 +63,67 @@
if (folder == 'modtrans') return 'Model transforms';
return _.startCase(folder);
}
async function handleUploadedFile(e) {
const files = [...e.target.files];
for (const file of files) {
const formData = new FormData();
formData.append('name', file.name);
formData.append('data', file);
const fetchOptions = {
method: 'POST',
body: formData,
headers: resolveApiHeaders(),
};
const apiBase = resolveApi();
const resp = await fetch(`${apiBase}/uploads/upload-data-file`, fetchOptions);
const fileData = await resp.json();
}
}
async function handleOpenElectronFile() {
const filePaths = await electron.showOpenDialog({
filters: [
{
name: `All supported files`,
extensions: ['sql'],
},
{ name: `SQL files`, extensions: ['sql'] },
],
properties: ['showHiddenFiles', 'openFile'],
});
const filePath = filePaths && filePaths[0];
await apiCall('uploads/save-data-file', { filePath });
}
</script>
<WidgetsInnerContainer>
<SearchBoxWrapper>
<SearchInput placeholder="Search saved files" bind:value={filter} />
<CloseSearchButton bind:filter />
<InlineButton on:click={handleRefreshFiles} title="Refresh files">
{#if electron}
<InlineButton on:click={handleOpenElectronFile} title="Add file" data-testid="SavedFileList_buttonAddFile">
<FontIcon icon="icon plus-thick" />
</InlineButton>
{:else}
<InlineButtonLabel
on:click={() => {}}
title="Add file"
data-testid="SavedFileList_buttonAddFile"
htmlFor="uploadSavedFileButton"
>
<FontIcon icon="icon plus-thick" />
</InlineButtonLabel>
{/if}
<InlineButton on:click={handleRefreshFiles} title="Refresh files" data-testid="SavedFileList_buttonRefresh">
<FontIcon icon="icon refresh" />
</InlineButton>
</SearchBoxWrapper>
<input type="file" id="uploadSavedFileButton" hidden on:change={handleUploadedFile} />
<AppObjectList list={files} module={savedFileAppObject} groupFunc={data => dataFolderTitle(data.folder)} {filter} />
</WidgetsInnerContainer>

View File

@@ -2,23 +2,36 @@
let domDiv;
export let hideContent = false;
export let fixedWidth = 0;
export function scrollTop() {
domDiv.scrollTop = 0;
}
</script>
<div on:drop bind:this={domDiv} class:hideContent data-testid={$$props['data-testid']}><slot /></div>
<div
on:drop
bind:this={domDiv}
class:hideContent
class:leftFixedWidth={!fixedWidth}
data-testid={$$props['data-testid']}
style:width={fixedWidth ? `${fixedWidth}px` : undefined}
>
<slot />
</div>
<style>
div {
flex: 1 1;
overflow-x: auto;
overflow-y: auto;
}
div.leftFixedWidth {
width: var(--dim-left-panel-width);
}
.hideContent {
div.hideContent {
visibility: hidden;
}
</style>

View File

@@ -36,7 +36,6 @@
"webpack-cli": "^5.1.4"
},
"dependencies": {
"antares-mysql-dumper": "^0.0.1",
"dbgate-query-splitter": "^4.11.3",
"dbgate-tools": "^6.0.0-alpha.1",
"lodash": "^4.17.21",

View File

@@ -5,7 +5,6 @@ const Analyser = require('./Analyser');
const mysql2 = require('mysql2');
const { getLogger, createBulkInsertStreamBase, makeUniqueColumnNames, extractErrorLogData } =
global.DBGATE_PACKAGES['dbgate-tools'];
const { MySqlDumper } = require('antares-mysql-dumper');
const logger = getLogger('mysqlDriver');
@@ -203,15 +202,6 @@ const drivers = driverBases.map(driverBase => ({
// @ts-ignore
return createBulkInsertStreamBase(this, stream, dbhan, name, options);
},
async createBackupDumper(dbhan, options) {
const { outputFile, databaseName, schemaName } = options;
const res = new MySqlDumper({
connection: dbhan.client,
schema: databaseName || schemaName,
outputFile,
});
return res;
},
getAuthTypes() {
const res = [
{

View File

@@ -178,7 +178,8 @@ const mysqlDriverBase = {
: mysqlSplitterOptions,
readOnlySessions: true,
supportsDatabaseDump: true,
supportsDatabaseBackup: true,
supportsDatabaseRestore: true,
authTypeLabel: 'Connection mode',
defaultAuthTypeName: 'hostPort',
defaultSocketPath: '/var/run/mysqld/mysqld.sock',
@@ -199,6 +200,122 @@ const mysqlDriverBase = {
},
];
},
getCliConnectionArgs(connection, externalTools) {
const args = [`--user=${connection.user}`, `--password=${connection.password}`, `--host=${connection.server}`];
if (connection.port) {
args.push(`--port=${connection.port}`);
}
if (externalTools.mysqlPlugins) {
args.push(`--plugin-dir=${externalTools.mysqlPlugins}`);
}
if (connection.server == 'localhost') {
args.push(`--protocol=tcp`);
}
return args;
},
backupDatabaseCommand(connection, settings, externalTools) {
const { outputFile, database, skippedTables, options } = settings;
const command = externalTools.mysqldump || 'mysqldump';
const args = this.getCliConnectionArgs(connection, externalTools);
args.push(`--result-file=${outputFile}`);
args.push('--verbose');
for (const table of skippedTables) {
args.push(`--ignore-table=${database}.${table.pureName}`);
}
if (options.noData) {
args.push('--no-data');
}
if (options.noStructure) {
args.push('--no-create-info');
}
if (options.includeEvents !== false && !options.noStructure) {
args.push('--events');
}
if (options.includeRoutines !== false && !options.noStructure) {
args.push('--routines');
}
if (options.includeTriggers !== false && !options.noStructure) {
args.push('--triggers');
}
if (options.force) {
args.push('--force');
}
args.push(database);
return { command, args };
},
restoreDatabaseCommand(connection, settings, externalTools) {
const { inputFile, database } = settings;
const command = externalTools.mysql || 'mysql';
const args = this.getCliConnectionArgs(connection, externalTools);
if (database) {
args.push(database);
}
return { command, args, stdinFilePath: inputFile };
},
transformNativeCommandMessage(message) {
if (message.message?.startsWith('--')) {
if (message.message.startsWith('-- Retrieving table structure for table')) {
return {
...message,
severity: 'info',
message: message.message.replace('-- Retrieving table structure for table', 'Processing table'),
};
} else {
return {
...message,
severity: 'debug',
message: message.message.replace('-- ', ''),
};
}
}
return message;
},
getNativeOperationFormArgs(operation) {
if (operation == 'backup') {
return [
{
type: 'checkbox',
label: 'No data (dump only structure)',
name: 'noData',
default: false,
},
{
type: 'checkbox',
label: 'No structure (dump only data)',
name: 'noStructure',
default: false,
},
{
type: 'checkbox',
label: 'Force (ignore all errors)',
name: 'force',
default: false,
},
{
type: 'checkbox',
label: 'Backup events',
name: 'includeEvents',
default: true,
disabledFn: values => values.noStructure,
},
{
type: 'checkbox',
label: 'Backup routines',
name: 'includeRoutines',
default: true,
disabledFn: values => values.noStructure,
},
{
type: 'checkbox',
label: 'Backup triggers',
name: 'includeTriggers',
default: true,
disabledFn: values => values.noStructure,
},
];
}
return null;
},
};
/** @type {import('dbgate-types').EngineDriver} */

View File

@@ -218,10 +218,135 @@ EXECUTE FUNCTION function_name();`,
defaultAuthTypeName: 'hostPort',
defaultSocketPath: '/var/run/postgresql',
supportsDatabaseBackup: true,
supportsDatabaseRestore: true,
adaptDataType(dataType) {
if (dataType?.toLowerCase() == 'datetime') return 'timestamp';
return dataType;
},
getCliConnectionArgs(connection) {
const args = [`--username=${connection.user}`, `--host=${connection.server}`];
if (connection.port) {
args.push(`--port=${connection.port}`);
}
return args;
},
getNativeOperationFormArgs(operation) {
if (operation == 'backup') {
return [
{
type: 'checkbox',
label: 'Dump only data (without structure)',
name: 'dataOnly',
default: false,
},
{
type: 'checkbox',
label: 'Dump schema only (no data)',
name: 'schemaOnly',
default: false,
},
{
type: 'checkbox',
label: 'Use SQL insert instead of COPY for rows',
name: 'insert',
default: false,
},
{
type: 'checkbox',
label: 'Prevent dumping of access privileges (grant/revoke)',
name: 'noPrivileges',
default: false,
},
{
type: 'checkbox',
label: 'Do not output commands to set ownership of objects ',
name: 'noOwner',
default: false,
},
];
}
return null;
},
backupDatabaseCommand(connection, settings, externalTools) {
const { outputFile, database, selectedTables, skippedTables, options, argsFormat } = settings;
const command = externalTools.pg_dump || 'pg_dump';
const args = this.getCliConnectionArgs(connection, externalTools);
args.push(`--file=${outputFile}`);
args.push('--verbose');
args.push(database);
if (options.dataOnly) {
args.push(`--data-only`);
}
if (options.schemaOnly) {
args.push(`--schema-only`);
}
if (options.insert) {
args.push(`--insert`);
}
if (options.noPrivileges) {
args.push(`--no-privileges`);
}
if (options.noOwner) {
args.push(`--no-owner`);
}
if (skippedTables.length > 0) {
for (const table of selectedTables) {
args.push(
argsFormat == 'spawn'
? `--table="${table.schemaName}"."${table.pureName}"`
: `--table='"${table.schemaName}"."${table.pureName}"'`
);
}
}
return {
command,
args,
env: { PGPASSWORD: connection.password },
};
},
restoreDatabaseCommand(connection, settings, externalTools) {
const { inputFile, database } = settings;
const command = externalTools.psql || 'psql';
const args = this.getCliConnectionArgs(connection, externalTools);
args.push(`--dbname=${database}`);
// args.push('--verbose');
args.push(`--file=${inputFile}`);
return {
command,
args,
env: { PGPASSWORD: connection.password },
};
},
transformNativeCommandMessage(message) {
if (message.message.startsWith('INSERT ') || message.message == 'SET') {
return null;
}
if (message.message.startsWith('pg_dump: processing data for table')) {
return {
...message,
severity: 'info',
message: message.message.replace('pg_dump: processing data for table', 'Processing table'),
};
} else if (message.message.toLowerCase().includes('error:')) {
return {
...message,
severity: 'error',
};
} else {
return {
...message,
severity: 'debug',
};
}
return message;
},
};
/** @type {import('dbgate-types').EngineDriver} */