SYNC: Merge branch 'feature/backup-restore'

packages/api/src/controllers/databaseConnections.js

@@ -1,4 +1,5 @@
 const connections = require('./connections');
+const runners = require('./runners');
 const archive = require('./archive');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
@@ -613,4 +614,129 @@ module.exports = {
 
     return res;
   },
+
+  async getNativeOpCommandArgs(
+    command,
+    { conid, database, outputFile, inputFile, options, selectedTables, skippedTables, argsFormat }
+  ) {
+    const connection = await connections.getCore({ conid });
+    const driver = requireEngineDriver(connection);
+
+    const settingsValue = await config.getSettings();
+
+    const externalTools = {};
+    for (const pair of Object.entries(settingsValue || {})) {
+      const [name, value] = pair;
+      if (name.startsWith('externalTools.')) {
+        externalTools[name.substring('externalTools.'.length)] = value;
+      }
+    }
+
+    return {
+      ...(command == 'backup'
+        ? driver.backupDatabaseCommand(
+            connection,
+            { outputFile, database, options, selectedTables, skippedTables, argsFormat },
+            // @ts-ignore
+            externalTools
+          )
+        : driver.restoreDatabaseCommand(
+            connection,
+            { inputFile, database, options, argsFormat },
+            // @ts-ignore
+            externalTools
+          )),
+      transformMessage: driver.transformNativeCommandMessage
+        ? message => driver.transformNativeCommandMessage(message, command)
+        : null,
+    };
+  },
+
+  commandArgsToCommandLine(commandArgs) {
+    const { command, args, stdinFilePath } = commandArgs;
+    let res = `${command} ${args.join(' ')}`;
+    if (stdinFilePath) {
+      res += ` < ${stdinFilePath}`;
+    }
+    return res;
+  },
+
+  nativeBackup_meta: true,
+  async nativeBackup({ conid, database, outputFile, runid, options, selectedTables, skippedTables }) {
+    const commandArgs = await this.getNativeOpCommandArgs('backup', {
+      conid,
+      database,
+      inputFile: undefined,
+      outputFile,
+      options,
+      selectedTables,
+      skippedTables,
+      argsFormat: 'spawn',
+    });
+
+    return runners.nativeRunCore(runid, {
+      ...commandArgs,
+      onFinished: () => {
+        socket.emitChanged(`files-changed`, { folder: 'sql' });
+        socket.emitChanged(`all-files-changed`);
+      },
+    });
+  },
+
+  nativeBackupCommand_meta: true,
+  async nativeBackupCommand({ conid, database, outputFile, options, selectedTables, skippedTables }) {
+    const commandArgs = await this.getNativeOpCommandArgs('backup', {
+      conid,
+      database,
+      outputFile,
+      inputFile: undefined,
+      options,
+      selectedTables,
+      skippedTables,
+      argsFormat: 'shell',
+    });
+
+    return {
+      ...commandArgs,
+      transformMessage: null,
+      commandLine: this.commandArgsToCommandLine(commandArgs),
+    };
+  },
+
+  nativeRestore_meta: true,
+  async nativeRestore({ conid, database, inputFile, runid }) {
+    const commandArgs = await this.getNativeOpCommandArgs('restore', {
+      conid,
+      database,
+      inputFile,
+      outputFile: undefined,
+      options: undefined,
+      argsFormat: 'spawn',
+    });
+
+    return runners.nativeRunCore(runid, {
+      ...commandArgs,
+      onFinished: () => {
+        this.syncModel({ conid, database, isFullRefresh: true });
+      },
+    });
+  },
+
+  nativeRestoreCommand_meta: true,
+  async nativeRestoreCommand({ conid, database, inputFile }) {
+    const commandArgs = await this.getNativeOpCommandArgs('restore', {
+      conid,
+      database,
+      inputFile,
+      outputFile: undefined,
+      options: undefined,
+      argsFormat: 'shell',
+    });
+
+    return {
+      ...commandArgs,
+      transformMessage: null,
+      commandLine: this.commandArgsToCommandLine(commandArgs),
+    };
+  },
 };

packages/api/src/controllers/runners.js

@@ -4,7 +4,7 @@ const path = require('path');
 const fs = require('fs-extra');
 const byline = require('byline');
 const socket = require('../utility/socket');
-const { fork } = require('child_process');
+const { fork, spawn } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
 const {
   extractShellApiPlugins,
@@ -13,6 +13,8 @@ const {
   getLogger,
   safeJsonParse,
   pinoLogRecordToMessageRecord,
+  extractErrorMessage,
+  extractErrorLogData,
 } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
@@ -80,6 +82,7 @@ module.exports = {
           }
         : {
             message,
+            severity: 'info',
             time: new Date(),
           };
 
@@ -173,7 +176,7 @@ module.exports = {
       // console.log('... ERROR subprocess', error);
       this.rejectRequest(runid, { message: error && (error.message || error.toString()) });
       console.error('... ERROR subprocess', error);
-      this.dispatchMessage({
+      this.dispatchMessage(runid, {
         severity: 'error',
         message: error.toString(),
       });
@@ -192,6 +195,75 @@ module.exports = {
     return _.pick(newOpened, ['runid']);
   },
 
+  nativeRunCore(runid, commandArgs) {
+    const { command, args, env, transformMessage, stdinFilePath, onFinished } = commandArgs;
+    const pipeDispatcher = severity => data => {
+      let messageObject = {
+        message: data.toString().trim(),
+        severity,
+      };
+      if (transformMessage) {
+        messageObject = transformMessage(messageObject);
+      }
+
+      if (messageObject) {
+        return this.dispatchMessage(runid, messageObject);
+      }
+    };
+
+    const subprocess = spawn(command, args, { env: { ...process.env, ...env } });
+
+    byline(subprocess.stdout).on('data', pipeDispatcher('info'));
+    byline(subprocess.stderr).on('data', pipeDispatcher('error'));
+
+    subprocess.on('exit', code => {
+      console.log('... EXITED', code);
+      logger.info({ code, pid: subprocess.pid }, 'Exited process');
+      this.dispatchMessage(runid, `Finished external process with code ${code}`);
+      socket.emit(`runner-done-${runid}`, code);
+      if (onFinished) {
+        onFinished();
+      }
+    });
+    subprocess.on('spawn', () => {
+      this.dispatchMessage(runid, `Started external process ${command}`);
+    });
+    subprocess.on('error', error => {
+      console.log('... ERROR subprocess', error);
+      this.dispatchMessage(runid, {
+        severity: 'error',
+        message: error.toString(),
+      });
+      if (error['code'] == 'ENOENT') {
+        this.dispatchMessage(runid, {
+          severity: 'error',
+          message: `Command ${command} not found, please install it and configure its location in DbGate settings, Settings/External tools, if ${command} is not in system PATH`,
+        });
+      }
+      socket.emit(`runner-done-${runid}`);
+    });
+
+    if (stdinFilePath) {
+      const inputStream = fs.createReadStream(stdinFilePath);
+      inputStream.pipe(subprocess.stdin);
+
+      subprocess.stdin.on('error', err => {
+        this.dispatchMessage(runid, {
+          severity: 'error',
+          message: extractErrorMessage(err),
+        });
+        logger.error(extractErrorLogData(err), 'Caught error on stdin');
+      });
+    }
+
+    const newOpened = {
+      runid,
+      subprocess,
+    };
+    this.opened.push(newOpened);
+    return _.pick(newOpened, ['runid']);
+  },
+
   start_meta: true,
   async start({ script }) {
     const runid = crypto.randomUUID();

packages/api/src/controllers/uploads.js

@@ -1,6 +1,6 @@
 const crypto = require('crypto');
 const path = require('path');
-const { uploadsdir, getLogsFilePath } = require('../utility/directories');
+const { uploadsdir, getLogsFilePath, filesdir } = require('../utility/directories');
 const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const logger = getLogger('uploads');
 const axios = require('axios');
@@ -13,6 +13,7 @@ const serverConnections = require('./serverConnections');
 const config = require('./config');
 const gistSecret = require('../gistSecret');
 const currentVersion = require('../currentVersion');
+const socket = require('../utility/socket');
 
 module.exports = {
   upload_meta: {
@@ -38,6 +39,52 @@ module.exports = {
     });
   },
 
+  uploadDataFile_meta: {
+    method: 'post',
+    raw: true,
+  },
+  uploadDataFile(req, res) {
+    const { data } = req.files || {};
+
+    if (!data) {
+      res.json(null);
+      return;
+    }
+
+    if (data.name.toLowerCase().endsWith('.sql')) {
+      logger.info(`Uploading SQL file ${data.name}, size=${data.size}`);
+      data.mv(path.join(filesdir(), 'sql', data.name), () => {
+        res.json({
+          name: data.name,
+          folder: 'sql',
+        });
+
+        socket.emitChanged(`files-changed`, { folder: 'sql' });
+        socket.emitChanged(`all-files-changed`);
+      });
+      return;
+    }
+
+    res.json(null);
+  },
+
+  saveDataFile_meta: true,
+  async saveDataFile({ filePath }) {
+    if (filePath.toLowerCase().endsWith('.sql')) {
+      logger.info(`Saving SQL file ${filePath}`);
+      await fs.copyFile(filePath, path.join(filesdir(), 'sql', path.basename(filePath)));
+
+      socket.emitChanged(`files-changed`, { folder: 'sql' });
+      socket.emitChanged(`all-files-changed`);
+      return {
+        name: path.basename(filePath),
+        folder: 'sql',
+      };
+    }
+
+    return null;
+  },
+
   get_meta: {
     method: 'get',
     raw: true,