Merge branch 'master' into feature/duckdb-2

packages/api/src/controllers/archive.js

@@ -2,14 +2,20 @@ const fs = require('fs-extra');
 const readline = require('readline');
 const crypto = require('crypto');
 const path = require('path');
-const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('../utility/directories');
+const { archivedir, clearArchiveLinksCache, resolveArchiveFolder, uploadsdir } = require('../utility/directories');
 const socket = require('../utility/socket');
 const loadFilesRecursive = require('../utility/loadFilesRecursive');
 const getJslFileName = require('../utility/getJslFileName');
-const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const { getLogger, extractErrorLogData, jsonLinesParse } = require('dbgate-tools');
 const dbgateApi = require('../shell');
 const jsldata = require('./jsldata');
 const platformInfo = require('../utility/platformInfo');
+const { isProApp } = require('../utility/checkLicense');
+const listZipEntries = require('../utility/listZipEntries');
+const unzipJsonLinesFile = require('../shell/unzipJsonLinesFile');
+const { zip } = require('lodash');
+const zipDirectory = require('../shell/zipDirectory');
+const unzipDirectory = require('../shell/unzipDirectory');
 
 const logger = getLogger('archive');
 
@@ -47,9 +53,31 @@ module.exports = {
     return folder;
   },
 
+  async getZipFiles({ file }) {
+    const entries = await listZipEntries(path.join(archivedir(), file));
+    const files = entries.map(entry => {
+      let name = entry.fileName;
+      if (isProApp() && entry.fileName.endsWith('.jsonl')) {
+        name = entry.fileName.slice(0, -6);
+      }
+      return {
+        name: name,
+        label: name,
+        type: isProApp() && entry.fileName.endsWith('.jsonl') ? 'jsonl' : 'other',
+      };
+    });
+    return files;
+  },
+
   files_meta: true,
   async files({ folder }) {
     try {
+      if (folder.endsWith('.zip')) {
+        if (await fs.exists(path.join(archivedir(), folder))) {
+          return this.getZipFiles({ file: folder });
+        }
+        return [];
+      }
       const dir = resolveArchiveFolder(folder);
       if (!(await fs.exists(dir))) return [];
       const files = await loadFilesRecursive(dir); // fs.readdir(dir);
@@ -91,6 +119,16 @@ module.exports = {
     return true;
   },
 
+  createFile_meta: true,
+  async createFile({ folder, file, fileType, tableInfo }) {
+    await fs.writeFile(
+      path.join(resolveArchiveFolder(folder), `${file}.${fileType}`),
+      tableInfo ? JSON.stringify({ __isStreamHeader: true, tableInfo }) : ''
+    );
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
   deleteFile_meta: true,
   async deleteFile({ folder, file, fileType }) {
     await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
@@ -158,7 +196,7 @@ module.exports = {
   deleteFolder_meta: true,
   async deleteFolder({ folder }) {
     if (!folder) throw new Error('Missing folder parameter');
-    if (folder.endsWith('.link')) {
+    if (folder.endsWith('.link') || folder.endsWith('.zip')) {
       await fs.unlink(path.join(archivedir(), folder));
     } else {
       await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
@@ -204,9 +242,10 @@ module.exports = {
   },
 
   async getNewArchiveFolder({ database }) {
-    const isLink = database.endsWith(database);
-    const name = isLink ? database.slice(0, -5) : database;
-    const suffix = isLink ? '.link' : '';
+    const isLink = database.endsWith('.link');
+    const isZip = database.endsWith('.zip');
+    const name = isLink ? database.slice(0, -5) : isZip ? database.slice(0, -4) : database;
+    const suffix = isLink ? '.link' : isZip ? '.zip' : '';
     if (!(await fs.exists(path.join(archivedir(), database)))) return database;
     let index = 2;
     while (await fs.exists(path.join(archivedir(), `${name}${index}${suffix}`))) {
@@ -214,4 +253,58 @@ module.exports = {
     }
     return `${name}${index}${suffix}`;
   },
+
+  getArchiveData_meta: true,
+  async getArchiveData({ folder, file }) {
+    let rows;
+    if (folder.endsWith('.zip')) {
+      rows = await unzipJsonLinesFile(path.join(archivedir(), folder), `${file}.jsonl`);
+    } else {
+      rows = jsonLinesParse(await fs.readFile(path.join(archivedir(), folder, `${file}.jsonl`), { encoding: 'utf8' }));
+    }
+    return rows.filter(x => !x.__isStreamHeader);
+  },
+
+  saveUploadedZip_meta: true,
+  async saveUploadedZip({ filePath, fileName }) {
+    if (!fileName?.endsWith('.zip')) {
+      throw new Error(`${fileName} is not a ZIP file`);
+    }
+
+    const folder = await this.getNewArchiveFolder({ database: fileName });
+    await fs.copyFile(filePath, path.join(archivedir(), folder));
+    socket.emitChanged(`archive-folders-changed`);
+
+    return null;
+  },
+
+  zip_meta: true,
+  async zip({ folder }) {
+    const newFolder = await this.getNewArchiveFolder({ database: folder + '.zip' });
+    await zipDirectory(path.join(archivedir(), folder), path.join(archivedir(), newFolder));
+    socket.emitChanged(`archive-folders-changed`);
+
+    return null;
+  },
+
+  unzip_meta: true,
+  async unzip({ folder }) {
+    const newFolder = await this.getNewArchiveFolder({ database: folder.slice(0, -4) });
+    await unzipDirectory(path.join(archivedir(), folder), path.join(archivedir(), newFolder));
+    socket.emitChanged(`archive-folders-changed`);
+
+    return null;
+  },
+
+  getZippedPath_meta: true,
+  async getZippedPath({ folder }) {
+    if (folder.endsWith('.zip')) {
+      return { filePath: path.join(archivedir(), folder) };
+    }
+
+    const uploadName = crypto.randomUUID();
+    const filePath = path.join(uploadsdir(), uploadName);
+    await zipDirectory(path.join(archivedir(), folder), filePath);
+    return { filePath };
+  },
 };

packages/api/src/controllers/auth.js

@@ -12,6 +12,7 @@ const {
   getAuthProviderById,
 } = require('../auth/authProvider');
 const storage = require('./storage');
+const { decryptPasswordString } = require('../utility/crypting');
 
 const logger = getLogger('auth');
 
@@ -44,6 +45,7 @@ function authMiddleware(req, res, next) {
     '/connections/dblogin-auth',
     '/connections/dblogin-auth-token',
     '/health',
+    '/__health',
   ];
 
   // console.log('********************* getAuthProvider()', getAuthProvider());
@@ -95,7 +97,7 @@ module.exports = {
       let adminPassword = process.env.ADMIN_PASSWORD;
       if (!adminPassword) {
         const adminConfig = await storage.readConfig({ group: 'admin' });
-        adminPassword = adminConfig?.adminPassword;
+        adminPassword = decryptPasswordString(adminConfig?.adminPassword);
       }
       if (adminPassword && adminPassword == password) {
         return {

packages/api/src/controllers/config.js

@@ -19,6 +19,14 @@ const storage = require('./storage');
 const { getAuthProxyUrl } = require('../utility/authProxy');
 const { getPublicHardwareFingerprint } = require('../utility/hardwareFingerprint');
 const { extractErrorMessage } = require('dbgate-tools');
+const {
+  generateTransportEncryptionKey,
+  createTransportEncryptor,
+  recryptConnection,
+  getInternalEncryptor,
+  recryptUser,
+  recryptObjectPasswordFieldInPlace,
+} = require('../utility/crypting');
 
 const lock = new AsyncLock();
 
@@ -107,6 +115,7 @@ module.exports = {
         datadir(),
         processArgs.runE2eTests ? 'connections-e2etests.jsonl' : 'connections.jsonl'
       ),
+      supportCloudAutoUpgrade: !!process.env.CLOUD_UPGRADE_FILE,
       ...currentVersion,
     };
 
@@ -144,7 +153,7 @@ module.exports = {
     const res = {
       ...value,
     };
-    if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+    if (platformInfo.isElectron && value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
       // res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
       res['app.useNativeMenu'] = false;
     }
@@ -161,14 +170,19 @@ module.exports = {
 
   async loadSettings() {
     try {
-      const settingsText = await fs.readFile(
-        path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
-        { encoding: 'utf-8' }
-      );
-      return {
-        ...this.fillMissingSettings(JSON.parse(settingsText)),
-        'other.licenseKey': platformInfo.isElectron ? await this.loadLicenseKey() : undefined,
-      };
+      if (process.env.STORAGE_DATABASE) {
+        const settings = await storage.readConfig({ group: 'settings' });
+        return this.fillMissingSettings(settings);
+      } else {
+        const settingsText = await fs.readFile(
+          path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
+          { encoding: 'utf-8' }
+        );
+        return {
+          ...this.fillMissingSettings(JSON.parse(settingsText)),
+          'other.licenseKey': platformInfo.isElectron ? await this.loadLicenseKey() : undefined,
+        };
+      }
     } catch (err) {
       return this.fillMissingSettings({});
     }
@@ -246,19 +260,31 @@ module.exports = {
     const res = await lock.acquire('settings', async () => {
       const currentValue = await this.loadSettings();
       try {
-        const updated = {
-          ...currentValue,
-          ..._.omit(values, ['other.licenseKey']),
-        };
-        await fs.writeFile(
-          path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
-          JSON.stringify(updated, undefined, 2)
-        );
-        // this.settingsValue = updated;
+        let updated = currentValue;
+        if (process.env.STORAGE_DATABASE) {
+          updated = {
+            ...currentValue,
+            ...values,
+          };
+          await storage.writeConfig({
+            group: 'settings',
+            config: updated,
+          });
+        } else {
+          updated = {
+            ...currentValue,
+            ..._.omit(values, ['other.licenseKey']),
+          };
+          await fs.writeFile(
+            path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
+            JSON.stringify(updated, undefined, 2)
+          );
+          // this.settingsValue = updated;
 
-        if (currentValue['other.licenseKey'] != values['other.licenseKey']) {
-          await this.saveLicenseKey({ licenseKey: values['other.licenseKey'] });
-          socket.emitChanged(`config-changed`);
+          if (currentValue['other.licenseKey'] != values['other.licenseKey']) {
+            await this.saveLicenseKey({ licenseKey: values['other.licenseKey'] });
+            socket.emitChanged(`config-changed`);
+          }
         }
 
         socket.emitChanged(`settings-changed`);
@@ -281,4 +307,91 @@ module.exports = {
     const resp = await checkLicenseKey(licenseKey);
     return resp;
   },
+
+  recryptDatabaseForExport(db) {
+    const encryptionKey = generateTransportEncryptionKey();
+    const transportEncryptor = createTransportEncryptor(encryptionKey);
+
+    const config = _.cloneDeep([
+      ...(db.config?.filter(c => !(c.group == 'admin' && c.key == 'encryptionKey')) || []),
+      { group: 'admin', key: 'encryptionKey', value: encryptionKey },
+    ]);
+    const adminPassword = config.find(c => c.group == 'admin' && c.key == 'adminPassword');
+    recryptObjectPasswordFieldInPlace(adminPassword, 'value', getInternalEncryptor(), transportEncryptor);
+
+    return {
+      ...db,
+      connections: db.connections?.map(conn => recryptConnection(conn, getInternalEncryptor(), transportEncryptor)),
+      users: db.users?.map(conn => recryptUser(conn, getInternalEncryptor(), transportEncryptor)),
+      config,
+    };
+  },
+
+  recryptDatabaseFromImport(db) {
+    const encryptionKey = db.config?.find(c => c.group == 'admin' && c.key == 'encryptionKey')?.value;
+    if (!encryptionKey) {
+      throw new Error('Missing encryption key in the database');
+    }
+    const config = _.cloneDeep(db.config || []).filter(c => !(c.group == 'admin' && c.key == 'encryptionKey'));
+    const transportEncryptor = createTransportEncryptor(encryptionKey);
+
+    const adminPassword = config.find(c => c.group == 'admin' && c.key == 'adminPassword');
+    recryptObjectPasswordFieldInPlace(adminPassword, 'value', transportEncryptor, getInternalEncryptor());
+
+    return {
+      ...db,
+      connections: db.connections?.map(conn => recryptConnection(conn, transportEncryptor, getInternalEncryptor())),
+      users: db.users?.map(conn => recryptUser(conn, transportEncryptor, getInternalEncryptor())),
+      config,
+    };
+  },
+
+  exportConnectionsAndSettings_meta: true,
+  async exportConnectionsAndSettings(_params, req) {
+    if (!hasPermission(`admin/config`, req)) {
+      throw new Error('Permission denied: admin/config');
+    }
+
+    if (connections.portalConnections) {
+      throw new Error('Not allowed');
+    }
+
+    if (process.env.STORAGE_DATABASE) {
+      const db = await storage.getExportedDatabase();
+      return this.recryptDatabaseForExport(db);
+    }
+
+    return this.recryptDatabaseForExport({
+      connections: (await connections.list(null, req)).map((conn, index) => ({
+        ..._.omit(conn, ['_id']),
+        id: index + 1,
+        conid: conn._id,
+      })),
+    });
+  },
+
+  importConnectionsAndSettings_meta: true,
+  async importConnectionsAndSettings({ db }, req) {
+    if (!hasPermission(`admin/config`, req)) {
+      throw new Error('Permission denied: admin/config');
+    }
+
+    if (connections.portalConnections) {
+      throw new Error('Not allowed');
+    }
+
+    const recryptedDb = this.recryptDatabaseFromImport(db);
+    if (process.env.STORAGE_DATABASE) {
+      await storage.replicateImportedDatabase(recryptedDb);
+    } else {
+      await connections.importFromArray(
+        recryptedDb.connections.map(conn => ({
+          ..._.omit(conn, ['conid', 'id']),
+          _id: conn.conid,
+        }))
+      );
+    }
+
+    return true;
+  },
 };

packages/api/src/controllers/connections.js

@@ -107,8 +107,8 @@ function getPortalCollections() {
       trustServerCertificate: process.env[`SSL_TRUST_CERTIFICATE_${id}`],
     }));
 
-    for(const conn of connections) {
-      for(const prop in process.env) {
+    for (const conn of connections) {
+      for (const prop in process.env) {
         if (prop.startsWith(`CONNECTION_${conn._id}_`)) {
           const name = prop.substring(`CONNECTION_${conn._id}_`.length);
           conn[name] = process.env[prop];
@@ -321,6 +321,18 @@ module.exports = {
     return res;
   },
 
+  importFromArray(list) {
+    this.datastore.transformAll(connections => {
+      const mapped = connections.map(x => {
+        const found = list.find(y => y._id == x._id);
+        if (found) return found;
+        return x;
+      });
+      return [...mapped, ...list.filter(x => !connections.find(y => y._id == x._id))];
+    });
+    socket.emitChanged('connection-list-changed');
+  },
+
   async checkUnsavedConnectionsLimit() {
     if (!this.datastore) {
       return;

packages/api/src/controllers/databaseConnections.js

@@ -37,6 +37,8 @@ const loadModelTransform = require('../utility/loadModelTransform');
 const exportDbModelSql = require('../utility/exportDbModelSql');
 const axios = require('axios');
 const { callTextToSqlApi, callCompleteOnCursorApi, callRefactorSqlQueryApi } = require('../utility/authProxy');
+const { decryptConnection } = require('../utility/crypting');
+const { getSshTunnel } = require('../utility/sshTunnel');
 
 const logger = getLogger('databaseConnections');
 
@@ -140,6 +142,11 @@ module.exports = {
       if (newOpened.disconnected) return;
       this.close(conid, database, false);
     });
+    subprocess.on('error', err => {
+      logger.error(extractErrorLogData(err), 'Error in database connection subprocess');
+      if (newOpened.disconnected) return;
+      this.close(conid, database, false);
+    });
 
     subprocess.send({
       msgtype: 'connect',
@@ -619,9 +626,26 @@ module.exports = {
     command,
     { conid, database, outputFile, inputFile, options, selectedTables, skippedTables, argsFormat }
   ) {
-    const connection = await connections.getCore({ conid });
+    const sourceConnection = await connections.getCore({ conid });
+    const connection = {
+      ...decryptConnection(sourceConnection),
+    };
     const driver = requireEngineDriver(connection);
 
+    if (!connection.port && driver.defaultPort) {
+      connection.port = driver.defaultPort.toString();
+    }
+
+    if (connection.useSshTunnel) {
+      const tunnel = await getSshTunnel(connection);
+      if (tunnel.state == 'error') {
+        throw new Error(tunnel.message);
+      }
+
+      connection.server = tunnel.localHost;
+      connection.port = tunnel.localPort;
+    }
+
     const settingsValue = await config.getSettings();
 
     const externalTools = {};

packages/api/src/controllers/files.js

@@ -9,6 +9,9 @@ const scheduler = require('./scheduler');
 const getDiagramExport = require('../utility/getDiagramExport');
 const apps = require('./apps');
 const getMapExport = require('../utility/getMapExport');
+const dbgateApi = require('../shell');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('files');
 
 function serialize(format, data) {
   if (format == 'text') return data;
@@ -219,4 +222,60 @@ module.exports = {
       return path.join(dir, file);
     }
   },
+
+  createZipFromJsons_meta: true,
+  async createZipFromJsons({ db, filePath }) {
+    logger.info(`Creating zip file from JSONS ${filePath}`);
+    await dbgateApi.zipJsonLinesData(db, filePath);
+    return true;
+  },
+
+  getJsonsFromZip_meta: true,
+  async getJsonsFromZip({ filePath }) {
+    const res = await dbgateApi.unzipJsonLinesData(filePath);
+    return res;
+  },
+
+  downloadText_meta: true,
+  async downloadText({ uri }, req) {
+    if (!uri) return null;
+    const filePath = await dbgateApi.download(uri);
+    const text = await fs.readFile(filePath, {
+      encoding: 'utf-8',
+    });
+    return text;
+  },
+
+  saveUploadedFile_meta: true,
+  async saveUploadedFile({ filePath, fileName }) {
+    const FOLDERS = ['sql', 'sqlite'];
+    for (const folder of FOLDERS) {
+      if (fileName.toLowerCase().endsWith('.' + folder)) {
+        logger.info(`Saving ${folder} file ${fileName}`);
+        await fs.copyFile(filePath, path.join(filesdir(), folder, fileName));
+
+        socket.emitChanged(`files-changed`, { folder: folder });
+        socket.emitChanged(`all-files-changed`);
+        return {
+          name: path.basename(filePath),
+          folder: folder,
+        };
+      }
+    }
+
+    throw new Error(`${fileName} doesn't have one of supported extensions: ${FOLDERS.join(', ')}`);
+  },
+
+  exportFile_meta: true,
+  async exportFile({ folder, file, filePath }, req) {
+    if (!hasPermission(`files/${folder}/read`, req)) return false;
+    await fs.copyFile(path.join(filesdir(), folder, file), filePath);
+    return true;
+  },
+
+  simpleCopy_meta: true,
+  async simpleCopy({ sourceFilePath, targetFilePath }, req) {
+    await fs.copyFile(sourceFilePath, targetFilePath);
+    return true;
+  },
 };

packages/api/src/controllers/jsldata.js

@@ -8,6 +8,8 @@ const getJslFileName = require('../utility/getJslFileName');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
 const requirePluginFunction = require('../utility/requirePluginFunction');
 const socket = require('../utility/socket');
+const crypto = require('crypto');
+const dbgateApi = require('../shell');
 
 function readFirstLine(file) {
   return new Promise((resolve, reject) => {
@@ -293,4 +295,11 @@ module.exports = {
       })),
     };
   },
+
+  downloadJslData_meta: true,
+  async downloadJslData({ uri }) {
+    const jslid = crypto.randomUUID();
+    await dbgateApi.download(uri, { targetFile: getJslFileName(jslid) });
+    return { jslid };
+  },
 };

packages/api/src/controllers/runners.js

@@ -96,9 +96,9 @@ module.exports = {
 
   handle_ping() {},
 
-  handle_freeData(runid, { freeData }) {
+  handle_dataResult(runid, { dataResult }) {
     const { resolve } = this.requests[runid];
-    resolve(freeData);
+    resolve(dataResult);
     delete this.requests[runid];
   },
 
@@ -328,4 +328,24 @@ module.exports = {
     });
     return promise;
   },
+
+  scriptResult_meta: true,
+  async scriptResult({ script }) {
+    if (script.type != 'json') {
+      return { errorMessage: 'Only JSON scripts are allowed' };
+    }
+
+    const promise = new Promise((resolve, reject) => {
+      const runid = crypto.randomUUID();
+      this.requests[runid] = { resolve, reject, exitOnStreamError: true };
+      const cloned = _.cloneDeepWith(script, node => {
+        if (node?.$replace == 'runid') {
+          return runid;
+        }
+      });
+      const js = jsonScriptToJavascript(cloned);
+      this.startCore(runid, scriptTemplate(js, false));
+    });
+    return promise;
+  },
 };

packages/api/src/controllers/serverConnections.js

@@ -98,6 +98,11 @@ module.exports = {
         if (newOpened.disconnected) return;
         this.close(conid, false);
       });
+      subprocess.on('error', err => {
+        logger.error(extractErrorLogData(err), 'Error in server connection subprocess');
+        if (newOpened.disconnected) return;
+        this.close(conid, false);
+      });
       subprocess.send({ msgtype: 'connect', ...connection, globalSettings: await config.getSettings() });
       return newOpened;
     });

packages/api/src/controllers/storage.js

@@ -4,6 +4,10 @@ module.exports = {
     return null;
   },
 
+  async getExportedDatabase() {
+    return {};
+  },
+
   getConnection_meta: true,
   async getConnection({ conid }) {
     return null;

packages/api/src/controllers/uploads.js

@@ -39,52 +39,6 @@ module.exports = {
     });
   },
 
-  uploadDataFile_meta: {
-    method: 'post',
-    raw: true,
-  },
-  uploadDataFile(req, res) {
-    const { data } = req.files || {};
-
-    if (!data) {
-      res.json(null);
-      return;
-    }
-
-    if (data.name.toLowerCase().endsWith('.sql')) {
-      logger.info(`Uploading SQL file ${data.name}, size=${data.size}`);
-      data.mv(path.join(filesdir(), 'sql', data.name), () => {
-        res.json({
-          name: data.name,
-          folder: 'sql',
-        });
-
-        socket.emitChanged(`files-changed`, { folder: 'sql' });
-        socket.emitChanged(`all-files-changed`);
-      });
-      return;
-    }
-
-    res.json(null);
-  },
-
-  saveDataFile_meta: true,
-  async saveDataFile({ filePath }) {
-    if (filePath.toLowerCase().endsWith('.sql')) {
-      logger.info(`Saving SQL file ${filePath}`);
-      await fs.copyFile(filePath, path.join(filesdir(), 'sql', path.basename(filePath)));
-
-      socket.emitChanged(`files-changed`, { folder: 'sql' });
-      socket.emitChanged(`all-files-changed`);
-      return {
-        name: path.basename(filePath),
-        folder: 'sql',
-      };
-    }
-
-    return null;
-  },
-
   get_meta: {
     method: 'get',
     raw: true,