Merge branch 'master' into feature/firebird

packages/api/src/controllers/auth.js

@@ -13,6 +13,8 @@ const {
 } = require('../auth/authProvider');
 const storage = require('./storage');
 const { decryptPasswordString } = require('../utility/crypting');
+const { createDbGateIdentitySession, startCloudTokenChecking } = require('../utility/cloudIntf');
+const socket = require('../utility/socket');
 
 const logger = getLogger('auth');
 
@@ -135,5 +137,14 @@ module.exports = {
     return getAuthProviderById(amoid).redirect(params);
   },
 
+  createCloudLoginSession_meta: true,
+  async createCloudLoginSession({ client }) {
+    const res = await createDbGateIdentitySession(client);
+    startCloudTokenChecking(res.sid, tokenHolder => {
+      socket.emit('got-cloud-token', tokenHolder);
+    });
+    return res;
+  },
+
   authMiddleware,
 };

packages/api/src/controllers/cloud.js

@@ -0,0 +1,250 @@
+const {
+  getPublicCloudFiles,
+  getPublicFileData,
+  refreshPublicFiles,
+  callCloudApiGet,
+  callCloudApiPost,
+  getCloudFolderEncryptor,
+  getCloudContent,
+  putCloudContent,
+  removeCloudCachedConnection,
+} = require('../utility/cloudIntf');
+const connections = require('./connections');
+const socket = require('../utility/socket');
+const { recryptConnection, getInternalEncryptor, encryptConnection } = require('../utility/crypting');
+const { getConnectionLabel, getLogger, extractErrorLogData } = require('dbgate-tools');
+const logger = getLogger('cloud');
+const _ = require('lodash');
+const fs = require('fs-extra');
+
+module.exports = {
+  publicFiles_meta: true,
+  async publicFiles() {
+    const res = await getPublicCloudFiles();
+    return res;
+  },
+
+  publicFileData_meta: true,
+  async publicFileData({ path }) {
+    const res = getPublicFileData(path);
+    return res;
+  },
+
+  refreshPublicFiles_meta: true,
+  async refreshPublicFiles({ isRefresh }) {
+    await refreshPublicFiles(isRefresh);
+    return {
+      status: 'ok',
+    };
+  },
+
+  contentList_meta: true,
+  async contentList() {
+    try {
+      const resp = await callCloudApiGet('content-list');
+      return resp;
+    } catch (err) {
+      logger.error(extractErrorLogData(err), 'Error getting cloud content list');
+
+      return [];
+    }
+  },
+
+  getContent_meta: true,
+  async getContent({ folid, cntid }) {
+    const resp = await getCloudContent(folid, cntid);
+    return resp;
+  },
+
+  putContent_meta: true,
+  async putContent({ folid, cntid, content, name, type }) {
+    const resp = await putCloudContent(folid, cntid, content, name, type);
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return resp;
+  },
+
+  createFolder_meta: true,
+  async createFolder({ name }) {
+    const resp = await callCloudApiPost(`folders/create`, { name });
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return resp;
+  },
+
+  grantFolder_meta: true,
+  async grantFolder({ inviteLink }) {
+    const m = inviteLink.match(/^dbgate\:\/\/folder\/v1\/([a-zA-Z0-9]+)\?mode=(read|write|admin)$/);
+    if (!m) {
+      throw new Error('Invalid invite link format');
+    }
+    const invite = m[1];
+    const mode = m[2];
+
+    const resp = await callCloudApiPost(`folders/grant/${mode}`, { invite });
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return resp;
+  },
+
+  renameFolder_meta: true,
+  async renameFolder({ folid, name }) {
+    const resp = await callCloudApiPost(`folders/rename`, { folid, name });
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return resp;
+  },
+
+  deleteFolder_meta: true,
+  async deleteFolder({ folid }) {
+    const resp = await callCloudApiPost(`folders/delete`, { folid });
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return resp;
+  },
+
+  getInviteToken_meta: true,
+  async getInviteToken({ folid, role }) {
+    const resp = await callCloudApiGet(`invite-token/${folid}/${role}`);
+    return resp;
+  },
+
+  refreshContent_meta: true,
+  async refreshContent() {
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return {
+      status: 'ok',
+    };
+  },
+
+  copyConnectionCloud_meta: true,
+  async copyConnectionCloud({ conid, folid }) {
+    const conn = await connections.getCore({ conid });
+    const folderEncryptor = await getCloudFolderEncryptor(folid);
+    const recryptedConn = recryptConnection(conn, getInternalEncryptor(), folderEncryptor);
+    const connToSend = _.omit(recryptedConn, ['_id']);
+    const resp = await putCloudContent(
+      folid,
+      undefined,
+      JSON.stringify(connToSend),
+      getConnectionLabel(conn),
+      'connection'
+    );
+    return resp;
+  },
+
+  saveConnection_meta: true,
+  async saveConnection({ folid, connection }) {
+    let cntid = undefined;
+    if (connection._id) {
+      const m = connection._id.match(/^cloud\:\/\/(.+)\/(.+)$/);
+      if (!m) {
+        throw new Error('Invalid cloud connection ID format');
+      }
+      folid = m[1];
+      cntid = m[2];
+    }
+
+    if (!folid) {
+      throw new Error('Missing cloud folder ID');
+    }
+
+    const folderEncryptor = await getCloudFolderEncryptor(folid);
+    const recryptedConn = encryptConnection(connection, folderEncryptor);
+    const resp = await putCloudContent(
+      folid,
+      cntid,
+      JSON.stringify(recryptedConn),
+      getConnectionLabel(recryptedConn),
+      'connection'
+    );
+
+    if (resp.apiErrorMessage) {
+      return resp;
+    }
+
+    removeCloudCachedConnection(folid, resp.cntid);
+    cntid = resp.cntid;
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return {
+      ...recryptedConn,
+      _id: `cloud://${folid}/${cntid}`,
+    };
+  },
+
+  duplicateConnection_meta: true,
+  async duplicateConnection({ conid }) {
+    const m = conid.match(/^cloud\:\/\/(.+)\/(.+)$/);
+    if (!m) {
+      throw new Error('Invalid cloud connection ID format');
+    }
+    const folid = m[1];
+    const cntid = m[2];
+    const respGet = await getCloudContent(folid, cntid);
+    const conn = JSON.parse(respGet.content);
+    const conn2 = {
+      ...conn,
+      displayName: getConnectionLabel(conn) + ' - copy',
+    };
+    const respPut = await putCloudContent(folid, undefined, JSON.stringify(conn2), conn2.displayName, 'connection');
+    return respPut;
+  },
+
+  deleteConnection_meta: true,
+  async deleteConnection({ conid }) {
+    const m = conid.match(/^cloud\:\/\/(.+)\/(.+)$/);
+    if (!m) {
+      throw new Error('Invalid cloud connection ID format');
+    }
+    const folid = m[1];
+    const cntid = m[2];
+    const resp = await callCloudApiPost(`content/delete/${folid}/${cntid}`);
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return resp;
+  },
+
+  deleteContent_meta: true,
+  async deleteContent({ folid, cntid }) {
+    const resp = await callCloudApiPost(`content/delete/${folid}/${cntid}`);
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return resp;
+  },
+
+  renameContent_meta: true,
+  async renameContent({ folid, cntid, name }) {
+    const resp = await callCloudApiPost(`content/rename/${folid}/${cntid}`, { name });
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return resp;
+  },
+
+  saveFile_meta: true,
+  async saveFile({ folid, cntid, fileName, data, contentFolder, format }) {
+    const resp = await putCloudContent(folid, cntid, data, fileName, 'file', contentFolder, format);
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return resp;
+  },
+
+  copyFile_meta: true,
+  async copyFile({ folid, cntid, name }) {
+    const resp = await callCloudApiPost(`content/duplicate/${folid}/${cntid}`, { name });
+    socket.emitChanged('cloud-content-changed');
+    socket.emit('cloud-content-updated');
+    return resp;
+  },
+
+  exportFile_meta: true,
+  async exportFile({ folid, cntid, filePath }, req) {
+    const { content } = await getCloudContent(folid, cntid);
+    if (!content) {
+      throw new Error('File not found');
+    }
+    await fs.writeFile(filePath, content);
+    return true;
+  },
+};

packages/api/src/controllers/config.js

@@ -298,8 +298,12 @@ module.exports = {
 
   changelog_meta: true,
   async changelog() {
-    const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
-    return resp.data;
+    try {
+      const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
+      return resp.data;
+    } catch (err) {
+      return ''
+    }
   },
 
   checkLicense_meta: true,

packages/api/src/controllers/connections.js

@@ -239,6 +239,19 @@ module.exports = {
     return (await this.datastore.find()).filter(x => connectionHasPermission(x, req));
   },
 
+  async getUsedEngines() {
+    const storage = require('./storage');
+
+    const storageEngines = await storage.getUsedEngines();
+    if (storageEngines) {
+      return storageEngines;
+    }
+    if (portalConnections) {
+      return _.uniq(_.compact(portalConnections.map(x => x.engine)));
+    }
+    return _.uniq((await this.datastore.find()).map(x => x.engine));
+  },
+
   test_meta: true,
   test({ connection, requestDbList = false }) {
     const subprocess = fork(
@@ -410,6 +423,13 @@ module.exports = {
       return volatile;
     }
 
+    const cloudMatch = conid.match(/^cloud\:\/\/(.+)\/(.+)$/);
+    if (cloudMatch) {
+      const { loadCachedCloudConnection } = require('../utility/cloudIntf');
+      const conn = await loadCachedCloudConnection(cloudMatch[1], cloudMatch[2]);
+      return conn;
+    }
+
     const storage = require('./storage');
 
     const storageConnection = await storage.getConnection({ conid });

packages/api/src/controllers/databaseConnections.js

@@ -148,6 +148,9 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
     const connection = await connections.getCore({ conid });
+    if (!connection) {
+      throw new Error(`databaseConnections: Connection with conid="${conid}" not found`);
+    }
     if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
       throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
     }
@@ -304,6 +307,12 @@ module.exports = {
     return this.loadDataCore('loadKeys', { conid, database, root, filter, limit });
   },
 
+  scanKeys_meta: true,
+  async scanKeys({ conid, database, root, pattern, cursor, count }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('scanKeys', { conid, database, root, pattern, cursor, count });
+  },
+
   exportKeys_meta: true,
   async exportKeys({ conid, database, options }, req) {
     testConnectionPermission(conid, req);

packages/api/src/controllers/jsldata.js

@@ -10,6 +10,7 @@ const requirePluginFunction = require('../utility/requirePluginFunction');
 const socket = require('../utility/socket');
 const crypto = require('crypto');
 const dbgateApi = require('../shell');
+const { ChartProcessor } = require('dbgate-datalib');
 
 function readFirstLine(file) {
   return new Promise((resolve, reject) => {
@@ -302,4 +303,29 @@ module.exports = {
     await dbgateApi.download(uri, { targetFile: getJslFileName(jslid) });
     return { jslid };
   },
+
+  buildChart_meta: true,
+  async buildChart({ jslid, definition }) {
+    const datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    const processor = new ChartProcessor(definition ? [definition] : undefined);
+    await datastore.enumRows(row => {
+      processor.addRow(row);
+      return true;
+    });
+    processor.finalize();
+    return processor.charts;
+  },
+
+  detectChartColumns_meta: true,
+  async detectChartColumns({ jslid }) {
+    const datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    const processor = new ChartProcessor();
+    processor.autoDetectCharts = false;
+    await datastore.enumRows(row => {
+      processor.addRow(row);
+      return true;
+    });
+    processor.finalize();
+    return processor.availableColumns;
+  },
 };

packages/api/src/controllers/serverConnections.js

@@ -52,7 +52,7 @@ module.exports = {
       if (existing) return existing;
       const connection = await connections.getCore({ conid });
       if (!connection) {
-        throw new Error(`Connection with conid="${conid}" not found`);
+        throw new Error(`serverConnections: Connection with conid="${conid}" not found`);
       }
       if (connection.singleDatabase) {
         return null;

packages/api/src/controllers/sessions.js

@@ -83,6 +83,11 @@ module.exports = {
     jsldata.notifyChangedStats(stats);
   },
 
+  handle_charts(sesid, props) {
+    const { jslid, charts, resultIndex } = props;
+    socket.emit(`session-charts-${sesid}`, { jslid, resultIndex, charts });
+  },
+
   handle_initializeFile(sesid, props) {
     const { jslid } = props;
     socket.emit(`session-initialize-file-${jslid}`);
@@ -141,7 +146,7 @@ module.exports = {
   },
 
   executeQuery_meta: true,
-  async executeQuery({ sesid, sql, autoCommit }) {
+  async executeQuery({ sesid, sql, autoCommit, limitRows, frontMatter }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
       throw new Error('Invalid session');
@@ -149,7 +154,7 @@ module.exports = {
 
     logger.info({ sesid, sql }, 'Processing query');
     this.dispatchMessage(sesid, 'Query execution started');
-    session.subprocess.send({ msgtype: 'executeQuery', sql, autoCommit });
+    session.subprocess.send({ msgtype: 'executeQuery', sql, autoCommit, limitRows, frontMatter });
 
     return { state: 'ok' };
   },

packages/api/src/controllers/storage.js

@@ -32,4 +32,8 @@ module.exports = {
   },
 
   startRefreshLicense() {},
+
+  async getUsedEngines() {
+    return null;
+  },
 };

packages/api/src/main.js

@@ -27,6 +27,7 @@ const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
 const scheduler = require('./controllers/scheduler');
 const queryHistory = require('./controllers/queryHistory');
+const cloud = require('./controllers/cloud');
 const onFinished = require('on-finished');
 const processArgs = require('./utility/processArgs');
 
@@ -39,6 +40,7 @@ const { getDefaultAuthProvider } = require('./auth/authProvider');
 const startCloudUpgradeTimer = require('./utility/cloudUpgrade');
 const { isProApp } = require('./utility/checkLicense');
 const { getHealthStatus, getHealthStatusSprinx } = require('./utility/healthStatus');
+const { startCloudFiles } = require('./utility/cloudIntf');
 
 const logger = getLogger('main');
 
@@ -200,6 +202,8 @@ function start() {
   if (process.env.CLOUD_UPGRADE_FILE) {
     startCloudUpgradeTimer();
   }
+
+  startCloudFiles();
 }
 
 function useAllControllers(app, electron) {
@@ -220,6 +224,7 @@ function useAllControllers(app, electron) {
   useController(app, electron, '/query-history', queryHistory);
   useController(app, electron, '/apps', apps);
   useController(app, electron, '/auth', auth);
+  useController(app, electron, '/cloud', cloud);
 }
 
 function setElectronSender(electronSender) {

packages/api/src/proc/connectProcess.js

@@ -28,14 +28,7 @@ function start() {
       let version = {
         version: 'Unknown',
       };
-      try {
-        version = await driver.getVersion(dbhan);
-      } catch (err) {
-        logger.error(extractErrorLogData(err), 'Error getting DB server version');
-        version = {
-          version: 'Unknown',
-        };
-      }
+      version = await driver.getVersion(dbhan);
       let databases = undefined;
       if (requestDbList) {
         databases = await driver.listDatabases(dbhan);

packages/api/src/proc/databaseConnectionProcess.js

@@ -12,6 +12,7 @@ const {
   ScriptWriterEval,
   SqlGenerator,
   playJsonScriptWriter,
+  serializeJsTypesForJsonStringify,
 } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { connectUtility } = require('../utility/connectUtility');
@@ -232,7 +233,7 @@ async function handleQueryData({ msgid, sql, range }, skipReadonlyCheck = false)
   try {
     if (!skipReadonlyCheck) ensureExecuteCustomScript(driver);
     const res = await driver.query(dbhan, sql, { range });
-    process.send({ msgtype: 'response', msgid, ...res });
+    process.send({ msgtype: 'response', msgid, ...serializeJsTypesForJsonStringify(res) });
   } catch (err) {
     process.send({
       msgtype: 'response',
@@ -254,7 +255,7 @@ async function handleDriverDataCore(msgid, callMethod, { logName }) {
   const driver = requireEngineDriver(storedConnection);
   try {
     const result = await callMethod(driver);
-    process.send({ msgtype: 'response', msgid, result });
+    process.send({ msgtype: 'response', msgid, result: serializeJsTypesForJsonStringify(result) });
   } catch (err) {
     logger.error(extractErrorLogData(err, { logName }), `Error when handling message ${logName}`);
     process.send({ msgtype: 'response', msgid, errorMessage: extractErrorMessage(err, 'Error executing DB data') });
@@ -274,6 +275,10 @@ async function handleLoadKeys({ msgid, root, filter, limit }) {
   return handleDriverDataCore(msgid, driver => driver.loadKeys(dbhan, root, filter, limit), { logName: 'loadKeys' });
 }
 
+async function handleScanKeys({ msgid, pattern, cursor, count }) {
+  return handleDriverDataCore(msgid, driver => driver.scanKeys(dbhan, pattern, cursor, count), { logName: 'scanKeys' });
+}
+
 async function handleExportKeys({ msgid, options }) {
   return handleDriverDataCore(msgid, driver => driver.exportKeys(dbhan, options), { logName: 'exportKeys' });
 }
@@ -452,6 +457,7 @@ const messageHandlers = {
   updateCollection: handleUpdateCollection,
   collectionData: handleCollectionData,
   loadKeys: handleLoadKeys,
+  scanKeys: handleScanKeys,
   loadKeyInfo: handleLoadKeyInfo,
   callMethod: handleCallMethod,
   loadKeyTableRange: handleLoadKeyTableRange,

packages/api/src/proc/sessionProcess.js

@@ -117,7 +117,7 @@ async function handleExecuteControlCommand({ command }) {
   }
 }
 
-async function handleExecuteQuery({ sql, autoCommit }) {
+async function handleExecuteQuery({ sql, autoCommit, limitRows, frontMatter }) {
   lastActivity = new Date().getTime();
 
   await waitConnected();
@@ -146,7 +146,7 @@ async function handleExecuteQuery({ sql, autoCommit }) {
       ...driver.getQuerySplitterOptions('stream'),
       returnRichInfo: true,
     })) {
-      await handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem);
+      await handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, undefined, limitRows, frontMatter);
       // const handler = new StreamHandler(resultIndex);
       // const stream = await driver.stream(systemConnection, sqlItem, handler);
       // handler.stream = stream;

packages/api/src/shell/executeQuery.js

@@ -15,6 +15,7 @@ const logger = getLogger('execQuery');
  * @param {string} [options.sqlFile] - SQL file
  * @param {boolean} [options.logScriptItems] - whether to log script items instead of whole script
  * @param {boolean} [options.useTransaction] - run query in transaction
+ * @param {boolean} [options.skipLogging] - whether to skip logging
  */
 async function executeQuery({
   connection = undefined,
@@ -23,9 +24,10 @@ async function executeQuery({
   sql,
   sqlFile = undefined,
   logScriptItems = false,
+  skipLogging = false,
   useTransaction,
 }) {
-  if (!logScriptItems) {
+  if (!logScriptItems && !skipLogging) {
     logger.info({ sql: getLimitedQuery(sql) }, `Execute query`);
   }
 
@@ -38,7 +40,9 @@ async function executeQuery({
   }
 
   try {
-    logger.debug(`Running SQL query, length: ${sql.length}`);
+    if (!skipLogging) {
+      logger.debug(`Running SQL query, length: ${sql.length}`);
+    }
 
     await driver.script(dbhan, sql, { logScriptItems, useTransaction });
   } finally {

packages/api/src/shell/generateDeploySql.js

@@ -52,7 +52,10 @@ async function generateDeploySql({
     dbdiffOptionsExtra?.['schemaMode'] !== 'ignore' &&
     dbdiffOptionsExtra?.['schemaMode'] !== 'ignoreImplicit'
   ) {
-    throw new Error('targetSchema is required for databases with multiple schemas');
+    if (!driver?.dialect?.defaultSchemaName) {
+      throw new Error('targetSchema is required for databases with multiple schemas');
+    }
+    targetSchema = driver.dialect.defaultSchemaName;
   }
 
   try {

packages/api/src/shell/queryReader.js

@@ -7,6 +7,8 @@ const logger = getLogger('queryReader');
  * Returns reader object for {@link copyStream} function. This reader object reads data from query.
  * @param {object} options
  * @param {connectionType} options.connection - connection object
+ * @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
+ * @param {object} options.driver - driver object. If not provided, it will be loaded from connection
  * @param {string} options.query - SQL query
  * @param {string} [options.queryType] - query type
  * @param {string} [options.sql] - SQL query. obsolete; use query instead
@@ -16,6 +18,8 @@ async function queryReader({
   connection,
   query,
   queryType,
+  systemConnection,
+  driver,
   // obsolete; use query instead
   sql,
 }) {
@@ -28,10 +32,13 @@ async function queryReader({
   logger.info({ sql: query || sql }, `Reading query`);
   // else console.log(`Reading query ${JSON.stringify(json)}`);
 
-  const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
+  if (!driver) {
+    driver = requireEngineDriver(connection);
+  }
+  const dbhan = systemConnection || (await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script'));
+
   const reader =
-    queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
+    queryType == 'json' ? await driver.readJsonQuery(dbhan, query) : await driver.readQuery(dbhan, query || sql);
   return reader;
 }
 

packages/api/src/utility/authProxy.js

@@ -36,6 +36,10 @@ async function callRefactorSqlQueryApi(query, task, structure, dialect) {
   return null;
 }
 
+function getLicenseHttpHeaders() {
+  return {};
+}
+
 module.exports = {
   isAuthProxySupported,
   authProxyGetRedirectUrl,
@@ -47,4 +51,5 @@ module.exports = {
   callTextToSqlApi,
   callCompleteOnCursorApi,
   callRefactorSqlQueryApi,
+  getLicenseHttpHeaders,
 };

packages/api/src/utility/cloudIntf.js

@@ -0,0 +1,380 @@
+const axios = require('axios');
+const fs = require('fs-extra');
+const _ = require('lodash');
+const path = require('path');
+const { getLicenseHttpHeaders } = require('./authProxy');
+const { getLogger, extractErrorLogData, jsonLinesParse } = require('dbgate-tools');
+const { datadir } = require('./directories');
+const platformInfo = require('./platformInfo');
+const connections = require('../controllers/connections');
+const { isProApp } = require('./checkLicense');
+const socket = require('./socket');
+const config = require('../controllers/config');
+const simpleEncryptor = require('simple-encryptor');
+const currentVersion = require('../currentVersion');
+const { getPublicIpInfo } = require('./hardwareFingerprint');
+
+const logger = getLogger('cloudIntf');
+
+let cloudFiles = null;
+
+const DBGATE_IDENTITY_URL = process.env.LOCAL_DBGATE_IDENTITY
+  ? 'http://localhost:3103'
+  : process.env.DEVWEB || process.env.DEVMODE
+  ? 'https://identity.dbgate.udolni.net'
+  : 'https://identity.dbgate.io';
+
+const DBGATE_CLOUD_URL = process.env.LOCAL_DBGATE_CLOUD
+  ? 'http://localhost:3110'
+  : process.env.DEVWEB || process.env.DEVMODE
+  ? 'https://cloud.dbgate.udolni.net'
+  : 'https://cloud.dbgate.io';
+
+async function createDbGateIdentitySession(client) {
+  const resp = await axios.default.post(
+    `${DBGATE_IDENTITY_URL}/api/create-session`,
+    {
+      client,
+    },
+    {
+      headers: {
+        ...getLicenseHttpHeaders(),
+        'Content-Type': 'application/json',
+      },
+    }
+  );
+  return {
+    sid: resp.data.sid,
+    url: `${DBGATE_IDENTITY_URL}/api/signin/${resp.data.sid}`,
+  };
+}
+
+function startCloudTokenChecking(sid, callback) {
+  const started = Date.now();
+  const interval = setInterval(async () => {
+    if (Date.now() - started > 60 * 1000) {
+      clearInterval(interval);
+      return;
+    }
+
+    try {
+      // console.log(`Checking cloud token for session: ${DBGATE_IDENTITY_URL}/api/get-token/${sid}`);
+      const resp = await axios.default.get(`${DBGATE_IDENTITY_URL}/api/get-token/${sid}`, {
+        headers: {
+          ...getLicenseHttpHeaders(),
+        },
+      });
+      // console.log('CHECK RESP:', resp.data);
+
+      if (resp.data.email) {
+        clearInterval(interval);
+        callback(resp.data);
+      }
+    } catch (err) {
+      logger.error(extractErrorLogData(err), 'Error checking cloud token');
+    }
+  }, 500);
+}
+
+async function loadCloudFiles() {
+  try {
+    const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files.jsonl'), 'utf-8');
+    const parsedJson = jsonLinesParse(fileContent);
+    cloudFiles = _.sortBy(parsedJson, x => `${x.folder}/${x.title}`);
+  } catch (err) {
+    cloudFiles = [];
+  }
+}
+
+async function collectCloudFilesSearchTags() {
+  const res = [];
+  if (platformInfo.isElectron) {
+    res.push('app');
+  } else {
+    res.push('web');
+  }
+  if (platformInfo.isWindows) {
+    res.push('windows');
+  }
+  if (platformInfo.isMac) {
+    res.push('mac');
+  }
+  if (platformInfo.isLinux) {
+    res.push('linux');
+  }
+  if (platformInfo.isAwsUbuntuLayout) {
+    res.push('aws');
+  }
+  if (platformInfo.isAzureUbuntuLayout) {
+    res.push('azure');
+  }
+  if (platformInfo.isSnap) {
+    res.push('snap');
+  }
+  if (platformInfo.isDocker) {
+    res.push('docker');
+  }
+  if (platformInfo.isNpmDist) {
+    res.push('npm');
+  }
+  const engines = await connections.getUsedEngines();
+  const engineTags = engines.map(engine => engine.split('@')[0]);
+  res.push(...engineTags);
+
+  // team-premium and trials will return the same cloud files as premium - no need to check
+  res.push(isProApp() ? 'premium' : 'community');
+
+  return res;
+}
+
+async function getCloudSigninHolder() {
+  const settingsValue = await config.getSettings();
+  const holder = settingsValue['cloudSigninTokenHolder'];
+  return holder;
+}
+
+async function getCloudSigninHeaders(holder = null) {
+  if (!holder) {
+    holder = await getCloudSigninHolder();
+  }
+  if (holder) {
+    return {
+      'x-cloud-login': holder.token,
+    };
+  }
+  return null;
+}
+
+async function updateCloudFiles(isRefresh) {
+  let lastCloudFilesTags;
+  try {
+    lastCloudFilesTags = await fs.readFile(path.join(datadir(), 'cloud-files-tags.txt'), 'utf-8');
+  } catch (err) {
+    lastCloudFilesTags = '';
+  }
+
+  const ipInfo = await getPublicIpInfo();
+
+  const tags = (await collectCloudFilesSearchTags()).join(',');
+  let lastCheckedTm = 0;
+  if (tags == lastCloudFilesTags && cloudFiles.length > 0) {
+    lastCheckedTm = _.max(cloudFiles.map(x => parseInt(x.modifiedTm)));
+  }
+
+  logger.info({ tags, lastCheckedTm }, 'Downloading cloud files');
+
+  const resp = await axios.default.get(
+    `${DBGATE_CLOUD_URL}/public-cloud-updates?lastCheckedTm=${lastCheckedTm}&tags=${tags}&isRefresh=${
+      isRefresh ? 1 : 0
+    }&country=${ipInfo?.country || ''}`,
+    {
+      headers: {
+        ...getLicenseHttpHeaders(),
+        ...(await getCloudSigninHeaders()),
+        'x-app-version': currentVersion.version,
+      },
+    }
+  );
+
+  logger.info(`Downloaded ${resp.data.length} cloud files`);
+
+  const filesByPath = lastCheckedTm == 0 ? {} : _.keyBy(cloudFiles, 'path');
+  for (const file of resp.data) {
+    if (file.isDeleted) {
+      delete filesByPath[file.path];
+    } else {
+      filesByPath[file.path] = file;
+    }
+  }
+
+  cloudFiles = Object.values(filesByPath);
+
+  await fs.writeFile(path.join(datadir(), 'cloud-files.jsonl'), cloudFiles.map(x => JSON.stringify(x)).join('\n'));
+  await fs.writeFile(path.join(datadir(), 'cloud-files-tags.txt'), tags);
+
+  socket.emitChanged(`public-cloud-changed`);
+}
+
+async function startCloudFiles() {
+  loadCloudFiles();
+}
+
+async function getPublicCloudFiles() {
+  if (!loadCloudFiles) {
+    await loadCloudFiles();
+  }
+  return cloudFiles;
+}
+
+async function getPublicFileData(path) {
+  const resp = await axios.default.get(`${DBGATE_CLOUD_URL}/public/${path}`, {
+    headers: {
+      ...getLicenseHttpHeaders(),
+    },
+  });
+  return resp.data;
+}
+
+async function refreshPublicFiles(isRefresh) {
+  if (!cloudFiles) {
+    await loadCloudFiles();
+  }
+  try {
+    await updateCloudFiles(isRefresh);
+  } catch (err) {
+    logger.error(extractErrorLogData(err), 'Error updating cloud files');
+  }
+}
+
+async function callCloudApiGet(endpoint, signinHolder = null, additionalHeaders = {}) {
+  if (!signinHolder) {
+    signinHolder = await getCloudSigninHolder();
+  }
+  if (!signinHolder) {
+    return null;
+  }
+  const signinHeaders = await getCloudSigninHeaders(signinHolder);
+
+  const resp = await axios.default.get(`${DBGATE_CLOUD_URL}/${endpoint}`, {
+    headers: {
+      ...getLicenseHttpHeaders(),
+      ...signinHeaders,
+      ...additionalHeaders,
+    },
+    validateStatus: status => status < 500,
+  });
+  const { errorMessage } = resp.data;
+  if (errorMessage) {
+    return { apiErrorMessage: errorMessage };
+  }
+  return resp.data;
+}
+
+async function callCloudApiPost(endpoint, body, signinHolder = null) {
+  if (!signinHolder) {
+    signinHolder = await getCloudSigninHolder();
+  }
+  if (!signinHolder) {
+    return null;
+  }
+  const signinHeaders = await getCloudSigninHeaders(signinHolder);
+
+  const resp = await axios.default.post(`${DBGATE_CLOUD_URL}/${endpoint}`, body, {
+    headers: {
+      ...getLicenseHttpHeaders(),
+      ...signinHeaders,
+    },
+    validateStatus: status => status < 500,
+  });
+  const { errorMessage, isLicenseLimit, limitedLicenseLimits } = resp.data;
+  if (errorMessage) {
+    return {
+      apiErrorMessage: errorMessage,
+      apiErrorIsLicenseLimit: isLicenseLimit,
+      apiErrorLimitedLicenseLimits: limitedLicenseLimits,
+    };
+  }
+  return resp.data;
+}
+
+async function getCloudFolderEncryptor(folid) {
+  const { encryptionKey } = await callCloudApiGet(`folder-key/${folid}`);
+  if (!encryptionKey) {
+    throw new Error('No encryption key for folder: ' + folid);
+  }
+  return simpleEncryptor.createEncryptor(encryptionKey);
+}
+
+async function getCloudContent(folid, cntid) {
+  const signinHolder = await getCloudSigninHolder();
+  if (!signinHolder) {
+    throw new Error('No signed in');
+  }
+
+  const encryptor = simpleEncryptor.createEncryptor(signinHolder.encryptionKey);
+
+  const { content, name, type, contentFolder, contentType, apiErrorMessage } = await callCloudApiGet(
+    `content/${folid}/${cntid}`,
+    signinHolder,
+    {
+      'x-kehid': signinHolder.kehid,
+    }
+  );
+
+  if (apiErrorMessage) {
+    return { apiErrorMessage };
+  }
+
+  return {
+    content: encryptor.decrypt(content),
+    name,
+    type,
+    contentFolder,
+    contentType,
+  };
+}
+
+/**
+ *
+ * @returns Promise<{ cntid: string } | { apiErrorMessage: string }>
+ */
+async function putCloudContent(folid, cntid, content, name, type, contentFolder = null, contentType = null) {
+  const signinHolder = await getCloudSigninHolder();
+  if (!signinHolder) {
+    throw new Error('No signed in');
+  }
+
+  const encryptor = simpleEncryptor.createEncryptor(signinHolder.encryptionKey);
+
+  const resp = await callCloudApiPost(
+    `put-content`,
+    {
+      folid,
+      cntid,
+      name,
+      type,
+      kehid: signinHolder.kehid,
+      content: encryptor.encrypt(content),
+      contentFolder,
+      contentType,
+    },
+    signinHolder
+  );
+  socket.emitChanged('cloud-content-changed');
+  socket.emit('cloud-content-updated');
+  return resp;
+}
+
+const cloudConnectionCache = {};
+async function loadCachedCloudConnection(folid, cntid) {
+  const cacheKey = `${folid}|${cntid}`;
+  if (!cloudConnectionCache[cacheKey]) {
+    const { content } = await getCloudContent(folid, cntid);
+    cloudConnectionCache[cacheKey] = {
+      ...JSON.parse(content),
+      _id: `cloud://${folid}/${cntid}`,
+    };
+  }
+  return cloudConnectionCache[cacheKey];
+}
+
+function removeCloudCachedConnection(folid, cntid) {
+  const cacheKey = `${folid}|${cntid}`;
+  delete cloudConnectionCache[cacheKey];
+}
+
+module.exports = {
+  createDbGateIdentitySession,
+  startCloudTokenChecking,
+  startCloudFiles,
+  getPublicCloudFiles,
+  getPublicFileData,
+  refreshPublicFiles,
+  callCloudApiGet,
+  callCloudApiPost,
+  getCloudFolderEncryptor,
+  getCloudContent,
+  loadCachedCloudConnection,
+  putCloudContent,
+  removeCloudCachedConnection,
+};

packages/api/src/utility/crypting.js

@@ -81,11 +81,11 @@ function decryptPasswordString(password) {
   return password;
 }
 
-function encryptObjectPasswordField(obj, field) {
+function encryptObjectPasswordField(obj, field, encryptor = null) {
   if (obj && obj[field] && !obj[field].startsWith('crypt:')) {
     return {
       ...obj,
-      [field]: 'crypt:' + getInternalEncryptor().encrypt(obj[field]),
+      [field]: 'crypt:' + (encryptor || getInternalEncryptor()).encrypt(obj[field]),
     };
   }
   return obj;
@@ -101,11 +101,11 @@ function decryptObjectPasswordField(obj, field) {
   return obj;
 }
 
-function encryptConnection(connection) {
+function encryptConnection(connection, encryptor = null) {
   if (connection.passwordMode != 'saveRaw') {
-    connection = encryptObjectPasswordField(connection, 'password');
-    connection = encryptObjectPasswordField(connection, 'sshPassword');
-    connection = encryptObjectPasswordField(connection, 'sshKeyfilePassword');
+    connection = encryptObjectPasswordField(connection, 'password', encryptor);
+    connection = encryptObjectPasswordField(connection, 'sshPassword', encryptor);
+    connection = encryptObjectPasswordField(connection, 'sshKeyfilePassword', encryptor);
   }
   return connection;
 }

packages/api/src/utility/handleQueryStream.js

@@ -4,6 +4,9 @@ const fs = require('fs');
 const _ = require('lodash');
 
 const { jsldir } = require('../utility/directories');
+const { serializeJsTypesReplacer } = require('dbgate-tools');
+const { ChartProcessor } = require('dbgate-datalib');
+const { isProApp } = require('./checkLicense');
 
 class QueryStreamTableWriter {
   constructor(sesid = undefined) {
@@ -11,9 +14,12 @@ class QueryStreamTableWriter {
     this.currentChangeIndex = 1;
     this.initializedFile = false;
     this.sesid = sesid;
+    if (isProApp()) {
+      this.chartProcessor = new ChartProcessor();
+    }
   }
 
-  initializeFromQuery(structure, resultIndex) {
+  initializeFromQuery(structure, resultIndex, chartDefinition) {
     this.jslid = crypto.randomUUID();
     this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     fs.writeFileSync(
@@ -27,6 +33,9 @@ class QueryStreamTableWriter {
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
     this.initializedFile = true;
+    if (isProApp() && chartDefinition) {
+      this.chartProcessor = new ChartProcessor([chartDefinition]);
+    }
     process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex, sesid: this.sesid });
   }
 
@@ -38,7 +47,16 @@ class QueryStreamTableWriter {
 
   row(row) {
     // console.log('ACCEPT ROW', row);
-    this.currentStream.write(JSON.stringify(row) + '\n');
+    this.currentStream.write(JSON.stringify(row, serializeJsTypesReplacer) + '\n');
+    try {
+      if (this.chartProcessor) {
+        this.chartProcessor.addRow(row);
+      }
+    } catch (e) {
+      console.error('Error processing chart row', e);
+      this.chartProcessor = null;
+    }
+
     this.currentRowCount += 1;
 
     if (!this.plannedStats) {
@@ -81,20 +99,52 @@ class QueryStreamTableWriter {
   }
 
   close(afterClose) {
-    if (this.currentStream) {
-      this.currentStream.end(() => {
-        this.writeCurrentStats(true, true);
-        if (afterClose) afterClose();
-      });
-    }
+    return new Promise(resolve => {
+      if (this.currentStream) {
+        this.currentStream.end(() => {
+          this.writeCurrentStats(true, true);
+          if (afterClose) afterClose();
+          if (this.chartProcessor) {
+            try {
+              this.chartProcessor.finalize();
+              if (this.chartProcessor.charts.length > 0) {
+                process.send({
+                  msgtype: 'charts',
+                  sesid: this.sesid,
+                  jslid: this.jslid,
+                  charts: this.chartProcessor.charts,
+                  resultIndex: this.resultIndex,
+                });
+              }
+            } catch (e) {
+              console.error('Error finalizing chart processor', e);
+              this.chartProcessor = null;
+            }
+          }
+          resolve();
+        });
+      } else {
+        resolve();
+      }
+    });
   }
 }
 
 class StreamHandler {
-  constructor(queryStreamInfoHolder, resolve, startLine, sesid = undefined) {
+  constructor(
+    queryStreamInfoHolder,
+    resolve,
+    startLine,
+    sesid = undefined,
+    limitRows = undefined,
+    frontMatter = undefined
+  ) {
     this.recordset = this.recordset.bind(this);
     this.startLine = startLine;
     this.sesid = sesid;
+    this.frontMatter = frontMatter;
+    this.limitRows = limitRows;
+    this.rowsLimitOverflow = false;
     this.row = this.row.bind(this);
     // this.error = this.error.bind(this);
     this.done = this.done.bind(this);
@@ -106,6 +156,7 @@ class StreamHandler {
     this.plannedStats = false;
     this.queryStreamInfoHolder = queryStreamInfoHolder;
     this.resolve = resolve;
+    this.rowCounter = 0;
     // currentHandlers = [...currentHandlers, this];
   }
 
@@ -117,13 +168,18 @@ class StreamHandler {
   }
 
   recordset(columns) {
+    if (this.rowsLimitOverflow) {
+      return;
+    }
     this.closeCurrentWriter();
     this.currentWriter = new QueryStreamTableWriter(this.sesid);
     this.currentWriter.initializeFromQuery(
       Array.isArray(columns) ? { columns } : columns,
-      this.queryStreamInfoHolder.resultIndex
+      this.queryStreamInfoHolder.resultIndex,
+      this.frontMatter?.[`chart-${this.queryStreamInfoHolder.resultIndex + 1}`]
     );
     this.queryStreamInfoHolder.resultIndex += 1;
+    this.rowCounter = 0;
 
     // this.writeCurrentStats();
 
@@ -134,8 +190,36 @@ class StreamHandler {
     // }, 500);
   }
   row(row) {
-    if (this.currentWriter) this.currentWriter.row(row);
-    else if (row.message) process.send({ msgtype: 'info', info: { message: row.message }, sesid: this.sesid });
+    if (this.rowsLimitOverflow) {
+      return;
+    }
+
+    if (this.limitRows && this.rowCounter >= this.limitRows) {
+      process.send({
+        msgtype: 'info',
+        info: { message: `Rows limit overflow, loaded ${this.rowCounter} rows, canceling query`, severity: 'error' },
+        sesid: this.sesid,
+      });
+      this.rowsLimitOverflow = true;
+
+      this.queryStreamInfoHolder.canceled = true;
+      if (this.currentWriter) {
+        this.currentWriter.close().then(() => {
+          process.exit(0);
+        });
+      } else {
+        process.exit(0);
+      }
+
+      return;
+    }
+
+    if (this.currentWriter) {
+      this.currentWriter.row(row);
+      this.rowCounter += 1;
+    } else if (row.message) {
+      process.send({ msgtype: 'info', info: { message: row.message }, sesid: this.sesid });
+    }
     // this.onRow(this.jslid);
   }
   // error(error) {
@@ -160,10 +244,25 @@ class StreamHandler {
   }
 }
 
-function handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, sesid = undefined) {
+function handleQueryStream(
+  dbhan,
+  driver,
+  queryStreamInfoHolder,
+  sqlItem,
+  sesid = undefined,
+  limitRows = undefined,
+  frontMatter = undefined
+) {
   return new Promise((resolve, reject) => {
     const start = sqlItem.trimStart || sqlItem.start;
-    const handler = new StreamHandler(queryStreamInfoHolder, resolve, start && start.line, sesid);
+    const handler = new StreamHandler(
+      queryStreamInfoHolder,
+      resolve,
+      start && start.line,
+      sesid,
+      limitRows,
+      frontMatter
+    );
     driver.stream(dbhan, sqlItem.text, handler);
   });
 }

packages/api/src/utility/hardwareFingerprint.js

@@ -87,4 +87,5 @@ module.exports = {
   getHardwareFingerprint,
   getHardwareFingerprintHash,
   getPublicHardwareFingerprint,
+  getPublicIpInfo,
 };