SYNC: Merge branch 'feature/audit-logs'

package.json

@@ -43,7 +43,7 @@
     "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
     "build:plugins:backend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:backend",
     "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch",
-    "storage-json": "dbmodel model-to-json storage-db packages/api/src/storageModel.js --commonjs",
+    "storage-json": "node packages/dbmodel/bin/dbmodel.js model-to-json storage-db packages/api/src/storageModel.js --commonjs",
     "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist",
     "build:app:local": "yarn plugins:copydist && cd app && yarn build:local",
     "start:app:local": "cd app && yarn start:local",

packages/api/src/auth/authProvider.js

@@ -11,7 +11,7 @@ const logger = getLogger('authProvider');
 class AuthProviderBase {
   amoid = 'none';
 
-  async login(login, password, options = undefined) {
+  async login(login, password, options = undefined, req = undefined) {
     return {
       accessToken: jwt.sign(
         {
@@ -23,7 +23,7 @@ class AuthProviderBase {
     };
   }
 
-  oauthToken(params) {
+  oauthToken(params, req) {
     return {};
   }
 

packages/api/src/controllers/auth.js

@@ -20,6 +20,7 @@ const {
   readCloudTestTokenHolder,
 } = require('../utility/cloudIntf');
 const socket = require('../utility/socket');
+const { sendToAuditLog } = require('../utility/auditlog');
 
 const logger = getLogger('auth');
 
@@ -92,12 +93,12 @@ function authMiddleware(req, res, next) {
 
 module.exports = {
   oauthToken_meta: true,
-  async oauthToken(params) {
+  async oauthToken(params, req) {
     const { amoid } = params;
-    return getAuthProviderById(amoid).oauthToken(params);
+    return getAuthProviderById(amoid).oauthToken(params, req);
   },
   login_meta: true,
-  async login(params) {
+  async login(params, req) {
     const { amoid, login, password, isAdminPage } = params;
 
     if (isAdminPage) {
@@ -107,6 +108,15 @@ module.exports = {
         adminPassword = decryptPasswordString(adminConfig?.adminPassword);
       }
       if (adminPassword && adminPassword == password) {
+        sendToAuditLog(req, {
+          category: 'auth',
+          component: 'AuthController',
+          action: 'login',
+          event: 'login.admin',
+          severity: 'info',
+          message: 'Administration login successful',
+        });
+
         return {
           accessToken: jwt.sign(
             {
@@ -122,10 +132,19 @@ module.exports = {
         };
       }
 
+      sendToAuditLog(req, {
+        category: 'auth',
+        component: 'AuthController',
+        action: 'loginFail',
+        event: 'login.adminFailed',
+        severity: 'warn',
+        message: 'Administraton login failed',
+      });
+
       return { error: 'Login failed' };
     }
 
-    return getAuthProviderById(amoid).login(login, password);
+    return getAuthProviderById(amoid).login(login, password, undefined, req);
   },
 
   getProviders_meta: true,

packages/api/src/controllers/config.js

@@ -29,6 +29,7 @@ const {
 } = require('../utility/crypting');
 
 const lock = new AsyncLock();
+let cachedSettingsValue = null;
 
 module.exports = {
   // settingsValue: {},
@@ -145,6 +146,13 @@ module.exports = {
     return res;
   },
 
+  async getCachedSettings() {
+    if (!cachedSettingsValue) {
+      cachedSettingsValue = await this.loadSettings();
+    }
+    return cachedSettingsValue;
+  },
+
   deleteSettings_meta: true,
   async deleteSettings() {
     await fs.unlink(path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'));
@@ -258,6 +266,7 @@ module.exports = {
   updateSettings_meta: true,
   async updateSettings(values, req) {
     if (!hasPermission(`settings/change`, req)) return false;
+    cachedSettingsValue = null;
 
     const res = await lock.acquire('settings', async () => {
       const currentValue = await this.loadSettings();
@@ -304,7 +313,7 @@ module.exports = {
       const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
       return resp.data;
     } catch (err) {
-      return ''
+      return '';
     }
   },
 

packages/api/src/controllers/connections.js

@@ -536,14 +536,14 @@ module.exports = {
   },
 
   dbloginAuthToken_meta: true,
-  async dbloginAuthToken({ amoid, code, conid, redirectUri, sid }) {
+  async dbloginAuthToken({ amoid, code, conid, redirectUri, sid }, req) {
     try {
       const connection = await this.getCore({ conid });
       const driver = requireEngineDriver(connection);
       const accessToken = await driver.getAuthTokenFromCode(connection, { code, redirectUri, sid });
       const volatile = await this.saveVolatile({ conid, accessToken });
       const authProvider = getAuthProviderById(amoid);
-      const resp = await authProvider.login(null, null, { conid: volatile._id });
+      const resp = await authProvider.login(null, null, { conid: volatile._id }, req);
       return resp;
     } catch (err) {
       logger.error(extractErrorLogData(err), 'Error getting DB token');
@@ -552,18 +552,18 @@ module.exports = {
   },
 
   dbloginAuth_meta: true,
-  async dbloginAuth({ amoid, conid, user, password }) {
+  async dbloginAuth({ amoid, conid, user, password }, req) {
     if (user || password) {
       const saveResp = await this.saveVolatile({ conid, user, password, test: true });
       if (saveResp.msgtype == 'connected') {
-        const loginResp = await getAuthProviderById(amoid).login(user, password, { conid: saveResp._id });
+        const loginResp = await getAuthProviderById(amoid).login(user, password, { conid: saveResp._id }, req);
         return loginResp;
       }
       return saveResp;
     }
 
     // user and password is stored in connection, volatile connection is not needed
-    const loginResp = await getAuthProviderById(amoid).login(null, null, { conid });
+    const loginResp = await getAuthProviderById(amoid).login(null, null, { conid }, req);
     return loginResp;
   },
 

packages/api/src/controllers/databaseConnections.js

@@ -41,6 +41,7 @@ const { decryptConnection } = require('../utility/crypting');
 const { getSshTunnel } = require('../utility/sshTunnel');
 const sessions = require('./sessions');
 const jsldata = require('./jsldata');
+const { sendToAuditLog } = require('../utility/auditlog');
 
 const logger = getLogger('databaseConnections');
 
@@ -83,8 +84,11 @@ module.exports = {
     }
   },
   handle_response(conid, database, { msgid, ...response }) {
-    const [resolve, reject] = this.requests[msgid];
+    const [resolve, reject, additionalData] = this.requests[msgid];
     resolve(response);
+    if (additionalData?.auditLogger) {
+      additionalData?.auditLogger(response);
+    }
     delete this.requests[msgid];
   },
   handle_status(conid, database, { status }) {
@@ -215,10 +219,10 @@ module.exports = {
   },
 
   /** @param {import('dbgate-types').OpenedDatabaseConnection} conn */
-  sendRequest(conn, message) {
+  sendRequest(conn, message, additionalData = {}) {
     const msgid = crypto.randomUUID();
     const promise = new Promise((resolve, reject) => {
-      this.requests[msgid] = [resolve, reject];
+      this.requests[msgid] = [resolve, reject, additionalData];
       try {
         conn.subprocess.send({ msgid, ...message });
       } catch (err) {
@@ -242,10 +246,35 @@ module.exports = {
   },
 
   sqlSelect_meta: true,
-  async sqlSelect({ conid, database, select }, req) {
+  async sqlSelect({ conid, database, select, auditLogSessionGroup }, req) {
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid, database);
-    const res = await this.sendRequest(opened, { msgtype: 'sqlSelect', select });
+    const res = await this.sendRequest(
+      opened,
+      { msgtype: 'sqlSelect', select },
+      {
+        auditLogger:
+          auditLogSessionGroup && select?.from?.name?.pureName
+            ? response => {
+                sendToAuditLog(req, {
+                  category: 'dbop',
+                  component: 'DatabaseConnectionsController',
+                  event: 'sql.select',
+                  action: 'select',
+                  severity: 'info',
+                  conid,
+                  database,
+                  schemaName: select?.from?.name?.schemaName,
+                  pureName: select?.from?.name?.pureName,
+                  sumint1: response?.rows?.length,
+                  sessionParam: `${select?.from?.name?.schemaName || '0'}::${select?.from?.name?.pureName}`,
+                  sessionGroup: auditLogSessionGroup,
+                  message: `Loaded table data from ${select?.from?.name?.pureName}`,
+                });
+              }
+            : null,
+      }
+    );
     return res;
   },
 
@@ -492,6 +521,20 @@ module.exports = {
     }
 
     const opened = await this.ensureOpened(conid, database);
+
+    sendToAuditLog(req, {
+      category: 'dbop',
+      component: 'DatabaseConnectionsController',
+      action: 'structure',
+      event: 'dbStructure.get',
+      severity: 'info',
+      conid,
+      database,
+      sessionParam: `${conid}::${database}`,
+      sessionGroup: 'getStructure',
+      message: `Loaded database structure for ${database}`
+    });
+
     return opened.structure;
     // const existing = this.opened.find((x) => x.conid == conid && x.database == database);
     // if (existing) return existing.status;

packages/api/src/controllers/runners.js

@@ -20,6 +20,7 @@ const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
 const platformInfo = require('../utility/platformInfo');
 const { checkSecureDirectories, checkSecureDirectoriesInScript } = require('../utility/security');
+const { sendToAuditLog, logJsonRunnerScript } = require('../utility/auditlog');
 const logger = getLogger('runners');
 
 function extractPlugins(script) {
@@ -270,7 +271,7 @@ module.exports = {
   },
 
   start_meta: true,
-  async start({ script }) {
+  async start({ script }, req) {
     const runid = crypto.randomUUID();
 
     if (script.type == 'json') {
@@ -280,14 +281,36 @@ module.exports = {
         }
       }
 
+      logJsonRunnerScript(req, script);
+
       const js = await jsonScriptToJavascript(script);
       return this.startCore(runid, scriptTemplate(js, false));
     }
 
     if (!platformInfo.allowShellScripting) {
+      sendToAuditLog(req, {
+        category: 'shell',
+        component: 'RunnersController',
+        event: 'script.runFailed',
+        action: 'script',
+        severity: 'warn',
+        detail: script,
+        message: 'Scripts are not allowed',
+      });
+
       return { errorMessage: 'Shell scripting is not allowed' };
     }
 
+    sendToAuditLog(req, {
+      category: 'shell',
+      component: 'RunnersController',
+      event: 'script.run.shell',
+      action: 'script',
+      severity: 'info',
+      detail: script,
+      message: 'Running JS script',
+    });
+
     return this.startCore(runid, scriptTemplate(script, false));
   },
 

packages/api/src/controllers/serverConnections.js

@@ -12,6 +12,7 @@ const { testConnectionPermission } = require('../utility/hasPermission');
 const { MissingCredentialsError } = require('../utility/exceptions');
 const pipeForkLogs = require('../utility/pipeForkLogs');
 const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const { sendToAuditLog } = require('../utility/auditlog');
 
 const logger = getLogger('serverConnection');
 
@@ -145,6 +146,17 @@ module.exports = {
     if (conid == '__model') return [];
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
+    sendToAuditLog(req, {
+      category: 'serverop',
+      component: 'ServerConnectionsController',
+      action: 'listDatabases',
+      event: 'databases.list',
+      severity: 'info',
+      conid,
+      sessionParam: `${conid}`,
+      sessionGroup: 'listDatabases',
+      message: `Loaded databases for connection`,
+    });
     return opened?.databases ?? [];
   },
 

packages/api/src/controllers/sessions.js

@@ -11,6 +11,7 @@ const { appdir } = require('../utility/directories');
 const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const pipeForkLogs = require('../utility/pipeForkLogs');
 const config = require('./config');
+const { sendToAuditLog } = require('../utility/auditlog');
 
 const logger = getLogger('sessions');
 
@@ -146,12 +147,24 @@ module.exports = {
   },
 
   executeQuery_meta: true,
-  async executeQuery({ sesid, sql, autoCommit, autoDetectCharts, limitRows, frontMatter }) {
+  async executeQuery({ sesid, sql, autoCommit, autoDetectCharts, limitRows, frontMatter }, req) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
       throw new Error('Invalid session');
     }
 
+    sendToAuditLog(req, {
+      category: 'dbop',
+      component: 'SessionController',
+      action: 'executeQuery',
+      event: 'query.execute',
+      severity: 'info',
+      detail: sql,
+      conid: session.conid,
+      database: session.database,
+      message: 'Executing query',
+    });
+
     logger.info({ sesid, sql }, 'Processing query');
     this.dispatchMessage(sesid, 'Query execution started');
     session.subprocess.send({

packages/api/src/storageModel.js

@@ -1,5 +1,192 @@
 module.exports = {
   "tables": [
+    {
+      "pureName": "audit_log",
+      "columns": [
+        {
+          "pureName": "audit_log",
+          "columnName": "id",
+          "dataType": "int",
+          "autoIncrement": true,
+          "notNull": true
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "created",
+          "dataType": "bigint",
+          "notNull": true
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "modified",
+          "dataType": "bigint",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "user_id",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "user_login",
+          "dataType": "varchar(250)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "category",
+          "dataType": "varchar(50)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "component",
+          "dataType": "varchar(50)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "action",
+          "dataType": "varchar(50)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "severity",
+          "dataType": "varchar(50)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "event",
+          "dataType": "varchar(100)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "message",
+          "dataType": "varchar(250)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "detail",
+          "dataType": "varchar(1000)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "detail_full_length",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "session_id",
+          "dataType": "varchar(200)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "session_group",
+          "dataType": "varchar(50)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "session_param",
+          "dataType": "varchar(200)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "conid",
+          "dataType": "varchar(100)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "connection_data",
+          "dataType": "varchar(1000)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "database",
+          "dataType": "varchar(200)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "schema_name",
+          "dataType": "varchar(100)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "pure_name",
+          "dataType": "varchar(100)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "sumint_1",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "sumint_2",
+          "dataType": "int",
+          "notNull": false
+        }
+      ],
+      "foreignKeys": [
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_audit_log_user_id",
+          "pureName": "audit_log",
+          "refTableName": "users",
+          "deleteAction": "SET NULL",
+          "columns": [
+            {
+              "columnName": "user_id",
+              "refColumnName": "id"
+            }
+          ]
+        }
+      ],
+      "indexes": [
+        {
+          "constraintName": "idx_audit_log_session",
+          "pureName": "audit_log",
+          "constraintType": "index",
+          "columns": [
+            {
+              "columnName": "session_group"
+            },
+            {
+              "columnName": "session_id"
+            },
+            {
+              "columnName": "session_param"
+            }
+          ]
+        }
+      ],
+      "primaryKey": {
+        "pureName": "audit_log",
+        "constraintType": "primaryKey",
+        "constraintName": "PK_audit_log",
+        "columns": [
+          {
+            "columnName": "id"
+          }
+        ]
+      }
+    },
     {
       "pureName": "auth_methods",
       "columns": [
@@ -50,6 +237,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "auth_methods",
         "constraintType": "primaryKey",
+        "constraintName": "PK_auth_methods",
         "columns": [
           {
             "columnName": "id"
@@ -103,6 +291,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_auth_methods_config_auth_method_id",
           "pureName": "auth_methods_config",
           "refTableName": "auth_methods",
           "deleteAction": "CASCADE",
@@ -114,9 +303,25 @@ module.exports = {
           ]
         }
       ],
+      "uniques": [
+        {
+          "constraintName": "UQ_auth_methods_config_auth_method_id_key",
+          "pureName": "auth_methods_config",
+          "constraintType": "unique",
+          "columns": [
+            {
+              "columnName": "auth_method_id"
+            },
+            {
+              "columnName": "key"
+            }
+          ]
+        }
+      ],
       "primaryKey": {
         "pureName": "auth_methods_config",
         "constraintType": "primaryKey",
+        "constraintName": "PK_auth_methods_config",
         "columns": [
           {
             "columnName": "id"
@@ -154,9 +359,25 @@ module.exports = {
         }
       ],
       "foreignKeys": [],
+      "uniques": [
+        {
+          "constraintName": "UQ_config_group_key",
+          "pureName": "config",
+          "constraintType": "unique",
+          "columns": [
+            {
+              "columnName": "group"
+            },
+            {
+              "columnName": "key"
+            }
+          ]
+        }
+      ],
       "primaryKey": {
         "pureName": "config",
         "constraintType": "primaryKey",
+        "constraintName": "PK_config",
         "columns": [
           {
             "columnName": "id"
@@ -449,6 +670,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "connections",
         "constraintType": "primaryKey",
+        "constraintName": "PK_connections",
         "columns": [
           {
             "columnName": "id"
@@ -477,6 +699,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "roles",
         "constraintType": "primaryKey",
+        "constraintName": "PK_roles",
         "columns": [
           {
             "columnName": "id"
@@ -524,6 +747,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_role_connections_role_id",
           "pureName": "role_connections",
           "refTableName": "roles",
           "deleteAction": "CASCADE",
@@ -536,6 +760,7 @@ module.exports = {
         },
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_role_connections_connection_id",
           "pureName": "role_connections",
           "refTableName": "connections",
           "deleteAction": "CASCADE",
@@ -550,6 +775,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "role_connections",
         "constraintType": "primaryKey",
+        "constraintName": "PK_role_connections",
         "columns": [
           {
             "columnName": "id"
@@ -583,6 +809,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_role_permissions_role_id",
           "pureName": "role_permissions",
           "refTableName": "roles",
           "deleteAction": "CASCADE",
@@ -597,6 +824,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "role_permissions",
         "constraintType": "primaryKey",
+        "constraintName": "PK_role_permissions",
         "columns": [
           {
             "columnName": "id"
@@ -637,6 +865,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "users",
         "constraintType": "primaryKey",
+        "constraintName": "PK_users",
         "columns": [
           {
             "columnName": "id"
@@ -670,6 +899,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_user_connections_user_id",
           "pureName": "user_connections",
           "refTableName": "users",
           "deleteAction": "CASCADE",
@@ -682,6 +912,7 @@ module.exports = {
         },
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_user_connections_connection_id",
           "pureName": "user_connections",
           "refTableName": "connections",
           "deleteAction": "CASCADE",
@@ -696,6 +927,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "user_connections",
         "constraintType": "primaryKey",
+        "constraintName": "PK_user_connections",
         "columns": [
           {
             "columnName": "id"
@@ -729,6 +961,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_user_permissions_user_id",
           "pureName": "user_permissions",
           "refTableName": "users",
           "deleteAction": "CASCADE",
@@ -743,6 +976,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "user_permissions",
         "constraintType": "primaryKey",
+        "constraintName": "PK_user_permissions",
         "columns": [
           {
             "columnName": "id"
@@ -776,6 +1010,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_user_roles_user_id",
           "pureName": "user_roles",
           "refTableName": "users",
           "deleteAction": "CASCADE",
@@ -788,6 +1023,7 @@ module.exports = {
         },
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_user_roles_role_id",
           "pureName": "user_roles",
           "refTableName": "roles",
           "deleteAction": "CASCADE",
@@ -802,6 +1038,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "user_roles",
         "constraintType": "primaryKey",
+        "constraintName": "PK_user_roles",
         "columns": [
           {
             "columnName": "id"
@@ -815,5 +1052,6 @@ module.exports = {
   "matviews": [],
   "functions": [],
   "procedures": [],
-  "triggers": []
+  "triggers": [],
+  "schedulerEvents": []
 };

packages/api/src/utility/auditlog.js

@@ -1,8 +1,251 @@
-// only in DbGate Premium
+// *** This file is part of DbGate Premium ***
 
-async function sendToAuditLog(req, props) {}
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
-async function logJsonRunnerScript(req, script) {}
+const { storageSqlCommandFmt, storageSelectFmt } = require('../controllers/storageDb');
+const logger = getLogger('auditLog');
+const _ = require('lodash');
 
+let auditLogQueue = [];
+let isProcessing = false;
+let isPlanned = false;
+
+function nullableSum(a, b) {
+  const res = (a || 0) + (b || 0);
+  return res == 0 ? null : res;
+}
+
+async function processAuditLogQueue() {
+  do {
+    isProcessing = true;
+    const elements = [...auditLogQueue];
+    auditLogQueue = [];
+
+    while (elements.length > 0) {
+      const element = elements.shift();
+      if (!element) continue;
+      if (element.sessionId && element.sessionGroup && element.sessionParam) {
+        const existingRows = await storageSelectFmt(
+          '^select ~id, ~sumint_1, ~sumint_2  from ~audit_log where ~session_id = %v and ~session_group = %v and ~session_param = %v',
+          element.sessionId,
+          element.sessionGroup,
+          element.sessionParam
+        );
+        if (existingRows && existingRows.length > 0) {
+          const existing = existingRows[0];
+          await storageSqlCommandFmt(
+            '^update ~audit_log set ~sumint_1 = %v, ~sumint_2 = %v, ~modified = %v where ~id = %v',
+            nullableSum(element.sumint1, existing.sumint_1),
+            nullableSum(element.sumint2, existing.sumint_2),
+            element.created,
+            existing.id
+          );
+          // only update existing session
+          continue;
+        }
+      }
+      try {
+        let connectionData = null;
+        if (element.conid) {
+          const connections = await storageSelectFmt('^select * from ~connections where ~conid = %v', element.conid);
+          if (connections[0])
+            connectionData = _.pick(connections[0], [
+              'displayName',
+              'engine',
+              'displayName',
+              'databaseUrl',
+              'singleDatabase',
+              'server',
+              'databaseFile',
+              'useSshTunnel',
+              'sshHost',
+              'defaultDatabase',
+            ]);
+        }
+
+        const detailText = _.isPlainObject(element.detail) ? JSON.stringify(element.detail) : element.detail || null;
+        const connectionDataText = connectionData ? JSON.stringify(connectionData) : null;
+        await storageSqlCommandFmt(
+          `^insert ^into ~audit_log (
+            ~user_id, ~user_login, ~created, ~category, ~component, ~event, ~detail, ~detail_full_length, ~action, ~severity, 
+            ~conid, ~database, ~schema_name, ~pure_name, ~sumint_1, ~sumint_2, ~session_id, ~session_group, ~session_param, ~connection_data, ~message) 
+            values (%v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v)`,
+          element.userId || null,
+          element.login || null,
+          element.created,
+          element.category || null,
+          element.component || null,
+          element.event || null,
+          detailText?.slice(0, 1000) || null,
+          detailText?.length || null,
+          element.action || null,
+          element.severity || 'info',
+          element.conid || null,
+          element.database || null,
+          element.schemaName || null,
+          element.pureName || null,
+          element.sumint1 || null,
+          element.sumint2 || null,
+          element.sessionId || null,
+          element.sessionGroup || null,
+          element.sessionParam || null,
+          connectionDataText?.slice(0, 1000) || null,
+          element.message || null
+        );
+      } catch (err) {
+        logger.error(extractErrorLogData(err), 'Error processing audit log entry');
+      }
+    }
+
+    isProcessing = false;
+  } while (auditLogQueue.length > 0);
+  isPlanned = false;
+}
+
+async function sendToAuditLog(
+  req,
+  {
+    category,
+    component,
+    event,
+    detail = null,
+    action,
+    severity = 'info',
+    conid = null,
+    database = null,
+    schemaName = null,
+    pureName = null,
+    sumint1 = null,
+    sumint2 = null,
+    sessionGroup = null,
+    sessionParam = null,
+    message = null,
+  }
+) {
+  if (!process.env.STORAGE_DATABASE) {
+    return;
+  }
+  const config = require('../controllers/config');
+  const settings = await config.getCachedSettings();
+  if (settings?.['storage.useAuditLog'] != 1) {
+    return;
+  }
+
+  const { login, userId } = req?.user || {};
+  const sessionId = req?.headers?.['x-api-session-id'];
+
+  auditLogQueue.push({
+    userId,
+    login,
+    created: new Date().getTime(),
+    category,
+    component,
+    event,
+    detail,
+    action,
+    severity,
+    conid,
+    database,
+    schemaName,
+    pureName,
+    sumint1,
+    sumint2,
+    sessionId,
+    sessionGroup,
+    sessionParam,
+    message,
+  });
+  if (!isProcessing && !isPlanned) {
+    setTimeout(() => {
+      isPlanned = true;
+      processAuditLogQueue();
+    }, 0);
+  }
+}
+
+function maskPasswords(script) {
+  return _.cloneDeepWith(script, (value, key) => {
+    if (_.isString(key) && key.toLowerCase().includes('password')) {
+      return '****';
+    }
+  });
+}
+
+function analyseJsonRunnerScript(script) {
+  const [assignSource, assignTarget, copyStream] = _.isArray(script?.commands) ? script?.commands : [];
+  if (assignSource?.type != 'assign') {
+    return null;
+  }
+  if (assignTarget?.type != 'assign') {
+    return null;
+  }
+  if (copyStream?.type != 'copyStream') {
+    return null;
+  }
+
+  if (assignTarget?.functionName == 'tableWriter') {
+    const pureName = assignTarget?.props?.pureName;
+    const schemaName = assignTarget?.props?.schemaName;
+    const connection = assignTarget?.props?.connection;
+    if (pureName && connection) {
+      return {
+        category: 'import',
+        component: 'RunnersController',
+        event: 'import.data',
+        action: 'import',
+        severity: 'info',
+        message: 'Importing data',
+        pureName: pureName,
+        conid: connection?.conid,
+        database: connection?.database,
+        schemaName: schemaName,
+        detail: maskPasswords(script),
+      };
+    }
+    return null;
+  }
+
+  if (assignSource?.functionName == 'tableReader') {
+    const pureName = assignSource?.props?.pureName;
+    const schemaName = assignSource?.props?.schemaName;
+    const connection = assignSource?.props?.connection;
+    if (pureName && connection) {
+      return {
+        category: 'export',
+        component: 'RunnersController',
+        event: 'export.data',
+        action: 'export',
+        severity: 'info',
+        message: 'Exporting data',
+        pureName: pureName,
+        conid: connection?.conid,
+        database: connection?.database,
+        schemaName: schemaName,
+        detail: maskPasswords(script),
+      };
+    }
+    return null;
+  }
+
+  return null;
+}
+
+function logJsonRunnerScript(req, script) {
+  const analysed = analyseJsonRunnerScript(script);
+
+  if (analysed) {
+    sendToAuditLog(req, analysed);
+  } else {
+    sendToAuditLog(req, {
+      category: 'shell',
+      component: 'RunnersController',
+      event: 'script.run.json',
+      action: 'script',
+      severity: 'info',
+      detail: maskPasswords(script),
+      message: 'Running JSON script',
+    });
+  }
+}
 
 module.exports = {
   sendToAuditLog,

packages/datalib/src/PerspectiveDataLoader.ts

@@ -106,6 +106,7 @@ export class PerspectiveDataLoader {
       conid: props.databaseConfig.conid,
       database: props.databaseConfig.database,
       select,
+      auditLogSessionGroup: 'perspective',
     });
 
     if (response.errorMessage) return response;
@@ -227,6 +228,7 @@ export class PerspectiveDataLoader {
       conid: props.databaseConfig.conid,
       database: props.databaseConfig.database,
       select,
+      auditLogSessionGroup: 'perspective',
     });
 
     if (response.errorMessage) return response;
@@ -330,6 +332,7 @@ export class PerspectiveDataLoader {
       conid: props.databaseConfig.conid,
       database: props.databaseConfig.database,
       select,
+      auditLogSessionGroup: 'perspective',
     });
 
     if (response.errorMessage) return response;

packages/tools/src/yamlModelConv.ts

@@ -31,6 +31,11 @@ export interface IndexInfoYaml {
   included?: string[];
 }
 
+export interface UniqueInfoYaml {
+  name: string;
+  columns: string[];
+}
+
 export interface TableInfoYaml {
   name: string;
   // schema?: string;
@@ -38,6 +43,7 @@ export interface TableInfoYaml {
   primaryKey?: string[];
   sortingKey?: string[];
   indexes?: IndexInfoYaml[];
+  uniques?: UniqueInfoYaml[];
 
   insertKey?: string[];
   insertOnly?: string[];
@@ -121,6 +127,12 @@ export function tableInfoToYaml(table: TableInfo): TableInfoYaml {
       return idx;
     });
   }
+  if (tableCopy.uniques?.length > 0) {
+    res.uniques = tableCopy.uniques.map(unique => ({
+      name: unique.constraintName,
+      columns: unique.columns.map(x => x.columnName),
+    }));
+  }
   return res;
 }
 
@@ -165,6 +177,12 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
         ...(index.included || []).map(columnName => ({ columnName, isIncludedColumn: true })),
       ],
     })),
+    uniques: table.uniques?.map(unique => ({
+      constraintName: unique.name,
+      pureName: table.name,
+      constraintType: 'unique',
+      columns: unique.columns.map(columnName => ({ columnName })),
+    })),
   };
   if (table.primaryKey) {
     res.primaryKey = {

packages/web/package.json

@@ -64,6 +64,7 @@
     "chartjs-plugin-zoom": "^1.2.0",
     "date-fns": "^4.1.0",
     "debug": "^4.3.4",
+    "flatpickr": "^4.6.13",
     "fuzzy": "^0.1.3",
     "highlight.js": "^11.11.1",
     "interval-operations": "^1.0.7",

packages/web/public/global.css

@@ -117,7 +117,7 @@ body {
   max-width: 16.6666%;
 }
 
-.largeFormMarker input[type='text'], .largeFormMarker input[type='number'], .largeFormMarker input[type='password'], .largeFormMarker textarea {
+.largeFormMarker input[type='text'], .largeFormMarker input[type='number'], .largeFormMarker input[type='password'], .largeFormMarker textarea  {
   width: 100%;
   padding: 10px 10px;
   font-size: 14px;
@@ -126,6 +126,13 @@ body {
   border: 1px solid var(--theme-border);
 }
 
+.input1 {
+  padding: 5px 5px;
+  font-size: 14px;
+  box-sizing: border-box;
+  border-radius: 4px;
+  border: 1px solid var(--theme-border);
+}
 
 .largeFormMarker select {
   width: 100%;

packages/web/src/App.svelte

@@ -60,10 +60,9 @@
         installNewCloudTokenListener();
         initializeAppUpdates();
         installCloudListeners();
+        refreshPublicCloudFiles();
       }
 
-      refreshPublicCloudFiles();
-
       loadedApi = loadedApiValue;
 
       if (!loadedApi) {

packages/web/src/datagrid/SqlDataGridCore.svelte

@@ -18,41 +18,6 @@
     testEnabled: () => getCurrentEditor() != null && hasPermission('dbops/export'),
     onClick: () => getCurrentEditor().exportGrid(),
   });
-
-  async function loadDataPage(props, offset, limit) {
-    const { display, conid, database } = props;
-
-    const select = display.getPageQuery(offset, limit);
-
-    const response = await apiCall('database-connections/sql-select', {
-      conid,
-      database,
-      select,
-    });
-
-    if (response.errorMessage) return response;
-    return response.rows;
-  }
-
-  function dataPageAvailable(props) {
-    const { display } = props;
-    const select = display.getPageQuery(0, 1);
-    return !!select;
-  }
-
-  async function loadRowCount(props) {
-    const { display, conid, database } = props;
-
-    const select = display.getCountQuery();
-
-    const response = await apiCall('database-connections/sql-select', {
-      conid,
-      database,
-      select,
-    });
-
-    return parseInt(response.rows[0].count);
-  }
 </script>
 
 <script lang="ts">
@@ -217,6 +182,42 @@
   function handleSetLoadedRows(rows) {
     loadedRows = rows;
   }
+
+  async function loadDataPage(props, offset, limit) {
+    const { display, conid, database } = props;
+
+    const select = display.getPageQuery(offset, limit);
+
+    const response = await apiCall('database-connections/sql-select', {
+      conid,
+      database,
+      select,
+      auditLogSessionGroup: 'data-grid',
+    });
+
+    if (response.errorMessage) return response;
+    return response.rows;
+  }
+
+  function dataPageAvailable(props) {
+    const { display } = props;
+    const select = display.getPageQuery(0, 1);
+    return !!select;
+  }
+
+  async function loadRowCount(props) {
+    const { display, conid, database } = props;
+
+    const select = display.getCountQuery();
+
+    const response = await apiCall('database-connections/sql-select', {
+      conid,
+      database,
+      select,
+    });
+
+    return parseInt(response.rows[0].count);
+  }
 </script>
 
 <LoadingDataGridCore

packages/web/src/elements/Chip.svelte

@@ -0,0 +1,33 @@
+<script lang="ts">
+  import FontIcon from "../icons/FontIcon.svelte";
+
+  export let onClose;
+</script>
+
+<div class="chip">
+  <slot />
+  {#if onClose}
+    <span class="close" on:click={onClose}><FontIcon icon="icon close" /></span>
+  {/if}
+</div>
+
+<style>
+  .chip {
+    display: inline-block;
+    padding: 0.25em 0.5em;
+    border-radius: 1em;
+    background-color: var(--theme-bg-2);
+    color: var(--theme-text-1);
+    font-size: 0.875em;
+    cursor: pointer;
+    margin: 2px;
+  }
+  .chip .close {
+    margin-left: 0.2em;
+    color: var(--theme-text-2);
+    cursor: pointer;
+  }
+  .chip .close:hover {
+    color: var(--theme-font-hover);
+  }
+</style>

packages/web/src/elements/DateRangeSelector.svelte

@@ -0,0 +1,32 @@
+<script lang="ts">
+  import flatpickr from 'flatpickr';
+  import 'flatpickr/dist/flatpickr.min.css';
+  import 'flatpickr/dist/themes/dark.css';
+
+  import { onMount } from 'svelte';
+
+  let flatpickrInstance;
+  let inputElement;
+
+  export let defaultValue = ['today', 'today'];
+  export let onChange: (value) => void;
+
+  onMount(() => {
+    flatpickrInstance = flatpickr(inputElement, {
+      mode: 'range',
+      maxDate: 'today',
+      dateFormat: 'Y-m-d',
+      defaultDate: defaultValue,
+
+      onClose: selectedDates => {
+        console.log('Selected dates:', selectedDates);
+        if (selectedDates.length === 1) {
+          flatpickrInstance.setDate([selectedDates[0], selectedDates[0]], true);
+        }
+        onChange(selectedDates.length == 1 ? [selectedDates[0], selectedDates[0]] : selectedDates);
+      },
+    });
+  });
+</script>
+
+<input bind:this={inputElement} type="text" class="input1" />

packages/web/src/formview/SqlFormView.svelte

@@ -8,6 +8,7 @@
       conid,
       database,
       select,
+      auditLogSessionGroup: 'data-form',
     });
 
     if (response.errorMessage) return response;

packages/web/src/icons/FontIcon.svelte

@@ -143,6 +143,7 @@
     'icon markdown': 'mdi mdi-application',
     'icon preview': 'mdi mdi-file-find',
     'icon eye': 'mdi mdi-eye',
+    'icon auditlog': 'mdi mdi-eye',
     'icon check-all': 'mdi mdi-check-all',
     'icon checkbox-blank': 'mdi mdi-checkbox-blank-outline',
     'icon checkbox-marked': 'mdi mdi-checkbox-marked-outline',
@@ -307,6 +308,7 @@
     'img filter': 'mdi mdi-filter',
     'img group': 'mdi mdi-group',
     'img perspective': 'mdi mdi-eye color-icon-yellow',
+    'img auditlog': 'mdi mdi-eye color-icon-blue',
     'img parent-filter': 'mdi mdi-home-alert color-icon-yellow',
 
     'img folder': 'mdi mdi-folder color-icon-yellow',

packages/web/src/modals/DictionaryLookupModal.svelte

@@ -104,7 +104,8 @@
     const response = await apiCall('database-connections/sql-select', {
       conid,
       database,
-      select
+      select,
+      auditLogSessionGroup: 'lookup',
     });
 
     rows = response.rows;

packages/web/src/utility/api.ts

@@ -185,6 +185,7 @@ export async function apiCall(
       cache: 'no-cache',
       headers: {
         'Content-Type': 'application/json',
+        'x-api-session-id': getApiSessionId(),
         ...resolveApiHeaders(),
       },
       body: JSON.stringify(args, serializeJsTypesReplacer),
@@ -318,6 +319,20 @@ export function refreshPublicCloudFiles(force = false) {
   sessionStorage.setItem('publicCloudFilesLoaded', 'true');
 }
 
+let apiSessionIdValue = null;
+function getApiSessionId() {
+  if (!apiSessionIdValue) {
+    apiSessionIdValue = uuidv1();
+  }
+  return apiSessionIdValue;
+
+  // if (!sessionStorage.getItem('apiSessionId')) {
+  //   const sessionId = uuidv1();
+  //   sessionStorage.setItem('apiSessionId', sessionId);
+  // }
+  // return sessionStorage.getItem('apiSessionId');
+}
+
 function enableApiLog() {
   apiLogging = true;
   console.log('API loggin enabled');

yarn.lock

@@ -5414,6 +5414,11 @@ flat@^5.0.2:
   resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241"
   integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==
 
+flatpickr@^4.6.13:
+  version "4.6.13"
+  resolved "https://registry.yarnpkg.com/flatpickr/-/flatpickr-4.6.13.tgz#8a029548187fd6e0d670908471e43abe9ad18d94"
+  integrity sha512-97PMG/aywoYpB4IvbvUJi0RQi8vearvU0oov1WW3k0WZPBMrTQVqekSX5CjSG/M4Q3i6A/0FKXC7RyAoAUUSPw==
+
 flatted@^2.0.0:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138"