diff --git a/package.json b/package.json index e79efd801..ab479de5d 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend", "build:plugins:backend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:backend", "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch", - "storage-json": "dbmodel model-to-json storage-db packages/api/src/storageModel.js --commonjs", + "storage-json": "node packages/dbmodel/bin/dbmodel.js model-to-json storage-db packages/api/src/storageModel.js --commonjs", "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist", "build:app:local": "yarn plugins:copydist && cd app && yarn build:local", "start:app:local": "cd app && yarn start:local", diff --git a/packages/api/src/auth/authProvider.js b/packages/api/src/auth/authProvider.js index 00fa04d81..075956b28 100644 --- a/packages/api/src/auth/authProvider.js +++ b/packages/api/src/auth/authProvider.js @@ -11,7 +11,7 @@ const logger = getLogger('authProvider'); class AuthProviderBase { amoid = 'none'; - async login(login, password, options = undefined) { + async login(login, password, options = undefined, req = undefined) { return { accessToken: jwt.sign( { @@ -23,7 +23,7 @@ class AuthProviderBase { }; } - oauthToken(params) { + oauthToken(params, req) { return {}; } diff --git a/packages/api/src/controllers/auth.js b/packages/api/src/controllers/auth.js index dd6953f77..2a052c8b8 100644 --- a/packages/api/src/controllers/auth.js +++ b/packages/api/src/controllers/auth.js @@ -20,6 +20,7 @@ const { readCloudTestTokenHolder, } = require('../utility/cloudIntf'); const socket = require('../utility/socket'); +const { sendToAuditLog } = require('../utility/auditlog'); const logger = getLogger('auth'); @@ -92,12 +93,12 @@ function authMiddleware(req, res, next) { module.exports = { oauthToken_meta: true, - async oauthToken(params) { + async oauthToken(params, req) { const { amoid } = params; - return getAuthProviderById(amoid).oauthToken(params); + return getAuthProviderById(amoid).oauthToken(params, req); }, login_meta: true, - async login(params) { + async login(params, req) { const { amoid, login, password, isAdminPage } = params; if (isAdminPage) { @@ -107,6 +108,15 @@ module.exports = { adminPassword = decryptPasswordString(adminConfig?.adminPassword); } if (adminPassword && adminPassword == password) { + sendToAuditLog(req, { + category: 'auth', + component: 'AuthController', + action: 'login', + event: 'login.admin', + severity: 'info', + message: 'Administration login successful', + }); + return { accessToken: jwt.sign( { @@ -122,10 +132,19 @@ module.exports = { }; } + sendToAuditLog(req, { + category: 'auth', + component: 'AuthController', + action: 'loginFail', + event: 'login.adminFailed', + severity: 'warn', + message: 'Administraton login failed', + }); + return { error: 'Login failed' }; } - return getAuthProviderById(amoid).login(login, password); + return getAuthProviderById(amoid).login(login, password, undefined, req); }, getProviders_meta: true, diff --git a/packages/api/src/controllers/config.js b/packages/api/src/controllers/config.js index 1eaeb893e..0ed593129 100644 --- a/packages/api/src/controllers/config.js +++ b/packages/api/src/controllers/config.js @@ -29,6 +29,7 @@ const { } = require('../utility/crypting'); const lock = new AsyncLock(); +let cachedSettingsValue = null; module.exports = { // settingsValue: {}, @@ -145,6 +146,13 @@ module.exports = { return res; }, + async getCachedSettings() { + if (!cachedSettingsValue) { + cachedSettingsValue = await this.loadSettings(); + } + return cachedSettingsValue; + }, + deleteSettings_meta: true, async deleteSettings() { await fs.unlink(path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json')); @@ -258,6 +266,7 @@ module.exports = { updateSettings_meta: true, async updateSettings(values, req) { if (!hasPermission(`settings/change`, req)) return false; + cachedSettingsValue = null; const res = await lock.acquire('settings', async () => { const currentValue = await this.loadSettings(); @@ -304,7 +313,7 @@ module.exports = { const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md'); return resp.data; } catch (err) { - return '' + return ''; } }, diff --git a/packages/api/src/controllers/connections.js b/packages/api/src/controllers/connections.js index f5243f0d0..56ecc25e0 100644 --- a/packages/api/src/controllers/connections.js +++ b/packages/api/src/controllers/connections.js @@ -536,14 +536,14 @@ module.exports = { }, dbloginAuthToken_meta: true, - async dbloginAuthToken({ amoid, code, conid, redirectUri, sid }) { + async dbloginAuthToken({ amoid, code, conid, redirectUri, sid }, req) { try { const connection = await this.getCore({ conid }); const driver = requireEngineDriver(connection); const accessToken = await driver.getAuthTokenFromCode(connection, { code, redirectUri, sid }); const volatile = await this.saveVolatile({ conid, accessToken }); const authProvider = getAuthProviderById(amoid); - const resp = await authProvider.login(null, null, { conid: volatile._id }); + const resp = await authProvider.login(null, null, { conid: volatile._id }, req); return resp; } catch (err) { logger.error(extractErrorLogData(err), 'Error getting DB token'); @@ -552,18 +552,18 @@ module.exports = { }, dbloginAuth_meta: true, - async dbloginAuth({ amoid, conid, user, password }) { + async dbloginAuth({ amoid, conid, user, password }, req) { if (user || password) { const saveResp = await this.saveVolatile({ conid, user, password, test: true }); if (saveResp.msgtype == 'connected') { - const loginResp = await getAuthProviderById(amoid).login(user, password, { conid: saveResp._id }); + const loginResp = await getAuthProviderById(amoid).login(user, password, { conid: saveResp._id }, req); return loginResp; } return saveResp; } // user and password is stored in connection, volatile connection is not needed - const loginResp = await getAuthProviderById(amoid).login(null, null, { conid }); + const loginResp = await getAuthProviderById(amoid).login(null, null, { conid }, req); return loginResp; }, diff --git a/packages/api/src/controllers/databaseConnections.js b/packages/api/src/controllers/databaseConnections.js index 4f50b1085..bfcfcd9ba 100644 --- a/packages/api/src/controllers/databaseConnections.js +++ b/packages/api/src/controllers/databaseConnections.js @@ -41,6 +41,7 @@ const { decryptConnection } = require('../utility/crypting'); const { getSshTunnel } = require('../utility/sshTunnel'); const sessions = require('./sessions'); const jsldata = require('./jsldata'); +const { sendToAuditLog } = require('../utility/auditlog'); const logger = getLogger('databaseConnections'); @@ -83,8 +84,11 @@ module.exports = { } }, handle_response(conid, database, { msgid, ...response }) { - const [resolve, reject] = this.requests[msgid]; + const [resolve, reject, additionalData] = this.requests[msgid]; resolve(response); + if (additionalData?.auditLogger) { + additionalData?.auditLogger(response); + } delete this.requests[msgid]; }, handle_status(conid, database, { status }) { @@ -215,10 +219,10 @@ module.exports = { }, /** @param {import('dbgate-types').OpenedDatabaseConnection} conn */ - sendRequest(conn, message) { + sendRequest(conn, message, additionalData = {}) { const msgid = crypto.randomUUID(); const promise = new Promise((resolve, reject) => { - this.requests[msgid] = [resolve, reject]; + this.requests[msgid] = [resolve, reject, additionalData]; try { conn.subprocess.send({ msgid, ...message }); } catch (err) { @@ -242,10 +246,35 @@ module.exports = { }, sqlSelect_meta: true, - async sqlSelect({ conid, database, select }, req) { + async sqlSelect({ conid, database, select, auditLogSessionGroup }, req) { testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid, database); - const res = await this.sendRequest(opened, { msgtype: 'sqlSelect', select }); + const res = await this.sendRequest( + opened, + { msgtype: 'sqlSelect', select }, + { + auditLogger: + auditLogSessionGroup && select?.from?.name?.pureName + ? response => { + sendToAuditLog(req, { + category: 'dbop', + component: 'DatabaseConnectionsController', + event: 'sql.select', + action: 'select', + severity: 'info', + conid, + database, + schemaName: select?.from?.name?.schemaName, + pureName: select?.from?.name?.pureName, + sumint1: response?.rows?.length, + sessionParam: `${select?.from?.name?.schemaName || '0'}::${select?.from?.name?.pureName}`, + sessionGroup: auditLogSessionGroup, + message: `Loaded table data from ${select?.from?.name?.pureName}`, + }); + } + : null, + } + ); return res; }, @@ -492,6 +521,20 @@ module.exports = { } const opened = await this.ensureOpened(conid, database); + + sendToAuditLog(req, { + category: 'dbop', + component: 'DatabaseConnectionsController', + action: 'structure', + event: 'dbStructure.get', + severity: 'info', + conid, + database, + sessionParam: `${conid}::${database}`, + sessionGroup: 'getStructure', + message: `Loaded database structure for ${database}` + }); + return opened.structure; // const existing = this.opened.find((x) => x.conid == conid && x.database == database); // if (existing) return existing.status; diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js index 3ee287953..3c412ae81 100644 --- a/packages/api/src/controllers/runners.js +++ b/packages/api/src/controllers/runners.js @@ -20,6 +20,7 @@ const { handleProcessCommunication } = require('../utility/processComm'); const processArgs = require('../utility/processArgs'); const platformInfo = require('../utility/platformInfo'); const { checkSecureDirectories, checkSecureDirectoriesInScript } = require('../utility/security'); +const { sendToAuditLog, logJsonRunnerScript } = require('../utility/auditlog'); const logger = getLogger('runners'); function extractPlugins(script) { @@ -270,7 +271,7 @@ module.exports = { }, start_meta: true, - async start({ script }) { + async start({ script }, req) { const runid = crypto.randomUUID(); if (script.type == 'json') { @@ -280,14 +281,36 @@ module.exports = { } } + logJsonRunnerScript(req, script); + const js = await jsonScriptToJavascript(script); return this.startCore(runid, scriptTemplate(js, false)); } if (!platformInfo.allowShellScripting) { + sendToAuditLog(req, { + category: 'shell', + component: 'RunnersController', + event: 'script.runFailed', + action: 'script', + severity: 'warn', + detail: script, + message: 'Scripts are not allowed', + }); + return { errorMessage: 'Shell scripting is not allowed' }; } + sendToAuditLog(req, { + category: 'shell', + component: 'RunnersController', + event: 'script.run.shell', + action: 'script', + severity: 'info', + detail: script, + message: 'Running JS script', + }); + return this.startCore(runid, scriptTemplate(script, false)); }, diff --git a/packages/api/src/controllers/serverConnections.js b/packages/api/src/controllers/serverConnections.js index 68e89e477..a14612432 100644 --- a/packages/api/src/controllers/serverConnections.js +++ b/packages/api/src/controllers/serverConnections.js @@ -12,6 +12,7 @@ const { testConnectionPermission } = require('../utility/hasPermission'); const { MissingCredentialsError } = require('../utility/exceptions'); const pipeForkLogs = require('../utility/pipeForkLogs'); const { getLogger, extractErrorLogData } = require('dbgate-tools'); +const { sendToAuditLog } = require('../utility/auditlog'); const logger = getLogger('serverConnection'); @@ -145,6 +146,17 @@ module.exports = { if (conid == '__model') return []; testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid); + sendToAuditLog(req, { + category: 'serverop', + component: 'ServerConnectionsController', + action: 'listDatabases', + event: 'databases.list', + severity: 'info', + conid, + sessionParam: `${conid}`, + sessionGroup: 'listDatabases', + message: `Loaded databases for connection`, + }); return opened?.databases ?? []; }, diff --git a/packages/api/src/controllers/sessions.js b/packages/api/src/controllers/sessions.js index 3b6c68100..559094004 100644 --- a/packages/api/src/controllers/sessions.js +++ b/packages/api/src/controllers/sessions.js @@ -11,6 +11,7 @@ const { appdir } = require('../utility/directories'); const { getLogger, extractErrorLogData } = require('dbgate-tools'); const pipeForkLogs = require('../utility/pipeForkLogs'); const config = require('./config'); +const { sendToAuditLog } = require('../utility/auditlog'); const logger = getLogger('sessions'); @@ -146,12 +147,24 @@ module.exports = { }, executeQuery_meta: true, - async executeQuery({ sesid, sql, autoCommit, autoDetectCharts, limitRows, frontMatter }) { + async executeQuery({ sesid, sql, autoCommit, autoDetectCharts, limitRows, frontMatter }, req) { const session = this.opened.find(x => x.sesid == sesid); if (!session) { throw new Error('Invalid session'); } + sendToAuditLog(req, { + category: 'dbop', + component: 'SessionController', + action: 'executeQuery', + event: 'query.execute', + severity: 'info', + detail: sql, + conid: session.conid, + database: session.database, + message: 'Executing query', + }); + logger.info({ sesid, sql }, 'Processing query'); this.dispatchMessage(sesid, 'Query execution started'); session.subprocess.send({ diff --git a/packages/api/src/storageModel.js b/packages/api/src/storageModel.js index f8d99b008..08407b002 100644 --- a/packages/api/src/storageModel.js +++ b/packages/api/src/storageModel.js @@ -1,5 +1,192 @@ module.exports = { "tables": [ + { + "pureName": "audit_log", + "columns": [ + { + "pureName": "audit_log", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "audit_log", + "columnName": "created", + "dataType": "bigint", + "notNull": true + }, + { + "pureName": "audit_log", + "columnName": "modified", + "dataType": "bigint", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "user_id", + "dataType": "int", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "user_login", + "dataType": "varchar(250)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "category", + "dataType": "varchar(50)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "component", + "dataType": "varchar(50)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "action", + "dataType": "varchar(50)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "severity", + "dataType": "varchar(50)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "event", + "dataType": "varchar(100)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "message", + "dataType": "varchar(250)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "detail", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "detail_full_length", + "dataType": "int", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "session_id", + "dataType": "varchar(200)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "session_group", + "dataType": "varchar(50)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "session_param", + "dataType": "varchar(200)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "conid", + "dataType": "varchar(100)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "connection_data", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "database", + "dataType": "varchar(200)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "schema_name", + "dataType": "varchar(100)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "pure_name", + "dataType": "varchar(100)", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "sumint_1", + "dataType": "int", + "notNull": false + }, + { + "pureName": "audit_log", + "columnName": "sumint_2", + "dataType": "int", + "notNull": false + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_audit_log_user_id", + "pureName": "audit_log", + "refTableName": "users", + "deleteAction": "SET NULL", + "columns": [ + { + "columnName": "user_id", + "refColumnName": "id" + } + ] + } + ], + "indexes": [ + { + "constraintName": "idx_audit_log_session", + "pureName": "audit_log", + "constraintType": "index", + "columns": [ + { + "columnName": "session_group" + }, + { + "columnName": "session_id" + }, + { + "columnName": "session_param" + } + ] + } + ], + "primaryKey": { + "pureName": "audit_log", + "constraintType": "primaryKey", + "constraintName": "PK_audit_log", + "columns": [ + { + "columnName": "id" + } + ] + } + }, { "pureName": "auth_methods", "columns": [ @@ -50,6 +237,7 @@ module.exports = { "primaryKey": { "pureName": "auth_methods", "constraintType": "primaryKey", + "constraintName": "PK_auth_methods", "columns": [ { "columnName": "id" @@ -103,6 +291,7 @@ module.exports = { "foreignKeys": [ { "constraintType": "foreignKey", + "constraintName": "FK_auth_methods_config_auth_method_id", "pureName": "auth_methods_config", "refTableName": "auth_methods", "deleteAction": "CASCADE", @@ -114,9 +303,25 @@ module.exports = { ] } ], + "uniques": [ + { + "constraintName": "UQ_auth_methods_config_auth_method_id_key", + "pureName": "auth_methods_config", + "constraintType": "unique", + "columns": [ + { + "columnName": "auth_method_id" + }, + { + "columnName": "key" + } + ] + } + ], "primaryKey": { "pureName": "auth_methods_config", "constraintType": "primaryKey", + "constraintName": "PK_auth_methods_config", "columns": [ { "columnName": "id" @@ -154,9 +359,25 @@ module.exports = { } ], "foreignKeys": [], + "uniques": [ + { + "constraintName": "UQ_config_group_key", + "pureName": "config", + "constraintType": "unique", + "columns": [ + { + "columnName": "group" + }, + { + "columnName": "key" + } + ] + } + ], "primaryKey": { "pureName": "config", "constraintType": "primaryKey", + "constraintName": "PK_config", "columns": [ { "columnName": "id" @@ -449,6 +670,7 @@ module.exports = { "primaryKey": { "pureName": "connections", "constraintType": "primaryKey", + "constraintName": "PK_connections", "columns": [ { "columnName": "id" @@ -477,6 +699,7 @@ module.exports = { "primaryKey": { "pureName": "roles", "constraintType": "primaryKey", + "constraintName": "PK_roles", "columns": [ { "columnName": "id" @@ -524,6 +747,7 @@ module.exports = { "foreignKeys": [ { "constraintType": "foreignKey", + "constraintName": "FK_role_connections_role_id", "pureName": "role_connections", "refTableName": "roles", "deleteAction": "CASCADE", @@ -536,6 +760,7 @@ module.exports = { }, { "constraintType": "foreignKey", + "constraintName": "FK_role_connections_connection_id", "pureName": "role_connections", "refTableName": "connections", "deleteAction": "CASCADE", @@ -550,6 +775,7 @@ module.exports = { "primaryKey": { "pureName": "role_connections", "constraintType": "primaryKey", + "constraintName": "PK_role_connections", "columns": [ { "columnName": "id" @@ -583,6 +809,7 @@ module.exports = { "foreignKeys": [ { "constraintType": "foreignKey", + "constraintName": "FK_role_permissions_role_id", "pureName": "role_permissions", "refTableName": "roles", "deleteAction": "CASCADE", @@ -597,6 +824,7 @@ module.exports = { "primaryKey": { "pureName": "role_permissions", "constraintType": "primaryKey", + "constraintName": "PK_role_permissions", "columns": [ { "columnName": "id" @@ -637,6 +865,7 @@ module.exports = { "primaryKey": { "pureName": "users", "constraintType": "primaryKey", + "constraintName": "PK_users", "columns": [ { "columnName": "id" @@ -670,6 +899,7 @@ module.exports = { "foreignKeys": [ { "constraintType": "foreignKey", + "constraintName": "FK_user_connections_user_id", "pureName": "user_connections", "refTableName": "users", "deleteAction": "CASCADE", @@ -682,6 +912,7 @@ module.exports = { }, { "constraintType": "foreignKey", + "constraintName": "FK_user_connections_connection_id", "pureName": "user_connections", "refTableName": "connections", "deleteAction": "CASCADE", @@ -696,6 +927,7 @@ module.exports = { "primaryKey": { "pureName": "user_connections", "constraintType": "primaryKey", + "constraintName": "PK_user_connections", "columns": [ { "columnName": "id" @@ -729,6 +961,7 @@ module.exports = { "foreignKeys": [ { "constraintType": "foreignKey", + "constraintName": "FK_user_permissions_user_id", "pureName": "user_permissions", "refTableName": "users", "deleteAction": "CASCADE", @@ -743,6 +976,7 @@ module.exports = { "primaryKey": { "pureName": "user_permissions", "constraintType": "primaryKey", + "constraintName": "PK_user_permissions", "columns": [ { "columnName": "id" @@ -776,6 +1010,7 @@ module.exports = { "foreignKeys": [ { "constraintType": "foreignKey", + "constraintName": "FK_user_roles_user_id", "pureName": "user_roles", "refTableName": "users", "deleteAction": "CASCADE", @@ -788,6 +1023,7 @@ module.exports = { }, { "constraintType": "foreignKey", + "constraintName": "FK_user_roles_role_id", "pureName": "user_roles", "refTableName": "roles", "deleteAction": "CASCADE", @@ -802,6 +1038,7 @@ module.exports = { "primaryKey": { "pureName": "user_roles", "constraintType": "primaryKey", + "constraintName": "PK_user_roles", "columns": [ { "columnName": "id" @@ -815,5 +1052,6 @@ module.exports = { "matviews": [], "functions": [], "procedures": [], - "triggers": [] + "triggers": [], + "schedulerEvents": [] }; \ No newline at end of file diff --git a/packages/api/src/utility/auditlog.js b/packages/api/src/utility/auditlog.js index b510fadc2..6ef1001e7 100644 --- a/packages/api/src/utility/auditlog.js +++ b/packages/api/src/utility/auditlog.js @@ -1,8 +1,251 @@ -// only in DbGate Premium +// *** This file is part of DbGate Premium *** -async function sendToAuditLog(req, props) {} -async function logJsonRunnerScript(req, script) {} +const { getLogger, extractErrorLogData } = require('dbgate-tools'); +const { storageSqlCommandFmt, storageSelectFmt } = require('../controllers/storageDb'); +const logger = getLogger('auditLog'); +const _ = require('lodash'); +let auditLogQueue = []; +let isProcessing = false; +let isPlanned = false; + +function nullableSum(a, b) { + const res = (a || 0) + (b || 0); + return res == 0 ? null : res; +} + +async function processAuditLogQueue() { + do { + isProcessing = true; + const elements = [...auditLogQueue]; + auditLogQueue = []; + + while (elements.length > 0) { + const element = elements.shift(); + if (!element) continue; + if (element.sessionId && element.sessionGroup && element.sessionParam) { + const existingRows = await storageSelectFmt( + '^select ~id, ~sumint_1, ~sumint_2 from ~audit_log where ~session_id = %v and ~session_group = %v and ~session_param = %v', + element.sessionId, + element.sessionGroup, + element.sessionParam + ); + if (existingRows && existingRows.length > 0) { + const existing = existingRows[0]; + await storageSqlCommandFmt( + '^update ~audit_log set ~sumint_1 = %v, ~sumint_2 = %v, ~modified = %v where ~id = %v', + nullableSum(element.sumint1, existing.sumint_1), + nullableSum(element.sumint2, existing.sumint_2), + element.created, + existing.id + ); + // only update existing session + continue; + } + } + try { + let connectionData = null; + if (element.conid) { + const connections = await storageSelectFmt('^select * from ~connections where ~conid = %v', element.conid); + if (connections[0]) + connectionData = _.pick(connections[0], [ + 'displayName', + 'engine', + 'displayName', + 'databaseUrl', + 'singleDatabase', + 'server', + 'databaseFile', + 'useSshTunnel', + 'sshHost', + 'defaultDatabase', + ]); + } + + const detailText = _.isPlainObject(element.detail) ? JSON.stringify(element.detail) : element.detail || null; + const connectionDataText = connectionData ? JSON.stringify(connectionData) : null; + await storageSqlCommandFmt( + `^insert ^into ~audit_log ( + ~user_id, ~user_login, ~created, ~category, ~component, ~event, ~detail, ~detail_full_length, ~action, ~severity, + ~conid, ~database, ~schema_name, ~pure_name, ~sumint_1, ~sumint_2, ~session_id, ~session_group, ~session_param, ~connection_data, ~message) + values (%v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v)`, + element.userId || null, + element.login || null, + element.created, + element.category || null, + element.component || null, + element.event || null, + detailText?.slice(0, 1000) || null, + detailText?.length || null, + element.action || null, + element.severity || 'info', + element.conid || null, + element.database || null, + element.schemaName || null, + element.pureName || null, + element.sumint1 || null, + element.sumint2 || null, + element.sessionId || null, + element.sessionGroup || null, + element.sessionParam || null, + connectionDataText?.slice(0, 1000) || null, + element.message || null + ); + } catch (err) { + logger.error(extractErrorLogData(err), 'Error processing audit log entry'); + } + } + + isProcessing = false; + } while (auditLogQueue.length > 0); + isPlanned = false; +} + +async function sendToAuditLog( + req, + { + category, + component, + event, + detail = null, + action, + severity = 'info', + conid = null, + database = null, + schemaName = null, + pureName = null, + sumint1 = null, + sumint2 = null, + sessionGroup = null, + sessionParam = null, + message = null, + } +) { + if (!process.env.STORAGE_DATABASE) { + return; + } + const config = require('../controllers/config'); + const settings = await config.getCachedSettings(); + if (settings?.['storage.useAuditLog'] != 1) { + return; + } + + const { login, userId } = req?.user || {}; + const sessionId = req?.headers?.['x-api-session-id']; + + auditLogQueue.push({ + userId, + login, + created: new Date().getTime(), + category, + component, + event, + detail, + action, + severity, + conid, + database, + schemaName, + pureName, + sumint1, + sumint2, + sessionId, + sessionGroup, + sessionParam, + message, + }); + if (!isProcessing && !isPlanned) { + setTimeout(() => { + isPlanned = true; + processAuditLogQueue(); + }, 0); + } +} + +function maskPasswords(script) { + return _.cloneDeepWith(script, (value, key) => { + if (_.isString(key) && key.toLowerCase().includes('password')) { + return '****'; + } + }); +} + +function analyseJsonRunnerScript(script) { + const [assignSource, assignTarget, copyStream] = _.isArray(script?.commands) ? script?.commands : []; + if (assignSource?.type != 'assign') { + return null; + } + if (assignTarget?.type != 'assign') { + return null; + } + if (copyStream?.type != 'copyStream') { + return null; + } + + if (assignTarget?.functionName == 'tableWriter') { + const pureName = assignTarget?.props?.pureName; + const schemaName = assignTarget?.props?.schemaName; + const connection = assignTarget?.props?.connection; + if (pureName && connection) { + return { + category: 'import', + component: 'RunnersController', + event: 'import.data', + action: 'import', + severity: 'info', + message: 'Importing data', + pureName: pureName, + conid: connection?.conid, + database: connection?.database, + schemaName: schemaName, + detail: maskPasswords(script), + }; + } + return null; + } + + if (assignSource?.functionName == 'tableReader') { + const pureName = assignSource?.props?.pureName; + const schemaName = assignSource?.props?.schemaName; + const connection = assignSource?.props?.connection; + if (pureName && connection) { + return { + category: 'export', + component: 'RunnersController', + event: 'export.data', + action: 'export', + severity: 'info', + message: 'Exporting data', + pureName: pureName, + conid: connection?.conid, + database: connection?.database, + schemaName: schemaName, + detail: maskPasswords(script), + }; + } + return null; + } + + return null; +} + +function logJsonRunnerScript(req, script) { + const analysed = analyseJsonRunnerScript(script); + + if (analysed) { + sendToAuditLog(req, analysed); + } else { + sendToAuditLog(req, { + category: 'shell', + component: 'RunnersController', + event: 'script.run.json', + action: 'script', + severity: 'info', + detail: maskPasswords(script), + message: 'Running JSON script', + }); + } +} module.exports = { sendToAuditLog, diff --git a/packages/datalib/src/PerspectiveDataLoader.ts b/packages/datalib/src/PerspectiveDataLoader.ts index 327b0478d..f6745d938 100644 --- a/packages/datalib/src/PerspectiveDataLoader.ts +++ b/packages/datalib/src/PerspectiveDataLoader.ts @@ -106,6 +106,7 @@ export class PerspectiveDataLoader { conid: props.databaseConfig.conid, database: props.databaseConfig.database, select, + auditLogSessionGroup: 'perspective', }); if (response.errorMessage) return response; @@ -227,6 +228,7 @@ export class PerspectiveDataLoader { conid: props.databaseConfig.conid, database: props.databaseConfig.database, select, + auditLogSessionGroup: 'perspective', }); if (response.errorMessage) return response; @@ -330,6 +332,7 @@ export class PerspectiveDataLoader { conid: props.databaseConfig.conid, database: props.databaseConfig.database, select, + auditLogSessionGroup: 'perspective', }); if (response.errorMessage) return response; diff --git a/packages/tools/src/yamlModelConv.ts b/packages/tools/src/yamlModelConv.ts index 4ed6e3b9b..f641964bf 100644 --- a/packages/tools/src/yamlModelConv.ts +++ b/packages/tools/src/yamlModelConv.ts @@ -31,6 +31,11 @@ export interface IndexInfoYaml { included?: string[]; } +export interface UniqueInfoYaml { + name: string; + columns: string[]; +} + export interface TableInfoYaml { name: string; // schema?: string; @@ -38,6 +43,7 @@ export interface TableInfoYaml { primaryKey?: string[]; sortingKey?: string[]; indexes?: IndexInfoYaml[]; + uniques?: UniqueInfoYaml[]; insertKey?: string[]; insertOnly?: string[]; @@ -121,6 +127,12 @@ export function tableInfoToYaml(table: TableInfo): TableInfoYaml { return idx; }); } + if (tableCopy.uniques?.length > 0) { + res.uniques = tableCopy.uniques.map(unique => ({ + name: unique.constraintName, + columns: unique.columns.map(x => x.columnName), + })); + } return res; } @@ -165,6 +177,12 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml ...(index.included || []).map(columnName => ({ columnName, isIncludedColumn: true })), ], })), + uniques: table.uniques?.map(unique => ({ + constraintName: unique.name, + pureName: table.name, + constraintType: 'unique', + columns: unique.columns.map(columnName => ({ columnName })), + })), }; if (table.primaryKey) { res.primaryKey = { diff --git a/packages/web/package.json b/packages/web/package.json index 8931a4d18..8874f20a8 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -64,6 +64,7 @@ "chartjs-plugin-zoom": "^1.2.0", "date-fns": "^4.1.0", "debug": "^4.3.4", + "flatpickr": "^4.6.13", "fuzzy": "^0.1.3", "highlight.js": "^11.11.1", "interval-operations": "^1.0.7", diff --git a/packages/web/public/global.css b/packages/web/public/global.css index 2977bfccf..0e2ab885a 100644 --- a/packages/web/public/global.css +++ b/packages/web/public/global.css @@ -117,7 +117,7 @@ body { max-width: 16.6666%; } -.largeFormMarker input[type='text'], .largeFormMarker input[type='number'], .largeFormMarker input[type='password'], .largeFormMarker textarea { +.largeFormMarker input[type='text'], .largeFormMarker input[type='number'], .largeFormMarker input[type='password'], .largeFormMarker textarea { width: 100%; padding: 10px 10px; font-size: 14px; @@ -126,6 +126,13 @@ body { border: 1px solid var(--theme-border); } +.input1 { + padding: 5px 5px; + font-size: 14px; + box-sizing: border-box; + border-radius: 4px; + border: 1px solid var(--theme-border); +} .largeFormMarker select { width: 100%; diff --git a/packages/web/src/App.svelte b/packages/web/src/App.svelte index 124be6c23..081f2a90f 100644 --- a/packages/web/src/App.svelte +++ b/packages/web/src/App.svelte @@ -60,10 +60,9 @@ installNewCloudTokenListener(); initializeAppUpdates(); installCloudListeners(); + refreshPublicCloudFiles(); } - refreshPublicCloudFiles(); - loadedApi = loadedApiValue; if (!loadedApi) { diff --git a/packages/web/src/datagrid/SqlDataGridCore.svelte b/packages/web/src/datagrid/SqlDataGridCore.svelte index 2acbfce70..f1134aca5 100644 --- a/packages/web/src/datagrid/SqlDataGridCore.svelte +++ b/packages/web/src/datagrid/SqlDataGridCore.svelte @@ -18,41 +18,6 @@ testEnabled: () => getCurrentEditor() != null && hasPermission('dbops/export'), onClick: () => getCurrentEditor().exportGrid(), }); - - async function loadDataPage(props, offset, limit) { - const { display, conid, database } = props; - - const select = display.getPageQuery(offset, limit); - - const response = await apiCall('database-connections/sql-select', { - conid, - database, - select, - }); - - if (response.errorMessage) return response; - return response.rows; - } - - function dataPageAvailable(props) { - const { display } = props; - const select = display.getPageQuery(0, 1); - return !!select; - } - - async function loadRowCount(props) { - const { display, conid, database } = props; - - const select = display.getCountQuery(); - - const response = await apiCall('database-connections/sql-select', { - conid, - database, - select, - }); - - return parseInt(response.rows[0].count); - } + import FontIcon from "../icons/FontIcon.svelte"; + + export let onClose; + + +
+ + {#if onClose} + + {/if} +
+ + diff --git a/packages/web/src/elements/DateRangeSelector.svelte b/packages/web/src/elements/DateRangeSelector.svelte new file mode 100644 index 000000000..3a546b611 --- /dev/null +++ b/packages/web/src/elements/DateRangeSelector.svelte @@ -0,0 +1,32 @@ + + + diff --git a/packages/web/src/formview/SqlFormView.svelte b/packages/web/src/formview/SqlFormView.svelte index 0c7af9fec..6a7f81333 100644 --- a/packages/web/src/formview/SqlFormView.svelte +++ b/packages/web/src/formview/SqlFormView.svelte @@ -8,6 +8,7 @@ conid, database, select, + auditLogSessionGroup: 'data-form', }); if (response.errorMessage) return response; diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index b210e61db..06918d8f3 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -143,6 +143,7 @@ 'icon markdown': 'mdi mdi-application', 'icon preview': 'mdi mdi-file-find', 'icon eye': 'mdi mdi-eye', + 'icon auditlog': 'mdi mdi-eye', 'icon check-all': 'mdi mdi-check-all', 'icon checkbox-blank': 'mdi mdi-checkbox-blank-outline', 'icon checkbox-marked': 'mdi mdi-checkbox-marked-outline', @@ -307,6 +308,7 @@ 'img filter': 'mdi mdi-filter', 'img group': 'mdi mdi-group', 'img perspective': 'mdi mdi-eye color-icon-yellow', + 'img auditlog': 'mdi mdi-eye color-icon-blue', 'img parent-filter': 'mdi mdi-home-alert color-icon-yellow', 'img folder': 'mdi mdi-folder color-icon-yellow', diff --git a/packages/web/src/modals/DictionaryLookupModal.svelte b/packages/web/src/modals/DictionaryLookupModal.svelte index e3dda4066..2333e5b39 100644 --- a/packages/web/src/modals/DictionaryLookupModal.svelte +++ b/packages/web/src/modals/DictionaryLookupModal.svelte @@ -104,7 +104,8 @@ const response = await apiCall('database-connections/sql-select', { conid, database, - select + select, + auditLogSessionGroup: 'lookup', }); rows = response.rows; diff --git a/packages/web/src/utility/api.ts b/packages/web/src/utility/api.ts index 31a8e23bc..4ba094c35 100644 --- a/packages/web/src/utility/api.ts +++ b/packages/web/src/utility/api.ts @@ -185,6 +185,7 @@ export async function apiCall( cache: 'no-cache', headers: { 'Content-Type': 'application/json', + 'x-api-session-id': getApiSessionId(), ...resolveApiHeaders(), }, body: JSON.stringify(args, serializeJsTypesReplacer), @@ -318,6 +319,20 @@ export function refreshPublicCloudFiles(force = false) { sessionStorage.setItem('publicCloudFilesLoaded', 'true'); } +let apiSessionIdValue = null; +function getApiSessionId() { + if (!apiSessionIdValue) { + apiSessionIdValue = uuidv1(); + } + return apiSessionIdValue; + + // if (!sessionStorage.getItem('apiSessionId')) { + // const sessionId = uuidv1(); + // sessionStorage.setItem('apiSessionId', sessionId); + // } + // return sessionStorage.getItem('apiSessionId'); +} + function enableApiLog() { apiLogging = true; console.log('API loggin enabled'); diff --git a/yarn.lock b/yarn.lock index 1c9a634e2..6c3637ee2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5414,6 +5414,11 @@ flat@^5.0.2: resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== +flatpickr@^4.6.13: + version "4.6.13" + resolved "https://registry.yarnpkg.com/flatpickr/-/flatpickr-4.6.13.tgz#8a029548187fd6e0d670908471e43abe9ad18d94" + integrity sha512-97PMG/aywoYpB4IvbvUJi0RQi8vearvU0oov1WW3k0WZPBMrTQVqekSX5CjSG/M4Q3i6A/0FKXC7RyAoAUUSPw== + flatted@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138"