dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-17 21:26:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

SYNC: Merge branch 'feature/audit-logs'

Browse Source
This commit is contained in:
SPRINX0\prochazka
2025-06-27 13:05:26 +02:00
committed by Diflow
parent e3c6d05a0a
commit 90bbdd563b
24 changed files with 781 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@@ -43,7 +43,7 @@
"build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
"build:plugins:backend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:backend",
"build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch",
"storage-json": "dbmodel model-to-json storage-db packages/api/src/storageModel.js --commonjs",
"storage-json": "node packages/dbmodel/bin/dbmodel.js model-to-json storage-db packages/api/src/storageModel.js --commonjs",
"plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist",
"build:app:local": "yarn plugins:copydist && cd app && yarn build:local",
"start:app:local": "cd app && yarn start:local",

4
packages/api/src/auth/authProvider.js
View File

@@ -11,7 +11,7 @@ const logger = getLogger('authProvider');
class AuthProviderBase {
amoid = 'none';
async login(login, password, options = undefined) {
async login(login, password, options = undefined, req = undefined) {
return {
accessToken: jwt.sign(
{
@@ -23,7 +23,7 @@ class AuthProviderBase {
};
}
oauthToken(params) {
oauthToken(params, req) {
return {};
}

27
packages/api/src/controllers/auth.js
View File

@@ -20,6 +20,7 @@ const {
readCloudTestTokenHolder,
} = require('../utility/cloudIntf');
const socket = require('../utility/socket');
const { sendToAuditLog } = require('../utility/auditlog');
const logger = getLogger('auth');
@@ -92,12 +93,12 @@ function authMiddleware(req, res, next) {
module.exports = {
oauthToken_meta: true,
async oauthToken(params) {
async oauthToken(params, req) {
const { amoid } = params;
return getAuthProviderById(amoid).oauthToken(params);
return getAuthProviderById(amoid).oauthToken(params, req);
},
login_meta: true,
async login(params) {
async login(params, req) {
const { amoid, login, password, isAdminPage } = params;
if (isAdminPage) {
@@ -107,6 +108,15 @@ module.exports = {
adminPassword = decryptPasswordString(adminConfig?.adminPassword);
}
if (adminPassword && adminPassword == password) {
sendToAuditLog(req, {
category: 'auth',
component: 'AuthController',
action: 'login',
event: 'login.admin',
severity: 'info',
message: 'Administration login successful',
});
return {
accessToken: jwt.sign(
{
@@ -122,10 +132,19 @@ module.exports = {
};
}
sendToAuditLog(req, {
category: 'auth',
component: 'AuthController',
action: 'loginFail',
event: 'login.adminFailed',
severity: 'warn',
message: 'Administraton login failed',
});
return { error: 'Login failed' };
}
return getAuthProviderById(amoid).login(login, password);
return getAuthProviderById(amoid).login(login, password, undefined, req);
},
getProviders_meta: true,

11
packages/api/src/controllers/config.js
View File

@@ -29,6 +29,7 @@ const {
} = require('../utility/crypting');
const lock = new AsyncLock();
let cachedSettingsValue = null;
module.exports = {
// settingsValue: {},
@@ -145,6 +146,13 @@ module.exports = {
return res;
},
async getCachedSettings() {
if (!cachedSettingsValue) {
cachedSettingsValue = await this.loadSettings();
}
return cachedSettingsValue;
},
deleteSettings_meta: true,
async deleteSettings() {
await fs.unlink(path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'));
@@ -258,6 +266,7 @@ module.exports = {
updateSettings_meta: true,
async updateSettings(values, req) {
if (!hasPermission(`settings/change`, req)) return false;
cachedSettingsValue = null;
const res = await lock.acquire('settings', async () => {
const currentValue = await this.loadSettings();
@@ -304,7 +313,7 @@ module.exports = {
const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
return resp.data;
} catch (err) {
return ''
return '';
}
},

10
packages/api/src/controllers/connections.js
View File

@@ -536,14 +536,14 @@ module.exports = {
},
dbloginAuthToken_meta: true,
async dbloginAuthToken({ amoid, code, conid, redirectUri, sid }) {
async dbloginAuthToken({ amoid, code, conid, redirectUri, sid }, req) {
try {
const connection = await this.getCore({ conid });
const driver = requireEngineDriver(connection);
const accessToken = await driver.getAuthTokenFromCode(connection, { code, redirectUri, sid });
const volatile = await this.saveVolatile({ conid, accessToken });
const authProvider = getAuthProviderById(amoid);
const resp = await authProvider.login(null, null, { conid: volatile._id });
const resp = await authProvider.login(null, null, { conid: volatile._id }, req);
return resp;
} catch (err) {
logger.error(extractErrorLogData(err), 'Error getting DB token');
@@ -552,18 +552,18 @@ module.exports = {
},
dbloginAuth_meta: true,
async dbloginAuth({ amoid, conid, user, password }) {
async dbloginAuth({ amoid, conid, user, password }, req) {
if (user || password) {
const saveResp = await this.saveVolatile({ conid, user, password, test: true });
if (saveResp.msgtype == 'connected') {
const loginResp = await getAuthProviderById(amoid).login(user, password, { conid: saveResp._id });
const loginResp = await getAuthProviderById(amoid).login(user, password, { conid: saveResp._id }, req);
return loginResp;
}
return saveResp;
}
// user and password is stored in connection, volatile connection is not needed
const loginResp = await getAuthProviderById(amoid).login(null, null, { conid });
const loginResp = await getAuthProviderById(amoid).login(null, null, { conid }, req);
return loginResp;
},

53
packages/api/src/controllers/databaseConnections.js
View File

@@ -41,6 +41,7 @@ const { decryptConnection } = require('../utility/crypting');
const { getSshTunnel } = require('../utility/sshTunnel');
const sessions = require('./sessions');
const jsldata = require('./jsldata');
const { sendToAuditLog } = require('../utility/auditlog');
const logger = getLogger('databaseConnections');
@@ -83,8 +84,11 @@ module.exports = {
}
},
handle_response(conid, database, { msgid, ...response }) {
const [resolve, reject] = this.requests[msgid];
const [resolve, reject, additionalData] = this.requests[msgid];
resolve(response);
if (additionalData?.auditLogger) {
additionalData?.auditLogger(response);
}
delete this.requests[msgid];
},
handle_status(conid, database, { status }) {
@@ -215,10 +219,10 @@ module.exports = {
},
/** @param {import('dbgate-types').OpenedDatabaseConnection} conn */
sendRequest(conn, message) {
sendRequest(conn, message, additionalData = {}) {
const msgid = crypto.randomUUID();
const promise = new Promise((resolve, reject) => {
this.requests[msgid] = [resolve, reject];
this.requests[msgid] = [resolve, reject, additionalData];
try {
conn.subprocess.send({ msgid, ...message });
} catch (err) {
@@ -242,10 +246,35 @@ module.exports = {
},
sqlSelect_meta: true,
async sqlSelect({ conid, database, select }, req) {
async sqlSelect({ conid, database, select, auditLogSessionGroup }, req) {
testConnectionPermission(conid, req);
const opened = await this.ensureOpened(conid, database);
const res = await this.sendRequest(opened, { msgtype: 'sqlSelect', select });
const res = await this.sendRequest(
opened,
{ msgtype: 'sqlSelect', select },
{
auditLogger:
auditLogSessionGroup && select?.from?.name?.pureName
? response => {
sendToAuditLog(req, {
category: 'dbop',
component: 'DatabaseConnectionsController',
event: 'sql.select',
action: 'select',
severity: 'info',
conid,
database,
schemaName: select?.from?.name?.schemaName,
pureName: select?.from?.name?.pureName,
sumint1: response?.rows?.length,
sessionParam: `${select?.from?.name?.schemaName || '0'}::${select?.from?.name?.pureName}`,
sessionGroup: auditLogSessionGroup,
message: `Loaded table data from ${select?.from?.name?.pureName}`,
});
}
: null,
}
);
return res;
},
@@ -492,6 +521,20 @@ module.exports = {
}
const opened = await this.ensureOpened(conid, database);
sendToAuditLog(req, {
category: 'dbop',
component: 'DatabaseConnectionsController',
action: 'structure',
event: 'dbStructure.get',
severity: 'info',
conid,
database,
sessionParam: `${conid}::${database}`,
sessionGroup: 'getStructure',
message: `Loaded database structure for ${database}`
});
return opened.structure;
// const existing = this.opened.find((x) => x.conid == conid && x.database == database);
// if (existing) return existing.status;

25
packages/api/src/controllers/runners.js
View File

@@ -20,6 +20,7 @@ const { handleProcessCommunication } = require('../utility/processComm');
const processArgs = require('../utility/processArgs');
const platformInfo = require('../utility/platformInfo');
const { checkSecureDirectories, checkSecureDirectoriesInScript } = require('../utility/security');
const { sendToAuditLog, logJsonRunnerScript } = require('../utility/auditlog');
const logger = getLogger('runners');
function extractPlugins(script) {
@@ -270,7 +271,7 @@ module.exports = {
},
start_meta: true,
async start({ script }) {
async start({ script }, req) {
const runid = crypto.randomUUID();
if (script.type == 'json') {
@@ -280,14 +281,36 @@ module.exports = {
}
}
logJsonRunnerScript(req, script);
const js = await jsonScriptToJavascript(script);
return this.startCore(runid, scriptTemplate(js, false));
}
if (!platformInfo.allowShellScripting) {
sendToAuditLog(req, {
category: 'shell',
component: 'RunnersController',
event: 'script.runFailed',
action: 'script',
severity: 'warn',
detail: script,
message: 'Scripts are not allowed',
});
return { errorMessage: 'Shell scripting is not allowed' };
}
sendToAuditLog(req, {
category: 'shell',
component: 'RunnersController',
event: 'script.run.shell',
action: 'script',
severity: 'info',
detail: script,
message: 'Running JS script',
});
return this.startCore(runid, scriptTemplate(script, false));
},

12
packages/api/src/controllers/serverConnections.js
View File

@@ -12,6 +12,7 @@ const { testConnectionPermission } = require('../utility/hasPermission');
const { MissingCredentialsError } = require('../utility/exceptions');
const pipeForkLogs = require('../utility/pipeForkLogs');
const { getLogger, extractErrorLogData } = require('dbgate-tools');
const { sendToAuditLog } = require('../utility/auditlog');
const logger = getLogger('serverConnection');
@@ -145,6 +146,17 @@ module.exports = {
if (conid == '__model') return [];
testConnectionPermission(conid, req);
const opened = await this.ensureOpened(conid);
sendToAuditLog(req, {
category: 'serverop',
component: 'ServerConnectionsController',
action: 'listDatabases',
event: 'databases.list',
severity: 'info',
conid,
sessionParam: `${conid}`,
sessionGroup: 'listDatabases',
message: `Loaded databases for connection`,
});
return opened?.databases ?? [];
},

15
packages/api/src/controllers/sessions.js
View File

@@ -11,6 +11,7 @@ const { appdir } = require('../utility/directories');
const { getLogger, extractErrorLogData } = require('dbgate-tools');
const pipeForkLogs = require('../utility/pipeForkLogs');
const config = require('./config');
const { sendToAuditLog } = require('../utility/auditlog');
const logger = getLogger('sessions');
@@ -146,12 +147,24 @@ module.exports = {
},
executeQuery_meta: true,
async executeQuery({ sesid, sql, autoCommit, autoDetectCharts, limitRows, frontMatter }) {
async executeQuery({ sesid, sql, autoCommit, autoDetectCharts, limitRows, frontMatter }, req) {
const session = this.opened.find(x => x.sesid == sesid);
if (!session) {
throw new Error('Invalid session');
}
sendToAuditLog(req, {
category: 'dbop',
component: 'SessionController',
action: 'executeQuery',
event: 'query.execute',
severity: 'info',
detail: sql,
conid: session.conid,
database: session.database,
message: 'Executing query',
});
logger.info({ sesid, sql }, 'Processing query');
this.dispatchMessage(sesid, 'Query execution started');
session.subprocess.send({

240
packages/api/src/storageModel.js
View File

@@ -1,5 +1,192 @@
module.exports = {
"tables": [
{
"pureName": "audit_log",
"columns": [
{
"pureName": "audit_log",
"columnName": "id",
"dataType": "int",
"autoIncrement": true,
"notNull": true
},
{
"pureName": "audit_log",
"columnName": "created",
"dataType": "bigint",
"notNull": true
},
{
"pureName": "audit_log",
"columnName": "modified",
"dataType": "bigint",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "user_id",
"dataType": "int",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "user_login",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "category",
"dataType": "varchar(50)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "component",
"dataType": "varchar(50)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "action",
"dataType": "varchar(50)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "severity",
"dataType": "varchar(50)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "event",
"dataType": "varchar(100)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "message",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "detail",
"dataType": "varchar(1000)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "detail_full_length",
"dataType": "int",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "session_id",
"dataType": "varchar(200)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "session_group",
"dataType": "varchar(50)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "session_param",
"dataType": "varchar(200)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "conid",
"dataType": "varchar(100)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "connection_data",
"dataType": "varchar(1000)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "database",
"dataType": "varchar(200)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "schema_name",
"dataType": "varchar(100)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "pure_name",
"dataType": "varchar(100)",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "sumint_1",
"dataType": "int",
"notNull": false
},
{
"pureName": "audit_log",
"columnName": "sumint_2",
"dataType": "int",
"notNull": false
}
],
"foreignKeys": [
{
"constraintType": "foreignKey",
"constraintName": "FK_audit_log_user_id",
"pureName": "audit_log",
"refTableName": "users",
"deleteAction": "SET NULL",
"columns": [
{
"columnName": "user_id",
"refColumnName": "id"
}
]
}
],
"indexes": [
{
"constraintName": "idx_audit_log_session",
"pureName": "audit_log",
"constraintType": "index",
"columns": [
{
"columnName": "session_group"
},
{
"columnName": "session_id"
},
{
"columnName": "session_param"
}
]
}
],
"primaryKey": {
"pureName": "audit_log",
"constraintType": "primaryKey",
"constraintName": "PK_audit_log",
"columns": [
{
"columnName": "id"
}
]
}
},
{
"pureName": "auth_methods",
"columns": [
@@ -50,6 +237,7 @@ module.exports = {
"primaryKey": {
"pureName": "auth_methods",
"constraintType": "primaryKey",
"constraintName": "PK_auth_methods",
"columns": [
{
"columnName": "id"
@@ -103,6 +291,7 @@ module.exports = {
"foreignKeys": [
{
"constraintType": "foreignKey",
"constraintName": "FK_auth_methods_config_auth_method_id",
"pureName": "auth_methods_config",
"refTableName": "auth_methods",
"deleteAction": "CASCADE",
@@ -114,9 +303,25 @@ module.exports = {
]
}
],
"uniques": [
{
"constraintName": "UQ_auth_methods_config_auth_method_id_key",
"pureName": "auth_methods_config",
"constraintType": "unique",
"columns": [
{
"columnName": "auth_method_id"
},
{
"columnName": "key"
}
]
}
],
"primaryKey": {
"pureName": "auth_methods_config",
"constraintType": "primaryKey",
"constraintName": "PK_auth_methods_config",
"columns": [
{
"columnName": "id"
@@ -154,9 +359,25 @@ module.exports = {
}
],
"foreignKeys": [],
"uniques": [
{
"constraintName": "UQ_config_group_key",
"pureName": "config",
"constraintType": "unique",
"columns": [
{
"columnName": "group"
},
{
"columnName": "key"
}
]
}
],
"primaryKey": {
"pureName": "config",
"constraintType": "primaryKey",
"constraintName": "PK_config",
"columns": [
{
"columnName": "id"
@@ -449,6 +670,7 @@ module.exports = {
"primaryKey": {
"pureName": "connections",
"constraintType": "primaryKey",
"constraintName": "PK_connections",
"columns": [
{
"columnName": "id"
@@ -477,6 +699,7 @@ module.exports = {
"primaryKey": {
"pureName": "roles",
"constraintType": "primaryKey",
"constraintName": "PK_roles",
"columns": [
{
"columnName": "id"
@@ -524,6 +747,7 @@ module.exports = {
"foreignKeys": [
{
"constraintType": "foreignKey",
"constraintName": "FK_role_connections_role_id",
"pureName": "role_connections",
"refTableName": "roles",
"deleteAction": "CASCADE",
@@ -536,6 +760,7 @@ module.exports = {
},
{
"constraintType": "foreignKey",
"constraintName": "FK_role_connections_connection_id",
"pureName": "role_connections",
"refTableName": "connections",
"deleteAction": "CASCADE",
@@ -550,6 +775,7 @@ module.exports = {
"primaryKey": {
"pureName": "role_connections",
"constraintType": "primaryKey",
"constraintName": "PK_role_connections",
"columns": [
{
"columnName": "id"
@@ -583,6 +809,7 @@ module.exports = {
"foreignKeys": [
{
"constraintType": "foreignKey",
"constraintName": "FK_role_permissions_role_id",
"pureName": "role_permissions",
"refTableName": "roles",
"deleteAction": "CASCADE",
@@ -597,6 +824,7 @@ module.exports = {
"primaryKey": {
"pureName": "role_permissions",
"constraintType": "primaryKey",
"constraintName": "PK_role_permissions",
"columns": [
{
"columnName": "id"
@@ -637,6 +865,7 @@ module.exports = {
"primaryKey": {
"pureName": "users",
"constraintType": "primaryKey",
"constraintName": "PK_users",
"columns": [
{
"columnName": "id"
@@ -670,6 +899,7 @@ module.exports = {
"foreignKeys": [
{
"constraintType": "foreignKey",
"constraintName": "FK_user_connections_user_id",
"pureName": "user_connections",
"refTableName": "users",
"deleteAction": "CASCADE",
@@ -682,6 +912,7 @@ module.exports = {
},
{
"constraintType": "foreignKey",
"constraintName": "FK_user_connections_connection_id",
"pureName": "user_connections",
"refTableName": "connections",
"deleteAction": "CASCADE",
@@ -696,6 +927,7 @@ module.exports = {
"primaryKey": {
"pureName": "user_connections",
"constraintType": "primaryKey",
"constraintName": "PK_user_connections",
"columns": [
{
"columnName": "id"
@@ -729,6 +961,7 @@ module.exports = {
"foreignKeys": [
{
"constraintType": "foreignKey",
"constraintName": "FK_user_permissions_user_id",
"pureName": "user_permissions",
"refTableName": "users",
"deleteAction": "CASCADE",
@@ -743,6 +976,7 @@ module.exports = {
"primaryKey": {
"pureName": "user_permissions",
"constraintType": "primaryKey",
"constraintName": "PK_user_permissions",
"columns": [
{
"columnName": "id"
@@ -776,6 +1010,7 @@ module.exports = {
"foreignKeys": [
{
"constraintType": "foreignKey",
"constraintName": "FK_user_roles_user_id",
"pureName": "user_roles",
"refTableName": "users",
"deleteAction": "CASCADE",
@@ -788,6 +1023,7 @@ module.exports = {
},
{
"constraintType": "foreignKey",
"constraintName": "FK_user_roles_role_id",
"pureName": "user_roles",
"refTableName": "roles",
"deleteAction": "CASCADE",
@@ -802,6 +1038,7 @@ module.exports = {
"primaryKey": {
"pureName": "user_roles",
"constraintType": "primaryKey",
"constraintName": "PK_user_roles",
"columns": [
{
"columnName": "id"
@@ -815,5 +1052,6 @@ module.exports = {
"matviews": [],
"functions": [],
"procedures": [],
"triggers": []
"triggers": [],
"schedulerEvents": []
};

249
packages/api/src/utility/auditlog.js
View File

@@ -1,8 +1,251 @@
// only in DbGate Premium
// *** This file is part of DbGate Premium ***
async function sendToAuditLog(req, props) {}
async function logJsonRunnerScript(req, script) {}
const { getLogger, extractErrorLogData } = require('dbgate-tools');
const { storageSqlCommandFmt, storageSelectFmt } = require('../controllers/storageDb');
const logger = getLogger('auditLog');
const _ = require('lodash');
let auditLogQueue = [];
let isProcessing = false;
let isPlanned = false;
function nullableSum(a, b) {
const res = (a || 0) + (b || 0);
return res == 0 ? null : res;
}
async function processAuditLogQueue() {
do {
isProcessing = true;
const elements = [...auditLogQueue];
auditLogQueue = [];
while (elements.length > 0) {
const element = elements.shift();
if (!element) continue;
if (element.sessionId && element.sessionGroup && element.sessionParam) {
const existingRows = await storageSelectFmt(
'^select ~id, ~sumint_1, ~sumint_2 from ~audit_log where ~session_id = %v and ~session_group = %v and ~session_param = %v',
element.sessionId,
element.sessionGroup,
element.sessionParam
);
if (existingRows && existingRows.length > 0) {
const existing = existingRows[0];
await storageSqlCommandFmt(
'^update ~audit_log set ~sumint_1 = %v, ~sumint_2 = %v, ~modified = %v where ~id = %v',
nullableSum(element.sumint1, existing.sumint_1),
nullableSum(element.sumint2, existing.sumint_2),
element.created,
existing.id
);
// only update existing session
continue;
}
}
try {
let connectionData = null;
if (element.conid) {
const connections = await storageSelectFmt('^select * from ~connections where ~conid = %v', element.conid);
if (connections[0])
connectionData = _.pick(connections[0], [
'displayName',
'engine',
'displayName',
'databaseUrl',
'singleDatabase',
'server',
'databaseFile',
'useSshTunnel',
'sshHost',
'defaultDatabase',
]);
}
const detailText = _.isPlainObject(element.detail) ? JSON.stringify(element.detail) : element.detail || null;
const connectionDataText = connectionData ? JSON.stringify(connectionData) : null;
await storageSqlCommandFmt(
`^insert ^into ~audit_log (
~user_id, ~user_login, ~created, ~category, ~component, ~event, ~detail, ~detail_full_length, ~action, ~severity,
~conid, ~database, ~schema_name, ~pure_name, ~sumint_1, ~sumint_2, ~session_id, ~session_group, ~session_param, ~connection_data, ~message)
values (%v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v)`,
element.userId || null,
element.login || null,
element.created,
element.category || null,
element.component || null,
element.event || null,
detailText?.slice(0, 1000) || null,
detailText?.length || null,
element.action || null,
element.severity || 'info',
element.conid || null,
element.database || null,
element.schemaName || null,
element.pureName || null,
element.sumint1 || null,
element.sumint2 || null,
element.sessionId || null,
element.sessionGroup || null,
element.sessionParam || null,
connectionDataText?.slice(0, 1000) || null,
element.message || null
);
} catch (err) {
logger.error(extractErrorLogData(err), 'Error processing audit log entry');
}
}
isProcessing = false;
} while (auditLogQueue.length > 0);
isPlanned = false;
}
async function sendToAuditLog(
req,
{
category,
component,
event,
detail = null,
action,
severity = 'info',
conid = null,
database = null,
schemaName = null,
pureName = null,
sumint1 = null,
sumint2 = null,
sessionGroup = null,
sessionParam = null,
message = null,
}
) {
if (!process.env.STORAGE_DATABASE) {
return;
}
const config = require('../controllers/config');
const settings = await config.getCachedSettings();
if (settings?.['storage.useAuditLog'] != 1) {
return;
}
const { login, userId } = req?.user || {};
const sessionId = req?.headers?.['x-api-session-id'];
auditLogQueue.push({
userId,
login,
created: new Date().getTime(),
category,
component,
event,
detail,
action,
severity,
conid,
database,
schemaName,
pureName,
sumint1,
sumint2,
sessionId,
sessionGroup,
sessionParam,
message,
});
if (!isProcessing && !isPlanned) {
setTimeout(() => {
isPlanned = true;
processAuditLogQueue();
}, 0);
}
}
function maskPasswords(script) {
return _.cloneDeepWith(script, (value, key) => {
if (_.isString(key) && key.toLowerCase().includes('password')) {
return '****';
}
});
}
function analyseJsonRunnerScript(script) {
const [assignSource, assignTarget, copyStream] = _.isArray(script?.commands) ? script?.commands : [];
if (assignSource?.type != 'assign') {
return null;
}
if (assignTarget?.type != 'assign') {
return null;
}
if (copyStream?.type != 'copyStream') {
return null;
}
if (assignTarget?.functionName == 'tableWriter') {
const pureName = assignTarget?.props?.pureName;
const schemaName = assignTarget?.props?.schemaName;
const connection = assignTarget?.props?.connection;
if (pureName && connection) {
return {
category: 'import',
component: 'RunnersController',
event: 'import.data',
action: 'import',
severity: 'info',
message: 'Importing data',
pureName: pureName,
conid: connection?.conid,
database: connection?.database,
schemaName: schemaName,
detail: maskPasswords(script),
};
}
return null;
}
if (assignSource?.functionName == 'tableReader') {
const pureName = assignSource?.props?.pureName;
const schemaName = assignSource?.props?.schemaName;
const connection = assignSource?.props?.connection;
if (pureName && connection) {
return {
category: 'export',
component: 'RunnersController',
event: 'export.data',
action: 'export',
severity: 'info',
message: 'Exporting data',
pureName: pureName,
conid: connection?.conid,
database: connection?.database,
schemaName: schemaName,
detail: maskPasswords(script),
};
}
return null;
}
return null;
}
function logJsonRunnerScript(req, script) {
const analysed = analyseJsonRunnerScript(script);
if (analysed) {
sendToAuditLog(req, analysed);
} else {
sendToAuditLog(req, {
category: 'shell',
component: 'RunnersController',
event: 'script.run.json',
action: 'script',
severity: 'info',
detail: maskPasswords(script),
message: 'Running JSON script',
});
}
}
module.exports = {
sendToAuditLog,

3
packages/datalib/src/PerspectiveDataLoader.ts
View File

@@ -106,6 +106,7 @@ export class PerspectiveDataLoader {
conid: props.databaseConfig.conid,
database: props.databaseConfig.database,
select,
auditLogSessionGroup: 'perspective',
});
if (response.errorMessage) return response;
@@ -227,6 +228,7 @@ export class PerspectiveDataLoader {
conid: props.databaseConfig.conid,
database: props.databaseConfig.database,
select,
auditLogSessionGroup: 'perspective',
});
if (response.errorMessage) return response;
@@ -330,6 +332,7 @@ export class PerspectiveDataLoader {
conid: props.databaseConfig.conid,
database: props.databaseConfig.database,
select,
auditLogSessionGroup: 'perspective',
});
if (response.errorMessage) return response;

18
packages/tools/src/yamlModelConv.ts
View File

@@ -31,6 +31,11 @@ export interface IndexInfoYaml {
included?: string[];
}
export interface UniqueInfoYaml {
name: string;
columns: string[];
}
export interface TableInfoYaml {
name: string;
// schema?: string;
@@ -38,6 +43,7 @@ export interface TableInfoYaml {
primaryKey?: string[];
sortingKey?: string[];
indexes?: IndexInfoYaml[];
uniques?: UniqueInfoYaml[];
insertKey?: string[];
insertOnly?: string[];
@@ -121,6 +127,12 @@ export function tableInfoToYaml(table: TableInfo): TableInfoYaml {
return idx;
});
}
if (tableCopy.uniques?.length > 0) {
res.uniques = tableCopy.uniques.map(unique => ({
name: unique.constraintName,
columns: unique.columns.map(x => x.columnName),
}));
}
return res;
}
@@ -165,6 +177,12 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
...(index.included || []).map(columnName => ({ columnName, isIncludedColumn: true })),
],
})),
uniques: table.uniques?.map(unique => ({
constraintName: unique.name,
pureName: table.name,
constraintType: 'unique',
columns: unique.columns.map(columnName => ({ columnName })),
})),
};
if (table.primaryKey) {
res.primaryKey = {

1
packages/web/package.json
View File

@@ -64,6 +64,7 @@
"chartjs-plugin-zoom": "^1.2.0",
"date-fns": "^4.1.0",
"debug": "^4.3.4",
"flatpickr": "^4.6.13",
"fuzzy": "^0.1.3",
"highlight.js": "^11.11.1",
"interval-operations": "^1.0.7",

9
packages/web/public/global.css
View File

@@ -117,7 +117,7 @@ body {
max-width: 16.6666%;
}
.largeFormMarker input[type='text'], .largeFormMarker input[type='number'], .largeFormMarker input[type='password'], .largeFormMarker textarea {
.largeFormMarker input[type='text'], .largeFormMarker input[type='number'], .largeFormMarker input[type='password'], .largeFormMarker textarea {
width: 100%;
padding: 10px 10px;
font-size: 14px;
@@ -126,6 +126,13 @@ body {
border: 1px solid var(--theme-border);
}
.input1 {
padding: 5px 5px;
font-size: 14px;
box-sizing: border-box;
border-radius: 4px;
border: 1px solid var(--theme-border);
}
.largeFormMarker select {
width: 100%;

3
packages/web/src/App.svelte
View File

@@ -60,10 +60,9 @@
installNewCloudTokenListener();
initializeAppUpdates();
installCloudListeners();
refreshPublicCloudFiles();
}
refreshPublicCloudFiles();
loadedApi = loadedApiValue;
if (!loadedApi) {

71
packages/web/src/datagrid/SqlDataGridCore.svelte
View File

@@ -18,41 +18,6 @@
testEnabled: () => getCurrentEditor() != null && hasPermission('dbops/export'),
onClick: () => getCurrentEditor().exportGrid(),
});
async function loadDataPage(props, offset, limit) {
const { display, conid, database } = props;
const select = display.getPageQuery(offset, limit);
const response = await apiCall('database-connections/sql-select', {
conid,
database,
select,
});
if (response.errorMessage) return response;
return response.rows;
}
function dataPageAvailable(props) {
const { display } = props;
const select = display.getPageQuery(0, 1);
return !!select;
}
async function loadRowCount(props) {
const { display, conid, database } = props;
const select = display.getCountQuery();
const response = await apiCall('database-connections/sql-select', {
conid,
database,
select,
});
return parseInt(response.rows[0].count);
}
</script>
<script lang="ts">
@@ -217,6 +182,42 @@
function handleSetLoadedRows(rows) {
loadedRows = rows;
}
async function loadDataPage(props, offset, limit) {
const { display, conid, database } = props;
const select = display.getPageQuery(offset, limit);
const response = await apiCall('database-connections/sql-select', {
conid,
database,
select,
auditLogSessionGroup: 'data-grid',
});
if (response.errorMessage) return response;
return response.rows;
}
function dataPageAvailable(props) {
const { display } = props;
const select = display.getPageQuery(0, 1);
return !!select;
}
async function loadRowCount(props) {
const { display, conid, database } = props;
const select = display.getCountQuery();
const response = await apiCall('database-connections/sql-select', {
conid,
database,
select,
});
return parseInt(response.rows[0].count);
}
</script>
<LoadingDataGridCore

33
packages/web/src/elements/Chip.svelte Normal file
View File

@@ -0,0 +1,33 @@
<script lang="ts">
import FontIcon from "../icons/FontIcon.svelte";
export let onClose;
</script>
<div class="chip">
<slot />
{#if onClose}
<span class="close" on:click={onClose}><FontIcon icon="icon close" /></span>
{/if}
</div>
<style>
.chip {
display: inline-block;
padding: 0.25em 0.5em;
border-radius: 1em;
background-color: var(--theme-bg-2);
color: var(--theme-text-1);
font-size: 0.875em;
cursor: pointer;
margin: 2px;
}
.chip .close {
margin-left: 0.2em;
color: var(--theme-text-2);
cursor: pointer;
}
.chip .close:hover {
color: var(--theme-font-hover);
}
</style>

32
packages/web/src/elements/DateRangeSelector.svelte Normal file
View File

@@ -0,0 +1,32 @@
<script lang="ts">
import flatpickr from 'flatpickr';
import 'flatpickr/dist/flatpickr.min.css';
import 'flatpickr/dist/themes/dark.css';
import { onMount } from 'svelte';
let flatpickrInstance;
let inputElement;
export let defaultValue = ['today', 'today'];
export let onChange: (value) => void;
onMount(() => {
flatpickrInstance = flatpickr(inputElement, {
mode: 'range',
maxDate: 'today',
dateFormat: 'Y-m-d',
defaultDate: defaultValue,
onClose: selectedDates => {
console.log('Selected dates:', selectedDates);
if (selectedDates.length === 1) {
flatpickrInstance.setDate([selectedDates[0], selectedDates[0]], true);
}
onChange(selectedDates.length == 1 ? [selectedDates[0], selectedDates[0]] : selectedDates);
},
});
});
</script>
<input bind:this={inputElement} type="text" class="input1" />

1
packages/web/src/formview/SqlFormView.svelte
View File

@@ -8,6 +8,7 @@
conid,
database,
select,
auditLogSessionGroup: 'data-form',
});
if (response.errorMessage) return response;

2
packages/web/src/icons/FontIcon.svelte
View File

@@ -143,6 +143,7 @@
'icon markdown': 'mdi mdi-application',
'icon preview': 'mdi mdi-file-find',
'icon eye': 'mdi mdi-eye',
'icon auditlog': 'mdi mdi-eye',
'icon check-all': 'mdi mdi-check-all',
'icon checkbox-blank': 'mdi mdi-checkbox-blank-outline',
'icon checkbox-marked': 'mdi mdi-checkbox-marked-outline',
@@ -307,6 +308,7 @@
'img filter': 'mdi mdi-filter',
'img group': 'mdi mdi-group',
'img perspective': 'mdi mdi-eye color-icon-yellow',
'img auditlog': 'mdi mdi-eye color-icon-blue',
'img parent-filter': 'mdi mdi-home-alert color-icon-yellow',
'img folder': 'mdi mdi-folder color-icon-yellow',

3
packages/web/src/modals/DictionaryLookupModal.svelte
View File

@@ -104,7 +104,8 @@
const response = await apiCall('database-connections/sql-select', {
conid,
database,
select
select,
auditLogSessionGroup: 'lookup',
});
rows = response.rows;

15
packages/web/src/utility/api.ts
View File

@@ -185,6 +185,7 @@ export async function apiCall(
cache: 'no-cache',
headers: {
'Content-Type': 'application/json',
'x-api-session-id': getApiSessionId(),
...resolveApiHeaders(),
},
body: JSON.stringify(args, serializeJsTypesReplacer),
@@ -318,6 +319,20 @@ export function refreshPublicCloudFiles(force = false) {
sessionStorage.setItem('publicCloudFilesLoaded', 'true');
}
let apiSessionIdValue = null;
function getApiSessionId() {
if (!apiSessionIdValue) {
apiSessionIdValue = uuidv1();
}
return apiSessionIdValue;
// if (!sessionStorage.getItem('apiSessionId')) {
// const sessionId = uuidv1();
// sessionStorage.setItem('apiSessionId', sessionId);
// }
// return sessionStorage.getItem('apiSessionId');
}
function enableApiLog() {
apiLogging = true;
console.log('API loggin enabled');

5
yarn.lock
View File

@@ -5414,6 +5414,11 @@ flat@^5.0.2:
resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241"
integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==
flatpickr@^4.6.13:
version "4.6.13"
resolved "https://registry.yarnpkg.com/flatpickr/-/flatpickr-4.6.13.tgz#8a029548187fd6e0d670908471e43abe9ad18d94"
integrity sha512-97PMG/aywoYpB4IvbvUJi0RQi8vearvU0oov1WW3k0WZPBMrTQVqekSX5CjSG/M4Q3i6A/0FKXC7RyAoAUUSPw==
flatted@^2.0.0:
version "2.0.2"
resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138"