From 0e819bcc45b2627f9816b58faa4996cd31de3b1f Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 16 Dec 2022 14:52:49 +0100 Subject: [PATCH 01/10] mongodb profiler --- packages/api/src/controllers/sessions.js | 25 ++++ packages/api/src/proc/sessionProcess.js | 28 +++++ packages/types/engines.d.ts | 3 + .../web/src/appobj/DatabaseAppObject.svelte | 15 ++- packages/web/src/icons/FontIcon.svelte | 4 + packages/web/src/tabs/ProfilerTab.svelte | 115 ++++++++++++++++++ packages/web/src/tabs/index.js | 2 + .../dbgate-plugin-mongo/src/backend/driver.js | 45 ++++++- .../src/frontend/driver.js | 1 + 9 files changed, 236 insertions(+), 2 deletions(-) create mode 100644 packages/web/src/tabs/ProfilerTab.svelte diff --git a/packages/api/src/controllers/sessions.js b/packages/api/src/controllers/sessions.js index 71e04bf84..a010dba4e 100644 --- a/packages/api/src/controllers/sessions.js +++ b/packages/api/src/controllers/sessions.js @@ -150,6 +150,31 @@ module.exports = { return true; }, + startProfiler_meta: true, + async startProfiler({ sesid }) { + const jslid = uuidv1(); + const session = this.opened.find(x => x.sesid == sesid); + if (!session) { + throw new Error('Invalid session'); + } + + console.log(`Starting profiler, sesid=${sesid}`); + session.loadingReader_jslid = jslid; + session.subprocess.send({ msgtype: 'startProfiler', jslid }); + + return { state: 'ok', jslid }; + }, + + stopProfiler_meta: true, + async stopProfiler({ sesid }) { + const session = this.opened.find(x => x.sesid == sesid); + if (!session) { + throw new Error('Invalid session'); + } + session.subprocess.send({ msgtype: 'stopProfiler' }); + return { state: 'ok' }; + }, + // cancel_meta: true, // async cancel({ sesid }) { // const session = this.opened.find((x) => x.sesid == sesid); diff --git a/packages/api/src/proc/sessionProcess.js b/packages/api/src/proc/sessionProcess.js index 4b7d140fe..4772e4404 100644 --- a/packages/api/src/proc/sessionProcess.js +++ b/packages/api/src/proc/sessionProcess.js @@ -16,6 +16,7 @@ let storedConnection; let afterConnectCallbacks = []; // let currentHandlers = []; let lastPing = null; +let currentProfiler = null; class TableWriter { constructor() { @@ -210,6 +211,31 @@ function waitConnected() { }); } +async function handleStartProfiler({ jslid }) { + await waitConnected(); + const driver = requireEngineDriver(storedConnection); + + if (!allowExecuteCustomScript(driver)) { + process.send({ msgtype: 'done' }); + return; + } + + const writer = new TableWriter(); + writer.initializeFromReader(jslid); + + currentProfiler = await driver.startProfiler(systemConnection, { + row: data => writer.rowFromReader(data), + }); + currentProfiler.writer = writer; +} + +async function handleStopProfiler({ jslid }) { + const driver = requireEngineDriver(storedConnection); + currentProfiler.writer.close(); + driver.stopProfiler(systemConnection, currentProfiler); + currentProfiler = null; +} + async function handleExecuteQuery({ sql }) { await waitConnected(); const driver = requireEngineDriver(storedConnection); @@ -280,6 +306,8 @@ const messageHandlers = { connect: handleConnect, executeQuery: handleExecuteQuery, executeReader: handleExecuteReader, + startProfiler: handleStartProfiler, + stopProfiler: handleStopProfiler, ping: handlePing, // cancel: handleCancel, }; diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index e2da64c8b..f3d5ce678 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -77,6 +77,7 @@ export interface EngineDriver { supportsDatabaseUrl?: boolean; supportsDatabaseDump?: boolean; supportsServerSummary?: boolean; + supportsDatabaseProfiler?: boolean; isElectronOnly?: boolean; supportedCreateDatabase?: boolean; showConnectionField?: (field: string, values: any) => boolean; @@ -130,6 +131,8 @@ export interface EngineDriver { callMethod(pool, method, args); serverSummary(pool): Promise; summaryCommand(pool, command, row): Promise; + startProfiler(pool, options): Promise; + stopProfiler(pool, profiler): Promise; analyserClass?: any; dumperClass?: any; diff --git a/packages/web/src/appobj/DatabaseAppObject.svelte b/packages/web/src/appobj/DatabaseAppObject.svelte index f660ad802..d1baed29a 100644 --- a/packages/web/src/appobj/DatabaseAppObject.svelte +++ b/packages/web/src/appobj/DatabaseAppObject.svelte @@ -254,6 +254,18 @@ }); }; + const handleDatabaseProfiler = () => { + openNewTab({ + title: 'Profiler', + icon: 'img profiler', + tabComponent: 'ProfilerTab', + props: { + conid: connection._id, + database: name, + }, + }); + }; + async function handleConfirmSql(sql) { saveScriptToDatabase({ conid: connection._id, database: name }, sql, false); } @@ -284,7 +296,8 @@ !connection.singleDatabase && { onClick: handleDropDatabase, text: 'Drop database' }, { divider: true }, driver?.databaseEngineTypes?.includes('sql') && { onClick: handleShowDiagram, text: 'Show diagram' }, - isSqlOrDoc && { onClick: handleSqlGenerator, text: 'SQL Generator' }, + driver?.databaseEngineTypes?.includes('sql') && { onClick: handleSqlGenerator, text: 'SQL Generator' }, + driver?.supportsDatabaseProfiler && { onClick: handleDatabaseProfiler, text: 'Database profiler' }, isSqlOrDoc && { onClick: handleOpenJsonModel, text: 'Open model as JSON' }, isSqlOrDoc && { onClick: handleExportModel, text: 'Export DB model - experimental' }, isSqlOrDoc && diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index 713eadf3a..4d34a7d26 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -49,6 +49,9 @@ 'icon close': 'mdi mdi-close', 'icon unsaved': 'mdi mdi-record', 'icon stop': 'mdi mdi-close-octagon', + 'icon play': 'mdi mdi-play', + 'icon play-stop': 'mdi mdi-stop', + 'icon pause': 'mdi mdi-pause', 'icon filter': 'mdi mdi-filter', 'icon filter-off': 'mdi mdi-filter-off', 'icon reload': 'mdi mdi-reload', @@ -176,6 +179,7 @@ 'img app-command': 'mdi mdi-flash color-icon-green', 'img app-query': 'mdi mdi-view-comfy color-icon-magenta', 'img connection': 'mdi mdi-connection color-icon-blue', + 'img profiler': 'mdi mdi-gauge color-icon-blue', 'img add': 'mdi mdi-plus-circle color-icon-green', 'img minus': 'mdi mdi-minus-circle color-icon-red', diff --git a/packages/web/src/tabs/ProfilerTab.svelte b/packages/web/src/tabs/ProfilerTab.svelte new file mode 100644 index 000000000..823b63c9a --- /dev/null +++ b/packages/web/src/tabs/ProfilerTab.svelte @@ -0,0 +1,115 @@ + + + + + + {#if jslid} + + {:else} + + {/if} + + + + + + + diff --git a/packages/web/src/tabs/index.js b/packages/web/src/tabs/index.js index db9ab5037..54c202cd2 100644 --- a/packages/web/src/tabs/index.js +++ b/packages/web/src/tabs/index.js @@ -27,6 +27,7 @@ import * as ConnectionTab from './ConnectionTab.svelte'; import * as MapTab from './MapTab.svelte'; import * as PerspectiveTab from './PerspectiveTab.svelte'; import * as ServerSummaryTab from './ServerSummaryTab.svelte'; +import * as ProfilerTab from './ProfilerTab.svelte'; export default { TableDataTab, @@ -58,4 +59,5 @@ export default { MapTab, PerspectiveTab, ServerSummaryTab, + ProfilerTab, }; diff --git a/plugins/dbgate-plugin-mongo/src/backend/driver.js b/plugins/dbgate-plugin-mongo/src/backend/driver.js index 0621fab16..b28ccde5e 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/backend/driver.js @@ -38,7 +38,7 @@ async function getScriptableDb(pool) { const db = pool.__getDatabase(); const collections = await db.listCollections().toArray(); for (const collection of collections) { - db[collection.name] = db.collection(collection.name); + _.set(db, collection.name, db.collection(collection.name)); } return db; } @@ -165,6 +165,49 @@ const driver = { options.done(); }, + async startProfiler(pool, options) { + const db = await getScriptableDb(pool); + const old = await db.command({ profile: -1 }); + await db.command({ profile: 2 }); + const cursor = await db.collection('system.profile').find({ + ns: /^((?!(admin\.\$cmd|\.system|\.tmp\.)).)*$/, + ts: { $gt: new Date() }, + 'command.profile': { $exists: false }, + 'command.collStats': { $exists: false }, + 'command.collstats': { $exists: false }, + 'command.createIndexes': { $exists: false }, + 'command.listIndexes': { $exists: false }, + // "command.cursor": {"$exists": false}, + 'command.create': { $exists: false }, + 'command.dbstats': { $exists: false }, + 'command.scale': { $exists: false }, + 'command.explain': { $exists: false }, + 'command.killCursors': { $exists: false }, + 'command.count': { $ne: 'system.profile' }, + op: /^((?!(getmore|killcursors)).)/i, + }); + + cursor.addCursorFlag('tailable', true); + cursor.addCursorFlag('awaitData', true); + + cursor + .forEach((row) => { + // console.log('ROW', row); + options.row(row); + }) + .catch((err) => { + console.error('Cursor stopped with error:', err.message); + }); + return { + cursor, + old, + }; + }, + async stopProfiler(pool, { cursor, old }) { + cursor.close(); + const db = await getScriptableDb(pool); + await db.command({ profile: old.was, slowms: old.slowms }); + }, async readQuery(pool, sql, structure) { try { const json = JSON.parse(sql); diff --git a/plugins/dbgate-plugin-mongo/src/frontend/driver.js b/plugins/dbgate-plugin-mongo/src/frontend/driver.js index ca50a0735..eb2fa8933 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/driver.js @@ -33,6 +33,7 @@ const driver = { defaultPort: 27017, supportsDatabaseUrl: true, supportsServerSummary: true, + supportsDatabaseProfiler: true, databaseUrlPlaceholder: 'e.g. mongodb://username:password@mongodb.mydomain.net/dbname', getQuerySplitterOptions: () => mongoSplitterOptions, From 34a4f9adbf7f829cb8860bef8e6f10db99335488 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 17 Dec 2022 08:57:16 +0100 Subject: [PATCH 02/10] save profiler output to archive --- packages/api/src/controllers/archive.js | 10 ++++++++ packages/web/src/tabs/ProfilerTab.svelte | 31 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/packages/api/src/controllers/archive.js b/packages/api/src/controllers/archive.js index 92c31c0fd..a9147169a 100644 --- a/packages/api/src/controllers/archive.js +++ b/packages/api/src/controllers/archive.js @@ -5,6 +5,7 @@ const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('.. const socket = require('../utility/socket'); const { saveFreeTableData } = require('../utility/freeTableStorage'); const loadFilesRecursive = require('../utility/loadFilesRecursive'); +const getJslFileName = require('../utility/getJslFileName'); module.exports = { folders_meta: true, @@ -150,6 +151,15 @@ module.exports = { return true; }, + saveJslData_meta: true, + async saveJslData({ folder, file, jslid }) { + const source = getJslFileName(jslid); + const target = path.join(resolveArchiveFolder(folder), `${file}.jsonl`); + await fs.copyFile(source, target); + socket.emitChanged(`archive-files-changed-${folder}`); + return true; + }, + async getNewArchiveFolder({ database }) { const isLink = database.endsWith(database); const name = isLink ? database.slice(0, -5) : database; diff --git a/packages/web/src/tabs/ProfilerTab.svelte b/packages/web/src/tabs/ProfilerTab.svelte index 823b63c9a..2e1c8a4ee 100644 --- a/packages/web/src/tabs/ProfilerTab.svelte +++ b/packages/web/src/tabs/ProfilerTab.svelte @@ -20,6 +20,15 @@ testEnabled: () => getCurrentEditor()?.isProfiling(), onClick: () => getCurrentEditor().stopProfiling(), }); + + registerCommand({ + id: 'profiler.save', + category: 'Profiler', + name: 'Save', + icon: 'icon save', + testEnabled: () => getCurrentEditor()?.saveEnabled(), + onClick: () => getCurrentEditor().save(), + }); @@ -111,5 +141,6 @@ + From 123e00ecbc3096194607c3d3a167f5666987aa59 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 17 Dec 2022 12:34:28 +0100 Subject: [PATCH 03/10] mongo profiler formatter --- packages/api/src/controllers/jsldata.js | 14 ++-- .../api/src/utility/JsonLinesDatastore.js | 12 +++- .../api/src/utility/requirePluginFunction.js | 16 +++++ packages/types/engines.d.ts | 1 + .../web/src/datagrid/JslDataGridCore.svelte | 7 +- packages/web/src/tabs/ProfilerTab.svelte | 9 ++- .../dbgate-plugin-mongo/src/backend/index.js | 4 ++ .../src/frontend/driver.js | 1 + .../src/frontend/formatProfilerEntry.js | 70 +++++++++++++++++++ .../dbgate-plugin-mongo/src/frontend/index.js | 4 ++ 10 files changed, 127 insertions(+), 11 deletions(-) create mode 100644 packages/api/src/utility/requirePluginFunction.js create mode 100644 plugins/dbgate-plugin-mongo/src/frontend/formatProfilerEntry.js diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index 0ab2f2ecf..dbace5a25 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -7,6 +7,7 @@ const DatastoreProxy = require('../utility/DatastoreProxy'); const { saveFreeTableData } = require('../utility/freeTableStorage'); const getJslFileName = require('../utility/getJslFileName'); const JsonLinesDatastore = require('../utility/JsonLinesDatastore'); +const requirePluginFunction = require('../utility/requirePluginFunction'); const socket = require('../utility/socket'); function readFirstLine(file) { @@ -99,10 +100,11 @@ module.exports = { // return readerInfo; // }, - async ensureDatastore(jslid) { + async ensureDatastore(jslid, formatterFunction) { + const rowFormatter = requirePluginFunction(formatterFunction); let datastore = this.datastores[jslid]; if (!datastore) { - datastore = new JsonLinesDatastore(getJslFileName(jslid)); + datastore = new JsonLinesDatastore(getJslFileName(jslid), rowFormatter); // datastore = new DatastoreProxy(getJslFileName(jslid)); this.datastores[jslid] = datastore; } @@ -131,8 +133,8 @@ module.exports = { }, getRows_meta: true, - async getRows({ jslid, offset, limit, filters }) { - const datastore = await this.ensureDatastore(jslid); + async getRows({ jslid, offset, limit, filters, formatterFunction }) { + const datastore = await this.ensureDatastore(jslid, formatterFunction); return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters); }, @@ -150,8 +152,8 @@ module.exports = { }, loadFieldValues_meta: true, - async loadFieldValues({ jslid, field, search }) { - const datastore = await this.ensureDatastore(jslid); + async loadFieldValues({ jslid, field, search, formatterFunction }) { + const datastore = await this.ensureDatastore(jslid, formatterFunction); const res = new Set(); await datastore.enumRows(row => { if (!filterName(search, row[field])) return true; diff --git a/packages/api/src/utility/JsonLinesDatastore.js b/packages/api/src/utility/JsonLinesDatastore.js index 68fec9455..b3a2005d8 100644 --- a/packages/api/src/utility/JsonLinesDatastore.js +++ b/packages/api/src/utility/JsonLinesDatastore.js @@ -22,8 +22,9 @@ function fetchNextLineFromReader(reader) { } class JsonLinesDatastore { - constructor(file) { + constructor(file, rowFormatter) { this.file = file; + this.rowFormatter = rowFormatter; this.reader = null; this.readedDataRowCount = 0; this.readedSchemaRow = false; @@ -62,6 +63,11 @@ class JsonLinesDatastore { ); } + parseLine(line) { + const res = JSON.parse(line); + return this.rowFormatter ? this.rowFormatter(res) : res; + } + async _readLine(parse) { // if (this.firstRowToBeReturned) { // const res = this.firstRowToBeReturned; @@ -84,14 +90,14 @@ class JsonLinesDatastore { } } if (this.currentFilter) { - const parsedLine = JSON.parse(line); + const parsedLine = this.parseLine(line); if (evaluateCondition(this.currentFilter, parsedLine)) { this.readedDataRowCount += 1; return parse ? parsedLine : true; } } else { this.readedDataRowCount += 1; - return parse ? JSON.parse(line) : true; + return parse ? this.parseLine(line) : true; } } diff --git a/packages/api/src/utility/requirePluginFunction.js b/packages/api/src/utility/requirePluginFunction.js new file mode 100644 index 000000000..11f9e33eb --- /dev/null +++ b/packages/api/src/utility/requirePluginFunction.js @@ -0,0 +1,16 @@ +const _ = require('lodash'); +const requirePlugin = require('../shell/requirePlugin'); + +function requirePluginFunction(functionName) { + if (!functionName) return null; + if (functionName.includes('@')) { + const [shortName, packageName] = functionName.split('@'); + const plugin = requirePlugin(packageName); + if (plugin.functions) { + return plugin.functions[shortName]; + } + } + return null; +} + +module.exports = requirePluginFunction; diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index f3d5ce678..1976e9d3b 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -78,6 +78,7 @@ export interface EngineDriver { supportsDatabaseDump?: boolean; supportsServerSummary?: boolean; supportsDatabaseProfiler?: boolean; + profilerFormatterFunction?: string; isElectronOnly?: boolean; supportedCreateDatabase?: boolean; showConnectionField?: (field: string, values: any) => boolean; diff --git a/packages/web/src/datagrid/JslDataGridCore.svelte b/packages/web/src/datagrid/JslDataGridCore.svelte index cb2afde3e..f3e95722b 100644 --- a/packages/web/src/datagrid/JslDataGridCore.svelte +++ b/packages/web/src/datagrid/JslDataGridCore.svelte @@ -12,12 +12,13 @@ }); async function loadDataPage(props, offset, limit) { - const { jslid, display } = props; + const { jslid, display, formatterFunction } = props; const response = await apiCall('jsldata/get-rows', { jslid, offset, limit, + formatterFunction, filters: display ? display.compileFilters() : null, }); @@ -34,6 +35,9 @@ const response = await apiCall('jsldata/get-stats', { jslid }); return response.rowCount; } + + export let formatterPlugin; + export let formatterFunction; diff --git a/packages/web/src/stores.ts b/packages/web/src/stores.ts index 382753290..fcfe0baf2 100644 --- a/packages/web/src/stores.ts +++ b/packages/web/src/stores.ts @@ -216,7 +216,7 @@ export const getCurrentDatabase = () => currentDatabaseValue; let currentSettingsValue = null; export const getCurrentSettings = () => currentSettingsValue || {}; -let extensionsValue = null; +let extensionsValue: ExtensionsDirectory = null; extensions.subscribe(value => { extensionsValue = value; }); diff --git a/packages/web/src/tabs/ProfilerTab.svelte b/packages/web/src/tabs/ProfilerTab.svelte index c57c2e9f4..5f2b7caf1 100644 --- a/packages/web/src/tabs/ProfilerTab.svelte +++ b/packages/web/src/tabs/ProfilerTab.svelte @@ -8,7 +8,7 @@ category: 'Profiler', name: 'Start profiling', icon: 'icon play', - testEnabled: () => getCurrentEditor() && !getCurrentEditor()?.isProfiling(), + testEnabled: () => getCurrentEditor()?.startProfilingEnabled(), onClick: () => getCurrentEditor().startProfiling(), }); @@ -17,7 +17,7 @@ category: 'Profiler', name: 'Stop profiling', icon: 'icon play-stop', - testEnabled: () => getCurrentEditor()?.isProfiling(), + testEnabled: () => getCurrentEditor()?.stopProfilingEnabled(), onClick: () => getCurrentEditor().stopProfiling(), }); @@ -55,9 +55,10 @@ export let conid; export let database; + export let jslid; + export let formatterFunction; let profiling = false; - let jslid; let sessionId; let intervalId; @@ -104,6 +105,10 @@ invalidateCommands(); } + export function startProfilingEnabled() { + return conid && database && !isProfiling; + } + export function stopProfiling() { profiling = false; apiCall('sessions/stop-profiler', { sesid: sessionId }); @@ -111,6 +116,10 @@ invalidateCommands(); } + export function stopProfilingEnabled() { + return conid && database && isProfiling; + } + export function saveEnabled() { return !!jslid; } @@ -132,7 +141,11 @@ {#if jslid} - + {:else} {/if} From cf3df9cda344e21fec020a3d9fabc7fa9432cdc1 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 17 Dec 2022 20:22:42 +0100 Subject: [PATCH 05/10] short json value shown in grid --- packages/web/src/datagrid/CellValue.svelte | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/web/src/datagrid/CellValue.svelte b/packages/web/src/datagrid/CellValue.svelte index e0e22b85a..e2e5eb1f2 100644 --- a/packages/web/src/datagrid/CellValue.svelte +++ b/packages/web/src/datagrid/CellValue.svelte @@ -75,11 +75,17 @@ {:else if value.$oid} ObjectId("{value.$oid}") {:else if _.isPlainObject(value)} - (JSON) + {@const svalue = JSON.stringify(value, undefined, 2)} + {#if svalue.length < 100}{JSON.stringify(value)}{:else}(JSON){/if} {:else if _.isArray(value)} JSON.stringify(x)).join('\n')}>[{value.length} items] {:else if _.isPlainObject(jsonParsedValue)} - (JSON) + {@const svalue = JSON.stringify(jsonParsedValue, undefined, 2)} + {#if svalue.length < 100}{JSON.stringify(jsonParsedValue)}{:else}(JSON){/if} {:else if _.isArray(jsonParsedValue)} JSON.stringify(x)).join('\n')} >[{jsonParsedValue.length} items] Date: Sun, 18 Dec 2022 09:08:03 +0100 Subject: [PATCH 06/10] jsonl filtering fixes --- packages/sqltree/src/evaluateExpression.ts | 2 +- packages/web/src/datagrid/DataFilterControl.svelte | 2 ++ packages/web/src/datagrid/DataGridCore.svelte | 2 ++ packages/web/src/modals/ValueLookupModal.svelte | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/sqltree/src/evaluateExpression.ts b/packages/sqltree/src/evaluateExpression.ts index 2f99943c6..e2d78f5e8 100644 --- a/packages/sqltree/src/evaluateExpression.ts +++ b/packages/sqltree/src/evaluateExpression.ts @@ -6,7 +6,7 @@ import { dumpSqlSourceRef } from './dumpSqlSource'; export function evaluateExpression(expr: Expression, values) { switch (expr.exprType) { case 'column': - return values[expr.columnName]; + return _.get(values, expr.columnName); case 'placeholder': return values.__placeholder; diff --git a/packages/web/src/datagrid/DataFilterControl.svelte b/packages/web/src/datagrid/DataFilterControl.svelte index 38e769859..39d68181b 100644 --- a/packages/web/src/datagrid/DataFilterControl.svelte +++ b/packages/web/src/datagrid/DataFilterControl.svelte @@ -33,6 +33,7 @@ export let customCommandIcon = null; export let onCustomCommand = null; export let customCommandTooltip = null; + export let formatterFunction = null; export let pureName = null; export let schemaName = null; @@ -276,6 +277,7 @@ schemaName, pureName, field: columnName || uniqueName, + formatterFunction, onConfirm: keys => setFilter(keys.map(x => getFilterValueExpression(x)).join(',')), }); } diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index 29adf299f..ef9013d7a 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -355,6 +355,7 @@ export let pureName = undefined; export let schemaName = undefined; export let allowDefineVirtualReferences = false; + export let formatterFunction; export let isLoadedAll; export let loadedTime; @@ -1743,6 +1744,7 @@ {conid} {database} {jslid} + {formatterFunction} driver={display?.driver} filterType={useEvalFilters ? 'eval' : col.filterType || getFilterType(col.dataType)} filter={display.getFilter(col.uniqueName)} diff --git a/packages/web/src/modals/ValueLookupModal.svelte b/packages/web/src/modals/ValueLookupModal.svelte index b235486e4..07d62aa89 100644 --- a/packages/web/src/modals/ValueLookupModal.svelte +++ b/packages/web/src/modals/ValueLookupModal.svelte @@ -25,6 +25,7 @@ export let driver; export let multiselect = false; export let jslid; + export let formatterFunction; // console.log('ValueLookupModal', conid, database, pureName, schemaName, columnName, driver); @@ -42,6 +43,7 @@ jslid, search, field, + formatterFunction, }); } else { rows = await apiCall('database-connections/load-field-values', { From 9a2631dc09d0038a2564b39a9703def943fde28c Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 18 Dec 2022 12:29:21 +0100 Subject: [PATCH 07/10] profiler charts --- packages/api/src/controllers/jsldata.js | 49 +++++++- packages/types/engines.d.ts | 2 + .../src/appobj/ArchiveFileAppObject.svelte | 4 +- packages/web/src/tabs/ProfilerTab.svelte | 118 +++++++++++++++--- .../dbgate-plugin-mongo/src/backend/index.js | 2 + .../src/frontend/driver.js | 5 + .../src/frontend/formatProfilerChartEntry.js | 14 +++ .../src/frontend/formatProfilerEntry.js | 2 + .../dbgate-plugin-mongo/src/frontend/index.js | 2 + 9 files changed, 178 insertions(+), 20 deletions(-) create mode 100644 plugins/dbgate-plugin-mongo/src/frontend/formatProfilerChartEntry.js diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index dbace5a25..697a561d3 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -102,11 +102,12 @@ module.exports = { async ensureDatastore(jslid, formatterFunction) { const rowFormatter = requirePluginFunction(formatterFunction); - let datastore = this.datastores[jslid]; + const dskey = `${jslid}||${formatterFunction}`; + let datastore = this.datastores[dskey]; if (!datastore) { datastore = new JsonLinesDatastore(getJslFileName(jslid), rowFormatter); // datastore = new DatastoreProxy(getJslFileName(jslid)); - this.datastores[jslid] = datastore; + this.datastores[dskey] = datastore; } return datastore; }, @@ -190,4 +191,48 @@ module.exports = { await fs.promises.writeFile(getJslFileName(jslid), text); return true; }, + + extractTimelineChart_meta: true, + async extractTimelineChart({ jslid, formatterFunction, measures }) { + const formater = requirePluginFunction(formatterFunction); + const datastore = new JsonLinesDatastore(getJslFileName(jslid), formater); + let mints = null; + let maxts = null; + // pass 1 - counts stats, time range + await datastore.enumRows(row => { + if (!mints || row.ts < mints) mints = row.ts; + if (!maxts || row.ts > maxts) maxts = row.ts; + return true; + }); + const minTime = new Date(mints).getTime(); + const maxTime = new Date(maxts).getTime(); + const duration = maxTime - minTime; + const STEPS = 100; + const step = duration / STEPS; + const labels = _.range(STEPS).map(i => new Date(minTime + step / 2 + step * i)); + + const datasets = measures.map(m => ({ + label: m.label, + data: Array(STEPS).fill(0), + })); + + // pass 2 - count measures + await datastore.enumRows(row => { + if (!mints || row.ts < mints) mints = row.ts; + if (!maxts || row.ts > maxts) maxts = row.ts; + + for (let i = 0; i < measures.length; i++) { + const part = Math.round((new Date(row.ts).getTime() - minTime) / step); + datasets[i].data[part] += row[measures[i].field]; + } + return true; + }); + + datastore._closeReader(); + + return { + labels, + datasets, + }; + }, }; diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 1976e9d3b..dae480ed6 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -79,6 +79,8 @@ export interface EngineDriver { supportsServerSummary?: boolean; supportsDatabaseProfiler?: boolean; profilerFormatterFunction?: string; + profilerChartFormatterFunction?: string; + profilerChartMeasures?: { label: string; field: string }[]; isElectronOnly?: boolean; supportedCreateDatabase?: boolean; showConnectionField?: (field: string, values: any) => boolean; diff --git a/packages/web/src/appobj/ArchiveFileAppObject.svelte b/packages/web/src/appobj/ArchiveFileAppObject.svelte index 6eb04de32..629782d52 100644 --- a/packages/web/src/appobj/ArchiveFileAppObject.svelte +++ b/packages/web/src/appobj/ArchiveFileAppObject.svelte @@ -214,7 +214,9 @@ tabComponent: 'ProfilerTab', props: { jslid: `archive://${data.folderName}/${data.fileName}`, - formatterFunction: eng.profilerFormatterFunction, + profilerFormatterFunction: eng.profilerFormatterFunction, + profilerChartFormatterFunction: eng.profilerChartFormatterFunction, + profilerChartMeasures: eng.profilerChartMeasures, }, }); }, diff --git a/packages/web/src/tabs/ProfilerTab.svelte b/packages/web/src/tabs/ProfilerTab.svelte index 5f2b7caf1..33ae37bc0 100644 --- a/packages/web/src/tabs/ProfilerTab.svelte +++ b/packages/web/src/tabs/ProfilerTab.svelte @@ -1,6 +1,4 @@ {#if jslid} - + + + + + + {#if isLoadingChart} + + {:else} + + {/if} + + {:else} {/if} - diff --git a/plugins/dbgate-plugin-mongo/src/backend/index.js b/plugins/dbgate-plugin-mongo/src/backend/index.js index 0f8fbd10c..7b6e0f5d9 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/index.js +++ b/plugins/dbgate-plugin-mongo/src/backend/index.js @@ -1,10 +1,12 @@ const driver = require('./driver'); const formatProfilerEntry = require('../frontend/formatProfilerEntry'); +const formatProfilerChartEntry = require('../frontend/formatProfilerChartEntry'); module.exports = { packageName: 'dbgate-plugin-mongo', drivers: [driver], functions: { formatProfilerEntry, + formatProfilerChartEntry, }, }; diff --git a/plugins/dbgate-plugin-mongo/src/frontend/driver.js b/plugins/dbgate-plugin-mongo/src/frontend/driver.js index 53887eca6..ab8787c20 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/driver.js @@ -35,6 +35,11 @@ const driver = { supportsServerSummary: true, supportsDatabaseProfiler: true, profilerFormatterFunction: 'formatProfilerEntry@dbgate-plugin-mongo', + profilerChartFormatterFunction: 'formatProfilerChartEntry@dbgate-plugin-mongo', + profilerChartMeasures: [ + { label: 'Req count', field: 'count' }, + { label: 'Duration', field: 'millis' }, + ], databaseUrlPlaceholder: 'e.g. mongodb://username:password@mongodb.mydomain.net/dbname', getQuerySplitterOptions: () => mongoSplitterOptions, diff --git a/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerChartEntry.js b/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerChartEntry.js new file mode 100644 index 000000000..8ee5a27a4 --- /dev/null +++ b/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerChartEntry.js @@ -0,0 +1,14 @@ +const _ = require('lodash'); +const formatProfilerEntry = require('./formatProfilerEntry'); + +function formatProfilerChartEntry(obj) { + const fmt = formatProfilerEntry(obj); + + return { + ts: fmt.ts, + millis: fmt.stats.millis, + count: 1, + }; +} + +module.exports = formatProfilerChartEntry; diff --git a/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerEntry.js b/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerEntry.js index cb57e8b3a..9b4e4cedb 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerEntry.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerEntry.js @@ -2,6 +2,7 @@ const _ = require('lodash'); function formatProfilerEntry(obj) { const ts = obj.ts; + const stats = { millis: obj.millis }; let op = obj.op; let doc; let query; @@ -64,6 +65,7 @@ function formatProfilerEntry(obj) { doc, query, ext, + stats, }; } diff --git a/plugins/dbgate-plugin-mongo/src/frontend/index.js b/plugins/dbgate-plugin-mongo/src/frontend/index.js index fbe5cc8d8..c68b8daa5 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/index.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/index.js @@ -1,10 +1,12 @@ import driver from './driver'; import formatProfilerEntry from './formatProfilerEntry'; +import formatProfilerChartEntry from './formatProfilerChartEntry'; export default { packageName: 'dbgate-plugin-mongo', drivers: [driver], functions: { formatProfilerEntry, + formatProfilerChartEntry, }, }; From 2e377884710a8cb978f15282a5bdfe4f67ad6a6f Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 18 Dec 2022 13:48:24 +0100 Subject: [PATCH 08/10] profiler charts --- packages/api/src/controllers/jsldata.js | 71 ++++++++++++++----- packages/types/engines.d.ts | 3 +- .../src/appobj/ArchiveFileAppObject.svelte | 3 +- packages/web/src/tabs/ProfilerTab.svelte | 6 +- .../dbgate-plugin-mongo/src/backend/index.js | 10 ++- .../src/frontend/driver.js | 11 ++- .../src/frontend/formatProfilerChartEntry.js | 14 ---- .../dbgate-plugin-mongo/src/frontend/index.js | 6 +- ...tProfilerEntry.js => profilerFunctions.js} | 31 +++++++- 9 files changed, 110 insertions(+), 45 deletions(-) delete mode 100644 plugins/dbgate-plugin-mongo/src/frontend/formatProfilerChartEntry.js rename plugins/dbgate-plugin-mongo/src/frontend/{formatProfilerEntry.js => profilerFunctions.js} (74%) diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index 697a561d3..8b0d29ffa 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -193,46 +193,83 @@ module.exports = { }, extractTimelineChart_meta: true, - async extractTimelineChart({ jslid, formatterFunction, measures }) { - const formater = requirePluginFunction(formatterFunction); - const datastore = new JsonLinesDatastore(getJslFileName(jslid), formater); + async extractTimelineChart({ jslid, timestampFunction, aggregateFunction, measures }) { + const timestamp = requirePluginFunction(timestampFunction); + const aggregate = requirePluginFunction(aggregateFunction); + const datastore = new JsonLinesDatastore(getJslFileName(jslid)); let mints = null; let maxts = null; // pass 1 - counts stats, time range await datastore.enumRows(row => { - if (!mints || row.ts < mints) mints = row.ts; - if (!maxts || row.ts > maxts) maxts = row.ts; + const ts = timestamp(row); + if (!mints || ts < mints) mints = ts; + if (!maxts || ts > maxts) maxts = ts; return true; }); const minTime = new Date(mints).getTime(); const maxTime = new Date(maxts).getTime(); const duration = maxTime - minTime; const STEPS = 100; - const step = duration / STEPS; - const labels = _.range(STEPS).map(i => new Date(minTime + step / 2 + step * i)); + let stepCount = duration > 100 * 1000 ? STEPS : Math.round((maxTime - minTime) / 1000); + if (stepCount < 2) { + stepCount = 2; + } + const stepDuration = duration / stepCount; + const labels = _.range(stepCount).map(i => new Date(minTime + stepDuration / 2 + stepDuration * i)); - const datasets = measures.map(m => ({ - label: m.label, - data: Array(STEPS).fill(0), + // const datasets = measures.map(m => ({ + // label: m.label, + // data: Array(stepCount).fill(0), + // })); + + const mproc = measures.map(m => ({ + ...m, })); + const data = Array(stepCount) + .fill(0) + .map(() => ({})); + // pass 2 - count measures await datastore.enumRows(row => { - if (!mints || row.ts < mints) mints = row.ts; - if (!maxts || row.ts > maxts) maxts = row.ts; - - for (let i = 0; i < measures.length; i++) { - const part = Math.round((new Date(row.ts).getTime() - minTime) / step); - datasets[i].data[part] += row[measures[i].field]; + const ts = timestamp(row); + let part = Math.round((new Date(ts).getTime() - minTime) / stepDuration); + if (part < 0) part = 0; + if (part >= stepCount) part - stepCount - 1; + if (data[part]) { + data[part] = aggregate(data[part], row, stepDuration); } return true; }); datastore._closeReader(); + // const measureByField = _.fromPairs(measures.map((m, i) => [m.field, i])); + + // for (let mindex = 0; mindex < measures.length; mindex++) { + // for (let stepIndex = 0; stepIndex < stepCount; stepIndex++) { + // const measure = measures[mindex]; + // if (measure.perSecond) { + // datasets[mindex].data[stepIndex] /= stepDuration / 1000; + // } + // if (measure.perField) { + // datasets[mindex].data[stepIndex] /= datasets[measureByField[measure.perField]].data[stepIndex]; + // } + // } + // } + + // for (let i = 0; i < measures.length; i++) { + // if (measures[i].hidden) { + // datasets[i] = null; + // } + // } + return { labels, - datasets, + datasets: mproc.map(m => ({ + label: m.label, + data: data.map(d => d[m.field] || 0), + })), }; }, }; diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index dae480ed6..fac146b74 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -79,7 +79,8 @@ export interface EngineDriver { supportsServerSummary?: boolean; supportsDatabaseProfiler?: boolean; profilerFormatterFunction?: string; - profilerChartFormatterFunction?: string; + profilerTimestampFunction?: string; + profilerChartAggregateFunction?: string; profilerChartMeasures?: { label: string; field: string }[]; isElectronOnly?: boolean; supportedCreateDatabase?: boolean; diff --git a/packages/web/src/appobj/ArchiveFileAppObject.svelte b/packages/web/src/appobj/ArchiveFileAppObject.svelte index 629782d52..0fcad808c 100644 --- a/packages/web/src/appobj/ArchiveFileAppObject.svelte +++ b/packages/web/src/appobj/ArchiveFileAppObject.svelte @@ -215,7 +215,8 @@ props: { jslid: `archive://${data.folderName}/${data.fileName}`, profilerFormatterFunction: eng.profilerFormatterFunction, - profilerChartFormatterFunction: eng.profilerChartFormatterFunction, + profilerTimestampFunction: eng.profilerTimestampFunction, + profilerChartAggregateFunction: eng.profilerChartAggregateFunction, profilerChartMeasures: eng.profilerChartMeasures, }, }); diff --git a/packages/web/src/tabs/ProfilerTab.svelte b/packages/web/src/tabs/ProfilerTab.svelte index 33ae37bc0..8bd0cb95a 100644 --- a/packages/web/src/tabs/ProfilerTab.svelte +++ b/packages/web/src/tabs/ProfilerTab.svelte @@ -58,7 +58,8 @@ export let database; export let jslid; export let profilerFormatterFunction; - export let profilerChartFormatterFunction; + export let profilerTimestampFunction; + export let profilerChartAggregateFunction; export let profilerChartMeasures; let profiling = false; @@ -128,7 +129,8 @@ const data = await apiCall('jsldata/extract-timeline-chart', { jslid, - formatterFunction: profilerChartFormatterFunction || engine.profilerChartFormatterFunction, + timestampFunction: profilerTimestampFunction || engine.profilerTimestampFunction, + aggregateFunction: profilerChartAggregateFunction || engine.profilerChartAggregateFunction, measures: profilerChartMeasures || engine.profilerChartMeasures, }); chartData = { diff --git a/plugins/dbgate-plugin-mongo/src/backend/index.js b/plugins/dbgate-plugin-mongo/src/backend/index.js index 7b6e0f5d9..2743dd1b3 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/index.js +++ b/plugins/dbgate-plugin-mongo/src/backend/index.js @@ -1,12 +1,16 @@ const driver = require('./driver'); -const formatProfilerEntry = require('../frontend/formatProfilerEntry'); -const formatProfilerChartEntry = require('../frontend/formatProfilerChartEntry'); +const { + formatProfilerEntry, + extractProfileTimestamp, + aggregateProfileChartEntry, +} = require('../frontend/profilerFunctions'); module.exports = { packageName: 'dbgate-plugin-mongo', drivers: [driver], functions: { formatProfilerEntry, - formatProfilerChartEntry, + extractProfileTimestamp, + aggregateProfileChartEntry, }, }; diff --git a/plugins/dbgate-plugin-mongo/src/frontend/driver.js b/plugins/dbgate-plugin-mongo/src/frontend/driver.js index ab8787c20..39f5de887 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/driver.js @@ -35,10 +35,15 @@ const driver = { supportsServerSummary: true, supportsDatabaseProfiler: true, profilerFormatterFunction: 'formatProfilerEntry@dbgate-plugin-mongo', - profilerChartFormatterFunction: 'formatProfilerChartEntry@dbgate-plugin-mongo', + profilerTimestampFunction: 'extractProfileTimestamp@dbgate-plugin-mongo', + profilerChartAggregateFunction: 'aggregateProfileChartEntry@dbgate-plugin-mongo', profilerChartMeasures: [ - { label: 'Req count', field: 'count' }, - { label: 'Duration', field: 'millis' }, + { label: 'Req count/s', field: 'countPerSec' }, + { label: 'Avg duration', field: 'avgDuration' }, + + // { label: 'Req count/s', field: 'countPerSec', perSecond: true }, + // { field: 'countAll', hidden: true }, + // { label: 'Avg duration', field: 'millis', perField: 'countAll' }, ], databaseUrlPlaceholder: 'e.g. mongodb://username:password@mongodb.mydomain.net/dbname', diff --git a/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerChartEntry.js b/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerChartEntry.js deleted file mode 100644 index 8ee5a27a4..000000000 --- a/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerChartEntry.js +++ /dev/null @@ -1,14 +0,0 @@ -const _ = require('lodash'); -const formatProfilerEntry = require('./formatProfilerEntry'); - -function formatProfilerChartEntry(obj) { - const fmt = formatProfilerEntry(obj); - - return { - ts: fmt.ts, - millis: fmt.stats.millis, - count: 1, - }; -} - -module.exports = formatProfilerChartEntry; diff --git a/plugins/dbgate-plugin-mongo/src/frontend/index.js b/plugins/dbgate-plugin-mongo/src/frontend/index.js index c68b8daa5..da0a77580 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/index.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/index.js @@ -1,12 +1,12 @@ import driver from './driver'; -import formatProfilerEntry from './formatProfilerEntry'; -import formatProfilerChartEntry from './formatProfilerChartEntry'; +import { formatProfilerEntry, extractProfileTimestamp, aggregateProfileChartEntry } from './profilerFunctions'; export default { packageName: 'dbgate-plugin-mongo', drivers: [driver], functions: { formatProfilerEntry, - formatProfilerChartEntry, + extractProfileTimestamp, + aggregateProfileChartEntry, }, }; diff --git a/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerEntry.js b/plugins/dbgate-plugin-mongo/src/frontend/profilerFunctions.js similarity index 74% rename from plugins/dbgate-plugin-mongo/src/frontend/formatProfilerEntry.js rename to plugins/dbgate-plugin-mongo/src/frontend/profilerFunctions.js index 9b4e4cedb..fdff1fbe1 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/formatProfilerEntry.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/profilerFunctions.js @@ -69,4 +69,33 @@ function formatProfilerEntry(obj) { }; } -module.exports = formatProfilerEntry; +function extractProfileTimestamp(obj) { + return obj.ts; +} + +function aggregateProfileChartEntry(aggr, obj, stepDuration) { + // const fmt = formatProfilerEntry(obj); + + const countAll = (aggr.countAll || 0) + 1; + const sumMillis = (aggr.sumMillis || 0) + obj.millis; + + return { + countAll, + sumMillis, + countPerSec: (countAll / stepDuration) * 1000, + avgDuration: sumMillis / countAll, + }; + + // return { + // ts: fmt.ts, + // millis: fmt.stats.millis, + // countAll: 1, + // countPerSec: 1, + // }; +} + +module.exports = { + formatProfilerEntry, + extractProfileTimestamp, + aggregateProfileChartEntry, +}; From 3bbdc563095138d9e284447fd5491e9d976a2c36 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 18 Dec 2022 16:18:56 +0100 Subject: [PATCH 09/10] max duration profiler measure --- packages/web/src/appobj/ArchiveFileAppObject.svelte | 1 + packages/web/src/tabs/ProfilerTab.svelte | 12 ++++++------ plugins/dbgate-plugin-mongo/src/frontend/driver.js | 5 +---- .../src/frontend/profilerFunctions.js | 2 ++ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/web/src/appobj/ArchiveFileAppObject.svelte b/packages/web/src/appobj/ArchiveFileAppObject.svelte index 0fcad808c..e77e42e31 100644 --- a/packages/web/src/appobj/ArchiveFileAppObject.svelte +++ b/packages/web/src/appobj/ArchiveFileAppObject.svelte @@ -214,6 +214,7 @@ tabComponent: 'ProfilerTab', props: { jslid: `archive://${data.folderName}/${data.fileName}`, + // engine: eng.engine, profilerFormatterFunction: eng.profilerFormatterFunction, profilerTimestampFunction: eng.profilerTimestampFunction, profilerChartAggregateFunction: eng.profilerChartAggregateFunction, diff --git a/packages/web/src/tabs/ProfilerTab.svelte b/packages/web/src/tabs/ProfilerTab.svelte index 8bd0cb95a..ae691bdb5 100644 --- a/packages/web/src/tabs/ProfilerTab.svelte +++ b/packages/web/src/tabs/ProfilerTab.svelte @@ -70,7 +70,7 @@ let chartData; $: connection = useConnectionInfo({ conid }); - $: engine = findEngineDriver($connection, $extensions); + $: driver = findEngineDriver($connection, $extensions); onMount(() => { intervalId = setInterval(() => { @@ -123,15 +123,15 @@ isLoadingChart = true; const colors = randomcolor({ - count: (profilerChartMeasures || engine.profilerChartMeasures).length, + count: (profilerChartMeasures || driver.profilerChartMeasures).length, seed: 5, }); const data = await apiCall('jsldata/extract-timeline-chart', { jslid, - timestampFunction: profilerTimestampFunction || engine.profilerTimestampFunction, - aggregateFunction: profilerChartAggregateFunction || engine.profilerChartAggregateFunction, - measures: profilerChartMeasures || engine.profilerChartMeasures, + timestampFunction: profilerTimestampFunction || driver.profilerTimestampFunction, + aggregateFunction: profilerChartAggregateFunction || driver.profilerChartAggregateFunction, + measures: profilerChartMeasures || driver.profilerChartMeasures, }); chartData = { ...data, @@ -202,7 +202,7 @@ diff --git a/plugins/dbgate-plugin-mongo/src/frontend/driver.js b/plugins/dbgate-plugin-mongo/src/frontend/driver.js index 39f5de887..931055634 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/driver.js @@ -40,10 +40,7 @@ const driver = { profilerChartMeasures: [ { label: 'Req count/s', field: 'countPerSec' }, { label: 'Avg duration', field: 'avgDuration' }, - - // { label: 'Req count/s', field: 'countPerSec', perSecond: true }, - // { field: 'countAll', hidden: true }, - // { label: 'Avg duration', field: 'millis', perField: 'countAll' }, + { label: 'Max duration', field: 'maxDuration' }, ], databaseUrlPlaceholder: 'e.g. mongodb://username:password@mongodb.mydomain.net/dbname', diff --git a/plugins/dbgate-plugin-mongo/src/frontend/profilerFunctions.js b/plugins/dbgate-plugin-mongo/src/frontend/profilerFunctions.js index fdff1fbe1..8eff17b72 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/profilerFunctions.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/profilerFunctions.js @@ -78,12 +78,14 @@ function aggregateProfileChartEntry(aggr, obj, stepDuration) { const countAll = (aggr.countAll || 0) + 1; const sumMillis = (aggr.sumMillis || 0) + obj.millis; + const maxDuration = obj.millis > (aggr.maxDuration || 0) ? obj.millis : aggr.maxDuration || 0; return { countAll, sumMillis, countPerSec: (countAll / stepDuration) * 1000, avgDuration: sumMillis / countAll, + maxDuration, }; // return { From 0ff4f0d7e9b4836265e3b72c375a1a6d9cf1a25e Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 18 Dec 2022 17:03:47 +0100 Subject: [PATCH 10/10] profile refactoring, fixes --- packages/api/src/controllers/jsldata.js | 13 ++--- .../api/src/utility/JsonLinesDatastore.js | 6 ++- .../src/appobj/ArchiveFileAppObject.svelte | 12 ++--- packages/web/src/tabs/ProfilerTab.svelte | 52 +++++++++---------- 4 files changed, 42 insertions(+), 41 deletions(-) diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index 8b0d29ffa..4778e2f59 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -101,13 +101,14 @@ module.exports = { // }, async ensureDatastore(jslid, formatterFunction) { - const rowFormatter = requirePluginFunction(formatterFunction); - const dskey = `${jslid}||${formatterFunction}`; - let datastore = this.datastores[dskey]; - if (!datastore) { - datastore = new JsonLinesDatastore(getJslFileName(jslid), rowFormatter); + let datastore = this.datastores[jslid]; + if (!datastore || datastore.formatterFunction != formatterFunction) { + if (datastore) { + datastore._closeReader(); + } + datastore = new JsonLinesDatastore(getJslFileName(jslid), formatterFunction); // datastore = new DatastoreProxy(getJslFileName(jslid)); - this.datastores[dskey] = datastore; + this.datastores[jslid] = datastore; } return datastore; }, diff --git a/packages/api/src/utility/JsonLinesDatastore.js b/packages/api/src/utility/JsonLinesDatastore.js index b3a2005d8..a5e12da47 100644 --- a/packages/api/src/utility/JsonLinesDatastore.js +++ b/packages/api/src/utility/JsonLinesDatastore.js @@ -3,6 +3,7 @@ const AsyncLock = require('async-lock'); const lock = new AsyncLock(); const stableStringify = require('json-stable-stringify'); const { evaluateCondition } = require('dbgate-sqltree'); +const requirePluginFunction = require('./requirePluginFunction'); function fetchNextLineFromReader(reader) { return new Promise((resolve, reject) => { @@ -22,15 +23,16 @@ function fetchNextLineFromReader(reader) { } class JsonLinesDatastore { - constructor(file, rowFormatter) { + constructor(file, formatterFunction) { this.file = file; - this.rowFormatter = rowFormatter; + this.formatterFunction = formatterFunction; this.reader = null; this.readedDataRowCount = 0; this.readedSchemaRow = false; // this.firstRowToBeReturned = null; this.notifyChangedCallback = null; this.currentFilter = null; + this.rowFormatter = requirePluginFunction(formatterFunction); } _closeReader() { diff --git a/packages/web/src/appobj/ArchiveFileAppObject.svelte b/packages/web/src/appobj/ArchiveFileAppObject.svelte index e77e42e31..4cecca45d 100644 --- a/packages/web/src/appobj/ArchiveFileAppObject.svelte +++ b/packages/web/src/appobj/ArchiveFileAppObject.svelte @@ -213,12 +213,12 @@ icon: 'img profiler', tabComponent: 'ProfilerTab', props: { - jslid: `archive://${data.folderName}/${data.fileName}`, - // engine: eng.engine, - profilerFormatterFunction: eng.profilerFormatterFunction, - profilerTimestampFunction: eng.profilerTimestampFunction, - profilerChartAggregateFunction: eng.profilerChartAggregateFunction, - profilerChartMeasures: eng.profilerChartMeasures, + jslidLoad: `archive://${data.folderName}/${data.fileName}`, + engine: eng.engine, + // profilerFormatterFunction: eng.profilerFormatterFunction, + // profilerTimestampFunction: eng.profilerTimestampFunction, + // profilerChartAggregateFunction: eng.profilerChartAggregateFunction, + // profilerChartMeasures: eng.profilerChartMeasures, }, }); }, diff --git a/packages/web/src/tabs/ProfilerTab.svelte b/packages/web/src/tabs/ProfilerTab.svelte index ae691bdb5..774f6d4de 100644 --- a/packages/web/src/tabs/ProfilerTab.svelte +++ b/packages/web/src/tabs/ProfilerTab.svelte @@ -56,13 +56,12 @@ export let conid; export let database; - export let jslid; - export let profilerFormatterFunction; - export let profilerTimestampFunction; - export let profilerChartAggregateFunction; - export let profilerChartMeasures; + export let engine; + export let jslidLoad; - let profiling = false; + let jslidSession; + + let isProfiling = false; let sessionId; let isLoadingChart = false; @@ -70,7 +69,8 @@ let chartData; $: connection = useConnectionInfo({ conid }); - $: driver = findEngineDriver($connection, $extensions); + $: driver = findEngineDriver(engine || $connection, $extensions); + $: jslid = jslidSession || jslidLoad; onMount(() => { intervalId = setInterval(() => { @@ -80,22 +80,20 @@ }); } }, 15 * 1000); + }); - if (jslid) { + $: { + if (jslidLoad && driver) { loadChart(); } - }); + } onDestroy(() => { clearInterval(intervalId); }); - export function isProfiling() { - return profiling; - } - export async function startProfiling() { - profiling = true; + isProfiling = true; let sesid = sessionId; if (!sesid) { @@ -110,7 +108,7 @@ const resp = await apiCall('sessions/start-profiler', { sesid, }); - jslid = resp.jslid; + jslidSession = resp.jslid; invalidateCommands(); } @@ -123,15 +121,15 @@ isLoadingChart = true; const colors = randomcolor({ - count: (profilerChartMeasures || driver.profilerChartMeasures).length, + count: driver.profilerChartMeasures.length, seed: 5, }); const data = await apiCall('jsldata/extract-timeline-chart', { jslid, - timestampFunction: profilerTimestampFunction || driver.profilerTimestampFunction, - aggregateFunction: profilerChartAggregateFunction || driver.profilerChartAggregateFunction, - measures: profilerChartMeasures || driver.profilerChartMeasures, + timestampFunction: driver.profilerTimestampFunction, + aggregateFunction: driver.profilerChartAggregateFunction, + measures: driver.profilerChartMeasures, }); chartData = { ...data, @@ -145,8 +143,10 @@ } export async function stopProfiling() { - profiling = false; - apiCall('sessions/stop-profiler', { sesid: sessionId }); + isProfiling = false; + await apiCall('sessions/stop-profiler', { sesid: sessionId }); + await apiCall('sessions/kill', { sesid: sessionId }); + sessionId = null; invalidateCommands(); @@ -158,7 +158,7 @@ } export function saveEnabled() { - return !!jslid; + return !!jslidSession; } async function doSave(folder, file) { @@ -199,11 +199,9 @@ {#if jslid} - + {#key jslid} + + {/key} {#if isLoadingChart}