From 0af38c6e0eb8842201dc2dee57218f4d62b3159c Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 13 May 2025 17:38:56 +0200 Subject: [PATCH] redis load key refactor #1062 --- .../src/controllers/databaseConnections.js | 6 + .../api/src/proc/databaseConnectionProcess.js | 5 + packages/tools/src/dbKeysLoader.ts | 226 +++++++++++++----- packages/types/engines.d.ts | 1 + packages/web/public/global.css | 3 + .../web/src/elements/SearchBoxWrapper.svelte | 10 +- packages/web/src/icons/FontIcon.svelte | 1 + packages/web/src/utility/metadataLoaders.ts | 6 - .../web/src/widgets/DatabaseWidget.svelte | 2 +- packages/web/src/widgets/DbKeysTree.svelte | 37 ++- .../dbgate-plugin-redis/src/backend/driver.js | 16 +- 11 files changed, 226 insertions(+), 87 deletions(-) diff --git a/packages/api/src/controllers/databaseConnections.js b/packages/api/src/controllers/databaseConnections.js index c4f77ce74..9bfe37013 100644 --- a/packages/api/src/controllers/databaseConnections.js +++ b/packages/api/src/controllers/databaseConnections.js @@ -304,6 +304,12 @@ module.exports = { return this.loadDataCore('loadKeys', { conid, database, root, filter, limit }); }, + scanKeys_meta: true, + async scanKeys({ conid, database, root, pattern, cursor, count }, req) { + testConnectionPermission(conid, req); + return this.loadDataCore('scanKeys', { conid, database, root, pattern, cursor, count }); + }, + exportKeys_meta: true, async exportKeys({ conid, database, options }, req) { testConnectionPermission(conid, req); diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index 2bb0c1375..4948eae99 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -275,6 +275,10 @@ async function handleLoadKeys({ msgid, root, filter, limit }) { return handleDriverDataCore(msgid, driver => driver.loadKeys(dbhan, root, filter, limit), { logName: 'loadKeys' }); } +async function handleScanKeys({ msgid, pattern, cursor, count }) { + return handleDriverDataCore(msgid, driver => driver.scanKeys(dbhan, pattern, cursor, count), { logName: 'scanKeys' }); +} + async function handleExportKeys({ msgid, options }) { return handleDriverDataCore(msgid, driver => driver.exportKeys(dbhan, options), { logName: 'exportKeys' }); } @@ -453,6 +457,7 @@ const messageHandlers = { updateCollection: handleUpdateCollection, collectionData: handleCollectionData, loadKeys: handleLoadKeys, + scanKeys: handleScanKeys, loadKeyInfo: handleLoadKeyInfo, callMethod: handleCallMethod, loadKeyTableRange: handleLoadKeyTableRange, diff --git a/packages/tools/src/dbKeysLoader.ts b/packages/tools/src/dbKeysLoader.ts index 42c36923d..76696a99d 100644 --- a/packages/tools/src/dbKeysLoader.ts +++ b/packages/tools/src/dbKeysLoader.ts @@ -4,35 +4,51 @@ const SHOW_INCREMENT = 100; export interface DbKeysNodeModelBase { text?: string; + key: string; count?: number; level: number; + keyPath: string[]; + parentKey: string; } export interface DbKeysLeafNodeModel extends DbKeysNodeModelBase { - key: string; - type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL'; } export interface DbKeysFolderNodeModel extends DbKeysNodeModelBase { - root: string; + // root: string; type: 'dir'; - maxShowCount?: number; + visibleCount?: number; isExpanded?: boolean; - shouldLoadNext?: boolean; - hasNext?: boolean; } export interface DbKeysTreeModel { + treeKeySeparator: string; root: DbKeysFolderNodeModel; dirsByKey: { [key: string]: DbKeysFolderNodeModel }; childrenByKey: { [key: string]: DbKeysNodeModel[] }; - refreshAll?: boolean; + keyObjectsByKey: { [key: string]: DbKeysNodeModel }; + scannedKeys: number; + cursor: string; + loadedAll: false; + // refreshAll?: boolean; } export type DbKeysNodeModel = DbKeysLeafNodeModel | DbKeysFolderNodeModel; -export type DbKeysLoadFunction = (root: string, limit: number) => Promise; +export interface DbKeyLoadedModel { + key: string; + + type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL'; + count?: number; +} + +export interface DbKeysLoadResult { + nextCursor: string; + keys: DbKeyLoadedModel[]; +} + +export type DbKeysLoadFunction = (root: string, limit: number) => Promise; export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKeysTreeModel) => void; @@ -73,54 +89,129 @@ export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKey // }; // } -export async function dbKeys_loadMissing(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise { - const childrenByKey = { ...tree.childrenByKey }; - const dirsByKey = { ...tree.dirsByKey }; +// export async function dbKeys_loadMissing(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise { +// const childrenByKey = { ...tree.childrenByKey }; +// const dirsByKey = { ...tree.dirsByKey }; - for (const root in tree.dirsByKey) { - const dir = tree.dirsByKey[root]; +// for (const root in tree.dirsByKey) { +// const dir = tree.dirsByKey[root]; - if (dir.isExpanded && dir.shouldLoadNext) { - if (!tree.childrenByKey[root] || dir.hasNext) { - const loadCount = dir.maxShowCount && dir.shouldLoadNext ? dir.maxShowCount + SHOW_INCREMENT : SHOW_INCREMENT; - const items = await loader(root, loadCount + 1); +// if (dir.isExpanded && dir.shouldLoadNext) { +// if (!tree.childrenByKey[root] || dir.hasNext) { +// const loadCount = dir.maxShowCount && dir.shouldLoadNext ? dir.maxShowCount + SHOW_INCREMENT : SHOW_INCREMENT; +// const items = await loader(root, loadCount + 1); - childrenByKey[root] = items.slice(0, loadCount); - dirsByKey[root] = { - ...dir, - shouldLoadNext: false, - maxShowCount: loadCount, - hasNext: items.length > loadCount, - }; +// childrenByKey[root] = items.slice(0, loadCount); +// dirsByKey[root] = { +// ...dir, +// shouldLoadNext: false, +// maxShowCount: loadCount, +// hasNext: items.length > loadCount, +// }; - for (const child of items.slice(0, loadCount)) { - if (child.type == 'dir' && !dirsByKey[child.root]) { - dirsByKey[child.root] = { - shouldLoadNext: false, - maxShowCount: null, - hasNext: false, - isExpanded: false, - type: 'dir', - level: dir.level + 1, - root: child.root, - text: child.text, - }; - } - } - } else { - dirsByKey[root] = { - ...dir, - shouldLoadNext: false, +// for (const child of items.slice(0, loadCount)) { +// if (child.type == 'dir' && !dirsByKey[child.root]) { +// dirsByKey[child.root] = { +// shouldLoadNext: false, +// maxShowCount: null, +// hasNext: false, +// isExpanded: false, +// type: 'dir', +// level: dir.level + 1, +// root: child.root, +// text: child.text, +// }; +// } +// } +// } else { +// dirsByKey[root] = { +// ...dir, +// shouldLoadNext: false, +// }; +// } +// } +// } + +// return { +// ...tree, +// dirsByKey, +// childrenByKey, +// refreshAll: false, +// }; +// } + +export async function dbKeys_loadNext(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise { + const count = 2000; + const keyObjectsByKey = { ...tree.keyObjectsByKey }; + + const loaded = await loader(tree.cursor, count); + + for (const keyObj of loaded.keys) { + const keyPath = keyObj.key.split(tree.treeKeySeparator); + keyObjectsByKey[keyObj.key] = { + ...keyObj, + level: keyPath.length, + text: keyPath[keyPath.length - 1], + keyPath, + parentKey: keyPath.slice(0, -1).join(tree.treeKeySeparator), + }; + } + + const dirsByKey: { [key: string]: DbKeysFolderNodeModel } = {}; + const childrenByKey: { [key: string]: DbKeysNodeModel[] } = {}; + + dirsByKey[''] = tree.root; + + for (const keyObj of Object.values(keyObjectsByKey)) { + const dirPath = keyObj.keyPath.slice(0, -1); + const dirKey = dirPath.join(tree.treeKeySeparator); + + let dirDepth = keyObj.keyPath.length - 1; + + while (dirDepth > 0) { + const newDirPath = keyObj.keyPath.slice(0, dirDepth); + const newDirKey = newDirPath.join(tree.treeKeySeparator); + if (!dirsByKey[newDirKey]) { + dirsByKey[newDirKey] = { + isExpanded: tree.dirsByKey[newDirKey]?.isExpanded ?? false, + level: keyObj.level - 1, + keyPath: newDirPath, + parentKey: newDirPath.slice(0, -1).join(tree.treeKeySeparator), + type: 'dir', + key: newDirKey, + visibleCount: tree.dirsByKey[newDirKey]?.visibleCount ?? SHOW_INCREMENT, + text: `${newDirPath[newDirPath.length - 1]}${tree.treeKeySeparator}*`, }; } + + dirDepth -= 1; } + + if (!childrenByKey[dirKey]) { + childrenByKey[dirKey] = []; + } + + childrenByKey[dirKey].push(keyObj); + } + + for (const dirObj of Object.values(dirsByKey)) { + if (dirObj.key == '') { + continue; + } + + if (!childrenByKey[dirObj.parentKey]) { + childrenByKey[dirObj.parentKey] = []; + } + childrenByKey[dirObj.parentKey].push(dirObj); } return { ...tree, + cursor: loaded.nextCursor, dirsByKey, childrenByKey, - refreshAll: false, + keyObjectsByKey, + scannedKeys: tree.scannedKeys + count, }; } @@ -136,45 +227,50 @@ export function dbKeys_markNodeExpanded(tree: DbKeysTreeModel, root: string, isE [root]: { ...node, isExpanded, - shouldLoadNext: isExpanded, }, }, }; } -export function dbKeys_refreshAll(tree?: DbKeysTreeModel): DbKeysTreeModel { +export function dbKeys_refreshAll(treeKeySeparator: string, tree?: DbKeysTreeModel): DbKeysTreeModel { const root: DbKeysFolderNodeModel = { isExpanded: true, level: 0, - root: '', type: 'dir', - shouldLoadNext: true, + keyPath: [], + parentKey: '', + key: '', + visibleCount: SHOW_INCREMENT, }; return { ...tree, + treeKeySeparator, childrenByKey: {}, + keyObjectsByKey: {}, dirsByKey: { '': root, }, - refreshAll: true, + scannedKeys: 0, + cursor: '0', root, + loadedAll: false, }; } -export function dbKeys_reloadFolder(tree: DbKeysTreeModel, root: string): DbKeysTreeModel { - return { - ...tree, - childrenByKey: _omit(tree.childrenByKey, root), - dirsByKey: { - ...tree.dirsByKey, - [root]: { - ...tree.dirsByKey[root], - shouldLoadNext: true, - hasNext: undefined, - }, - }, - }; -} +// export function dbKeys_reloadFolder(tree: DbKeysTreeModel, root: string): DbKeysTreeModel { +// return { +// ...tree, +// childrenByKey: _omit(tree.childrenByKey, root), +// dirsByKey: { +// ...tree.dirsByKey, +// [root]: { +// ...tree.dirsByKey[root], +// shouldLoadNext: true, +// hasNext: undefined, +// }, +// }, +// }; +// } function addFlatItems(tree: DbKeysTreeModel, root: string, res: DbKeysNodeModel[], visitedRoots: string[] = []) { const item = tree.dirsByKey[root]; @@ -185,11 +281,11 @@ function addFlatItems(tree: DbKeysTreeModel, root: string, res: DbKeysNodeModel[ for (const child of children) { res.push(child); if (child.type == 'dir') { - if (visitedRoots.includes(child.root)) { - console.warn('Redis: preventing infinite loop for root', child.root); + if (visitedRoots.includes(child.key)) { + console.warn('Redis: preventing infinite loop for root', child.key); return false; } - addFlatItems(tree, child.root, res, [...visitedRoots, root]); + addFlatItems(tree, child.key, res, [...visitedRoots, root]); } } } diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index af5fd29df..9a99625a1 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -239,6 +239,7 @@ export interface EngineDriver extends FilterBehaviourProvider { }[] >; loadKeys(dbhan: DatabaseHandle, root: string, filter?: string): Promise; + scanKeys(dbhan: DatabaseHandle, root: string, pattern: string, cursor: string, count: number): Promise; exportKeys(dbhan: DatabaseHandle, options: {}): Promise; loadKeyInfo(dbhan: DatabaseHandle, key): Promise; loadKeyTableRange(dbhan: DatabaseHandle, key, cursor, count): Promise; diff --git a/packages/web/public/global.css b/packages/web/public/global.css index d39a035f9..2977bfccf 100644 --- a/packages/web/public/global.css +++ b/packages/web/public/global.css @@ -36,6 +36,9 @@ body { display: flex; justify-content: space-between; } +.align-items-center { + align-items: center; +} .flex { display: flex; } diff --git a/packages/web/src/elements/SearchBoxWrapper.svelte b/packages/web/src/elements/SearchBoxWrapper.svelte index d71019413..b6c31ff96 100644 --- a/packages/web/src/elements/SearchBoxWrapper.svelte +++ b/packages/web/src/elements/SearchBoxWrapper.svelte @@ -1,4 +1,8 @@ -
+ + +
diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index bd9d0df51..86bee43eb 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -151,6 +151,7 @@ 'icon text': 'mdi mdi-text', 'icon ai': 'mdi mdi-head-lightbulb', 'icon wait': 'mdi mdi-timer-sand', + 'icon more': 'mdi mdi-more', 'icon run': 'mdi mdi-play', 'icon chevron-down': 'mdi mdi-chevron-down', diff --git a/packages/web/src/utility/metadataLoaders.ts b/packages/web/src/utility/metadataLoaders.ts index 67f9d7465..2fc374a02 100644 --- a/packages/web/src/utility/metadataLoaders.ts +++ b/packages/web/src/utility/metadataLoaders.ts @@ -83,12 +83,6 @@ const databaseListLoader = ({ conid }) => ({ errorValue: [], }); -// const databaseKeysLoader = ({ conid, database, root }) => ({ -// url: 'database-connections/load-keys', -// params: { conid, database, root }, -// reloadTrigger: `database-keys-changed-${conid}-${database}`, -// }); - const serverVersionLoader = ({ conid }) => ({ url: 'server-connections/version', params: { conid }, diff --git a/packages/web/src/widgets/DatabaseWidget.svelte b/packages/web/src/widgets/DatabaseWidget.svelte index 7a295761e..7e525c090 100644 --- a/packages/web/src/widgets/DatabaseWidget.svelte +++ b/packages/web/src/widgets/DatabaseWidget.svelte @@ -80,7 +80,7 @@ storageName="dbObjectsWidget" skip={!(conid && (database || singleDatabase) && driver?.databaseEngineTypes?.includes('keyvalue'))} > - + import { dbKeys_getFlatList, - dbKeys_loadMissing, + dbKeys_loadNext, dbKeys_markNodeExpanded, dbKeys_refreshAll, findEngineDriver, @@ -36,6 +36,7 @@ export let conid; export let database; + export let treeKeySeparator = ':'; let domListHandler; let domContainer = null; @@ -43,10 +44,10 @@ let filter; - let model = dbKeys_refreshAll(); + let model = dbKeys_refreshAll(treeKeySeparator); function handleRefreshDatabase() { - changeModel(model => dbKeys_refreshAll(model)); + changeModel(model => dbKeys_refreshAll(treeKeySeparator, model)); } function handleAddKey() { @@ -80,22 +81,26 @@ $: connection = useConnectionInfo({ conid }); - async function changeModel(modelUpdate) { + function changeModel(modelUpdate) { model = modelUpdate(model); - model = await dbKeys_loadMissing(model, async (root, limit) => { - const result = await apiCall('database-connections/load-keys', { + } + + async function loadNextPage() { + model = await dbKeys_loadNext(model, async (cursor, count) => { + const result = await apiCall('database-connections/scan-keys', { conid, database, - root, - filter, - limit, + pattern: filter, + cursor, + count, }); return result; }); } function reloadModel() { - changeModel(model => dbKeys_refreshAll(model)); + changeModel(model => dbKeys_refreshAll(treeKeySeparator, model)); + loadNextPage(); } $: { @@ -104,11 +109,13 @@ filter; reloadModel(); } + + $: console.log('DbKeysTree MODEL', model); - + +
+
Scanned 10/20 keys
+ + Scan more + +
{#if differentFocusedDb} {/if} diff --git a/plugins/dbgate-plugin-redis/src/backend/driver.js b/plugins/dbgate-plugin-redis/src/backend/driver.js index b1a7d1b98..b2baf6eb5 100644 --- a/plugins/dbgate-plugin-redis/src/backend/driver.js +++ b/plugins/dbgate-plugin-redis/src/backend/driver.js @@ -201,6 +201,18 @@ const driver = { return _.range(16).map((index) => ({ name: `db${index}`, extInfo: info[`db${index}`], sortOrder: index })); }, + async scanKeys(dbhan, pattern, cursor = 0, count) { + const [nextCursor, keys] = await dbhan.client.scan(cursor, 'MATCH', pattern || '*', 'COUNT', count); + const keysMapped = keys.map((key) => ({ + key, + })); + await this.enrichKeyInfo(dbhan, keysMapped); + return { + nextCursor, + keys: keysMapped, + }; + }, + async loadKeys(dbhan, root = '', filter = null, limit = null) { const keys = await this.getKeys(dbhan, root ? `${root}${dbhan.treeKeySeparator}*` : '*'); const keysFiltered = keys.filter((x) => filterName(filter, x)); @@ -310,9 +322,9 @@ const driver = { item.count = await this.getKeyCardinality(dbhan, item.key, item.type); }, - async enrichKeyInfo(dbhan, levelInfo) { + async enrichKeyInfo(dbhan, keyObjects) { await async.eachLimit( - levelInfo.filter((x) => x.key), + keyObjects.filter((x) => x.key), 10, async (item) => await this.enrichOneKeyInfo(dbhan, item) );