From 0af38c6e0eb8842201dc2dee57218f4d62b3159c Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 13 May 2025 17:38:56 +0200 Subject: [PATCH 1/6] redis load key refactor #1062 --- .../src/controllers/databaseConnections.js | 6 + .../api/src/proc/databaseConnectionProcess.js | 5 + packages/tools/src/dbKeysLoader.ts | 226 +++++++++++++----- packages/types/engines.d.ts | 1 + packages/web/public/global.css | 3 + .../web/src/elements/SearchBoxWrapper.svelte | 10 +- packages/web/src/icons/FontIcon.svelte | 1 + packages/web/src/utility/metadataLoaders.ts | 6 - .../web/src/widgets/DatabaseWidget.svelte | 2 +- packages/web/src/widgets/DbKeysTree.svelte | 37 ++- .../dbgate-plugin-redis/src/backend/driver.js | 16 +- 11 files changed, 226 insertions(+), 87 deletions(-) diff --git a/packages/api/src/controllers/databaseConnections.js b/packages/api/src/controllers/databaseConnections.js index c4f77ce74..9bfe37013 100644 --- a/packages/api/src/controllers/databaseConnections.js +++ b/packages/api/src/controllers/databaseConnections.js @@ -304,6 +304,12 @@ module.exports = { return this.loadDataCore('loadKeys', { conid, database, root, filter, limit }); }, + scanKeys_meta: true, + async scanKeys({ conid, database, root, pattern, cursor, count }, req) { + testConnectionPermission(conid, req); + return this.loadDataCore('scanKeys', { conid, database, root, pattern, cursor, count }); + }, + exportKeys_meta: true, async exportKeys({ conid, database, options }, req) { testConnectionPermission(conid, req); diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index 2bb0c1375..4948eae99 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -275,6 +275,10 @@ async function handleLoadKeys({ msgid, root, filter, limit }) { return handleDriverDataCore(msgid, driver => driver.loadKeys(dbhan, root, filter, limit), { logName: 'loadKeys' }); } +async function handleScanKeys({ msgid, pattern, cursor, count }) { + return handleDriverDataCore(msgid, driver => driver.scanKeys(dbhan, pattern, cursor, count), { logName: 'scanKeys' }); +} + async function handleExportKeys({ msgid, options }) { return handleDriverDataCore(msgid, driver => driver.exportKeys(dbhan, options), { logName: 'exportKeys' }); } @@ -453,6 +457,7 @@ const messageHandlers = { updateCollection: handleUpdateCollection, collectionData: handleCollectionData, loadKeys: handleLoadKeys, + scanKeys: handleScanKeys, loadKeyInfo: handleLoadKeyInfo, callMethod: handleCallMethod, loadKeyTableRange: handleLoadKeyTableRange, diff --git a/packages/tools/src/dbKeysLoader.ts b/packages/tools/src/dbKeysLoader.ts index 42c36923d..76696a99d 100644 --- a/packages/tools/src/dbKeysLoader.ts +++ b/packages/tools/src/dbKeysLoader.ts @@ -4,35 +4,51 @@ const SHOW_INCREMENT = 100; export interface DbKeysNodeModelBase { text?: string; + key: string; count?: number; level: number; + keyPath: string[]; + parentKey: string; } export interface DbKeysLeafNodeModel extends DbKeysNodeModelBase { - key: string; - type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL'; } export interface DbKeysFolderNodeModel extends DbKeysNodeModelBase { - root: string; + // root: string; type: 'dir'; - maxShowCount?: number; + visibleCount?: number; isExpanded?: boolean; - shouldLoadNext?: boolean; - hasNext?: boolean; } export interface DbKeysTreeModel { + treeKeySeparator: string; root: DbKeysFolderNodeModel; dirsByKey: { [key: string]: DbKeysFolderNodeModel }; childrenByKey: { [key: string]: DbKeysNodeModel[] }; - refreshAll?: boolean; + keyObjectsByKey: { [key: string]: DbKeysNodeModel }; + scannedKeys: number; + cursor: string; + loadedAll: false; + // refreshAll?: boolean; } export type DbKeysNodeModel = DbKeysLeafNodeModel | DbKeysFolderNodeModel; -export type DbKeysLoadFunction = (root: string, limit: number) => Promise; +export interface DbKeyLoadedModel { + key: string; + + type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL'; + count?: number; +} + +export interface DbKeysLoadResult { + nextCursor: string; + keys: DbKeyLoadedModel[]; +} + +export type DbKeysLoadFunction = (root: string, limit: number) => Promise; export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKeysTreeModel) => void; @@ -73,54 +89,129 @@ export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKey // }; // } -export async function dbKeys_loadMissing(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise { - const childrenByKey = { ...tree.childrenByKey }; - const dirsByKey = { ...tree.dirsByKey }; +// export async function dbKeys_loadMissing(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise { +// const childrenByKey = { ...tree.childrenByKey }; +// const dirsByKey = { ...tree.dirsByKey }; - for (const root in tree.dirsByKey) { - const dir = tree.dirsByKey[root]; +// for (const root in tree.dirsByKey) { +// const dir = tree.dirsByKey[root]; - if (dir.isExpanded && dir.shouldLoadNext) { - if (!tree.childrenByKey[root] || dir.hasNext) { - const loadCount = dir.maxShowCount && dir.shouldLoadNext ? dir.maxShowCount + SHOW_INCREMENT : SHOW_INCREMENT; - const items = await loader(root, loadCount + 1); +// if (dir.isExpanded && dir.shouldLoadNext) { +// if (!tree.childrenByKey[root] || dir.hasNext) { +// const loadCount = dir.maxShowCount && dir.shouldLoadNext ? dir.maxShowCount + SHOW_INCREMENT : SHOW_INCREMENT; +// const items = await loader(root, loadCount + 1); - childrenByKey[root] = items.slice(0, loadCount); - dirsByKey[root] = { - ...dir, - shouldLoadNext: false, - maxShowCount: loadCount, - hasNext: items.length > loadCount, - }; +// childrenByKey[root] = items.slice(0, loadCount); +// dirsByKey[root] = { +// ...dir, +// shouldLoadNext: false, +// maxShowCount: loadCount, +// hasNext: items.length > loadCount, +// }; - for (const child of items.slice(0, loadCount)) { - if (child.type == 'dir' && !dirsByKey[child.root]) { - dirsByKey[child.root] = { - shouldLoadNext: false, - maxShowCount: null, - hasNext: false, - isExpanded: false, - type: 'dir', - level: dir.level + 1, - root: child.root, - text: child.text, - }; - } - } - } else { - dirsByKey[root] = { - ...dir, - shouldLoadNext: false, +// for (const child of items.slice(0, loadCount)) { +// if (child.type == 'dir' && !dirsByKey[child.root]) { +// dirsByKey[child.root] = { +// shouldLoadNext: false, +// maxShowCount: null, +// hasNext: false, +// isExpanded: false, +// type: 'dir', +// level: dir.level + 1, +// root: child.root, +// text: child.text, +// }; +// } +// } +// } else { +// dirsByKey[root] = { +// ...dir, +// shouldLoadNext: false, +// }; +// } +// } +// } + +// return { +// ...tree, +// dirsByKey, +// childrenByKey, +// refreshAll: false, +// }; +// } + +export async function dbKeys_loadNext(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise { + const count = 2000; + const keyObjectsByKey = { ...tree.keyObjectsByKey }; + + const loaded = await loader(tree.cursor, count); + + for (const keyObj of loaded.keys) { + const keyPath = keyObj.key.split(tree.treeKeySeparator); + keyObjectsByKey[keyObj.key] = { + ...keyObj, + level: keyPath.length, + text: keyPath[keyPath.length - 1], + keyPath, + parentKey: keyPath.slice(0, -1).join(tree.treeKeySeparator), + }; + } + + const dirsByKey: { [key: string]: DbKeysFolderNodeModel } = {}; + const childrenByKey: { [key: string]: DbKeysNodeModel[] } = {}; + + dirsByKey[''] = tree.root; + + for (const keyObj of Object.values(keyObjectsByKey)) { + const dirPath = keyObj.keyPath.slice(0, -1); + const dirKey = dirPath.join(tree.treeKeySeparator); + + let dirDepth = keyObj.keyPath.length - 1; + + while (dirDepth > 0) { + const newDirPath = keyObj.keyPath.slice(0, dirDepth); + const newDirKey = newDirPath.join(tree.treeKeySeparator); + if (!dirsByKey[newDirKey]) { + dirsByKey[newDirKey] = { + isExpanded: tree.dirsByKey[newDirKey]?.isExpanded ?? false, + level: keyObj.level - 1, + keyPath: newDirPath, + parentKey: newDirPath.slice(0, -1).join(tree.treeKeySeparator), + type: 'dir', + key: newDirKey, + visibleCount: tree.dirsByKey[newDirKey]?.visibleCount ?? SHOW_INCREMENT, + text: `${newDirPath[newDirPath.length - 1]}${tree.treeKeySeparator}*`, }; } + + dirDepth -= 1; } + + if (!childrenByKey[dirKey]) { + childrenByKey[dirKey] = []; + } + + childrenByKey[dirKey].push(keyObj); + } + + for (const dirObj of Object.values(dirsByKey)) { + if (dirObj.key == '') { + continue; + } + + if (!childrenByKey[dirObj.parentKey]) { + childrenByKey[dirObj.parentKey] = []; + } + childrenByKey[dirObj.parentKey].push(dirObj); } return { ...tree, + cursor: loaded.nextCursor, dirsByKey, childrenByKey, - refreshAll: false, + keyObjectsByKey, + scannedKeys: tree.scannedKeys + count, }; } @@ -136,45 +227,50 @@ export function dbKeys_markNodeExpanded(tree: DbKeysTreeModel, root: string, isE [root]: { ...node, isExpanded, - shouldLoadNext: isExpanded, }, }, }; } -export function dbKeys_refreshAll(tree?: DbKeysTreeModel): DbKeysTreeModel { +export function dbKeys_refreshAll(treeKeySeparator: string, tree?: DbKeysTreeModel): DbKeysTreeModel { const root: DbKeysFolderNodeModel = { isExpanded: true, level: 0, - root: '', type: 'dir', - shouldLoadNext: true, + keyPath: [], + parentKey: '', + key: '', + visibleCount: SHOW_INCREMENT, }; return { ...tree, + treeKeySeparator, childrenByKey: {}, + keyObjectsByKey: {}, dirsByKey: { '': root, }, - refreshAll: true, + scannedKeys: 0, + cursor: '0', root, + loadedAll: false, }; } -export function dbKeys_reloadFolder(tree: DbKeysTreeModel, root: string): DbKeysTreeModel { - return { - ...tree, - childrenByKey: _omit(tree.childrenByKey, root), - dirsByKey: { - ...tree.dirsByKey, - [root]: { - ...tree.dirsByKey[root], - shouldLoadNext: true, - hasNext: undefined, - }, - }, - }; -} +// export function dbKeys_reloadFolder(tree: DbKeysTreeModel, root: string): DbKeysTreeModel { +// return { +// ...tree, +// childrenByKey: _omit(tree.childrenByKey, root), +// dirsByKey: { +// ...tree.dirsByKey, +// [root]: { +// ...tree.dirsByKey[root], +// shouldLoadNext: true, +// hasNext: undefined, +// }, +// }, +// }; +// } function addFlatItems(tree: DbKeysTreeModel, root: string, res: DbKeysNodeModel[], visitedRoots: string[] = []) { const item = tree.dirsByKey[root]; @@ -185,11 +281,11 @@ function addFlatItems(tree: DbKeysTreeModel, root: string, res: DbKeysNodeModel[ for (const child of children) { res.push(child); if (child.type == 'dir') { - if (visitedRoots.includes(child.root)) { - console.warn('Redis: preventing infinite loop for root', child.root); + if (visitedRoots.includes(child.key)) { + console.warn('Redis: preventing infinite loop for root', child.key); return false; } - addFlatItems(tree, child.root, res, [...visitedRoots, root]); + addFlatItems(tree, child.key, res, [...visitedRoots, root]); } } } diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index af5fd29df..9a99625a1 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -239,6 +239,7 @@ export interface EngineDriver extends FilterBehaviourProvider { }[] >; loadKeys(dbhan: DatabaseHandle, root: string, filter?: string): Promise; + scanKeys(dbhan: DatabaseHandle, root: string, pattern: string, cursor: string, count: number): Promise; exportKeys(dbhan: DatabaseHandle, options: {}): Promise; loadKeyInfo(dbhan: DatabaseHandle, key): Promise; loadKeyTableRange(dbhan: DatabaseHandle, key, cursor, count): Promise; diff --git a/packages/web/public/global.css b/packages/web/public/global.css index d39a035f9..2977bfccf 100644 --- a/packages/web/public/global.css +++ b/packages/web/public/global.css @@ -36,6 +36,9 @@ body { display: flex; justify-content: space-between; } +.align-items-center { + align-items: center; +} .flex { display: flex; } diff --git a/packages/web/src/elements/SearchBoxWrapper.svelte b/packages/web/src/elements/SearchBoxWrapper.svelte index d71019413..b6c31ff96 100644 --- a/packages/web/src/elements/SearchBoxWrapper.svelte +++ b/packages/web/src/elements/SearchBoxWrapper.svelte @@ -1,4 +1,8 @@ -
+ + +
diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index bd9d0df51..86bee43eb 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -151,6 +151,7 @@ 'icon text': 'mdi mdi-text', 'icon ai': 'mdi mdi-head-lightbulb', 'icon wait': 'mdi mdi-timer-sand', + 'icon more': 'mdi mdi-more', 'icon run': 'mdi mdi-play', 'icon chevron-down': 'mdi mdi-chevron-down', diff --git a/packages/web/src/utility/metadataLoaders.ts b/packages/web/src/utility/metadataLoaders.ts index 67f9d7465..2fc374a02 100644 --- a/packages/web/src/utility/metadataLoaders.ts +++ b/packages/web/src/utility/metadataLoaders.ts @@ -83,12 +83,6 @@ const databaseListLoader = ({ conid }) => ({ errorValue: [], }); -// const databaseKeysLoader = ({ conid, database, root }) => ({ -// url: 'database-connections/load-keys', -// params: { conid, database, root }, -// reloadTrigger: `database-keys-changed-${conid}-${database}`, -// }); - const serverVersionLoader = ({ conid }) => ({ url: 'server-connections/version', params: { conid }, diff --git a/packages/web/src/widgets/DatabaseWidget.svelte b/packages/web/src/widgets/DatabaseWidget.svelte index 7a295761e..7e525c090 100644 --- a/packages/web/src/widgets/DatabaseWidget.svelte +++ b/packages/web/src/widgets/DatabaseWidget.svelte @@ -80,7 +80,7 @@ storageName="dbObjectsWidget" skip={!(conid && (database || singleDatabase) && driver?.databaseEngineTypes?.includes('keyvalue'))} > - + import { dbKeys_getFlatList, - dbKeys_loadMissing, + dbKeys_loadNext, dbKeys_markNodeExpanded, dbKeys_refreshAll, findEngineDriver, @@ -36,6 +36,7 @@ export let conid; export let database; + export let treeKeySeparator = ':'; let domListHandler; let domContainer = null; @@ -43,10 +44,10 @@ let filter; - let model = dbKeys_refreshAll(); + let model = dbKeys_refreshAll(treeKeySeparator); function handleRefreshDatabase() { - changeModel(model => dbKeys_refreshAll(model)); + changeModel(model => dbKeys_refreshAll(treeKeySeparator, model)); } function handleAddKey() { @@ -80,22 +81,26 @@ $: connection = useConnectionInfo({ conid }); - async function changeModel(modelUpdate) { + function changeModel(modelUpdate) { model = modelUpdate(model); - model = await dbKeys_loadMissing(model, async (root, limit) => { - const result = await apiCall('database-connections/load-keys', { + } + + async function loadNextPage() { + model = await dbKeys_loadNext(model, async (cursor, count) => { + const result = await apiCall('database-connections/scan-keys', { conid, database, - root, - filter, - limit, + pattern: filter, + cursor, + count, }); return result; }); } function reloadModel() { - changeModel(model => dbKeys_refreshAll(model)); + changeModel(model => dbKeys_refreshAll(treeKeySeparator, model)); + loadNextPage(); } $: { @@ -104,11 +109,13 @@ filter; reloadModel(); } + + $: console.log('DbKeysTree MODEL', model); - + +
+
Scanned 10/20 keys
+ + Scan more + +
{#if differentFocusedDb} {/if} diff --git a/plugins/dbgate-plugin-redis/src/backend/driver.js b/plugins/dbgate-plugin-redis/src/backend/driver.js index b1a7d1b98..b2baf6eb5 100644 --- a/plugins/dbgate-plugin-redis/src/backend/driver.js +++ b/plugins/dbgate-plugin-redis/src/backend/driver.js @@ -201,6 +201,18 @@ const driver = { return _.range(16).map((index) => ({ name: `db${index}`, extInfo: info[`db${index}`], sortOrder: index })); }, + async scanKeys(dbhan, pattern, cursor = 0, count) { + const [nextCursor, keys] = await dbhan.client.scan(cursor, 'MATCH', pattern || '*', 'COUNT', count); + const keysMapped = keys.map((key) => ({ + key, + })); + await this.enrichKeyInfo(dbhan, keysMapped); + return { + nextCursor, + keys: keysMapped, + }; + }, + async loadKeys(dbhan, root = '', filter = null, limit = null) { const keys = await this.getKeys(dbhan, root ? `${root}${dbhan.treeKeySeparator}*` : '*'); const keysFiltered = keys.filter((x) => filterName(filter, x)); @@ -310,9 +322,9 @@ const driver = { item.count = await this.getKeyCardinality(dbhan, item.key, item.type); }, - async enrichKeyInfo(dbhan, levelInfo) { + async enrichKeyInfo(dbhan, keyObjects) { await async.eachLimit( - levelInfo.filter((x) => x.key), + keyObjects.filter((x) => x.key), 10, async (item) => await this.enrichOneKeyInfo(dbhan, item) ); From b16b02c3f188c4049871bc94b21a9852eb3a58c9 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Wed, 14 May 2025 10:32:42 +0200 Subject: [PATCH 2/6] tree loader --- packages/tools/src/dbKeysLoader.ts | 29 +++++++++------ packages/web/src/widgets/DbKeysSubTree.svelte | 14 ++++---- packages/web/src/widgets/DbKeysTree.svelte | 35 ++++++++++--------- .../web/src/widgets/DbKeysTreeNode.svelte | 18 +++++----- .../dbgate-plugin-redis/src/backend/driver.js | 2 ++ 5 files changed, 56 insertions(+), 42 deletions(-) diff --git a/packages/tools/src/dbKeysLoader.ts b/packages/tools/src/dbKeysLoader.ts index 76696a99d..4aa5c711b 100644 --- a/packages/tools/src/dbKeysLoader.ts +++ b/packages/tools/src/dbKeysLoader.ts @@ -1,4 +1,5 @@ import _omit from 'lodash/omit'; +import _sortBy from 'lodash/sortBy'; const SHOW_INCREMENT = 100; @@ -29,8 +30,10 @@ export interface DbKeysTreeModel { childrenByKey: { [key: string]: DbKeysNodeModel[] }; keyObjectsByKey: { [key: string]: DbKeysNodeModel }; scannedKeys: number; + loadCount: number; + dbsize: number; cursor: string; - loadedAll: false; + loadedAll: boolean; // refreshAll?: boolean; } @@ -46,9 +49,10 @@ export interface DbKeyLoadedModel { export interface DbKeysLoadResult { nextCursor: string; keys: DbKeyLoadedModel[]; + dbsize: number; } -export type DbKeysLoadFunction = (root: string, limit: number) => Promise; +// export type DbKeysLoadFunction = (root: string, limit: number) => Promise; export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKeysTreeModel) => void; @@ -140,13 +144,10 @@ export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKey // }; // } -export async function dbKeys_loadNext(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise { - const count = 2000; +export function dbKeys_mergeNextPage(tree: DbKeysTreeModel, nextPage: DbKeysLoadResult): DbKeysTreeModel { const keyObjectsByKey = { ...tree.keyObjectsByKey }; - const loaded = await loader(tree.cursor, count); - - for (const keyObj of loaded.keys) { + for (const keyObj of nextPage.keys) { const keyPath = keyObj.key.split(tree.treeKeySeparator); keyObjectsByKey[keyObj.key] = { ...keyObj, @@ -198,20 +199,26 @@ export async function dbKeys_loadNext(tree: DbKeysTreeModel, loader: DbKeysLoadF if (dirObj.key == '') { continue; } - + if (!childrenByKey[dirObj.parentKey]) { childrenByKey[dirObj.parentKey] = []; } childrenByKey[dirObj.parentKey].push(dirObj); } + for (const key in childrenByKey) { + childrenByKey[key] = _sortBy(childrenByKey[key], 'text'); + } + return { ...tree, - cursor: loaded.nextCursor, + cursor: nextPage.nextCursor, dirsByKey, childrenByKey, keyObjectsByKey, - scannedKeys: tree.scannedKeys + count, + scannedKeys: tree.scannedKeys + tree.loadCount, + loadedAll: nextPage.nextCursor == '0', + dbsize: nextPage.dbsize, }; } @@ -251,6 +258,8 @@ export function dbKeys_refreshAll(treeKeySeparator: string, tree?: DbKeysTreeMod '': root, }, scannedKeys: 0, + dbsize: 0, + loadCount: 2000, cursor: '0', root, loadedAll: false, diff --git a/packages/web/src/widgets/DbKeysSubTree.svelte b/packages/web/src/widgets/DbKeysSubTree.svelte index b8e026839..6a1f3959b 100644 --- a/packages/web/src/widgets/DbKeysSubTree.svelte +++ b/packages/web/src/widgets/DbKeysSubTree.svelte @@ -6,7 +6,7 @@ import DbKeysTreeNode from './DbKeysTreeNode.svelte'; import { dbKeys_markNodeExpanded, DbKeysChangeModelFunction, DbKeysTreeModel } from 'dbgate-tools'; - export let root; + export let key; export let connection; export let database; export let conid; @@ -19,34 +19,34 @@ export let parentRoots = []; - $: items = model.childrenByKey[root] ?? []; + $: items = model.childrenByKey[key] ?? []; {#each items as item} {/each} -{#if model.dirsByKey[root]?.shouldLoadNext} +{#if model.dirsByKey[key]?.shouldLoadNext} -{:else if model.dirsByKey[root]?.hasNext} +{:else if model.dirsByKey[key]?.hasNext} { - changeModel(model => dbKeys_markNodeExpanded(model, root, true)); + changeModel(model => dbKeys_markNodeExpanded(model, key, true)); }} /> {/if} diff --git a/packages/web/src/widgets/DbKeysTree.svelte b/packages/web/src/widgets/DbKeysTree.svelte index ed39398d1..3e25d2ea9 100644 --- a/packages/web/src/widgets/DbKeysTree.svelte +++ b/packages/web/src/widgets/DbKeysTree.svelte @@ -1,8 +1,8 @@ @@ -126,20 +125,32 @@ - + -
- {#if model} -
- Scanned {Math.min(model?.scannedKeys, model?.dbsize) ?? '???'}/{model?.dbsize ?? '???'} -
- {/if} - - Scan more - -
+{#if !model?.loadedAll} +
+ {#if model} +
+ {#if isLoading} + Loading... + {:else} + Scanned {Math.min(model?.scannedKeys, model?.dbsize) ?? '???'}/{model?.dbsize ?? '???'} + {/if} +
+ {/if} + {#if isLoading} +
+ +
+ {:else} + + Scan more + + {/if} +
+{/if} {#if differentFocusedDb} {/if} @@ -172,11 +183,11 @@ }; } if (data.key && clickAction == 'keyEnter') { - changeModel(model => dbKeys_markNodeExpanded(model, data.key, !model.dirsByKey[data.key]?.isExpanded)); + changeModel(model => dbKeys_markNodeExpanded(model, data.key, !model.dirsByKey[data.key]?.isExpanded), false); } }} handleExpansion={(data, value) => { - changeModel(model => dbKeys_markNodeExpanded(model, data.key, value)); + changeModel(model => dbKeys_markNodeExpanded(model, data.key, value), false); }} onScrollTop={() => { domContainer?.scrollTop(); diff --git a/packages/web/src/widgets/DbKeysTreeNode.svelte b/packages/web/src/widgets/DbKeysTreeNode.svelte index ed00b0f1e..162fbf820 100644 --- a/packages/web/src/widgets/DbKeysTreeNode.svelte +++ b/packages/web/src/widgets/DbKeysTreeNode.svelte @@ -1,7 +1,7 @@ -{#each items as item} +{#each items.slice(0, visibleCount) as item} {/each} -{#if model.dirsByKey[key]?.shouldLoadNext} - -{:else if model.dirsByKey[key]?.hasNext} +{#if model.childrenByKey[key]?.length > visibleCount} { - changeModel(model => dbKeys_markNodeExpanded(model, key, true), false); + changeModel(model => dbKeys_showNextItems(model, key), false); }} /> {/if} From 1b297fed9013f0e674602b0d2ebe867e3728c8cc Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Wed, 14 May 2025 13:00:08 +0200 Subject: [PATCH 5/6] fix sorting --- packages/tools/src/dbKeysLoader.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/tools/src/dbKeysLoader.ts b/packages/tools/src/dbKeysLoader.ts index 80c93c9c0..b7556db57 100644 --- a/packages/tools/src/dbKeysLoader.ts +++ b/packages/tools/src/dbKeysLoader.ts @@ -5,6 +5,7 @@ export const DB_KEYS_SHOW_INCREMENT = 100; export interface DbKeysNodeModelBase { text?: string; + sortKey: string; key: string; count?: number; level: number; @@ -163,6 +164,7 @@ export function dbKeys_mergeNextPage(tree: DbKeysTreeModel, nextPage: DbKeysLoad ...keyObj, level: keyPath.length, text: keyPath[keyPath.length - 1], + sortKey: keyPath[keyPath.length - 1], keyPath, parentKey: keyPath.slice(0, -1).join(tree.treeKeySeparator), }; @@ -190,6 +192,7 @@ export function dbKeys_mergeNextPage(tree: DbKeysTreeModel, nextPage: DbKeysLoad type: 'dir', key: newDirKey, text: `${newDirPath[newDirPath.length - 1]}${tree.treeKeySeparator}*`, + sortKey: newDirPath[newDirPath.length - 1], }; } @@ -218,7 +221,7 @@ export function dbKeys_mergeNextPage(tree: DbKeysTreeModel, nextPage: DbKeysLoad } for (const key in childrenByKey) { - childrenByKey[key] = _sortBy(childrenByKey[key], 'text'); + childrenByKey[key] = _sortBy(childrenByKey[key], 'sortKey'); } return { From 9d924f8d1c13c5d4b74dc97bf43a3bc99628d498 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Wed, 14 May 2025 13:27:19 +0200 Subject: [PATCH 6/6] optimalized loading redis keys info - using pipeline --- .../dbgate-plugin-redis/src/backend/driver.js | 57 ++++++++++++++++--- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/plugins/dbgate-plugin-redis/src/backend/driver.js b/plugins/dbgate-plugin-redis/src/backend/driver.js index a24e0e272..ea8991c40 100644 --- a/plugins/dbgate-plugin-redis/src/backend/driver.js +++ b/plugins/dbgate-plugin-redis/src/backend/driver.js @@ -320,17 +320,56 @@ const driver = { } }, - async enrichOneKeyInfo(dbhan, item) { - item.type = await dbhan.client.type(item.key); - item.count = await this.getKeyCardinality(dbhan, item.key, item.type); - }, + // async enrichOneKeyInfo(dbhan, item) { + // item.type = await dbhan.client.type(item.key); + // item.count = await this.getKeyCardinality(dbhan, item.key, item.type); + // }, async enrichKeyInfo(dbhan, keyObjects) { - await async.eachLimit( - keyObjects.filter((x) => x.key), - 10, - async (item) => await this.enrichOneKeyInfo(dbhan, item) - ); + // 1. get type + const typePipeline = dbhan.client.pipeline(); + for (const item of keyObjects) { + typePipeline.type(item.key); + } + const resultType = await typePipeline.exec(); + for (let i = 0; i < resultType.length; i++) { + if (resultType[i][0] == null) { + keyObjects[i].type = resultType[i][1]; + } + } + + // 2. get cardinality + const cardinalityPipeline = dbhan.client.pipeline(); + for (const item of keyObjects) { + switch (item.type) { + case 'list': + cardinalityPipeline.llen(item.key); + case 'set': + cardinalityPipeline.scard(item.key); + case 'zset': + cardinalityPipeline.zcard(item.key); + case 'stream': + cardinalityPipeline.xlen(item.key); + case 'hash': + cardinalityPipeline.hlen(item.key); + } + } + const resultCardinality = await cardinalityPipeline.exec(); + let resIndex = 0; + for (const item of keyObjects) { + if ( + item.type == 'list' || + item.type == 'set' || + item.type == 'zset' || + item.type == 'stream' || + item.type == 'hash' + ) { + if (resultCardinality[resIndex][0] == null) { + item.count = resultCardinality[resIndex][1]; + resIndex++; + } + } + } }, async loadKeyInfo(dbhan, key) {