diff --git a/packages/api/src/controllers/databaseConnections.js b/packages/api/src/controllers/databaseConnections.js index c4f77ce74..9bfe37013 100644 --- a/packages/api/src/controllers/databaseConnections.js +++ b/packages/api/src/controllers/databaseConnections.js @@ -304,6 +304,12 @@ module.exports = { return this.loadDataCore('loadKeys', { conid, database, root, filter, limit }); }, + scanKeys_meta: true, + async scanKeys({ conid, database, root, pattern, cursor, count }, req) { + testConnectionPermission(conid, req); + return this.loadDataCore('scanKeys', { conid, database, root, pattern, cursor, count }); + }, + exportKeys_meta: true, async exportKeys({ conid, database, options }, req) { testConnectionPermission(conid, req); diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index 2bb0c1375..4948eae99 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -275,6 +275,10 @@ async function handleLoadKeys({ msgid, root, filter, limit }) { return handleDriverDataCore(msgid, driver => driver.loadKeys(dbhan, root, filter, limit), { logName: 'loadKeys' }); } +async function handleScanKeys({ msgid, pattern, cursor, count }) { + return handleDriverDataCore(msgid, driver => driver.scanKeys(dbhan, pattern, cursor, count), { logName: 'scanKeys' }); +} + async function handleExportKeys({ msgid, options }) { return handleDriverDataCore(msgid, driver => driver.exportKeys(dbhan, options), { logName: 'exportKeys' }); } @@ -453,6 +457,7 @@ const messageHandlers = { updateCollection: handleUpdateCollection, collectionData: handleCollectionData, loadKeys: handleLoadKeys, + scanKeys: handleScanKeys, loadKeyInfo: handleLoadKeyInfo, callMethod: handleCallMethod, loadKeyTableRange: handleLoadKeyTableRange, diff --git a/packages/tools/src/dbKeysLoader.ts b/packages/tools/src/dbKeysLoader.ts index 42c36923d..b7556db57 100644 --- a/packages/tools/src/dbKeysLoader.ts +++ b/packages/tools/src/dbKeysLoader.ts @@ -1,40 +1,71 @@ import _omit from 'lodash/omit'; +import _sortBy from 'lodash/sortBy'; -const SHOW_INCREMENT = 100; +export const DB_KEYS_SHOW_INCREMENT = 100; export interface DbKeysNodeModelBase { text?: string; + sortKey: string; + key: string; count?: number; level: number; + keyPath: string[]; + parentKey: string; } export interface DbKeysLeafNodeModel extends DbKeysNodeModelBase { - key: string; - type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL'; } export interface DbKeysFolderNodeModel extends DbKeysNodeModelBase { - root: string; + // root: string; type: 'dir'; - maxShowCount?: number; + // visibleCount?: number; + // isExpanded?: boolean; +} + +export interface DbKeysFolderStateMode { + key: string; + visibleCount?: number; isExpanded?: boolean; - shouldLoadNext?: boolean; - hasNext?: boolean; } export interface DbKeysTreeModel { + treeKeySeparator: string; root: DbKeysFolderNodeModel; dirsByKey: { [key: string]: DbKeysFolderNodeModel }; + dirStateByKey: { [key: string]: DbKeysFolderStateMode }; childrenByKey: { [key: string]: DbKeysNodeModel[] }; - refreshAll?: boolean; + keyObjectsByKey: { [key: string]: DbKeysNodeModel }; + scannedKeys: number; + loadCount: number; + dbsize: number; + cursor: string; + loadedAll: boolean; + // refreshAll?: boolean; } export type DbKeysNodeModel = DbKeysLeafNodeModel | DbKeysFolderNodeModel; -export type DbKeysLoadFunction = (root: string, limit: number) => Promise; +export interface DbKeyLoadedModel { + key: string; -export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKeysTreeModel) => void; + type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL'; + count?: number; +} + +export interface DbKeysLoadResult { + nextCursor: string; + keys: DbKeyLoadedModel[]; + dbsize: number; +} + +// export type DbKeysLoadFunction = (root: string, limit: number) => Promise; + +export type DbKeysChangeModelFunction = ( + func: (model: DbKeysTreeModel) => DbKeysTreeModel, + loadNextPage: boolean +) => void; // function dbKeys_findFolderNode(node: DbKeysNodeModel, root: string) { // if (node.type != 'dir') { @@ -73,123 +104,241 @@ export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKey // }; // } -export async function dbKeys_loadMissing(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise { - const childrenByKey = { ...tree.childrenByKey }; - const dirsByKey = { ...tree.dirsByKey }; +// export async function dbKeys_loadMissing(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise { +// const childrenByKey = { ...tree.childrenByKey }; +// const dirsByKey = { ...tree.dirsByKey }; - for (const root in tree.dirsByKey) { - const dir = tree.dirsByKey[root]; +// for (const root in tree.dirsByKey) { +// const dir = tree.dirsByKey[root]; - if (dir.isExpanded && dir.shouldLoadNext) { - if (!tree.childrenByKey[root] || dir.hasNext) { - const loadCount = dir.maxShowCount && dir.shouldLoadNext ? dir.maxShowCount + SHOW_INCREMENT : SHOW_INCREMENT; - const items = await loader(root, loadCount + 1); +// if (dir.isExpanded && dir.shouldLoadNext) { +// if (!tree.childrenByKey[root] || dir.hasNext) { +// const loadCount = dir.maxShowCount && dir.shouldLoadNext ? dir.maxShowCount + SHOW_INCREMENT : SHOW_INCREMENT; +// const items = await loader(root, loadCount + 1); - childrenByKey[root] = items.slice(0, loadCount); - dirsByKey[root] = { - ...dir, - shouldLoadNext: false, - maxShowCount: loadCount, - hasNext: items.length > loadCount, - }; +// childrenByKey[root] = items.slice(0, loadCount); +// dirsByKey[root] = { +// ...dir, +// shouldLoadNext: false, +// maxShowCount: loadCount, +// hasNext: items.length > loadCount, +// }; - for (const child of items.slice(0, loadCount)) { - if (child.type == 'dir' && !dirsByKey[child.root]) { - dirsByKey[child.root] = { - shouldLoadNext: false, - maxShowCount: null, - hasNext: false, - isExpanded: false, - type: 'dir', - level: dir.level + 1, - root: child.root, - text: child.text, - }; - } - } - } else { - dirsByKey[root] = { - ...dir, - shouldLoadNext: false, +// for (const child of items.slice(0, loadCount)) { +// if (child.type == 'dir' && !dirsByKey[child.root]) { +// dirsByKey[child.root] = { +// shouldLoadNext: false, +// maxShowCount: null, +// hasNext: false, +// isExpanded: false, +// type: 'dir', +// level: dir.level + 1, +// root: child.root, +// text: child.text, +// }; +// } +// } +// } else { +// dirsByKey[root] = { +// ...dir, +// shouldLoadNext: false, +// }; +// } +// } +// } + +// return { +// ...tree, +// dirsByKey, +// childrenByKey, +// refreshAll: false, +// }; +// } + +export function dbKeys_mergeNextPage(tree: DbKeysTreeModel, nextPage: DbKeysLoadResult): DbKeysTreeModel { + const keyObjectsByKey = { ...tree.keyObjectsByKey }; + + for (const keyObj of nextPage.keys) { + const keyPath = keyObj.key.split(tree.treeKeySeparator); + keyObjectsByKey[keyObj.key] = { + ...keyObj, + level: keyPath.length, + text: keyPath[keyPath.length - 1], + sortKey: keyPath[keyPath.length - 1], + keyPath, + parentKey: keyPath.slice(0, -1).join(tree.treeKeySeparator), + }; + } + + const dirsByKey: { [key: string]: DbKeysFolderNodeModel } = {}; + const childrenByKey: { [key: string]: DbKeysNodeModel[] } = {}; + + dirsByKey[''] = tree.root; + + for (const keyObj of Object.values(keyObjectsByKey)) { + const dirPath = keyObj.keyPath.slice(0, -1); + const dirKey = dirPath.join(tree.treeKeySeparator); + + let dirDepth = keyObj.keyPath.length - 1; + + while (dirDepth > 0) { + const newDirPath = keyObj.keyPath.slice(0, dirDepth); + const newDirKey = newDirPath.join(tree.treeKeySeparator); + if (!dirsByKey[newDirKey]) { + dirsByKey[newDirKey] = { + level: keyObj.level - 1, + keyPath: newDirPath, + parentKey: newDirPath.slice(0, -1).join(tree.treeKeySeparator), + type: 'dir', + key: newDirKey, + text: `${newDirPath[newDirPath.length - 1]}${tree.treeKeySeparator}*`, + sortKey: newDirPath[newDirPath.length - 1], }; } + + dirDepth -= 1; } + + if (!childrenByKey[dirKey]) { + childrenByKey[dirKey] = []; + } + + childrenByKey[dirKey].push(keyObj); + } + + for (const dirObj of Object.values(dirsByKey)) { + if (dirObj.key == '') { + continue; + } + + if (!childrenByKey[dirObj.parentKey]) { + childrenByKey[dirObj.parentKey] = []; + } + childrenByKey[dirObj.parentKey].push(dirObj); + + // set key count + dirsByKey[dirObj.key].count = childrenByKey[dirObj.key].length; + } + + for (const key in childrenByKey) { + childrenByKey[key] = _sortBy(childrenByKey[key], 'sortKey'); } return { ...tree, + cursor: nextPage.nextCursor, dirsByKey, childrenByKey, - refreshAll: false, + keyObjectsByKey, + scannedKeys: tree.scannedKeys + tree.loadCount, + loadedAll: nextPage.nextCursor == '0', + dbsize: nextPage.dbsize, }; } export function dbKeys_markNodeExpanded(tree: DbKeysTreeModel, root: string, isExpanded: boolean): DbKeysTreeModel { - const node = tree.dirsByKey[root]; - if (!node) { - return tree; - } + const node = tree.dirStateByKey[root]; return { ...tree, - dirsByKey: { - ...tree.dirsByKey, + dirStateByKey: { + ...tree.dirStateByKey, [root]: { ...node, isExpanded, - shouldLoadNext: isExpanded, }, }, }; } -export function dbKeys_refreshAll(tree?: DbKeysTreeModel): DbKeysTreeModel { - const root: DbKeysFolderNodeModel = { - isExpanded: true, - level: 0, - root: '', - type: 'dir', - shouldLoadNext: true, - }; +export function dbKeys_showNextItems(tree: DbKeysTreeModel, root: string): DbKeysTreeModel { + const node = tree.dirStateByKey[root]; return { ...tree, + dirStateByKey: { + ...tree.dirStateByKey, + [root]: { + ...node, + visibleCount: (node?.visibleCount ?? DB_KEYS_SHOW_INCREMENT) + DB_KEYS_SHOW_INCREMENT, + }, + }, + }; +} + +export function dbKeys_createNewModel(treeKeySeparator: string): DbKeysTreeModel { + const root: DbKeysFolderNodeModel = { + level: 0, + type: 'dir', + keyPath: [], + parentKey: '', + key: '', + }; + return { + treeKeySeparator, childrenByKey: {}, + keyObjectsByKey: {}, dirsByKey: { '': root, }, - refreshAll: true, - root, - }; -} - -export function dbKeys_reloadFolder(tree: DbKeysTreeModel, root: string): DbKeysTreeModel { - return { - ...tree, - childrenByKey: _omit(tree.childrenByKey, root), - dirsByKey: { - ...tree.dirsByKey, - [root]: { - ...tree.dirsByKey[root], - shouldLoadNext: true, - hasNext: undefined, + dirStateByKey: { + '': { + key: '', + visibleCount: DB_KEYS_SHOW_INCREMENT, + isExpanded: true, }, }, + scannedKeys: 0, + dbsize: 0, + loadCount: 2000, + cursor: '0', + root, + loadedAll: false, }; } +export function dbKeys_clearLoadedData(tree: DbKeysTreeModel): DbKeysTreeModel { + return { + ...tree, + childrenByKey: {}, + keyObjectsByKey: {}, + dirsByKey: { + '': tree.root, + }, + scannedKeys: 0, + dbsize: 0, + cursor: '0', + loadedAll: false, + }; +} + +// export function dbKeys_reloadFolder(tree: DbKeysTreeModel, root: string): DbKeysTreeModel { +// return { +// ...tree, +// childrenByKey: _omit(tree.childrenByKey, root), +// dirsByKey: { +// ...tree.dirsByKey, +// [root]: { +// ...tree.dirsByKey[root], +// shouldLoadNext: true, +// hasNext: undefined, +// }, +// }, +// }; +// } + function addFlatItems(tree: DbKeysTreeModel, root: string, res: DbKeysNodeModel[], visitedRoots: string[] = []) { - const item = tree.dirsByKey[root]; - if (!item.isExpanded) { + const item = tree.dirStateByKey[root]; + if (!item?.isExpanded) { return false; } const children = tree.childrenByKey[root] || []; for (const child of children) { res.push(child); if (child.type == 'dir') { - if (visitedRoots.includes(child.root)) { - console.warn('Redis: preventing infinite loop for root', child.root); + if (visitedRoots.includes(child.key)) { + console.warn('Redis: preventing infinite loop for root', child.key); return false; } - addFlatItems(tree, child.root, res, [...visitedRoots, root]); + addFlatItems(tree, child.key, res, [...visitedRoots, root]); } } } diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index af5fd29df..9a99625a1 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -239,6 +239,7 @@ export interface EngineDriver extends FilterBehaviourProvider { }[] >; loadKeys(dbhan: DatabaseHandle, root: string, filter?: string): Promise; + scanKeys(dbhan: DatabaseHandle, root: string, pattern: string, cursor: string, count: number): Promise; exportKeys(dbhan: DatabaseHandle, options: {}): Promise; loadKeyInfo(dbhan: DatabaseHandle, key): Promise; loadKeyTableRange(dbhan: DatabaseHandle, key, cursor, count): Promise; diff --git a/packages/web/public/global.css b/packages/web/public/global.css index d39a035f9..2977bfccf 100644 --- a/packages/web/public/global.css +++ b/packages/web/public/global.css @@ -36,6 +36,9 @@ body { display: flex; justify-content: space-between; } +.align-items-center { + align-items: center; +} .flex { display: flex; } diff --git a/packages/web/src/elements/SearchBoxWrapper.svelte b/packages/web/src/elements/SearchBoxWrapper.svelte index d71019413..b6c31ff96 100644 --- a/packages/web/src/elements/SearchBoxWrapper.svelte +++ b/packages/web/src/elements/SearchBoxWrapper.svelte @@ -1,4 +1,8 @@ -
+ + +
diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index bd9d0df51..86bee43eb 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -151,6 +151,7 @@ 'icon text': 'mdi mdi-text', 'icon ai': 'mdi mdi-head-lightbulb', 'icon wait': 'mdi mdi-timer-sand', + 'icon more': 'mdi mdi-more', 'icon run': 'mdi mdi-play', 'icon chevron-down': 'mdi mdi-chevron-down', diff --git a/packages/web/src/utility/metadataLoaders.ts b/packages/web/src/utility/metadataLoaders.ts index 67f9d7465..2fc374a02 100644 --- a/packages/web/src/utility/metadataLoaders.ts +++ b/packages/web/src/utility/metadataLoaders.ts @@ -83,12 +83,6 @@ const databaseListLoader = ({ conid }) => ({ errorValue: [], }); -// const databaseKeysLoader = ({ conid, database, root }) => ({ -// url: 'database-connections/load-keys', -// params: { conid, database, root }, -// reloadTrigger: `database-keys-changed-${conid}-${database}`, -// }); - const serverVersionLoader = ({ conid }) => ({ url: 'server-connections/version', params: { conid }, diff --git a/packages/web/src/widgets/DatabaseWidget.svelte b/packages/web/src/widgets/DatabaseWidget.svelte index 7a295761e..7e525c090 100644 --- a/packages/web/src/widgets/DatabaseWidget.svelte +++ b/packages/web/src/widgets/DatabaseWidget.svelte @@ -80,7 +80,7 @@ storageName="dbObjectsWidget" skip={!(conid && (database || singleDatabase) && driver?.databaseEngineTypes?.includes('keyvalue'))} > - + -{#each items as item} +{#each items.slice(0, visibleCount) as item} {/each} -{#if model.dirsByKey[root]?.shouldLoadNext} - -{:else if model.dirsByKey[root]?.hasNext} +{#if model.childrenByKey[key]?.length > visibleCount} { - changeModel(model => dbKeys_markNodeExpanded(model, root, true)); + changeModel(model => dbKeys_showNextItems(model, key), false); }} /> {/if} diff --git a/packages/web/src/widgets/DbKeysTree.svelte b/packages/web/src/widgets/DbKeysTree.svelte index 97c798b28..8512a9c71 100644 --- a/packages/web/src/widgets/DbKeysTree.svelte +++ b/packages/web/src/widgets/DbKeysTree.svelte @@ -1,9 +1,10 @@ - + - + +{#if !model?.loadedAll} +
+ {#if model} +
+ {#if isLoading} + Loading... + {:else} + Scanned {Math.min(model?.scannedKeys, model?.dbsize) ?? '???'}/{model?.dbsize ?? '???'} + {/if} +
+ {/if} + {#if isLoading} +
+ +
+ {:else} + + Scan more + + {/if} +
+{/if} {#if differentFocusedDb} {/if} @@ -133,7 +160,7 @@ list={dbKeys_getFlatList(model)} selectedObjectStore={focusedTreeDbKey} getSelectedObject={getFocusedTreeDbKey} - selectedObjectMatcher={(o1, o2) => o1?.key == o2?.key && o1?.type == o2?.type && o1?.root == o2?.root} + selectedObjectMatcher={(o1, o2) => o1?.key == o2?.key && o1?.type == o2?.type} handleObjectClick={(data, clickAction) => { focusedTreeDbKey.set(data); @@ -155,12 +182,12 @@ [`${conid}:${database}`]: data.key, }; } - if (data.root && clickAction == 'keyEnter') { - changeModel(model => dbKeys_markNodeExpanded(model, data.root, !model.dirsByKey[data.root]?.isExpanded)); + if (data.key && clickAction == 'keyEnter') { + changeModel(model => dbKeys_markNodeExpanded(model, data.key, !model.dirsByKey[data.key]?.isExpanded), false); } }} handleExpansion={(data, value) => { - changeModel(model => dbKeys_markNodeExpanded(model, data.root, value)); + changeModel(model => dbKeys_markNodeExpanded(model, data.key, value), false); }} onScrollTop={() => { domContainer?.scrollTop(); @@ -169,6 +196,6 @@ domFilter?.focus(text); }} > - + diff --git a/packages/web/src/widgets/DbKeysTreeNode.svelte b/packages/web/src/widgets/DbKeysTreeNode.svelte index 10b869b59..162fbf820 100644 --- a/packages/web/src/widgets/DbKeysTreeNode.svelte +++ b/packages/web/src/widgets/DbKeysTreeNode.svelte @@ -1,7 +1,7 @@