Merge branch 'feature/scan-redis-keys'

packages/api/src/controllers/databaseConnections.js

@@ -304,6 +304,12 @@ module.exports = {
     return this.loadDataCore('loadKeys', { conid, database, root, filter, limit });
   },
 
+  scanKeys_meta: true,
+  async scanKeys({ conid, database, root, pattern, cursor, count }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('scanKeys', { conid, database, root, pattern, cursor, count });
+  },
+
   exportKeys_meta: true,
   async exportKeys({ conid, database, options }, req) {
     testConnectionPermission(conid, req);

packages/api/src/proc/databaseConnectionProcess.js

@@ -275,6 +275,10 @@ async function handleLoadKeys({ msgid, root, filter, limit }) {
   return handleDriverDataCore(msgid, driver => driver.loadKeys(dbhan, root, filter, limit), { logName: 'loadKeys' });
 }
 
+async function handleScanKeys({ msgid, pattern, cursor, count }) {
+  return handleDriverDataCore(msgid, driver => driver.scanKeys(dbhan, pattern, cursor, count), { logName: 'scanKeys' });
+}
+
 async function handleExportKeys({ msgid, options }) {
   return handleDriverDataCore(msgid, driver => driver.exportKeys(dbhan, options), { logName: 'exportKeys' });
 }
@@ -453,6 +457,7 @@ const messageHandlers = {
   updateCollection: handleUpdateCollection,
   collectionData: handleCollectionData,
   loadKeys: handleLoadKeys,
+  scanKeys: handleScanKeys,
   loadKeyInfo: handleLoadKeyInfo,
   callMethod: handleCallMethod,
   loadKeyTableRange: handleLoadKeyTableRange,

packages/tools/src/dbKeysLoader.ts

@@ -1,40 +1,71 @@
 import _omit from 'lodash/omit';
+import _sortBy from 'lodash/sortBy';
 
-const SHOW_INCREMENT = 100;
+export const DB_KEYS_SHOW_INCREMENT = 100;
 
 export interface DbKeysNodeModelBase {
   text?: string;
+  sortKey: string;
+  key: string;
   count?: number;
   level: number;
+  keyPath: string[];
+  parentKey: string;
 }
 
 export interface DbKeysLeafNodeModel extends DbKeysNodeModelBase {
-  key: string;
-
   type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL';
 }
 
 export interface DbKeysFolderNodeModel extends DbKeysNodeModelBase {
-  root: string;
+  // root: string;
   type: 'dir';
-  maxShowCount?: number;
+  // visibleCount?: number;
+  // isExpanded?: boolean;
+}
+
+export interface DbKeysFolderStateMode {
+  key: string;
+  visibleCount?: number;
   isExpanded?: boolean;
-  shouldLoadNext?: boolean;
-  hasNext?: boolean;
 }
 
 export interface DbKeysTreeModel {
+  treeKeySeparator: string;
   root: DbKeysFolderNodeModel;
   dirsByKey: { [key: string]: DbKeysFolderNodeModel };
+  dirStateByKey: { [key: string]: DbKeysFolderStateMode };
   childrenByKey: { [key: string]: DbKeysNodeModel[] };
-  refreshAll?: boolean;
+  keyObjectsByKey: { [key: string]: DbKeysNodeModel };
+  scannedKeys: number;
+  loadCount: number;
+  dbsize: number;
+  cursor: string;
+  loadedAll: boolean;
+  // refreshAll?: boolean;
 }
 
 export type DbKeysNodeModel = DbKeysLeafNodeModel | DbKeysFolderNodeModel;
 
-export type DbKeysLoadFunction = (root: string, limit: number) => Promise<DbKeysNodeModel[]>;
+export interface DbKeyLoadedModel {
+  key: string;
 
-export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKeysTreeModel) => void;
+  type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL';
+  count?: number;
+}
+
+export interface DbKeysLoadResult {
+  nextCursor: string;
+  keys: DbKeyLoadedModel[];
+  dbsize: number;
+}
+
+// export type DbKeysLoadFunction = (root: string, limit: number) => Promise<DbKeysLoadResult>;
+
+export type DbKeysChangeModelFunction = (
+  func: (model: DbKeysTreeModel) => DbKeysTreeModel,
+  loadNextPage: boolean
+) => void;
 
 // function dbKeys_findFolderNode(node: DbKeysNodeModel, root: string) {
 //   if (node.type != 'dir') {
@@ -73,123 +104,241 @@ export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKey
 //   };
 // }
 
-export async function dbKeys_loadMissing(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise<DbKeysTreeModel> {
+// export async function dbKeys_loadMissing(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise<DbKeysTreeModel> {
-  const childrenByKey = { ...tree.childrenByKey };
+//   const childrenByKey = { ...tree.childrenByKey };
-  const dirsByKey = { ...tree.dirsByKey };
+//   const dirsByKey = { ...tree.dirsByKey };
 
-  for (const root in tree.dirsByKey) {
+//   for (const root in tree.dirsByKey) {
-    const dir = tree.dirsByKey[root];
+//     const dir = tree.dirsByKey[root];
 
-    if (dir.isExpanded && dir.shouldLoadNext) {
+//     if (dir.isExpanded && dir.shouldLoadNext) {
-      if (!tree.childrenByKey[root] || dir.hasNext) {
+//       if (!tree.childrenByKey[root] || dir.hasNext) {
-        const loadCount = dir.maxShowCount && dir.shouldLoadNext ? dir.maxShowCount + SHOW_INCREMENT : SHOW_INCREMENT;
+//         const loadCount = dir.maxShowCount && dir.shouldLoadNext ? dir.maxShowCount + SHOW_INCREMENT : SHOW_INCREMENT;
-        const items = await loader(root, loadCount + 1);
+//         const items = await loader(root, loadCount + 1);
 
-        childrenByKey[root] = items.slice(0, loadCount);
+//         childrenByKey[root] = items.slice(0, loadCount);
-        dirsByKey[root] = {
+//         dirsByKey[root] = {
-          ...dir,
+//           ...dir,
-          shouldLoadNext: false,
+//           shouldLoadNext: false,
-          maxShowCount: loadCount,
+//           maxShowCount: loadCount,
-          hasNext: items.length > loadCount,
+//           hasNext: items.length > loadCount,
+//         };
+
+//         for (const child of items.slice(0, loadCount)) {
+//           if (child.type == 'dir' && !dirsByKey[child.root]) {
+//             dirsByKey[child.root] = {
+//               shouldLoadNext: false,
+//               maxShowCount: null,
+//               hasNext: false,
+//               isExpanded: false,
+//               type: 'dir',
+//               level: dir.level + 1,
+//               root: child.root,
+//               text: child.text,
+//             };
+//           }
+//         }
+//       } else {
+//         dirsByKey[root] = {
+//           ...dir,
+//           shouldLoadNext: false,
+//         };
+//       }
+//     }
+//   }
+
+//   return {
+//     ...tree,
+//     dirsByKey,
+//     childrenByKey,
+//     refreshAll: false,
+//   };
+// }
+
+export function dbKeys_mergeNextPage(tree: DbKeysTreeModel, nextPage: DbKeysLoadResult): DbKeysTreeModel {
+  const keyObjectsByKey = { ...tree.keyObjectsByKey };
+
+  for (const keyObj of nextPage.keys) {
+    const keyPath = keyObj.key.split(tree.treeKeySeparator);
+    keyObjectsByKey[keyObj.key] = {
+      ...keyObj,
+      level: keyPath.length,
+      text: keyPath[keyPath.length - 1],
+      sortKey: keyPath[keyPath.length - 1],
+      keyPath,
+      parentKey: keyPath.slice(0, -1).join(tree.treeKeySeparator),
     };
+  }
 
-        for (const child of items.slice(0, loadCount)) {
+  const dirsByKey: { [key: string]: DbKeysFolderNodeModel } = {};
-          if (child.type == 'dir' && !dirsByKey[child.root]) {
+  const childrenByKey: { [key: string]: DbKeysNodeModel[] } = {};
-            dirsByKey[child.root] = {
+
-              shouldLoadNext: false,
+  dirsByKey[''] = tree.root;
-              maxShowCount: null,
+
-              hasNext: false,
+  for (const keyObj of Object.values(keyObjectsByKey)) {
-              isExpanded: false,
+    const dirPath = keyObj.keyPath.slice(0, -1);
+    const dirKey = dirPath.join(tree.treeKeySeparator);
+
+    let dirDepth = keyObj.keyPath.length - 1;
+
+    while (dirDepth > 0) {
+      const newDirPath = keyObj.keyPath.slice(0, dirDepth);
+      const newDirKey = newDirPath.join(tree.treeKeySeparator);
+      if (!dirsByKey[newDirKey]) {
+        dirsByKey[newDirKey] = {
+          level: keyObj.level - 1,
+          keyPath: newDirPath,
+          parentKey: newDirPath.slice(0, -1).join(tree.treeKeySeparator),
           type: 'dir',
-              level: dir.level + 1,
+          key: newDirKey,
-              root: child.root,
+          text: `${newDirPath[newDirPath.length - 1]}${tree.treeKeySeparator}*`,
-              text: child.text,
+          sortKey: newDirPath[newDirPath.length - 1],
         };
       }
+
+      dirDepth -= 1;
     }
-      } else {
+
-        dirsByKey[root] = {
+    if (!childrenByKey[dirKey]) {
-          ...dir,
+      childrenByKey[dirKey] = [];
-          shouldLoadNext: false,
-        };
     }
+
+    childrenByKey[dirKey].push(keyObj);
   }
+
+  for (const dirObj of Object.values(dirsByKey)) {
+    if (dirObj.key == '') {
+      continue;
+    }
+
+    if (!childrenByKey[dirObj.parentKey]) {
+      childrenByKey[dirObj.parentKey] = [];
+    }
+    childrenByKey[dirObj.parentKey].push(dirObj);
+
+    // set key count
+    dirsByKey[dirObj.key].count = childrenByKey[dirObj.key].length;
+  }
+
+  for (const key in childrenByKey) {
+    childrenByKey[key] = _sortBy(childrenByKey[key], 'sortKey');
   }
 
   return {
     ...tree,
+    cursor: nextPage.nextCursor,
     dirsByKey,
     childrenByKey,
-    refreshAll: false,
+    keyObjectsByKey,
+    scannedKeys: tree.scannedKeys + tree.loadCount,
+    loadedAll: nextPage.nextCursor == '0',
+    dbsize: nextPage.dbsize,
   };
 }
 
 export function dbKeys_markNodeExpanded(tree: DbKeysTreeModel, root: string, isExpanded: boolean): DbKeysTreeModel {
-  const node = tree.dirsByKey[root];
+  const node = tree.dirStateByKey[root];
-  if (!node) {
-    return tree;
-  }
   return {
     ...tree,
-    dirsByKey: {
+    dirStateByKey: {
-      ...tree.dirsByKey,
+      ...tree.dirStateByKey,
       [root]: {
         ...node,
         isExpanded,
-        shouldLoadNext: isExpanded,
       },
     },
   };
 }
 
-export function dbKeys_refreshAll(tree?: DbKeysTreeModel): DbKeysTreeModel {
+export function dbKeys_showNextItems(tree: DbKeysTreeModel, root: string): DbKeysTreeModel {
-  const root: DbKeysFolderNodeModel = {
+  const node = tree.dirStateByKey[root];
-    isExpanded: true,
-    level: 0,
-    root: '',
-    type: 'dir',
-    shouldLoadNext: true,
-  };
   return {
     ...tree,
+    dirStateByKey: {
+      ...tree.dirStateByKey,
+      [root]: {
+        ...node,
+        visibleCount: (node?.visibleCount ?? DB_KEYS_SHOW_INCREMENT) + DB_KEYS_SHOW_INCREMENT,
+      },
+    },
+  };
+}
+
+export function dbKeys_createNewModel(treeKeySeparator: string): DbKeysTreeModel {
+  const root: DbKeysFolderNodeModel = {
+    level: 0,
+    type: 'dir',
+    keyPath: [],
+    parentKey: '',
+    key: '',
+  };
+  return {
+    treeKeySeparator,
     childrenByKey: {},
+    keyObjectsByKey: {},
     dirsByKey: {
       '': root,
     },
-    refreshAll: true,
+    dirStateByKey: {
+      '': {
+        key: '',
+        visibleCount: DB_KEYS_SHOW_INCREMENT,
+        isExpanded: true,
+      },
+    },
+    scannedKeys: 0,
+    dbsize: 0,
+    loadCount: 2000,
+    cursor: '0',
     root,
+    loadedAll: false,
   };
 }
 
-export function dbKeys_reloadFolder(tree: DbKeysTreeModel, root: string): DbKeysTreeModel {
+export function dbKeys_clearLoadedData(tree: DbKeysTreeModel): DbKeysTreeModel {
   return {
     ...tree,
-    childrenByKey: _omit(tree.childrenByKey, root),
+    childrenByKey: {},
+    keyObjectsByKey: {},
     dirsByKey: {
-      ...tree.dirsByKey,
+      '': tree.root,
-      [root]: {
-        ...tree.dirsByKey[root],
-        shouldLoadNext: true,
-        hasNext: undefined,
-      },
     },
+    scannedKeys: 0,
+    dbsize: 0,
+    cursor: '0',
+    loadedAll: false,
   };
 }
 
+// export function dbKeys_reloadFolder(tree: DbKeysTreeModel, root: string): DbKeysTreeModel {
+//   return {
+//     ...tree,
+//     childrenByKey: _omit(tree.childrenByKey, root),
+//     dirsByKey: {
+//       ...tree.dirsByKey,
+//       [root]: {
+//         ...tree.dirsByKey[root],
+//         shouldLoadNext: true,
+//         hasNext: undefined,
+//       },
+//     },
+//   };
+// }
+
 function addFlatItems(tree: DbKeysTreeModel, root: string, res: DbKeysNodeModel[], visitedRoots: string[] = []) {
-  const item = tree.dirsByKey[root];
+  const item = tree.dirStateByKey[root];
-  if (!item.isExpanded) {
+  if (!item?.isExpanded) {
     return false;
   }
   const children = tree.childrenByKey[root] || [];
   for (const child of children) {
     res.push(child);
     if (child.type == 'dir') {
-      if (visitedRoots.includes(child.root)) {
+      if (visitedRoots.includes(child.key)) {
-        console.warn('Redis: preventing infinite loop for root', child.root);
+        console.warn('Redis: preventing infinite loop for root', child.key);
         return false;
       }
-      addFlatItems(tree, child.root, res, [...visitedRoots, root]);
+      addFlatItems(tree, child.key, res, [...visitedRoots, root]);
     }
   }
 }

packages/types/engines.d.ts

@@ -239,6 +239,7 @@ export interface EngineDriver<TClient = any> extends FilterBehaviourProvider {
     }[]
   >;
   loadKeys(dbhan: DatabaseHandle<TClient>, root: string, filter?: string): Promise;
+  scanKeys(dbhan: DatabaseHandle<TClient>, root: string, pattern: string, cursor: string, count: number): Promise;
   exportKeys(dbhan: DatabaseHandle<TClient>, options: {}): Promise;
   loadKeyInfo(dbhan: DatabaseHandle<TClient>, key): Promise;
   loadKeyTableRange(dbhan: DatabaseHandle<TClient>, key, cursor, count): Promise;

packages/web/public/global.css

@@ -36,6 +36,9 @@ body {
   display: flex;
   justify-content: space-between;
 }
+.align-items-center {
+  align-items: center;
+}
 .flex {
   display: flex;
 }

packages/web/src/elements/SearchBoxWrapper.svelte

@@ -1,4 +1,8 @@
-<div><slot /></div>
+<script lang="ts">
+  export let noMargin = false;
+</script>
+
+<div class:noMargin><slot /></div>
 
 <style>
   div {
@@ -6,4 +10,8 @@
     border-bottom: 1px solid var(--theme-border);
     margin-bottom: 5px;
   }
+
+  div.noMargin {
+    margin-bottom: 0;
+  }
 </style>

packages/web/src/icons/FontIcon.svelte

@@ -151,6 +151,7 @@
     'icon text': 'mdi mdi-text',
     'icon ai': 'mdi mdi-head-lightbulb',
     'icon wait': 'mdi mdi-timer-sand',
+    'icon more': 'mdi mdi-more',
 
     'icon run': 'mdi mdi-play',
     'icon chevron-down': 'mdi mdi-chevron-down',

packages/web/src/utility/metadataLoaders.ts

@@ -83,12 +83,6 @@ const databaseListLoader = ({ conid }) => ({
   errorValue: [],
 });
 
-// const databaseKeysLoader = ({ conid, database, root }) => ({
-//   url: 'database-connections/load-keys',
-//   params: { conid, database, root },
-//   reloadTrigger: `database-keys-changed-${conid}-${database}`,
-// });
-
 const serverVersionLoader = ({ conid }) => ({
   url: 'server-connections/version',
   params: { conid },

packages/web/src/widgets/DatabaseWidget.svelte

@@ -80,7 +80,7 @@
     storageName="dbObjectsWidget"
     skip={!(conid && (database || singleDatabase) && driver?.databaseEngineTypes?.includes('keyvalue'))}
   >
-    <DbKeysTree {conid} {database} />
+    <DbKeysTree {conid} {database} treeKeySeparator={$connection?.treeKeySeparator || ':'} />
   </WidgetColumnBarItem>
 
   <WidgetColumnBarItem

packages/web/src/widgets/DbKeysSubTree.svelte

@@ -4,9 +4,14 @@
   import AppObjectCore from '../appobj/AppObjectCore.svelte';
 
   import DbKeysTreeNode from './DbKeysTreeNode.svelte';
-  import { dbKeys_markNodeExpanded, DbKeysChangeModelFunction, DbKeysTreeModel } from 'dbgate-tools';
+  import {
+    DB_KEYS_SHOW_INCREMENT,
+    dbKeys_showNextItems,
+    DbKeysChangeModelFunction,
+    DbKeysTreeModel,
+  } from 'dbgate-tools';
 
-  export let root;
+  export let key;
   export let connection;
   export let database;
   export let conid;
@@ -19,34 +24,33 @@
 
   export let parentRoots = [];
 
-  $: items = model.childrenByKey[root] ?? [];
+  $: items = model.childrenByKey[key] ?? [];
+  $: visibleCount = model.dirStateByKey[key]?.visibleCount ?? DB_KEYS_SHOW_INCREMENT;
 </script>
 
-{#each items as item}
+{#each items.slice(0, visibleCount) as item}
   <DbKeysTreeNode
     {conid}
     {database}
-    {root}
+    {key}
     {connection}
     {item}
     {filter}
     {indentLevel}
     {model}
     {changeModel}
-    parentRoots={[...parentRoots, root]}
+    parentRoots={[...parentRoots, key]}
   />
 {/each}
 
-{#if model.dirsByKey[root]?.shouldLoadNext}
+{#if model.childrenByKey[key]?.length > visibleCount}
-  <AppObjectCore {indentLevel} title="Loading keys..." icon="icon loading" expandIcon="icon invisible-box" />
-{:else if model.dirsByKey[root]?.hasNext}
   <AppObjectCore
     {indentLevel}
     title="Show more..."
     icon="icon dots-horizontal"
     expandIcon="icon invisible-box"
     on:click={() => {
-      changeModel(model => dbKeys_markNodeExpanded(model, root, true));
+      changeModel(model => dbKeys_showNextItems(model, key), false);
     }}
   />
 {/if}

packages/web/src/widgets/DbKeysTree.svelte

@@ -1,9 +1,10 @@
 <script lang="ts">
   import {
+    dbKeys_clearLoadedData,
+    dbKeys_createNewModel,
     dbKeys_getFlatList,
-    dbKeys_loadMissing,
     dbKeys_markNodeExpanded,
-    dbKeys_refreshAll,
+    dbKeys_mergeNextPage,
     findEngineDriver,
   } from 'dbgate-tools';
 
@@ -32,22 +33,19 @@
   import AppObjectListHandler from './AppObjectListHandler.svelte';
   import { getOpenDetailOnArrowsSettings } from '../settings/settingsTools';
   import openNewTab from '../utility/openNewTab';
-  import clickOutside from '../utility/clickOutside';
 
   export let conid;
   export let database;
+  export let treeKeySeparator = ':';
 
   let domListHandler;
   let domContainer = null;
   let domFilter = null;
 
   let filter;
+  let isLoading = false;
 
-  let model = dbKeys_refreshAll();
+  let model = dbKeys_createNewModel(treeKeySeparator);
-
-  function handleRefreshDatabase() {
-    changeModel(model => dbKeys_refreshAll(model));
-  }
 
   function handleAddKey() {
     const connection = $currentDatabase?.connection;
@@ -68,7 +66,7 @@
           args: [item.keyName, ...type.dbKeyFields.map(fld => item[fld.name])],
         });
 
-        handleRefreshDatabase();
+        reloadModel();
       },
     });
   }
@@ -80,22 +78,27 @@
 
   $: connection = useConnectionInfo({ conid });
 
-  async function changeModel(modelUpdate) {
+  function changeModel(modelUpdate, loadNext) {
-    model = modelUpdate(model);
+    if (modelUpdate) model = modelUpdate(model);
-    model = await dbKeys_loadMissing(model, async (root, limit) => {
+    if (loadNext) loadNextPage();
-      const result = await apiCall('database-connections/load-keys', {
+  }
+
+  async function loadNextPage() {
+    isLoading = true;
+    const nextScan = await apiCall('database-connections/scan-keys', {
       conid,
       database,
-        root,
+      pattern: filter,
-        filter,
+      cursor: model.cursor,
-        limit,
+      count: model.loadCount,
-      });
-      return result;
     });
+
+    model = dbKeys_mergeNextPage(model, nextScan);
+    isLoading = false;
   }
 
   function reloadModel() {
-    changeModel(model => dbKeys_refreshAll(model));
+    changeModel(model => dbKeys_clearLoadedData(model), true);
   }
 
   $: {
@@ -104,11 +107,13 @@
     filter;
     reloadModel();
   }
+
+  // $: console.log('DbKeysTree MODEL', model);
 </script>
 
-<SearchBoxWrapper>
+<SearchBoxWrapper noMargin>
   <SearchInput
-    placeholder="Search keys"
+    placeholder="Redis pattern or key part"
     bind:value={filter}
     isDebounced
     bind:this={domFilter}
@@ -120,10 +125,32 @@
   <InlineButton on:click={handleAddKey} title="Add new key">
     <FontIcon icon="icon plus-thick" />
   </InlineButton>
-  <InlineButton on:click={handleRefreshDatabase} title="Refresh key list">
+  <InlineButton on:click={reloadModel} title="Refresh key list">
     <FontIcon icon="icon refresh" />
   </InlineButton>
 </SearchBoxWrapper>
+{#if !model?.loadedAll}
+  <div class="space-between align-items-center ml-1">
+    {#if model}
+      <div>
+        {#if isLoading}
+          Loading...
+        {:else}
+          Scanned {Math.min(model?.scannedKeys, model?.dbsize) ?? '???'}/{model?.dbsize ?? '???'}
+        {/if}
+      </div>
+    {/if}
+    {#if isLoading}
+      <div style="margin: 3px; margin-bottom: 2px">
+        <FontIcon icon="icon loading" />
+      </div>
+    {:else}
+      <InlineButton on:click={loadNextPage} title="Scan more keys">
+        <FontIcon icon="icon more" /> Scan more
+      </InlineButton>
+    {/if}
+  </div>
+{/if}
 {#if differentFocusedDb}
   <FocusedConnectionInfoWidget {conid} {database} connection={$connection} />
 {/if}
@@ -133,7 +160,7 @@
     list={dbKeys_getFlatList(model)}
     selectedObjectStore={focusedTreeDbKey}
     getSelectedObject={getFocusedTreeDbKey}
-    selectedObjectMatcher={(o1, o2) => o1?.key == o2?.key && o1?.type == o2?.type && o1?.root == o2?.root}
+    selectedObjectMatcher={(o1, o2) => o1?.key == o2?.key && o1?.type == o2?.type}
     handleObjectClick={(data, clickAction) => {
       focusedTreeDbKey.set(data);
 
@@ -155,12 +182,12 @@
           [`${conid}:${database}`]: data.key,
         };
       }
-      if (data.root && clickAction == 'keyEnter') {
+      if (data.key && clickAction == 'keyEnter') {
-        changeModel(model => dbKeys_markNodeExpanded(model, data.root, !model.dirsByKey[data.root]?.isExpanded));
+        changeModel(model => dbKeys_markNodeExpanded(model, data.key, !model.dirsByKey[data.key]?.isExpanded), false);
       }
     }}
     handleExpansion={(data, value) => {
-      changeModel(model => dbKeys_markNodeExpanded(model, data.root, value));
+      changeModel(model => dbKeys_markNodeExpanded(model, data.key, value), false);
     }}
     onScrollTop={() => {
       domContainer?.scrollTop();
@@ -169,6 +196,6 @@
       domFilter?.focus(text);
     }}
   >
-    <DbKeysSubTree root="" {filter} {model} {changeModel} {conid} {database} {connection} />
+    <DbKeysSubTree key="" {filter} {model} {changeModel} {conid} {database} {connection} />
   </AppObjectListHandler>
 </WidgetsInnerContainer>

packages/web/src/widgets/DbKeysTreeNode.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
   import {
+    dbKeys_clearLoadedData,
     dbKeys_markNodeExpanded,
-    dbKeys_reloadFolder,
     DbKeysChangeModelFunction,
     DbKeysTreeModel,
     getIconForRedisType,
@@ -26,7 +26,7 @@
   export let database;
   export let connection;
 
-  export let root;
+  export let key;
 
   export let item;
   export let indentLevel = 0;
@@ -36,7 +36,7 @@
   export let model: DbKeysTreeModel;
   export let changeModel: DbKeysChangeModelFunction;
 
-  $: isExpanded = model.dirsByKey[item.root]?.isExpanded;
+  $: isExpanded = model.dirStateByKey[item.key]?.isExpanded;
 
   // $: console.log(item.text, indentLevel);
   function createMenu() {
@@ -55,7 +55,7 @@
                   args: [item.key],
                 });
 
-                changeModel(m => dbKeys_reloadFolder(m, root));
+                changeModel(m => dbKeys_clearLoadedData(m), true);
               },
             });
           },
@@ -76,23 +76,23 @@
                   args: [item.key, newName],
                 });
 
-                changeModel(m => dbKeys_reloadFolder(m, root));
+                changeModel(m => dbKeys_clearLoadedData(m), true);
               },
             });
           },
         },
-      item.type == 'dir' &&
+      // item.type == 'dir' &&
-        !connection?.isReadOnly && {
+      //   !connection?.isReadOnly && {
-          label: 'Reload',
+      //     label: 'Reload',
-          onClick: () => {
+      //     onClick: () => {
-            changeModel(m => dbKeys_reloadFolder(m, root));
+      //       changeModel(m => dbKeys_clearLoadedData(m), true);
-          },
+      //     },
-        },
+      //   },
       item.type == 'dir' &&
         !connection?.isReadOnly && {
           label: 'Delete branch',
           onClick: () => {
-            const branch = `${item.root}:*`;
+            const branch = `${item.key}:*`;
             showModal(ConfirmModal, {
               message: `Really delete branch ${branch} with all keys?`,
               onConfirm: async () => {
@@ -103,7 +103,7 @@
                   args: [branch],
                 });
 
-                changeModel(m => dbKeys_reloadFolder(m, root));
+                changeModel(m => dbKeys_clearLoadedData(m), true);
               },
             });
           },
@@ -141,12 +141,12 @@
   expandIcon={item.type == 'dir' ? plusExpandIcon(isExpanded) : 'icon invisible-box'}
   on:expand={() => {
     if (item.type == 'dir') {
-      changeModel(tree => dbKeys_markNodeExpanded(tree, item.root, !isExpanded));
+      changeModel(tree => dbKeys_markNodeExpanded(tree, item.key, !isExpanded), false);
     }
   }}
   on:click={() => {
     if (item.type == 'dir') {
-      changeModel(tree => dbKeys_markNodeExpanded(tree, item.root, !isExpanded));
+      changeModel(tree => dbKeys_markNodeExpanded(tree, item.key, !isExpanded), false);
     } else {
       openNewTab({
         tabComponent: 'DbKeyDetailTab',
@@ -184,7 +184,7 @@
   <DbKeysSubTree
     {conid}
     {database}
-    root={item.root}
+    key={item.key}
     indentLevel={indentLevel + 1}
     {connection}
     {filter}

plugins/dbgate-plugin-redis/src/backend/driver.js

@@ -201,6 +201,21 @@ const driver = {
     return _.range(16).map((index) => ({ name: `db${index}`, extInfo: info[`db${index}`], sortOrder: index }));
   },
 
+  async scanKeys(dbhan, pattern, cursor = 0, count) {
+    const match = pattern?.match(/[\?\[\{]/) ? pattern : pattern ? `*${pattern}*` : '*';
+    const [nextCursor, keys] = await dbhan.client.scan(cursor, 'MATCH', match, 'COUNT', count);
+    const dbsize = await dbhan.client.dbsize();
+    const keysMapped = keys.map((key) => ({
+      key,
+    }));
+    await this.enrichKeyInfo(dbhan, keysMapped);
+    return {
+      nextCursor,
+      keys: keysMapped,
+      dbsize,
+    };
+  },
+
   async loadKeys(dbhan, root = '', filter = null, limit = null) {
     const keys = await this.getKeys(dbhan, root ? `${root}${dbhan.treeKeySeparator}*` : '*');
     const keysFiltered = keys.filter((x) => filterName(filter, x));
@@ -305,17 +320,56 @@ const driver = {
     }
   },
 
-  async enrichOneKeyInfo(dbhan, item) {
+  // async enrichOneKeyInfo(dbhan, item) {
-    item.type = await dbhan.client.type(item.key);
+  //   item.type = await dbhan.client.type(item.key);
-    item.count = await this.getKeyCardinality(dbhan, item.key, item.type);
+  //   item.count = await this.getKeyCardinality(dbhan, item.key, item.type);
-  },
+  // },
 
-  async enrichKeyInfo(dbhan, levelInfo) {
+  async enrichKeyInfo(dbhan, keyObjects) {
-    await async.eachLimit(
+    // 1. get type
-      levelInfo.filter((x) => x.key),
+    const typePipeline = dbhan.client.pipeline();
-      10,
+    for (const item of keyObjects) {
-      async (item) => await this.enrichOneKeyInfo(dbhan, item)
+      typePipeline.type(item.key);
-    );
+    }
+    const resultType = await typePipeline.exec();
+    for (let i = 0; i < resultType.length; i++) {
+      if (resultType[i][0] == null) {
+        keyObjects[i].type = resultType[i][1];
+      }
+    }
+
+    // 2. get cardinality
+    const cardinalityPipeline = dbhan.client.pipeline();
+    for (const item of keyObjects) {
+      switch (item.type) {
+        case 'list':
+          cardinalityPipeline.llen(item.key);
+        case 'set':
+          cardinalityPipeline.scard(item.key);
+        case 'zset':
+          cardinalityPipeline.zcard(item.key);
+        case 'stream':
+          cardinalityPipeline.xlen(item.key);
+        case 'hash':
+          cardinalityPipeline.hlen(item.key);
+      }
+    }
+    const resultCardinality = await cardinalityPipeline.exec();
+    let resIndex = 0;
+    for (const item of keyObjects) {
+      if (
+        item.type == 'list' ||
+        item.type == 'set' ||
+        item.type == 'zset' ||
+        item.type == 'stream' ||
+        item.type == 'hash'
+      ) {
+        if (resultCardinality[resIndex][0] == null) {
+          item.count = resultCardinality[resIndex][1];
+          resIndex++;
+        }
+      }
+    }
   },
 
   async loadKeyInfo(dbhan, key) {