redis load key refactor #1062

packages/api/src/controllers/databaseConnections.js

@@ -304,6 +304,12 @@ module.exports = {
     return this.loadDataCore('loadKeys', { conid, database, root, filter, limit });
   },
 
+  scanKeys_meta: true,
+  async scanKeys({ conid, database, root, pattern, cursor, count }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('scanKeys', { conid, database, root, pattern, cursor, count });
+  },
+
   exportKeys_meta: true,
   async exportKeys({ conid, database, options }, req) {
     testConnectionPermission(conid, req);

packages/api/src/proc/databaseConnectionProcess.js

@@ -275,6 +275,10 @@ async function handleLoadKeys({ msgid, root, filter, limit }) {
   return handleDriverDataCore(msgid, driver => driver.loadKeys(dbhan, root, filter, limit), { logName: 'loadKeys' });
 }
 
+async function handleScanKeys({ msgid, pattern, cursor, count }) {
+  return handleDriverDataCore(msgid, driver => driver.scanKeys(dbhan, pattern, cursor, count), { logName: 'scanKeys' });
+}
+
 async function handleExportKeys({ msgid, options }) {
   return handleDriverDataCore(msgid, driver => driver.exportKeys(dbhan, options), { logName: 'exportKeys' });
 }
@@ -453,6 +457,7 @@ const messageHandlers = {
   updateCollection: handleUpdateCollection,
   collectionData: handleCollectionData,
   loadKeys: handleLoadKeys,
+  scanKeys: handleScanKeys,
   loadKeyInfo: handleLoadKeyInfo,
   callMethod: handleCallMethod,
   loadKeyTableRange: handleLoadKeyTableRange,

packages/tools/src/dbKeysLoader.ts

@@ -4,35 +4,51 @@ const SHOW_INCREMENT = 100;
 
 export interface DbKeysNodeModelBase {
   text?: string;
+  key: string;
   count?: number;
   level: number;
+  keyPath: string[];
+  parentKey: string;
 }
 
 export interface DbKeysLeafNodeModel extends DbKeysNodeModelBase {
-  key: string;
-
   type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL';
 }
 
 export interface DbKeysFolderNodeModel extends DbKeysNodeModelBase {
-  root: string;
+  // root: string;
   type: 'dir';
-  maxShowCount?: number;
+  visibleCount?: number;
   isExpanded?: boolean;
-  shouldLoadNext?: boolean;
-  hasNext?: boolean;
 }
 
 export interface DbKeysTreeModel {
+  treeKeySeparator: string;
   root: DbKeysFolderNodeModel;
   dirsByKey: { [key: string]: DbKeysFolderNodeModel };
   childrenByKey: { [key: string]: DbKeysNodeModel[] };
-  refreshAll?: boolean;
+  keyObjectsByKey: { [key: string]: DbKeysNodeModel };
+  scannedKeys: number;
+  cursor: string;
+  loadedAll: false;
+  // refreshAll?: boolean;
 }
 
 export type DbKeysNodeModel = DbKeysLeafNodeModel | DbKeysFolderNodeModel;
 
-export type DbKeysLoadFunction = (root: string, limit: number) => Promise<DbKeysNodeModel[]>;
+export interface DbKeyLoadedModel {
+  key: string;
+
+  type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL';
+  count?: number;
+}
+
+export interface DbKeysLoadResult {
+  nextCursor: string;
+  keys: DbKeyLoadedModel[];
+}
+
+export type DbKeysLoadFunction = (root: string, limit: number) => Promise<DbKeysLoadResult>;
 
 export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKeysTreeModel) => void;
 
@@ -73,54 +89,129 @@ export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKey
 //   };
 // }
 
-export async function dbKeys_loadMissing(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise<DbKeysTreeModel> {
-  const childrenByKey = { ...tree.childrenByKey };
-  const dirsByKey = { ...tree.dirsByKey };
+// export async function dbKeys_loadMissing(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise<DbKeysTreeModel> {
+//   const childrenByKey = { ...tree.childrenByKey };
+//   const dirsByKey = { ...tree.dirsByKey };
 
-  for (const root in tree.dirsByKey) {
-    const dir = tree.dirsByKey[root];
+//   for (const root in tree.dirsByKey) {
+//     const dir = tree.dirsByKey[root];
 
-    if (dir.isExpanded && dir.shouldLoadNext) {
-      if (!tree.childrenByKey[root] || dir.hasNext) {
-        const loadCount = dir.maxShowCount && dir.shouldLoadNext ? dir.maxShowCount + SHOW_INCREMENT : SHOW_INCREMENT;
-        const items = await loader(root, loadCount + 1);
+//     if (dir.isExpanded && dir.shouldLoadNext) {
+//       if (!tree.childrenByKey[root] || dir.hasNext) {
+//         const loadCount = dir.maxShowCount && dir.shouldLoadNext ? dir.maxShowCount + SHOW_INCREMENT : SHOW_INCREMENT;
+//         const items = await loader(root, loadCount + 1);
 
-        childrenByKey[root] = items.slice(0, loadCount);
-        dirsByKey[root] = {
-          ...dir,
-          shouldLoadNext: false,
-          maxShowCount: loadCount,
-          hasNext: items.length > loadCount,
-        };
+//         childrenByKey[root] = items.slice(0, loadCount);
+//         dirsByKey[root] = {
+//           ...dir,
+//           shouldLoadNext: false,
+//           maxShowCount: loadCount,
+//           hasNext: items.length > loadCount,
+//         };
 
-        for (const child of items.slice(0, loadCount)) {
-          if (child.type == 'dir' && !dirsByKey[child.root]) {
-            dirsByKey[child.root] = {
-              shouldLoadNext: false,
-              maxShowCount: null,
-              hasNext: false,
-              isExpanded: false,
-              type: 'dir',
-              level: dir.level + 1,
-              root: child.root,
-              text: child.text,
-            };
-          }
-        }
-      } else {
-        dirsByKey[root] = {
-          ...dir,
-          shouldLoadNext: false,
+//         for (const child of items.slice(0, loadCount)) {
+//           if (child.type == 'dir' && !dirsByKey[child.root]) {
+//             dirsByKey[child.root] = {
+//               shouldLoadNext: false,
+//               maxShowCount: null,
+//               hasNext: false,
+//               isExpanded: false,
+//               type: 'dir',
+//               level: dir.level + 1,
+//               root: child.root,
+//               text: child.text,
+//             };
+//           }
+//         }
+//       } else {
+//         dirsByKey[root] = {
+//           ...dir,
+//           shouldLoadNext: false,
+//         };
+//       }
+//     }
+//   }
+
+//   return {
+//     ...tree,
+//     dirsByKey,
+//     childrenByKey,
+//     refreshAll: false,
+//   };
+// }
+
+export async function dbKeys_loadNext(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise<DbKeysTreeModel> {
+  const count = 2000;
+  const keyObjectsByKey = { ...tree.keyObjectsByKey };
+
+  const loaded = await loader(tree.cursor, count);
+
+  for (const keyObj of loaded.keys) {
+    const keyPath = keyObj.key.split(tree.treeKeySeparator);
+    keyObjectsByKey[keyObj.key] = {
+      ...keyObj,
+      level: keyPath.length,
+      text: keyPath[keyPath.length - 1],
+      keyPath,
+      parentKey: keyPath.slice(0, -1).join(tree.treeKeySeparator),
+    };
+  }
+
+  const dirsByKey: { [key: string]: DbKeysFolderNodeModel } = {};
+  const childrenByKey: { [key: string]: DbKeysNodeModel[] } = {};
+
+  dirsByKey[''] = tree.root;
+
+  for (const keyObj of Object.values(keyObjectsByKey)) {
+    const dirPath = keyObj.keyPath.slice(0, -1);
+    const dirKey = dirPath.join(tree.treeKeySeparator);
+
+    let dirDepth = keyObj.keyPath.length - 1;
+
+    while (dirDepth > 0) {
+      const newDirPath = keyObj.keyPath.slice(0, dirDepth);
+      const newDirKey = newDirPath.join(tree.treeKeySeparator);
+      if (!dirsByKey[newDirKey]) {
+        dirsByKey[newDirKey] = {
+          isExpanded: tree.dirsByKey[newDirKey]?.isExpanded ?? false,
+          level: keyObj.level - 1,
+          keyPath: newDirPath,
+          parentKey: newDirPath.slice(0, -1).join(tree.treeKeySeparator),
+          type: 'dir',
+          key: newDirKey,
+          visibleCount: tree.dirsByKey[newDirKey]?.visibleCount ?? SHOW_INCREMENT,
+          text: `${newDirPath[newDirPath.length - 1]}${tree.treeKeySeparator}*`,
         };
       }
+
+      dirDepth -= 1;
     }
+
+    if (!childrenByKey[dirKey]) {
+      childrenByKey[dirKey] = [];
+    }
+
+    childrenByKey[dirKey].push(keyObj);
+  }
+
+  for (const dirObj of Object.values(dirsByKey)) {
+    if (dirObj.key == '') {
+      continue;
+    }
+    
+    if (!childrenByKey[dirObj.parentKey]) {
+      childrenByKey[dirObj.parentKey] = [];
+    }
+    childrenByKey[dirObj.parentKey].push(dirObj);
   }
 
   return {
     ...tree,
+    cursor: loaded.nextCursor,
     dirsByKey,
     childrenByKey,
-    refreshAll: false,
+    keyObjectsByKey,
+    scannedKeys: tree.scannedKeys + count,
   };
 }
 
@@ -136,45 +227,50 @@ export function dbKeys_markNodeExpanded(tree: DbKeysTreeModel, root: string, isE
       [root]: {
         ...node,
         isExpanded,
-        shouldLoadNext: isExpanded,
       },
     },
   };
 }
 
-export function dbKeys_refreshAll(tree?: DbKeysTreeModel): DbKeysTreeModel {
+export function dbKeys_refreshAll(treeKeySeparator: string, tree?: DbKeysTreeModel): DbKeysTreeModel {
   const root: DbKeysFolderNodeModel = {
     isExpanded: true,
     level: 0,
-    root: '',
     type: 'dir',
-    shouldLoadNext: true,
+    keyPath: [],
+    parentKey: '',
+    key: '',
+    visibleCount: SHOW_INCREMENT,
   };
   return {
     ...tree,
+    treeKeySeparator,
     childrenByKey: {},
+    keyObjectsByKey: {},
     dirsByKey: {
       '': root,
     },
-    refreshAll: true,
+    scannedKeys: 0,
+    cursor: '0',
     root,
+    loadedAll: false,
   };
 }
 
-export function dbKeys_reloadFolder(tree: DbKeysTreeModel, root: string): DbKeysTreeModel {
-  return {
-    ...tree,
-    childrenByKey: _omit(tree.childrenByKey, root),
-    dirsByKey: {
-      ...tree.dirsByKey,
-      [root]: {
-        ...tree.dirsByKey[root],
-        shouldLoadNext: true,
-        hasNext: undefined,
-      },
-    },
-  };
-}
+// export function dbKeys_reloadFolder(tree: DbKeysTreeModel, root: string): DbKeysTreeModel {
+//   return {
+//     ...tree,
+//     childrenByKey: _omit(tree.childrenByKey, root),
+//     dirsByKey: {
+//       ...tree.dirsByKey,
+//       [root]: {
+//         ...tree.dirsByKey[root],
+//         shouldLoadNext: true,
+//         hasNext: undefined,
+//       },
+//     },
+//   };
+// }
 
 function addFlatItems(tree: DbKeysTreeModel, root: string, res: DbKeysNodeModel[], visitedRoots: string[] = []) {
   const item = tree.dirsByKey[root];
@@ -185,11 +281,11 @@ function addFlatItems(tree: DbKeysTreeModel, root: string, res: DbKeysNodeModel[
   for (const child of children) {
     res.push(child);
     if (child.type == 'dir') {
-      if (visitedRoots.includes(child.root)) {
-        console.warn('Redis: preventing infinite loop for root', child.root);
+      if (visitedRoots.includes(child.key)) {
+        console.warn('Redis: preventing infinite loop for root', child.key);
         return false;
       }
-      addFlatItems(tree, child.root, res, [...visitedRoots, root]);
+      addFlatItems(tree, child.key, res, [...visitedRoots, root]);
     }
   }
 }

packages/types/engines.d.ts

@@ -239,6 +239,7 @@ export interface EngineDriver<TClient = any> extends FilterBehaviourProvider {
     }[]
   >;
   loadKeys(dbhan: DatabaseHandle<TClient>, root: string, filter?: string): Promise;
+  scanKeys(dbhan: DatabaseHandle<TClient>, root: string, pattern: string, cursor: string, count: number): Promise;
   exportKeys(dbhan: DatabaseHandle<TClient>, options: {}): Promise;
   loadKeyInfo(dbhan: DatabaseHandle<TClient>, key): Promise;
   loadKeyTableRange(dbhan: DatabaseHandle<TClient>, key, cursor, count): Promise;

packages/web/public/global.css

@@ -36,6 +36,9 @@ body {
   display: flex;
   justify-content: space-between;
 }
+.align-items-center {
+  align-items: center;
+}
 .flex {
   display: flex;
 }

packages/web/src/elements/SearchBoxWrapper.svelte

@@ -1,4 +1,8 @@
-<div><slot /></div>
+<script lang="ts">
+  export let noMargin = false;
+</script>
+
+<div class:noMargin><slot /></div>
 
 <style>
   div {
@@ -6,4 +10,8 @@
     border-bottom: 1px solid var(--theme-border);
     margin-bottom: 5px;
   }
+
+  div.noMargin {
+    margin-bottom: 0;
+  }
 </style>

packages/web/src/icons/FontIcon.svelte

@@ -151,6 +151,7 @@
     'icon text': 'mdi mdi-text',
     'icon ai': 'mdi mdi-head-lightbulb',
     'icon wait': 'mdi mdi-timer-sand',
+    'icon more': 'mdi mdi-more',
 
     'icon run': 'mdi mdi-play',
     'icon chevron-down': 'mdi mdi-chevron-down',

packages/web/src/utility/metadataLoaders.ts

@@ -83,12 +83,6 @@ const databaseListLoader = ({ conid }) => ({
   errorValue: [],
 });
 
-// const databaseKeysLoader = ({ conid, database, root }) => ({
-//   url: 'database-connections/load-keys',
-//   params: { conid, database, root },
-//   reloadTrigger: `database-keys-changed-${conid}-${database}`,
-// });
-
 const serverVersionLoader = ({ conid }) => ({
   url: 'server-connections/version',
   params: { conid },

packages/web/src/widgets/DatabaseWidget.svelte

@@ -80,7 +80,7 @@
     storageName="dbObjectsWidget"
     skip={!(conid && (database || singleDatabase) && driver?.databaseEngineTypes?.includes('keyvalue'))}
   >
-    <DbKeysTree {conid} {database} />
+    <DbKeysTree {conid} {database} treeKeySeparator={$connection?.treeKeySeparator || ':'} />
   </WidgetColumnBarItem>
 
   <WidgetColumnBarItem

packages/web/src/widgets/DbKeysTree.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
   import {
     dbKeys_getFlatList,
-    dbKeys_loadMissing,
+    dbKeys_loadNext,
     dbKeys_markNodeExpanded,
     dbKeys_refreshAll,
     findEngineDriver,
@@ -36,6 +36,7 @@
 
   export let conid;
   export let database;
+  export let treeKeySeparator = ':';
 
   let domListHandler;
   let domContainer = null;
@@ -43,10 +44,10 @@
 
   let filter;
 
-  let model = dbKeys_refreshAll();
+  let model = dbKeys_refreshAll(treeKeySeparator);
 
   function handleRefreshDatabase() {
-    changeModel(model => dbKeys_refreshAll(model));
+    changeModel(model => dbKeys_refreshAll(treeKeySeparator, model));
   }
 
   function handleAddKey() {
@@ -80,22 +81,26 @@
 
   $: connection = useConnectionInfo({ conid });
 
-  async function changeModel(modelUpdate) {
+  function changeModel(modelUpdate) {
     model = modelUpdate(model);
-    model = await dbKeys_loadMissing(model, async (root, limit) => {
-      const result = await apiCall('database-connections/load-keys', {
+  }
+
+  async function loadNextPage() {
+    model = await dbKeys_loadNext(model, async (cursor, count) => {
+      const result = await apiCall('database-connections/scan-keys', {
         conid,
         database,
-        root,
-        filter,
-        limit,
+        pattern: filter,
+        cursor,
+        count,
       });
       return result;
     });
   }
 
   function reloadModel() {
-    changeModel(model => dbKeys_refreshAll(model));
+    changeModel(model => dbKeys_refreshAll(treeKeySeparator, model));
+    loadNextPage();
   }
 
   $: {
@@ -104,11 +109,13 @@
     filter;
     reloadModel();
   }
+
+  $: console.log('DbKeysTree MODEL', model);
 </script>
 
-<SearchBoxWrapper>
+<SearchBoxWrapper noMargin>
   <SearchInput
-    placeholder="Search keys"
+    placeholder="Redis pattern or key part"
     bind:value={filter}
     isDebounced
     bind:this={domFilter}
@@ -124,6 +131,12 @@
     <FontIcon icon="icon refresh" />
   </InlineButton>
 </SearchBoxWrapper>
+<div class="space-between align-items-center ml-1">
+  <div>Scanned 10/20 keys</div>
+  <InlineButton on:click={loadNextPage} title="Scan more keys">
+    <FontIcon icon="icon more" /> Scan more
+  </InlineButton>
+</div>
 {#if differentFocusedDb}
   <FocusedConnectionInfoWidget {conid} {database} connection={$connection} />
 {/if}

plugins/dbgate-plugin-redis/src/backend/driver.js

@@ -201,6 +201,18 @@ const driver = {
     return _.range(16).map((index) => ({ name: `db${index}`, extInfo: info[`db${index}`], sortOrder: index }));
   },
 
+  async scanKeys(dbhan, pattern, cursor = 0, count) {
+    const [nextCursor, keys] = await dbhan.client.scan(cursor, 'MATCH', pattern || '*', 'COUNT', count);
+    const keysMapped = keys.map((key) => ({
+      key,
+    }));
+    await this.enrichKeyInfo(dbhan, keysMapped);
+    return {
+      nextCursor,
+      keys: keysMapped,
+    };
+  },
+
   async loadKeys(dbhan, root = '', filter = null, limit = null) {
     const keys = await this.getKeys(dbhan, root ? `${root}${dbhan.treeKeySeparator}*` : '*');
     const keysFiltered = keys.filter((x) => filterName(filter, x));
@@ -310,9 +322,9 @@ const driver = {
     item.count = await this.getKeyCardinality(dbhan, item.key, item.type);
   },
 
-  async enrichKeyInfo(dbhan, levelInfo) {
+  async enrichKeyInfo(dbhan, keyObjects) {
     await async.eachLimit(
-      levelInfo.filter((x) => x.key),
+      keyObjects.filter((x) => x.key),
       10,
       async (item) => await this.enrichOneKeyInfo(dbhan, item)
     );