tree loader

packages/tools/src/dbKeysLoader.ts

@@ -1,4 +1,5 @@
 import _omit from 'lodash/omit';
+import _sortBy from 'lodash/sortBy';
 
 const SHOW_INCREMENT = 100;
 
@@ -29,8 +30,10 @@ export interface DbKeysTreeModel {
   childrenByKey: { [key: string]: DbKeysNodeModel[] };
   keyObjectsByKey: { [key: string]: DbKeysNodeModel };
   scannedKeys: number;
+  loadCount: number;
+  dbsize: number;
   cursor: string;
-  loadedAll: false;
+  loadedAll: boolean;
   // refreshAll?: boolean;
 }
 
@@ -46,9 +49,10 @@ export interface DbKeyLoadedModel {
 export interface DbKeysLoadResult {
   nextCursor: string;
   keys: DbKeyLoadedModel[];
+  dbsize: number;
 }
 
-export type DbKeysLoadFunction = (root: string, limit: number) => Promise<DbKeysLoadResult>;
+// export type DbKeysLoadFunction = (root: string, limit: number) => Promise<DbKeysLoadResult>;
 
 export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKeysTreeModel) => void;
 
@@ -140,13 +144,10 @@ export type DbKeysChangeModelFunction = (func: (model: DbKeysTreeModel) => DbKey
 //   };
 // }
 
-export async function dbKeys_loadNext(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise<DbKeysTreeModel> {
+export function dbKeys_mergeNextPage(tree: DbKeysTreeModel, nextPage: DbKeysLoadResult): DbKeysTreeModel {
-  const count = 2000;
   const keyObjectsByKey = { ...tree.keyObjectsByKey };
 
-  const loaded = await loader(tree.cursor, count);
+  for (const keyObj of nextPage.keys) {
-
-  for (const keyObj of loaded.keys) {
     const keyPath = keyObj.key.split(tree.treeKeySeparator);
     keyObjectsByKey[keyObj.key] = {
       ...keyObj,
@@ -205,13 +206,19 @@ export async function dbKeys_loadNext(tree: DbKeysTreeModel, loader: DbKeysLoadF
     childrenByKey[dirObj.parentKey].push(dirObj);
   }
 
+  for (const key in childrenByKey) {
+    childrenByKey[key] = _sortBy(childrenByKey[key], 'text');
+  }
+
   return {
     ...tree,
-    cursor: loaded.nextCursor,
+    cursor: nextPage.nextCursor,
     dirsByKey,
     childrenByKey,
     keyObjectsByKey,
-    scannedKeys: tree.scannedKeys + count,
+    scannedKeys: tree.scannedKeys + tree.loadCount,
+    loadedAll: nextPage.nextCursor == '0',
+    dbsize: nextPage.dbsize,
   };
 }
 
@@ -251,6 +258,8 @@ export function dbKeys_refreshAll(treeKeySeparator: string, tree?: DbKeysTreeMod
       '': root,
     },
     scannedKeys: 0,
+    dbsize: 0,
+    loadCount: 2000,
     cursor: '0',
     root,
     loadedAll: false,

packages/web/src/widgets/DbKeysSubTree.svelte

@@ -6,7 +6,7 @@
   import DbKeysTreeNode from './DbKeysTreeNode.svelte';
   import { dbKeys_markNodeExpanded, DbKeysChangeModelFunction, DbKeysTreeModel } from 'dbgate-tools';
 
-  export let root;
+  export let key;
   export let connection;
   export let database;
   export let conid;
@@ -19,34 +19,34 @@
 
   export let parentRoots = [];
 
-  $: items = model.childrenByKey[root] ?? [];
+  $: items = model.childrenByKey[key] ?? [];
 </script>
 
 {#each items as item}
   <DbKeysTreeNode
     {conid}
     {database}
-    {root}
+    {key}
     {connection}
     {item}
     {filter}
     {indentLevel}
     {model}
     {changeModel}
-    parentRoots={[...parentRoots, root]}
+    parentRoots={[...parentRoots, key]}
   />
 {/each}
 
-{#if model.dirsByKey[root]?.shouldLoadNext}
+{#if model.dirsByKey[key]?.shouldLoadNext}
   <AppObjectCore {indentLevel} title="Loading keys..." icon="icon loading" expandIcon="icon invisible-box" />
-{:else if model.dirsByKey[root]?.hasNext}
+{:else if model.dirsByKey[key]?.hasNext}
   <AppObjectCore
     {indentLevel}
     title="Show more..."
     icon="icon dots-horizontal"
     expandIcon="icon invisible-box"
     on:click={() => {
-      changeModel(model => dbKeys_markNodeExpanded(model, root, true));
+      changeModel(model => dbKeys_markNodeExpanded(model, key, true));
     }}
   />
 {/if}

packages/web/src/widgets/DbKeysTree.svelte

@@ -1,8 +1,8 @@
 <script lang="ts">
   import {
     dbKeys_getFlatList,
-    dbKeys_loadNext,
     dbKeys_markNodeExpanded,
+    dbKeys_mergeNextPage,
     dbKeys_refreshAll,
     findEngineDriver,
   } from 'dbgate-tools';
@@ -86,16 +86,15 @@
   }
 
   async function loadNextPage() {
-    model = await dbKeys_loadNext(model, async (cursor, count) => {
+    const nextScan = await apiCall('database-connections/scan-keys', {
-      const result = await apiCall('database-connections/scan-keys', {
+      conid,
-        conid,
+      database,
-        database,
+      pattern: filter,
-        pattern: filter,
+      cursor: model.cursor,
-        cursor,
+      count: model.loadCount,
-        count,
-      });
-      return result;
     });
+
+    model = dbKeys_mergeNextPage(model, nextScan);
   }
 
   function reloadModel() {
@@ -132,7 +131,11 @@
   </InlineButton>
 </SearchBoxWrapper>
 <div class="space-between align-items-center ml-1">
-  <div>Scanned 10/20 keys</div>
+  {#if model}
+    <div>
+      Scanned {Math.min(model?.scannedKeys, model?.dbsize) ?? '???'}/{model?.dbsize ?? '???'}
+    </div>
+  {/if}
   <InlineButton on:click={loadNextPage} title="Scan more keys">
     <FontIcon icon="icon more" /> Scan more
   </InlineButton>
@@ -146,7 +149,7 @@
     list={dbKeys_getFlatList(model)}
     selectedObjectStore={focusedTreeDbKey}
     getSelectedObject={getFocusedTreeDbKey}
-    selectedObjectMatcher={(o1, o2) => o1?.key == o2?.key && o1?.type == o2?.type && o1?.root == o2?.root}
+    selectedObjectMatcher={(o1, o2) => o1?.key == o2?.key && o1?.type == o2?.type}
     handleObjectClick={(data, clickAction) => {
       focusedTreeDbKey.set(data);
 
@@ -168,12 +171,12 @@
           [`${conid}:${database}`]: data.key,
         };
       }
-      if (data.root && clickAction == 'keyEnter') {
+      if (data.key && clickAction == 'keyEnter') {
-        changeModel(model => dbKeys_markNodeExpanded(model, data.root, !model.dirsByKey[data.root]?.isExpanded));
+        changeModel(model => dbKeys_markNodeExpanded(model, data.key, !model.dirsByKey[data.key]?.isExpanded));
       }
     }}
     handleExpansion={(data, value) => {
-      changeModel(model => dbKeys_markNodeExpanded(model, data.root, value));
+      changeModel(model => dbKeys_markNodeExpanded(model, data.key, value));
     }}
     onScrollTop={() => {
       domContainer?.scrollTop();
@@ -182,6 +185,6 @@
       domFilter?.focus(text);
     }}
   >
-    <DbKeysSubTree root="" {filter} {model} {changeModel} {conid} {database} {connection} />
+    <DbKeysSubTree key="" {filter} {model} {changeModel} {conid} {database} {connection} />
   </AppObjectListHandler>
 </WidgetsInnerContainer>

packages/web/src/widgets/DbKeysTreeNode.svelte

@@ -26,7 +26,7 @@
   export let database;
   export let connection;
 
-  export let root;
+  export let key;
 
   export let item;
   export let indentLevel = 0;
@@ -36,7 +36,7 @@
   export let model: DbKeysTreeModel;
   export let changeModel: DbKeysChangeModelFunction;
 
-  $: isExpanded = model.dirsByKey[item.root]?.isExpanded;
+  $: isExpanded = model.dirsByKey[item.key]?.isExpanded;
 
   // $: console.log(item.text, indentLevel);
   function createMenu() {
@@ -55,7 +55,7 @@
                   args: [item.key],
                 });
 
-                changeModel(m => dbKeys_reloadFolder(m, root));
+                changeModel(m => dbKeys_reloadFolder(m, key));
               },
             });
           },
@@ -76,7 +76,7 @@
                   args: [item.key, newName],
                 });
 
-                changeModel(m => dbKeys_reloadFolder(m, root));
+                changeModel(m => dbKeys_reloadFolder(m, key));
               },
             });
           },
@@ -85,7 +85,7 @@
         !connection?.isReadOnly && {
           label: 'Reload',
           onClick: () => {
-            changeModel(m => dbKeys_reloadFolder(m, root));
+            changeModel(m => dbKeys_reloadFolder(m, key));
           },
         },
       item.type == 'dir' &&
@@ -103,7 +103,7 @@
                   args: [branch],
                 });
 
-                changeModel(m => dbKeys_reloadFolder(m, root));
+                changeModel(m => dbKeys_reloadFolder(m, key));
               },
             });
           },
@@ -141,12 +141,12 @@
   expandIcon={item.type == 'dir' ? plusExpandIcon(isExpanded) : 'icon invisible-box'}
   on:expand={() => {
     if (item.type == 'dir') {
-      changeModel(tree => dbKeys_markNodeExpanded(tree, item.root, !isExpanded));
+      changeModel(tree => dbKeys_markNodeExpanded(tree, item.key, !isExpanded));
     }
   }}
   on:click={() => {
     if (item.type == 'dir') {
-      changeModel(tree => dbKeys_markNodeExpanded(tree, item.root, !isExpanded));
+      changeModel(tree => dbKeys_markNodeExpanded(tree, item.key, !isExpanded));
     } else {
       openNewTab({
         tabComponent: 'DbKeyDetailTab',
@@ -184,7 +184,7 @@
   <DbKeysSubTree
     {conid}
     {database}
-    root={item.root}
+    key={item.key}
     indentLevel={indentLevel + 1}
     {connection}
     {filter}

plugins/dbgate-plugin-redis/src/backend/driver.js

@@ -203,6 +203,7 @@ const driver = {
 
   async scanKeys(dbhan, pattern, cursor = 0, count) {
     const [nextCursor, keys] = await dbhan.client.scan(cursor, 'MATCH', pattern || '*', 'COUNT', count);
+    const dbsize = await dbhan.client.dbsize();
     const keysMapped = keys.map((key) => ({
       key,
     }));
@@ -210,6 +211,7 @@ const driver = {
     return {
       nextCursor,
       keys: keysMapped,
+      dbsize,
     };
   },