Merge branch 'feature/redis'

packages/api/src/controllers/databaseConnections.js

@@ -229,9 +229,9 @@ module.exports = {
   },
 
   loadKeys_meta: true,
-  async loadKeys({ conid, database, root, filter }, req) {
+  async loadKeys({ conid, database, root, filter, limit }, req) {
     testConnectionPermission(conid, req);
-    return this.loadDataCore('loadKeys', { conid, database, root, filter });
+    return this.loadDataCore('loadKeys', { conid, database, root, filter, limit });
   },
 
   exportKeys_meta: true,

packages/api/src/proc/databaseConnectionProcess.js

@@ -258,8 +258,8 @@ async function handleCollectionData({ msgid, options }) {
   return handleDriverDataCore(msgid, driver => driver.readCollection(dbhan, options), { logName: 'readCollection' });
 }
 
-async function handleLoadKeys({ msgid, root, filter }) {
+async function handleLoadKeys({ msgid, root, filter, limit }) {
-  return handleDriverDataCore(msgid, driver => driver.loadKeys(dbhan, root, filter), { logName: 'loadKeys' });
+  return handleDriverDataCore(msgid, driver => driver.loadKeys(dbhan, root, filter, limit), { logName: 'loadKeys' });
 }
 
 async function handleExportKeys({ msgid, options }) {

packages/web/src/widgets/DbKeysSubTree.svelte

@@ -5,7 +5,7 @@
   import LoadingInfo from '../elements/LoadingInfo.svelte';
   import { apiCall } from '../utility/api';
 
-  const SHOW_INCREMENT = 500;
+  const SHOW_INCREMENT = 100;
 
   import DbKeysTreeNode from './DbKeysTreeNode.svelte';
 
@@ -20,18 +20,46 @@
   export let filter;
 
   let reloadToken2 = 0;
-
   let maxShowCount = SHOW_INCREMENT;
+  let loading = false;
+  let loadingWhole = false;
+  let items = [];
 
-  // $: items = useDatabaseKeys({ conid, database, root, reloadToken });
+  async function loadData() {
+    loading = true;
+    const result = await apiCall('database-connections/load-keys', {
+      conid,
+      database,
+      root,
+      filter,
+      limit: maxShowCount + 1,
+    });
+    items = result;
+    loading = false;
+    loadingWhole = false;
+  }
+
+  $: {
+    conid;
+    database;
+    root;
+    filter;
+    reloadToken;
+    reloadToken2;
+    maxShowCount;
+    loadData();
+  }
+
+  $: {
+    reloadToken;
+    loadingWhole = true;
+  }
 </script>
 
-{#await apiCall('database-connections/load-keys', { conid, database, root, filter, reloadToken, reloadToken2 })}
+{#if loadingWhole}
   <LoadingInfo message="Loading key list" wrapper />
-{:then items}
+{:else}
-  {@const itemsSorted = _.sortBy(items || [], 'text')}
+  {#each items.slice(0, maxShowCount) as item}
-
-  {#each itemsSorted.slice(0, maxShowCount) as item}
     <DbKeysTreeNode
       {conid}
       {database}
@@ -46,7 +74,9 @@
     />
   {/each}
 
-  {#if itemsSorted.length > maxShowCount}
+  {#if loading}
+    <AppObjectCore {indentLevel} title="Loading keys..." icon="icon loading" expandIcon="icon invisible-box" />
+  {:else if items.length > maxShowCount}
     <AppObjectCore
       {indentLevel}
       title="Show more..."
@@ -57,4 +87,4 @@
       }}
     />
   {/if}
-{/await}
+{/if}

plugins/dbgate-plugin-redis/src/backend/driver.js

@@ -169,12 +169,14 @@ const driver = {
     return _.range(16).map((index) => ({ name: `db${index}`, extInfo: info[`db${index}`], sortOrder: index }));
   },
 
-  async loadKeys(dbhan, root = '', filter = null) {
+  async loadKeys(dbhan, root = '', filter = null, limit = null) {
     const keys = await this.getKeys(dbhan, root ? `${root}${dbhan.treeKeySeparator}*` : '*');
     const keysFiltered = keys.filter((x) => filterName(filter, x));
-    const res = this.extractKeysFromLevel(dbhan, root, keysFiltered);
+    const keysSorted = _.sortBy(keysFiltered, 'text');
-    await this.enrichKeyInfo(dbhan, res);
+    const res = this.extractKeysFromLevel(dbhan, root, keysSorted);
-    return res;
+    const resLimited = limit ? res.slice(0, limit) : res;
+    await this.enrichKeyInfo(dbhan, resLimited);
+    return resLimited;
   },
 
   async exportKeys(dbhan, options) {
@@ -192,14 +194,36 @@ const driver = {
   },
 
   async getKeys(dbhan, keyQuery = '*') {
-    const res = [];
+    const stream = dbhan.client.scanStream({
-    let cursor = 0;
+      match: keyQuery,
-    do {
+      count: 1000,
-      const [strCursor, keys] = await dbhan.client.scan(cursor, 'MATCH', keyQuery, 'COUNT', 100);
+    });
-      res.push(...keys);
+
-      cursor = parseInt(strCursor);
+    const keys = [];
-    } while (cursor > 0);
+
-    return res;
+    stream.on('data', (resultKeys) => {
+      for (const key of resultKeys) {
+        keys.push(key);
+      }
+    });
+
+    return new Promise((resolve, reject) => {
+      stream.on('end', () => {
+        resolve(keys);
+      });
+      stream.on('error', (err) => {
+        reject(err);
+      });
+    });
+
+    // const res = [];
+    // let cursor = 0;
+    // do {
+    //   const [strCursor, keys] = await dbhan.client.scan(cursor, 'MATCH', keyQuery, 'COUNT', 100);
+    //   res.push(...keys);
+    //   cursor = parseInt(strCursor);
+    // } while (cursor > 0);
+    // return res;
   },
 
   extractKeysFromLevel(dbhan, root, keys) {