incremental loading

packages/datalib/src/PerspectiveDisplay.ts

@@ -3,6 +3,9 @@ import _max from 'lodash/max';
 import _range from 'lodash/max';
 import _fill from 'lodash/fill';
 import _findIndex from 'lodash/findIndex';
+import debug from 'debug';
+
+const dbg = debug('dbgate:PerspectiveDisplay');
 
 export class PerspectiveDisplayColumn {
   title: string;
@@ -53,6 +56,7 @@ interface CollectedPerspectiveDisplayRow {
   rowData: any[];
   // rowSpans: number[] = null;
   subRowCollections: PerspectiveSubRowCollection[];
+  incompleteRowsIndicator?: string[];
 }
 
 export class PerspectiveDisplayRow {
@@ -73,6 +77,7 @@ export class PerspectiveDisplayRow {
 
   rowData: any[] = [];
   rowSpans: number[] = null;
+  incompleteRowsIndicator: string[] = null;
 }
 
 export class PerspectiveDisplay {
@@ -81,12 +86,15 @@ export class PerspectiveDisplay {
   readonly columnLevelCount: number;
 
   constructor(public root: PerspectiveTreeNode, rows: any[]) {
+    // dbg('source rows', rows);
     this.fillColumns(root.childNodes, []);
     this.columnLevelCount = _max(this.columns.map(x => x.parentNodes.length)) + 1;
     const collectedRows = this.collectRows(rows, root.childNodes);
+    // dbg('collected rows', collectedRows);
     // console.log('COLLECTED', collectedRows);
     // this.mergeRows(collectedRows);
     this.mergeRows(collectedRows);
+    // dbg('merged rows', this.rows);
     // console.log('MERGED', this.rows);
   }
 
@@ -163,6 +171,7 @@ export class PerspectiveDisplay {
         rowData,
         columnIndexes,
         subRowCollections,
+        incompleteRowsIndicator: sourceRow.incompleteRowsIndicator,
       });
     }
 
@@ -212,6 +221,7 @@ export class PerspectiveDisplay {
     for (let i = 0; i < collectedRow.columnIndexes.length; i++) {
       resultRow.rowData[collectedRow.columnIndexes[i]] = collectedRow.rowData[i];
     }
+    resultRow.incompleteRowsIndicator = collectedRow.incompleteRowsIndicator;
 
     for (const subrows of collectedRow.subRowCollections) {
       let rowIndex = 0;

packages/web/src/perspectives/PerspectiveIntersectionObserver.svelte

@@ -0,0 +1,33 @@
+<script lang="ts">
+  import { onMount } from 'svelte';
+
+  export let rootNode;
+  export let onLoadNext;
+
+  let domObserved;
+
+  function handleObserver(entries) {
+    // // console.log('HANDLE OBSERVER', loadedRows.length);
+    // if (isLoading || loadedRows.length == 0) return;
+    // loadNextData();
+    const [entry] = entries;
+
+    if (entry.isIntersecting) {
+      onLoadNext();
+    }
+  }
+
+  onMount(() => {
+    const observer = new IntersectionObserver(handleObserver, {
+      root: rootNode,
+      rootMargin: '100px',
+      threshold: 0.1,
+    });
+    observer.observe(domObserved);
+    return () => {
+      observer.disconnect();
+    };
+  });
+</script>
+
+<div bind:this={domObserved}>... data to be loaded</div>

packages/web/src/perspectives/PerspectiveTable.svelte

@@ -11,23 +11,37 @@
   import { prop_dev, tick } from 'svelte/internal';
   import { sleep } from '../utility/common';
   import resizeObserver from '../utility/resizeObserver';
+  import PerspectiveIntersectionObserver from './PerspectiveIntersectionObserver.svelte';
+  import debug from 'debug';
+
+  const dbg = debug('dbgate:PerspectivaTable');
 
   export let root: PerspectiveTreeNode;
+  export let loadedCounts;
   let dataRows;
   let domWrapper;
   let domTableHead;
   let domHeaderWrap;
   let theadClone;
 
-  async function loadLevelData(node: PerspectiveTreeNode, parentRows: any[]) {
+  async function loadLevelData(node: PerspectiveTreeNode, parentRows: any[], counts) {
+    dbg('load level data', counts);
     // const loadProps: PerspectiveDataLoadPropsWithNode[] = [];
     const loadChildNodes = [];
     const loadChildRows = [];
     const loadProps = node.getNodeLoadProps(parentRows);
-    const { rows, incomplete } = await node.dataProvider.loadData({
+    let { rows, incomplete } = await node.dataProvider.loadData({
       ...loadProps,
-      topCount: 100,
+      topCount: counts[node.uniqueName] || 100,
     });
+    if (incomplete) {
+      rows = [
+        ...rows,
+        {
+          incompleteRowsIndicator: [node.uniqueName],
+        },
+      ];
+    }
     // console.log('ROWS', rows, node.isRoot);
 
     if (node.isRoot) {
@@ -42,7 +56,7 @@
 
     for (const child of node.childNodes) {
       if (child.isExpandable && child.isChecked) {
-        await loadLevelData(child, rows);
+        await loadLevelData(child, rows, counts);
         // loadProps.push(child.getNodeLoadProps());
       }
     }
@@ -63,11 +77,11 @@
     // }
   }
 
-  async function loadData(node: PerspectiveTreeNode) {
+  async function loadData(node: PerspectiveTreeNode, counts) {
     // console.log('LOADING', node);
     if (!node) return;
     const rows = [];
-    await loadLevelData(node, rows);
+    await loadLevelData(node, rows, counts);
     dataRows = rows;
 
     // console.log('DISPLAY ROWS', rows);
@@ -102,7 +116,7 @@
 
   onMount(() => {});
 
-  $: loadData(root);
+  $: loadData(root, $loadedCounts);
   $: display = root && dataRows ? new PerspectiveDisplay(root, dataRows) : null;
 
   $: {
@@ -137,12 +151,30 @@
       <tbody>
         {#each display.rows as row}
           <tr>
+            {#if row.incompleteRowsIndicator}
+              <td colspan={display.columns.length}
+                ><PerspectiveIntersectionObserver
+                  rootNode={domWrapper}
+                  onLoadNext={() => {
+                    dbg('load next', row.incompleteRowsIndicator);
+                    loadedCounts.update(counts => {
+                      const res = { ...counts };
+                      for (const id of row.incompleteRowsIndicator) {
+                        res[id] = (res[id] || 100) + 100;
+                      }
+                      return res;
+                    });
+                  }}
+                /></td
+              >
+            {:else}
               {#each display.columns as column}
                 <!-- <td>{row.rowSpans[column.columnIndex]} {row.rowData[column.columnIndex]}</td> -->
                 {#if row.rowData[column.columnIndex] !== undefined}
                   <td rowspan={row.rowSpans[column.columnIndex]}>{row.rowData[column.columnIndex]}</td>
                 {/if}
               {/each}
+            {/if}
           </tr>
         {/each}
       </tbody>

packages/web/src/perspectives/PerspectiveView.svelte

@@ -35,6 +35,7 @@
 
   export let config;
   export let setConfig;
+  export let loadedCounts;
 
   export let cache;
 
@@ -58,7 +59,15 @@
   $: dataProvider = new PerspectiveDataProvider(cache, loader);
   $: loader = new PerspectiveDataLoader(apiCall);
   $: root = $tableInfo
-    ? new PerspectiveTableNode($tableInfo, $dbInfo, config, setConfig, dataProvider, { conid, database }, null)
+    ? new PerspectiveTableNode(
+        $tableInfo,
+        $dbInfo,
+        config,
+        setConfig,
+        dataProvider,
+        { conid, database },
+        null
+      )
     : null;
 </script>
 
@@ -76,7 +85,7 @@
   </div>
 
   <svelte:fragment slot="2">
-    <PerspectiveTable {root} />
+    <PerspectiveTable {root} {loadedCounts} />
   </svelte:fragment>
 </HorizontalSplitter>
 

packages/web/src/tabs/PerspectiveTab.svelte

@@ -4,6 +4,7 @@
   import PerspectiveView from '../perspectives/PerspectiveView.svelte';
   import usePerspectiveConfig from '../utility/usePerspectiveConfig';
   import stableStringify from 'json-stable-stringify';
+  import { writable } from 'svelte/store';
 
   export let tabid;
   export let conid;
@@ -13,6 +14,16 @@
 
   const config = usePerspectiveConfig(tabid);
   const cache = new PerspectiveCache(stableStringify);
+  const loadedCounts = writable({});
 </script>
 
-<PerspectiveView {conid} {database} {schemaName} {pureName} config={$config} setConfig={config.update} {cache} />
+<PerspectiveView
+  {conid}
+  {database}
+  {schemaName}
+  {pureName}
+  config={$config}
+  setConfig={config.update}
+  {cache}
+  {loadedCounts}
+/>

packages/web/src/utility/usePerspectiveConfig.ts

@@ -25,10 +25,3 @@ export default function usePerspectiveConfig(tabid) {
   onDestroy(unsubscribe);
   return config;
 }
-
-// export function usePerspectiveCache() {
-//   const cache = writable({
-//     tables: {},
-//   });
-//   return cache;
-// }