perspctives: nested incremental loading

packages/datalib/src/PerspectiveDataProvider.ts

@@ -1,7 +1,10 @@
 import { RangeDefinition } from 'dbgate-types';
-import { PerspectiveCache } from './PerspectiveCache';
+import { format } from 'path';
+import { PerspectiveBindingGroup, PerspectiveCache } from './PerspectiveCache';
 import { PerspectiveDataLoader } from './PerspectiveDataLoader';
 
+const PERSPECTIVE_PAGE_SIZE = 100;
+
 export interface PerspectiveDatabaseConfig {
   conid: string;
   database: string;
@@ -22,20 +25,140 @@ export interface PerspectiveDataLoadProps {
 export class PerspectiveDataProvider {
   constructor(public cache: PerspectiveCache, public loader: PerspectiveDataLoader) {}
   async loadData(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
+    // console.log('LOAD DATA', props);
+    if (props.bindingColumns) {
+      return this.loadDataNested(props);
+    } else {
+      return this.loadDataFlat(props);
+    }
+  }
+
+  async loadDataNested(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
     const tableCache = this.cache.getTableCache(props);
 
-    if (props.bindingColumns) {
-      const { cached, uncached } = tableCache.getBindingGroups(props);
+    const uncached = tableCache.getUncachedBindingGroups(props);
+    if (uncached.length > 0) {
       const counts = await this.loader.loadGrouping({
         ...props,
         bindingValues: uncached,
       });
+      // console.log('COUNTS', counts);
+      for (const resetItem of uncached) {
+        tableCache.storeGroupSize(props, resetItem, 0);
+      }
       for (const countItem of counts) {
         const { _perspective_group_size_, ...fields } = countItem;
-        tableCache.storeGroupSize(props, fields, _perspective_group_size_);
+        tableCache.storeGroupSize(
+          props,
+          props.bindingColumns.map(col => fields[col]),
+          _perspective_group_size_
+        );
       }
     }
 
+    const rows = [];
+
+    // console.log('CACHE', tableCache.bindingGroups);
+
+    let groupIndex = 0;
+    for (; groupIndex < props.bindingValues.length; groupIndex++) {
+      const groupValues = props.bindingValues[groupIndex];
+      const group = tableCache.getBindingGroup(groupValues);
+
+      if (!group.loadedAll) {
+        // wee need to load next data
+        await this.loadNextGroup(props, groupIndex);
+      }
+
+      // console.log('GRP', groupValues, group);
+      rows.push(...group.loadedRows);
+      if (rows.length >= props.topCount) {
+        return {
+          rows: rows.slice(0, props.topCount),
+          incomplete: props.topCount < rows.length || !group.loadedAll || groupIndex < props.bindingValues.length - 1,
+        };
+      }
+    }
+    if (groupIndex >= props.bindingValues.length) {
+      // all groups are fully loaded
+      return { rows, incomplete: false };
+    }
+  }
+
+  async loadNextGroup(props: PerspectiveDataLoadProps, groupIndex: number) {
+    const tableCache = this.cache.getTableCache(props);
+
+    const planLoadingGroupIndexes: number[] = [];
+    const planLoadingGroups: PerspectiveBindingGroup[] = [];
+    let planLoadRowCount = 0;
+
+    const loadPlanned = async () => {
+      // console.log(
+      //   'LOAD PLANNED',
+      //   planLoadingGroupIndexes,
+      //   planLoadingGroupIndexes.map(idx => props.bindingValues[idx])
+      // );
+      const rows = await this.loader.loadData({
+        ...props,
+        bindingValues: planLoadingGroupIndexes.map(idx => props.bindingValues[idx]),
+      });
+      // console.log('LOADED PLANNED', rows);
+      // distribute rows into groups
+      for (const row of rows) {
+        const group = planLoadingGroups.find(x => x.matchRow(row));
+        if (group) {
+          group.loadedRows.push(row);
+        }
+      }
+      for (const group of planLoadingGroups) {
+        group.loadedAll = true;
+      }
+    };
+
+    for (; groupIndex < props.bindingValues.length; groupIndex++) {
+      const groupValues = props.bindingValues[groupIndex];
+      const group = tableCache.getBindingGroup(groupValues);
+      if (group.loadedAll) continue;
+      if (group.groupSize == 0) {
+        group.loadedAll = true;
+        continue;
+      }
+      if (group.groupSize >= PERSPECTIVE_PAGE_SIZE) {
+        if (planLoadingGroupIndexes.length > 0) {
+          await loadPlanned();
+          return;
+        }
+        const nextRows = await this.loader.loadData({
+          ...props,
+          topCount: null,
+          range: {
+            offset: group.loadedRows.length,
+            limit: PERSPECTIVE_PAGE_SIZE,
+          },
+          bindingValues: [group.bindingValues],
+        });
+        group.loadedRows = [...group.loadedRows, nextRows];
+        group.loadedAll = nextRows.length < PERSPECTIVE_PAGE_SIZE;
+        return;
+      } else {
+        if (planLoadRowCount + group.groupSize > PERSPECTIVE_PAGE_SIZE) {
+          await loadPlanned();
+          return;
+        }
+        planLoadingGroupIndexes.push(groupIndex);
+        planLoadingGroups.push(group);
+        planLoadRowCount += group.groupSize;
+      }
+    }
+
+    if (planLoadingGroupIndexes.length > 0) {
+      await loadPlanned();
+    }
+  }
+
+  async loadDataFlat(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
+    const tableCache = this.cache.getTableCache(props);
+
     if (props.topCount <= tableCache.loadedCount) {
       return tableCache.getRowsResult(props);
     }
@@ -58,10 +181,5 @@ export class PerspectiveDataProvider {
     // const rows=tableCache.getRows(props);
 
     return tableCache.getRowsResult(props);
-
-    // return {
-    //   rows: await this.loader.loadData(props),
-    //   incomplete: true,
-    // };
   }
 }