perspctives: nested incremental loading

packages/datalib/src/PerspectiveCache.ts

@@ -14,6 +14,10 @@ export class PerspectiveBindingGroup {
   loadedAll: boolean;
   loadedRows: any[] = [];
   bindingValues: any[];
+
+  matchRow(row) {
+    return this.table.bindingColumns.every((column, index) => row[column] == this.bindingValues[index]);
+  }
 }
 
 export class PerspectiveCacheTable {
@@ -44,28 +48,34 @@ export class PerspectiveCacheTable {
     };
   }
 
-  getBindingGroups(props: PerspectiveDataLoadProps): { cached: PerspectiveBindingGroup[]; uncached: any[][] } {
-    const cached: PerspectiveBindingGroup[] = [];
+  getBindingGroup(groupValues: any[]) {
+    const key = this.cache.stableStringify(groupValues);
+    return this.bindingGroups[key];
+  }
+
+  getUncachedBindingGroups(props: PerspectiveDataLoadProps): any[][] {
     const uncached = [];
-    for (const group in props.bindingValues) {
+    for (const group of props.bindingValues) {
       const key = this.cache.stableStringify(group);
       const item = this.bindingGroups[key];
-      if (item) {
-        cached.push(item);
-      } else {
+      if (!item) {
         uncached.push(group);
       }
     }
-    return { cached, uncached };
+    return uncached;
   }
 
   storeGroupSize(props: PerspectiveDataLoadProps, bindingValues: any[], count: number) {
     const originalBindingValue = props.bindingValues.find(v => _zip(v, bindingValues).every(([x, y]) => x == y));
     if (originalBindingValue) {
       const key = this.cache.stableStringify(originalBindingValue);
+      // console.log('SET SIZE', originalBindingValue, bindingValues, key, count);
       const group = new PerspectiveBindingGroup(this);
+      group.bindingValues = bindingValues;
       group.groupSize = count;
       this.bindingGroups[key] = group;
+    } else {
+      dbg('Group not found', bindingValues);
     }
   }
 }

packages/datalib/src/PerspectiveDataLoader.ts

@@ -9,6 +9,18 @@ export class PerspectiveDataLoader {
 
   async loadGrouping(props: PerspectiveDataLoadProps) {
     const { schemaName, pureName, bindingColumns, bindingValues, dataColumns } = props;
+
+    const bindingColumnExpressions = bindingColumns.map(
+      columnName =>
+        ({
+          exprType: 'column',
+          columnName,
+          source: {
+            name: { schemaName, pureName },
+          },
+        } as Expression)
+    );
+
     const select: Select = {
       commandType: 'select',
       from: {
@@ -26,16 +38,7 @@ export class PerspectiveDataLoader {
           ],
           alias: '_perspective_group_size_',
         },
-        ...bindingColumns.map(
-          columnName =>
-            ({
-              exprType: 'column',
-              columnName,
-              source: {
-                name: { schemaName, pureName },
-              },
-            } as Expression)
-        ),
+        ...bindingColumnExpressions,
       ],
     };
     if (bindingColumns?.length == 1) {
@@ -48,10 +51,12 @@ export class PerspectiveDataLoader {
             name: { schemaName, pureName },
           },
         },
-        values: bindingValues,
+        values: bindingValues.map(x => x[0]),
       };
     }
 
+    select.groupBy = bindingColumnExpressions;
+
     if (dbg?.enabled) {
       dbg(`LOAD COUNTS, table=${props.pureName}, columns=${props.dataColumns?.join(',')}`);
     }
@@ -63,7 +68,10 @@ export class PerspectiveDataLoader {
     });
 
     if (response.errorMessage) return response;
-    return response.rows;
+    return response.rows.map(row => ({
+      ...row,
+      _perspective_group_size_: parseInt(row._perspective_group_size_),
+    }));
   }
 
   async loadData(props: PerspectiveDataLoadProps) {
@@ -101,12 +109,16 @@ export class PerspectiveDataLoader {
             name: { schemaName, pureName },
           },
         },
-        values: bindingValues,
+        values: bindingValues.map(x => x[0]),
       };
     }
 
     if (dbg?.enabled) {
-      dbg(`LOAD DATA, table=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${props.range?.offset},${props.range?.limit}`);
+      dbg(
+        `LOAD DATA, table=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${props.range?.offset},${
+          props.range?.limit
+        }`
+      );
     }
 
     const response = await this.apiCall('database-connections/sql-select', {

packages/datalib/src/PerspectiveDataProvider.ts

@@ -1,7 +1,10 @@
 import { RangeDefinition } from 'dbgate-types';
-import { PerspectiveCache } from './PerspectiveCache';
+import { format } from 'path';
+import { PerspectiveBindingGroup, PerspectiveCache } from './PerspectiveCache';
 import { PerspectiveDataLoader } from './PerspectiveDataLoader';
 
+const PERSPECTIVE_PAGE_SIZE = 100;
+
 export interface PerspectiveDatabaseConfig {
   conid: string;
   database: string;
@@ -22,20 +25,140 @@ export interface PerspectiveDataLoadProps {
 export class PerspectiveDataProvider {
   constructor(public cache: PerspectiveCache, public loader: PerspectiveDataLoader) {}
   async loadData(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
+    // console.log('LOAD DATA', props);
+    if (props.bindingColumns) {
+      return this.loadDataNested(props);
+    } else {
+      return this.loadDataFlat(props);
+    }
+  }
+
+  async loadDataNested(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
     const tableCache = this.cache.getTableCache(props);
 
-    if (props.bindingColumns) {
-      const { cached, uncached } = tableCache.getBindingGroups(props);
+    const uncached = tableCache.getUncachedBindingGroups(props);
+    if (uncached.length > 0) {
       const counts = await this.loader.loadGrouping({
         ...props,
         bindingValues: uncached,
       });
+      // console.log('COUNTS', counts);
+      for (const resetItem of uncached) {
+        tableCache.storeGroupSize(props, resetItem, 0);
+      }
       for (const countItem of counts) {
         const { _perspective_group_size_, ...fields } = countItem;
-        tableCache.storeGroupSize(props, fields, _perspective_group_size_);
+        tableCache.storeGroupSize(
+          props,
+          props.bindingColumns.map(col => fields[col]),
+          _perspective_group_size_
+        );
       }
     }
 
+    const rows = [];
+
+    // console.log('CACHE', tableCache.bindingGroups);
+
+    let groupIndex = 0;
+    for (; groupIndex < props.bindingValues.length; groupIndex++) {
+      const groupValues = props.bindingValues[groupIndex];
+      const group = tableCache.getBindingGroup(groupValues);
+
+      if (!group.loadedAll) {
+        // wee need to load next data
+        await this.loadNextGroup(props, groupIndex);
+      }
+
+      // console.log('GRP', groupValues, group);
+      rows.push(...group.loadedRows);
+      if (rows.length >= props.topCount) {
+        return {
+          rows: rows.slice(0, props.topCount),
+          incomplete: props.topCount < rows.length || !group.loadedAll || groupIndex < props.bindingValues.length - 1,
+        };
+      }
+    }
+    if (groupIndex >= props.bindingValues.length) {
+      // all groups are fully loaded
+      return { rows, incomplete: false };
+    }
+  }
+
+  async loadNextGroup(props: PerspectiveDataLoadProps, groupIndex: number) {
+    const tableCache = this.cache.getTableCache(props);
+
+    const planLoadingGroupIndexes: number[] = [];
+    const planLoadingGroups: PerspectiveBindingGroup[] = [];
+    let planLoadRowCount = 0;
+
+    const loadPlanned = async () => {
+      // console.log(
+      //   'LOAD PLANNED',
+      //   planLoadingGroupIndexes,
+      //   planLoadingGroupIndexes.map(idx => props.bindingValues[idx])
+      // );
+      const rows = await this.loader.loadData({
+        ...props,
+        bindingValues: planLoadingGroupIndexes.map(idx => props.bindingValues[idx]),
+      });
+      // console.log('LOADED PLANNED', rows);
+      // distribute rows into groups
+      for (const row of rows) {
+        const group = planLoadingGroups.find(x => x.matchRow(row));
+        if (group) {
+          group.loadedRows.push(row);
+        }
+      }
+      for (const group of planLoadingGroups) {
+        group.loadedAll = true;
+      }
+    };
+
+    for (; groupIndex < props.bindingValues.length; groupIndex++) {
+      const groupValues = props.bindingValues[groupIndex];
+      const group = tableCache.getBindingGroup(groupValues);
+      if (group.loadedAll) continue;
+      if (group.groupSize == 0) {
+        group.loadedAll = true;
+        continue;
+      }
+      if (group.groupSize >= PERSPECTIVE_PAGE_SIZE) {
+        if (planLoadingGroupIndexes.length > 0) {
+          await loadPlanned();
+          return;
+        }
+        const nextRows = await this.loader.loadData({
+          ...props,
+          topCount: null,
+          range: {
+            offset: group.loadedRows.length,
+            limit: PERSPECTIVE_PAGE_SIZE,
+          },
+          bindingValues: [group.bindingValues],
+        });
+        group.loadedRows = [...group.loadedRows, nextRows];
+        group.loadedAll = nextRows.length < PERSPECTIVE_PAGE_SIZE;
+        return;
+      } else {
+        if (planLoadRowCount + group.groupSize > PERSPECTIVE_PAGE_SIZE) {
+          await loadPlanned();
+          return;
+        }
+        planLoadingGroupIndexes.push(groupIndex);
+        planLoadingGroups.push(group);
+        planLoadRowCount += group.groupSize;
+      }
+    }
+
+    if (planLoadingGroupIndexes.length > 0) {
+      await loadPlanned();
+    }
+  }
+
+  async loadDataFlat(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
+    const tableCache = this.cache.getTableCache(props);
+
     if (props.topCount <= tableCache.loadedCount) {
       return tableCache.getRowsResult(props);
     }
@@ -58,10 +181,5 @@ export class PerspectiveDataProvider {
     // const rows=tableCache.getRows(props);
 
     return tableCache.getRowsResult(props);
-
-    // return {
-    //   rows: await this.loader.loadData(props),
-    //   incomplete: true,
-    // };
   }
 }

packages/datalib/src/PerspectiveTreeNode.ts

@@ -171,7 +171,7 @@ export class PerspectiveTableColumnNode extends PerspectiveTreeNode {
       schemaName: this.foreignKey.refSchemaName,
       pureName: this.foreignKey.refTableName,
       bindingColumns: [this.foreignKey.columns[0].refColumnName],
-      bindingValues: parentRows.map(row => row[this.foreignKey.columns[0].columnName]),
+      bindingValues: parentRows.map(row => [row[this.foreignKey.columns[0].columnName]]),
       dataColumns: this.getDataLoadColumns(),
       databaseConfig: this.databaseConfig,
       orderBy: this.table.primaryKey?.columns.map(x => x.columnName) || [this.table.columns[0].columnName],
@@ -304,7 +304,7 @@ export class PerspectiveTableReferenceNode extends PerspectiveTableNode {
       schemaName: this.table.schemaName,
       pureName: this.table.pureName,
       bindingColumns: [this.foreignKey.columns[0].columnName],
-      bindingValues: parentRows.map(row => row[this.foreignKey.columns[0].refColumnName]),
+      bindingValues: parentRows.map(row => [row[this.foreignKey.columns[0].refColumnName]]),
       dataColumns: this.getDataLoadColumns(),
       databaseConfig: this.databaseConfig,
       orderBy: this.table.primaryKey?.columns.map(x => x.columnName) || [this.table.columns[0].columnName],