perspective cache - basic design

packages/datalib/src/PerspectiveCache.ts

@@ -0,0 +1,63 @@
+import { RangeDefinition } from 'dbgate-types';
+import { PerspectiveDataLoadProps } from './PerspectiveDataProvider';
+import _pick from 'lodash/pick';
+import _omit from 'lodash/omit';
+import _difference from 'lodash/difference';
+import debug from 'debug';
+
+const dbg = debug('dbgate:PerspectiveCache');
+
+export class PerspectiveCacheTable {
+  constructor(props: PerspectiveDataLoadProps) {
+    this.schemaName = props.schemaName;
+    this.pureName = props.pureName;
+    this.bindingColumns = props.bindingColumns;
+    this.dataColumns = props.dataColumns;
+    this.loadedAll = false;
+  }
+
+  schemaName: string;
+  pureName: string;
+  bindingColumns?: string[];
+  dataColumns: string[];
+  loadedAll: boolean;
+  loadedRows: any[] = [];
+  get loadedCount() {
+    return this.loadedRows.length;
+  }
+
+  getRowsResult(props: PerspectiveDataLoadProps): { rows: any[]; incomplete: boolean } {
+    return {
+      rows: this.loadedRows.slice(0, props.topCount),
+      incomplete: props.topCount < this.loadedCount || !this.loadedAll,
+    };
+  }
+}
+
+export class PerspectiveCache {
+  constructor(public stableStringify) {}
+
+  tables: { [tableKey: string]: PerspectiveCacheTable } = {};
+
+  getTableCache(props: PerspectiveDataLoadProps) {
+    const tableKey = this.stableStringify(_omit(props, ['range', 'bindingValues', 'dataColumns']));
+    let res = this.tables[tableKey];
+
+    if (res && _difference(props.dataColumns, res.dataColumns).length > 0) {
+      dbg('Delete cache because incomplete columns', props.pureName, res.dataColumns);
+
+      // we have incomplete cache
+      delete this.tables[tableKey];
+      res = null;
+    }
+
+    if (!res) {
+      res = new PerspectiveCacheTable(props);
+      this.tables[tableKey] = res;
+      return res;
+    }
+
+    // cache could be used
+    return res;
+  }
+}

packages/datalib/src/PerspectiveDataLoader.ts

@@ -1,16 +1,73 @@
-import { Select } from 'dbgate-sqltree';
-import { PerspectiveDataLoadProps } from './PerspectiveTreeNode';
+import { Expression, Select } from 'dbgate-sqltree';
+import { PerspectiveDataLoadProps } from './PerspectiveDataProvider';
+import debug from 'debug';
 
-export interface PerspectiveDatabaseConfig {
-  conid: string;
-  database: string;
-}
+const dbg = debug('dbgate:PerspectiveDataLoader');
 
 export class PerspectiveDataLoader {
-  constructor(public apiCall, public dbg) {}
+  constructor(public apiCall) {}
+
+  async loadGrouping(props: PerspectiveDataLoadProps) {
+    const { schemaName, pureName, bindingColumns, bindingValues, dataColumns } = props;
+    const select: Select = {
+      commandType: 'select',
+      from: {
+        name: { schemaName, pureName },
+      },
+      columns: [
+        {
+          exprType: 'call',
+          func: 'COUNT',
+          args: [
+            {
+              exprType: 'raw',
+              sql: '*',
+            },
+          ],
+          alias: '_perspective_group_size_',
+        },
+        ...bindingColumns.map(
+          columnName =>
+            ({
+              exprType: 'column',
+              columnName,
+              source: {
+                name: { schemaName, pureName },
+              },
+            } as Expression)
+        ),
+      ],
+    };
+    if (bindingColumns?.length == 1) {
+      select.where = {
+        conditionType: 'in',
+        expr: {
+          exprType: 'column',
+          columnName: bindingColumns[0],
+          source: {
+            name: { schemaName, pureName },
+          },
+        },
+        values: bindingValues,
+      };
+    }
+
+    if (dbg?.enabled) {
+      dbg(`LOAD COUNTS, table=${props.pureName}, columns=${props.dataColumns?.join(',')}`);
+    }
+
+    const response = await this.apiCall('database-connections/sql-select', {
+      conid: props.databaseConfig.conid,
+      database: props.databaseConfig.database,
+      select,
+    });
+
+    if (response.errorMessage) return response;
+    return response.rows;
+  }
 
   async loadData(props: PerspectiveDataLoadProps) {
-    const { schemaName, pureName, bindingColumns, bindingValues, dataColumns } = props;
+    const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy } = props;
     const select: Select = {
       commandType: 'select',
       from: {
@@ -24,18 +81,14 @@ export class PerspectiveDataLoader {
         },
       })),
       selectAll: !dataColumns,
-      orderBy: dataColumns
-        ? [
-            {
-              exprType: 'column',
-              direction: 'ASC',
-              columnName: dataColumns[0],
-              source: {
-                name: { schemaName, pureName },
-              },
-            },
-          ]
-        : null,
+      orderBy: dataColumns?.map(columnName => ({
+        exprType: 'column',
+        columnName,
+        direction: 'ASC',
+        source: {
+          name: { schemaName, pureName },
+        },
+      })),
       range: props.range,
     };
     if (bindingColumns?.length == 1) {
@@ -52,8 +105,8 @@ export class PerspectiveDataLoader {
       };
     }
 
-    if (this.dbg?.enabled) {
-      this.dbg(`LOAD DATA, table=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${props.range}}`);
+    if (dbg?.enabled) {
+      dbg(`LOAD DATA, table=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${props.range}}`);
     }
 
     const response = await this.apiCall('database-connections/sql-select', {

packages/datalib/src/PerspectiveDataProvider.ts

@@ -1,18 +1,55 @@
+import { RangeDefinition } from 'dbgate-types';
+import { PerspectiveCache } from './PerspectiveCache';
 import { PerspectiveDataLoader } from './PerspectiveDataLoader';
-import { PerspectiveDataLoadProps } from './PerspectiveTreeNode';
 
-export interface PerspectiveDataCache {}
+export interface PerspectiveDatabaseConfig {
+  conid: string;
+  database: string;
+}
+
+export interface PerspectiveDataLoadProps {
+  databaseConfig: PerspectiveDatabaseConfig;
+  schemaName: string;
+  pureName: string;
+  dataColumns: string[];
+  orderBy: string[];
+  bindingColumns?: string[];
+  bindingValues?: any[][];
+  range?: RangeDefinition;
+  topCount?: number;
+}
 
 export class PerspectiveDataProvider {
-  constructor(
-    public cache: PerspectiveDataCache,
-    public setCache: (value: PerspectiveDataCache) => void,
-    public loader: PerspectiveDataLoader
-  ) {}
+  constructor(public cache: PerspectiveCache, public loader: PerspectiveDataLoader) {}
   async loadData(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
-    return {
-      rows: await this.loader.loadData(props),
-      incomplete: true,
-    };
+    const tableCache = this.cache.getTableCache(props);
+
+    if (props.topCount <= tableCache.loadedCount) {
+      return tableCache.getRowsResult(props);
+    }
+
+    // load missing rows
+    tableCache.dataColumns = props.dataColumns;
+
+    const nextRows = await this.loader.loadData({
+      ...props,
+      topCount: null,
+      range: {
+        offset: tableCache.loadedCount,
+        limit: props.topCount - tableCache.loadedCount,
+      },
+    });
+
+    tableCache.loadedRows = [...tableCache.loadedRows, ...nextRows];
+    tableCache.loadedAll = nextRows.length < props.topCount - tableCache.loadedCount;
+
+    // const rows=tableCache.getRows(props);
+
+    return tableCache.getRowsResult(props);
+
+    // return {
+    //   rows: await this.loader.loadData(props),
+    //   incomplete: true,
+    // };
   }
 }

packages/datalib/src/PerspectiveTreeNode.ts

@@ -6,19 +6,11 @@ import _cloneDeep from 'lodash/cloneDeep';
 import _compact from 'lodash/compact';
 import _uniq from 'lodash/uniq';
 import _flatten from 'lodash/flatten';
-import { PerspectiveDataProvider } from './PerspectiveDataProvider';
-import { PerspectiveDatabaseConfig } from './PerspectiveDataLoader';
-
-export interface PerspectiveDataLoadProps {
-  databaseConfig: PerspectiveDatabaseConfig;
-  schemaName: string;
-  pureName: string;
-  dataColumns: string[];
-  bindingColumns?: string[];
-  bindingValues?: any[][];
-  range?: RangeDefinition;
-  loadMore?: boolean;
-}
+import {
+  PerspectiveDatabaseConfig,
+  PerspectiveDataLoadProps,
+  PerspectiveDataProvider,
+} from './PerspectiveDataProvider';
 
 export interface PerspectiveDataLoadPropsWithNode {
   props: PerspectiveDataLoadProps;
@@ -182,6 +174,7 @@ export class PerspectiveTableColumnNode extends PerspectiveTreeNode {
       bindingValues: parentRows.map(row => row[this.foreignKey.columns[0].columnName]),
       dataColumns: this.getDataLoadColumns(),
       databaseConfig: this.databaseConfig,
+      orderBy: this.table.primaryKey?.columns.map(x => x.columnName) || [this.table.columns[0].columnName],
     };
   }
 
@@ -243,6 +236,7 @@ export class PerspectiveTableNode extends PerspectiveTreeNode {
       pureName: this.table.pureName,
       dataColumns: this.getDataLoadColumns(),
       databaseConfig: this.databaseConfig,
+      orderBy: this.table.primaryKey?.columns.map(x => x.columnName) || [this.table.columns[0].columnName],
     };
   }
 
@@ -313,6 +307,7 @@ export class PerspectiveTableReferenceNode extends PerspectiveTableNode {
       bindingValues: parentRows.map(row => row[this.foreignKey.columns[0].refColumnName]),
       dataColumns: this.getDataLoadColumns(),
       databaseConfig: this.databaseConfig,
+      orderBy: this.table.primaryKey?.columns.map(x => x.columnName) || [this.table.columns[0].columnName],
     };
   }
 

packages/datalib/src/index.ts

@@ -16,3 +16,4 @@ export * from './CollectionGridDisplay';
 export * from './deleteCascade';
 export * from './PerspectiveDisplay';
 export * from './PerspectiveDataProvider';
+export * from './PerspectiveCache';

packages/tools/package.json

@@ -33,6 +33,7 @@
   "dependencies": {
     "dbgate-query-splitter": "^4.9.0",
     "dbgate-sqltree": "^5.0.0-alpha.1",
+    "debug": "^4.3.4",
     "json-stable-stringify": "^1.0.1",
     "lodash": "^4.17.21",
     "uuid": "^3.4.0"

packages/web/src/perspectives/PerspectiveTable.svelte

@@ -24,7 +24,10 @@
     const loadChildNodes = [];
     const loadChildRows = [];
     const loadProps = node.getNodeLoadProps(parentRows);
-    const { rows, incomplete } = await node.dataProvider.loadData(loadProps);
+    const { rows, incomplete } = await node.dataProvider.loadData({
+      ...loadProps,
+      topCount: 100,
+    });
     // console.log('ROWS', rows, node.isRoot);
 
     if (node.isRoot) {

packages/web/src/perspectives/PerspectiveView.svelte

@@ -22,6 +22,9 @@
   import { Select } from 'dbgate-sqltree';
   import ManagerInnerContainer from '../elements/ManagerInnerContainer.svelte';
   import { PerspectiveDataLoader } from 'dbgate-datalib/lib/PerspectiveDataLoader';
+  import stableStringify from 'json-stable-stringify';
+  import createRef from '../utility/createRef';
+  import { tick } from 'svelte';
 
   const dbg = debug('dbgate:PerspectiveView');
 
@@ -34,9 +37,9 @@
   export let setConfig;
 
   export let cache;
-  export let setCache;
 
   let managerSize;
+  let nextCacheRef = createRef(null);
 
   $: if (managerSize) setLocalStorage('perspectiveManagerWidth', managerSize);
 
@@ -52,8 +55,8 @@
   const tableInfo = useTableInfo({ conid, database, schemaName, pureName });
   const viewInfo = useViewInfo({ conid, database, schemaName, pureName });
 
-  $: loader = new PerspectiveDataLoader(apiCall, dbg);
-  $: dataProvider = new PerspectiveDataProvider(cache, setCache, loader);
+  $: dataProvider = new PerspectiveDataProvider(cache, loader);
+  $: loader = new PerspectiveDataLoader(apiCall);
   $: root = $tableInfo
     ? new PerspectiveTableNode($tableInfo, $dbInfo, config, setConfig, dataProvider, { conid, database }, null)
     : null;

packages/web/src/tabs/PerspectiveTab.svelte

@@ -1,6 +1,9 @@
 <script lang="ts">
+  import { PerspectiveCache } from 'dbgate-datalib';
+
   import PerspectiveView from '../perspectives/PerspectiveView.svelte';
-  import usePerspectiveConfig, { usePerspectiveCache } from '../utility/usePerspectiveConfig';
+  import usePerspectiveConfig from '../utility/usePerspectiveConfig';
+  import stableStringify from 'json-stable-stringify';
 
   export let tabid;
   export let conid;
@@ -9,16 +12,7 @@
   export let pureName;
 
   const config = usePerspectiveConfig(tabid);
-  const cache = usePerspectiveCache();
+  const cache = new PerspectiveCache(stableStringify);
 </script>
 
-<PerspectiveView
-  {conid}
-  {database}
-  {schemaName}
-  {pureName}
-  config={$config}
-  setConfig={config.update}
-  {cache}
-  setCache={cache.update}
-/>
+<PerspectiveView {conid} {database} {schemaName} {pureName} config={$config} setConfig={config.update} {cache} />

packages/web/src/utility/usePerspectiveConfig.ts

@@ -26,7 +26,9 @@ export default function usePerspectiveConfig(tabid) {
   return config;
 }
 
-export function usePerspectiveCache() {
-  const cache = writable({});
-  return cache;
-}
+// export function usePerspectiveCache() {
+//   const cache = writable({
+//     tables: {},
+//   });
+//   return cache;
+// }