diff --git a/packages/datalib/src/PerspectiveCache.ts b/packages/datalib/src/PerspectiveCache.ts new file mode 100644 index 000000000..d8f2450d1 --- /dev/null +++ b/packages/datalib/src/PerspectiveCache.ts @@ -0,0 +1,63 @@ +import { RangeDefinition } from 'dbgate-types'; +import { PerspectiveDataLoadProps } from './PerspectiveDataProvider'; +import _pick from 'lodash/pick'; +import _omit from 'lodash/omit'; +import _difference from 'lodash/difference'; +import debug from 'debug'; + +const dbg = debug('dbgate:PerspectiveCache'); + +export class PerspectiveCacheTable { + constructor(props: PerspectiveDataLoadProps) { + this.schemaName = props.schemaName; + this.pureName = props.pureName; + this.bindingColumns = props.bindingColumns; + this.dataColumns = props.dataColumns; + this.loadedAll = false; + } + + schemaName: string; + pureName: string; + bindingColumns?: string[]; + dataColumns: string[]; + loadedAll: boolean; + loadedRows: any[] = []; + get loadedCount() { + return this.loadedRows.length; + } + + getRowsResult(props: PerspectiveDataLoadProps): { rows: any[]; incomplete: boolean } { + return { + rows: this.loadedRows.slice(0, props.topCount), + incomplete: props.topCount < this.loadedCount || !this.loadedAll, + }; + } +} + +export class PerspectiveCache { + constructor(public stableStringify) {} + + tables: { [tableKey: string]: PerspectiveCacheTable } = {}; + + getTableCache(props: PerspectiveDataLoadProps) { + const tableKey = this.stableStringify(_omit(props, ['range', 'bindingValues', 'dataColumns'])); + let res = this.tables[tableKey]; + + if (res && _difference(props.dataColumns, res.dataColumns).length > 0) { + dbg('Delete cache because incomplete columns', props.pureName, res.dataColumns); + + // we have incomplete cache + delete this.tables[tableKey]; + res = null; + } + + if (!res) { + res = new PerspectiveCacheTable(props); + this.tables[tableKey] = res; + return res; + } + + // cache could be used + return res; + } +} diff --git a/packages/datalib/src/PerspectiveDataLoader.ts b/packages/datalib/src/PerspectiveDataLoader.ts index 320639a95..3020682bd 100644 --- a/packages/datalib/src/PerspectiveDataLoader.ts +++ b/packages/datalib/src/PerspectiveDataLoader.ts @@ -1,16 +1,73 @@ -import { Select } from 'dbgate-sqltree'; -import { PerspectiveDataLoadProps } from './PerspectiveTreeNode'; +import { Expression, Select } from 'dbgate-sqltree'; +import { PerspectiveDataLoadProps } from './PerspectiveDataProvider'; +import debug from 'debug'; -export interface PerspectiveDatabaseConfig { - conid: string; - database: string; -} +const dbg = debug('dbgate:PerspectiveDataLoader'); export class PerspectiveDataLoader { - constructor(public apiCall, public dbg) {} + constructor(public apiCall) {} + + async loadGrouping(props: PerspectiveDataLoadProps) { + const { schemaName, pureName, bindingColumns, bindingValues, dataColumns } = props; + const select: Select = { + commandType: 'select', + from: { + name: { schemaName, pureName }, + }, + columns: [ + { + exprType: 'call', + func: 'COUNT', + args: [ + { + exprType: 'raw', + sql: '*', + }, + ], + alias: '_perspective_group_size_', + }, + ...bindingColumns.map( + columnName => + ({ + exprType: 'column', + columnName, + source: { + name: { schemaName, pureName }, + }, + } as Expression) + ), + ], + }; + if (bindingColumns?.length == 1) { + select.where = { + conditionType: 'in', + expr: { + exprType: 'column', + columnName: bindingColumns[0], + source: { + name: { schemaName, pureName }, + }, + }, + values: bindingValues, + }; + } + + if (dbg?.enabled) { + dbg(`LOAD COUNTS, table=${props.pureName}, columns=${props.dataColumns?.join(',')}`); + } + + const response = await this.apiCall('database-connections/sql-select', { + conid: props.databaseConfig.conid, + database: props.databaseConfig.database, + select, + }); + + if (response.errorMessage) return response; + return response.rows; + } async loadData(props: PerspectiveDataLoadProps) { - const { schemaName, pureName, bindingColumns, bindingValues, dataColumns } = props; + const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy } = props; const select: Select = { commandType: 'select', from: { @@ -24,18 +81,14 @@ export class PerspectiveDataLoader { }, })), selectAll: !dataColumns, - orderBy: dataColumns - ? [ - { - exprType: 'column', - direction: 'ASC', - columnName: dataColumns[0], - source: { - name: { schemaName, pureName }, - }, - }, - ] - : null, + orderBy: dataColumns?.map(columnName => ({ + exprType: 'column', + columnName, + direction: 'ASC', + source: { + name: { schemaName, pureName }, + }, + })), range: props.range, }; if (bindingColumns?.length == 1) { @@ -52,8 +105,8 @@ export class PerspectiveDataLoader { }; } - if (this.dbg?.enabled) { - this.dbg(`LOAD DATA, table=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${props.range}}`); + if (dbg?.enabled) { + dbg(`LOAD DATA, table=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${props.range}}`); } const response = await this.apiCall('database-connections/sql-select', { diff --git a/packages/datalib/src/PerspectiveDataProvider.ts b/packages/datalib/src/PerspectiveDataProvider.ts index 96eb1ba1e..b137e5224 100644 --- a/packages/datalib/src/PerspectiveDataProvider.ts +++ b/packages/datalib/src/PerspectiveDataProvider.ts @@ -1,18 +1,55 @@ +import { RangeDefinition } from 'dbgate-types'; +import { PerspectiveCache } from './PerspectiveCache'; import { PerspectiveDataLoader } from './PerspectiveDataLoader'; -import { PerspectiveDataLoadProps } from './PerspectiveTreeNode'; -export interface PerspectiveDataCache {} +export interface PerspectiveDatabaseConfig { + conid: string; + database: string; +} + +export interface PerspectiveDataLoadProps { + databaseConfig: PerspectiveDatabaseConfig; + schemaName: string; + pureName: string; + dataColumns: string[]; + orderBy: string[]; + bindingColumns?: string[]; + bindingValues?: any[][]; + range?: RangeDefinition; + topCount?: number; +} export class PerspectiveDataProvider { - constructor( - public cache: PerspectiveDataCache, - public setCache: (value: PerspectiveDataCache) => void, - public loader: PerspectiveDataLoader - ) {} + constructor(public cache: PerspectiveCache, public loader: PerspectiveDataLoader) {} async loadData(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> { - return { - rows: await this.loader.loadData(props), - incomplete: true, - }; + const tableCache = this.cache.getTableCache(props); + + if (props.topCount <= tableCache.loadedCount) { + return tableCache.getRowsResult(props); + } + + // load missing rows + tableCache.dataColumns = props.dataColumns; + + const nextRows = await this.loader.loadData({ + ...props, + topCount: null, + range: { + offset: tableCache.loadedCount, + limit: props.topCount - tableCache.loadedCount, + }, + }); + + tableCache.loadedRows = [...tableCache.loadedRows, ...nextRows]; + tableCache.loadedAll = nextRows.length < props.topCount - tableCache.loadedCount; + + // const rows=tableCache.getRows(props); + + return tableCache.getRowsResult(props); + + // return { + // rows: await this.loader.loadData(props), + // incomplete: true, + // }; } } diff --git a/packages/datalib/src/PerspectiveTreeNode.ts b/packages/datalib/src/PerspectiveTreeNode.ts index 674f4f805..7e37fa584 100644 --- a/packages/datalib/src/PerspectiveTreeNode.ts +++ b/packages/datalib/src/PerspectiveTreeNode.ts @@ -6,19 +6,11 @@ import _cloneDeep from 'lodash/cloneDeep'; import _compact from 'lodash/compact'; import _uniq from 'lodash/uniq'; import _flatten from 'lodash/flatten'; -import { PerspectiveDataProvider } from './PerspectiveDataProvider'; -import { PerspectiveDatabaseConfig } from './PerspectiveDataLoader'; - -export interface PerspectiveDataLoadProps { - databaseConfig: PerspectiveDatabaseConfig; - schemaName: string; - pureName: string; - dataColumns: string[]; - bindingColumns?: string[]; - bindingValues?: any[][]; - range?: RangeDefinition; - loadMore?: boolean; -} +import { + PerspectiveDatabaseConfig, + PerspectiveDataLoadProps, + PerspectiveDataProvider, +} from './PerspectiveDataProvider'; export interface PerspectiveDataLoadPropsWithNode { props: PerspectiveDataLoadProps; @@ -182,6 +174,7 @@ export class PerspectiveTableColumnNode extends PerspectiveTreeNode { bindingValues: parentRows.map(row => row[this.foreignKey.columns[0].columnName]), dataColumns: this.getDataLoadColumns(), databaseConfig: this.databaseConfig, + orderBy: this.table.primaryKey?.columns.map(x => x.columnName) || [this.table.columns[0].columnName], }; } @@ -243,6 +236,7 @@ export class PerspectiveTableNode extends PerspectiveTreeNode { pureName: this.table.pureName, dataColumns: this.getDataLoadColumns(), databaseConfig: this.databaseConfig, + orderBy: this.table.primaryKey?.columns.map(x => x.columnName) || [this.table.columns[0].columnName], }; } @@ -313,6 +307,7 @@ export class PerspectiveTableReferenceNode extends PerspectiveTableNode { bindingValues: parentRows.map(row => row[this.foreignKey.columns[0].refColumnName]), dataColumns: this.getDataLoadColumns(), databaseConfig: this.databaseConfig, + orderBy: this.table.primaryKey?.columns.map(x => x.columnName) || [this.table.columns[0].columnName], }; } diff --git a/packages/datalib/src/index.ts b/packages/datalib/src/index.ts index 76113dfb5..9ec457219 100644 --- a/packages/datalib/src/index.ts +++ b/packages/datalib/src/index.ts @@ -16,3 +16,4 @@ export * from './CollectionGridDisplay'; export * from './deleteCascade'; export * from './PerspectiveDisplay'; export * from './PerspectiveDataProvider'; +export * from './PerspectiveCache'; diff --git a/packages/tools/package.json b/packages/tools/package.json index 6dfaab9ea..17001d95a 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -33,6 +33,7 @@ "dependencies": { "dbgate-query-splitter": "^4.9.0", "dbgate-sqltree": "^5.0.0-alpha.1", + "debug": "^4.3.4", "json-stable-stringify": "^1.0.1", "lodash": "^4.17.21", "uuid": "^3.4.0" diff --git a/packages/web/src/perspectives/PerspectiveTable.svelte b/packages/web/src/perspectives/PerspectiveTable.svelte index 79b759400..4e35a07cf 100644 --- a/packages/web/src/perspectives/PerspectiveTable.svelte +++ b/packages/web/src/perspectives/PerspectiveTable.svelte @@ -24,7 +24,10 @@ const loadChildNodes = []; const loadChildRows = []; const loadProps = node.getNodeLoadProps(parentRows); - const { rows, incomplete } = await node.dataProvider.loadData(loadProps); + const { rows, incomplete } = await node.dataProvider.loadData({ + ...loadProps, + topCount: 100, + }); // console.log('ROWS', rows, node.isRoot); if (node.isRoot) { diff --git a/packages/web/src/perspectives/PerspectiveView.svelte b/packages/web/src/perspectives/PerspectiveView.svelte index 64be17156..4d3135fce 100644 --- a/packages/web/src/perspectives/PerspectiveView.svelte +++ b/packages/web/src/perspectives/PerspectiveView.svelte @@ -22,6 +22,9 @@ import { Select } from 'dbgate-sqltree'; import ManagerInnerContainer from '../elements/ManagerInnerContainer.svelte'; import { PerspectiveDataLoader } from 'dbgate-datalib/lib/PerspectiveDataLoader'; + import stableStringify from 'json-stable-stringify'; + import createRef from '../utility/createRef'; + import { tick } from 'svelte'; const dbg = debug('dbgate:PerspectiveView'); @@ -34,9 +37,9 @@ export let setConfig; export let cache; - export let setCache; let managerSize; + let nextCacheRef = createRef(null); $: if (managerSize) setLocalStorage('perspectiveManagerWidth', managerSize); @@ -52,8 +55,8 @@ const tableInfo = useTableInfo({ conid, database, schemaName, pureName }); const viewInfo = useViewInfo({ conid, database, schemaName, pureName }); - $: loader = new PerspectiveDataLoader(apiCall, dbg); - $: dataProvider = new PerspectiveDataProvider(cache, setCache, loader); + $: dataProvider = new PerspectiveDataProvider(cache, loader); + $: loader = new PerspectiveDataLoader(apiCall); $: root = $tableInfo ? new PerspectiveTableNode($tableInfo, $dbInfo, config, setConfig, dataProvider, { conid, database }, null) : null; diff --git a/packages/web/src/tabs/PerspectiveTab.svelte b/packages/web/src/tabs/PerspectiveTab.svelte index ee63c3a2e..4e94a4e7f 100644 --- a/packages/web/src/tabs/PerspectiveTab.svelte +++ b/packages/web/src/tabs/PerspectiveTab.svelte @@ -1,6 +1,9 @@ - + diff --git a/packages/web/src/utility/usePerspectiveConfig.ts b/packages/web/src/utility/usePerspectiveConfig.ts index 40512050e..f8ae08fc7 100644 --- a/packages/web/src/utility/usePerspectiveConfig.ts +++ b/packages/web/src/utility/usePerspectiveConfig.ts @@ -26,7 +26,9 @@ export default function usePerspectiveConfig(tabid) { return config; } -export function usePerspectiveCache() { - const cache = writable({}); - return cache; -} +// export function usePerspectiveCache() { +// const cache = writable({ +// tables: {}, +// }); +// return cache; +// }