diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index 2e7740b5c..68a6252d6 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -174,6 +174,7 @@ async function handleQueryData({ msgid, sql }, skipReadonlyCheck = false) { try { if (!skipReadonlyCheck) ensureExecuteCustomScript(driver); const res = await driver.query(systemConnection, sql); + // console.log(sql); process.send({ msgtype: 'response', msgid, ...res }); } catch (err) { process.send({ msgtype: 'response', msgid, errorMessage: err.message }); diff --git a/packages/datalib/src/PerspectiveCache.ts b/packages/datalib/src/PerspectiveCache.ts index 27b41b58a..f14d3f336 100644 --- a/packages/datalib/src/PerspectiveCache.ts +++ b/packages/datalib/src/PerspectiveCache.ts @@ -14,6 +14,10 @@ export class PerspectiveBindingGroup { loadedAll: boolean; loadedRows: any[] = []; bindingValues: any[]; + + matchRow(row) { + return this.table.bindingColumns.every((column, index) => row[column] == this.bindingValues[index]); + } } export class PerspectiveCacheTable { @@ -44,28 +48,34 @@ export class PerspectiveCacheTable { }; } - getBindingGroups(props: PerspectiveDataLoadProps): { cached: PerspectiveBindingGroup[]; uncached: any[][] } { - const cached: PerspectiveBindingGroup[] = []; + getBindingGroup(groupValues: any[]) { + const key = this.cache.stableStringify(groupValues); + return this.bindingGroups[key]; + } + + getUncachedBindingGroups(props: PerspectiveDataLoadProps): any[][] { const uncached = []; - for (const group in props.bindingValues) { + for (const group of props.bindingValues) { const key = this.cache.stableStringify(group); const item = this.bindingGroups[key]; - if (item) { - cached.push(item); - } else { + if (!item) { uncached.push(group); } } - return { cached, uncached }; + return uncached; } storeGroupSize(props: PerspectiveDataLoadProps, bindingValues: any[], count: number) { const originalBindingValue = props.bindingValues.find(v => _zip(v, bindingValues).every(([x, y]) => x == y)); if (originalBindingValue) { const key = this.cache.stableStringify(originalBindingValue); + // console.log('SET SIZE', originalBindingValue, bindingValues, key, count); const group = new PerspectiveBindingGroup(this); + group.bindingValues = bindingValues; group.groupSize = count; this.bindingGroups[key] = group; + } else { + dbg('Group not found', bindingValues); } } } diff --git a/packages/datalib/src/PerspectiveDataLoader.ts b/packages/datalib/src/PerspectiveDataLoader.ts index 6ddc49301..d50a76fdf 100644 --- a/packages/datalib/src/PerspectiveDataLoader.ts +++ b/packages/datalib/src/PerspectiveDataLoader.ts @@ -9,6 +9,18 @@ export class PerspectiveDataLoader { async loadGrouping(props: PerspectiveDataLoadProps) { const { schemaName, pureName, bindingColumns, bindingValues, dataColumns } = props; + + const bindingColumnExpressions = bindingColumns.map( + columnName => + ({ + exprType: 'column', + columnName, + source: { + name: { schemaName, pureName }, + }, + } as Expression) + ); + const select: Select = { commandType: 'select', from: { @@ -26,16 +38,7 @@ export class PerspectiveDataLoader { ], alias: '_perspective_group_size_', }, - ...bindingColumns.map( - columnName => - ({ - exprType: 'column', - columnName, - source: { - name: { schemaName, pureName }, - }, - } as Expression) - ), + ...bindingColumnExpressions, ], }; if (bindingColumns?.length == 1) { @@ -48,10 +51,12 @@ export class PerspectiveDataLoader { name: { schemaName, pureName }, }, }, - values: bindingValues, + values: bindingValues.map(x => x[0]), }; } + select.groupBy = bindingColumnExpressions; + if (dbg?.enabled) { dbg(`LOAD COUNTS, table=${props.pureName}, columns=${props.dataColumns?.join(',')}`); } @@ -63,7 +68,10 @@ export class PerspectiveDataLoader { }); if (response.errorMessage) return response; - return response.rows; + return response.rows.map(row => ({ + ...row, + _perspective_group_size_: parseInt(row._perspective_group_size_), + })); } async loadData(props: PerspectiveDataLoadProps) { @@ -101,12 +109,16 @@ export class PerspectiveDataLoader { name: { schemaName, pureName }, }, }, - values: bindingValues, + values: bindingValues.map(x => x[0]), }; } if (dbg?.enabled) { - dbg(`LOAD DATA, table=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${props.range?.offset},${props.range?.limit}`); + dbg( + `LOAD DATA, table=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${props.range?.offset},${ + props.range?.limit + }` + ); } const response = await this.apiCall('database-connections/sql-select', { diff --git a/packages/datalib/src/PerspectiveDataProvider.ts b/packages/datalib/src/PerspectiveDataProvider.ts index 739540f9b..5f98c5d2a 100644 --- a/packages/datalib/src/PerspectiveDataProvider.ts +++ b/packages/datalib/src/PerspectiveDataProvider.ts @@ -1,7 +1,10 @@ import { RangeDefinition } from 'dbgate-types'; -import { PerspectiveCache } from './PerspectiveCache'; +import { format } from 'path'; +import { PerspectiveBindingGroup, PerspectiveCache } from './PerspectiveCache'; import { PerspectiveDataLoader } from './PerspectiveDataLoader'; +const PERSPECTIVE_PAGE_SIZE = 100; + export interface PerspectiveDatabaseConfig { conid: string; database: string; @@ -22,20 +25,140 @@ export interface PerspectiveDataLoadProps { export class PerspectiveDataProvider { constructor(public cache: PerspectiveCache, public loader: PerspectiveDataLoader) {} async loadData(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> { + // console.log('LOAD DATA', props); + if (props.bindingColumns) { + return this.loadDataNested(props); + } else { + return this.loadDataFlat(props); + } + } + + async loadDataNested(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> { const tableCache = this.cache.getTableCache(props); - if (props.bindingColumns) { - const { cached, uncached } = tableCache.getBindingGroups(props); + const uncached = tableCache.getUncachedBindingGroups(props); + if (uncached.length > 0) { const counts = await this.loader.loadGrouping({ ...props, bindingValues: uncached, }); + // console.log('COUNTS', counts); + for (const resetItem of uncached) { + tableCache.storeGroupSize(props, resetItem, 0); + } for (const countItem of counts) { const { _perspective_group_size_, ...fields } = countItem; - tableCache.storeGroupSize(props, fields, _perspective_group_size_); + tableCache.storeGroupSize( + props, + props.bindingColumns.map(col => fields[col]), + _perspective_group_size_ + ); } } + const rows = []; + + // console.log('CACHE', tableCache.bindingGroups); + + let groupIndex = 0; + for (; groupIndex < props.bindingValues.length; groupIndex++) { + const groupValues = props.bindingValues[groupIndex]; + const group = tableCache.getBindingGroup(groupValues); + + if (!group.loadedAll) { + // wee need to load next data + await this.loadNextGroup(props, groupIndex); + } + + // console.log('GRP', groupValues, group); + rows.push(...group.loadedRows); + if (rows.length >= props.topCount) { + return { + rows: rows.slice(0, props.topCount), + incomplete: props.topCount < rows.length || !group.loadedAll || groupIndex < props.bindingValues.length - 1, + }; + } + } + if (groupIndex >= props.bindingValues.length) { + // all groups are fully loaded + return { rows, incomplete: false }; + } + } + + async loadNextGroup(props: PerspectiveDataLoadProps, groupIndex: number) { + const tableCache = this.cache.getTableCache(props); + + const planLoadingGroupIndexes: number[] = []; + const planLoadingGroups: PerspectiveBindingGroup[] = []; + let planLoadRowCount = 0; + + const loadPlanned = async () => { + // console.log( + // 'LOAD PLANNED', + // planLoadingGroupIndexes, + // planLoadingGroupIndexes.map(idx => props.bindingValues[idx]) + // ); + const rows = await this.loader.loadData({ + ...props, + bindingValues: planLoadingGroupIndexes.map(idx => props.bindingValues[idx]), + }); + // console.log('LOADED PLANNED', rows); + // distribute rows into groups + for (const row of rows) { + const group = planLoadingGroups.find(x => x.matchRow(row)); + if (group) { + group.loadedRows.push(row); + } + } + for (const group of planLoadingGroups) { + group.loadedAll = true; + } + }; + + for (; groupIndex < props.bindingValues.length; groupIndex++) { + const groupValues = props.bindingValues[groupIndex]; + const group = tableCache.getBindingGroup(groupValues); + if (group.loadedAll) continue; + if (group.groupSize == 0) { + group.loadedAll = true; + continue; + } + if (group.groupSize >= PERSPECTIVE_PAGE_SIZE) { + if (planLoadingGroupIndexes.length > 0) { + await loadPlanned(); + return; + } + const nextRows = await this.loader.loadData({ + ...props, + topCount: null, + range: { + offset: group.loadedRows.length, + limit: PERSPECTIVE_PAGE_SIZE, + }, + bindingValues: [group.bindingValues], + }); + group.loadedRows = [...group.loadedRows, nextRows]; + group.loadedAll = nextRows.length < PERSPECTIVE_PAGE_SIZE; + return; + } else { + if (planLoadRowCount + group.groupSize > PERSPECTIVE_PAGE_SIZE) { + await loadPlanned(); + return; + } + planLoadingGroupIndexes.push(groupIndex); + planLoadingGroups.push(group); + planLoadRowCount += group.groupSize; + } + } + + if (planLoadingGroupIndexes.length > 0) { + await loadPlanned(); + } + } + + async loadDataFlat(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> { + const tableCache = this.cache.getTableCache(props); + if (props.topCount <= tableCache.loadedCount) { return tableCache.getRowsResult(props); } @@ -58,10 +181,5 @@ export class PerspectiveDataProvider { // const rows=tableCache.getRows(props); return tableCache.getRowsResult(props); - - // return { - // rows: await this.loader.loadData(props), - // incomplete: true, - // }; } } diff --git a/packages/datalib/src/PerspectiveTreeNode.ts b/packages/datalib/src/PerspectiveTreeNode.ts index 7e37fa584..91cc1f6f9 100644 --- a/packages/datalib/src/PerspectiveTreeNode.ts +++ b/packages/datalib/src/PerspectiveTreeNode.ts @@ -171,7 +171,7 @@ export class PerspectiveTableColumnNode extends PerspectiveTreeNode { schemaName: this.foreignKey.refSchemaName, pureName: this.foreignKey.refTableName, bindingColumns: [this.foreignKey.columns[0].refColumnName], - bindingValues: parentRows.map(row => row[this.foreignKey.columns[0].columnName]), + bindingValues: parentRows.map(row => [row[this.foreignKey.columns[0].columnName]]), dataColumns: this.getDataLoadColumns(), databaseConfig: this.databaseConfig, orderBy: this.table.primaryKey?.columns.map(x => x.columnName) || [this.table.columns[0].columnName], @@ -304,7 +304,7 @@ export class PerspectiveTableReferenceNode extends PerspectiveTableNode { schemaName: this.table.schemaName, pureName: this.table.pureName, bindingColumns: [this.foreignKey.columns[0].columnName], - bindingValues: parentRows.map(row => row[this.foreignKey.columns[0].refColumnName]), + bindingValues: parentRows.map(row => [row[this.foreignKey.columns[0].refColumnName]]), dataColumns: this.getDataLoadColumns(), databaseConfig: this.databaseConfig, orderBy: this.table.primaryKey?.columns.map(x => x.columnName) || [this.table.columns[0].columnName], diff --git a/packages/web/src/perspectives/PerspectiveTable.svelte b/packages/web/src/perspectives/PerspectiveTable.svelte index 669ada76e..ee6a62bfd 100644 --- a/packages/web/src/perspectives/PerspectiveTable.svelte +++ b/packages/web/src/perspectives/PerspectiveTable.svelte @@ -84,6 +84,7 @@ await loadLevelData(node, rows, counts); dataRows = rows; + dbg('display rows', rows); // console.log('DISPLAY ROWS', rows); // const rows = await node.loadLevelData(); // for (const child of node.childNodes) {