import { Condition } from 'dbgate-sqltree'; import { RangeDefinition } from 'dbgate-types'; import { format } from 'path'; import { PerspectiveBindingGroup, PerspectiveCache } from './PerspectiveCache'; import { PerspectiveDataLoader } from './PerspectiveDataLoader'; const PERSPECTIVE_PAGE_SIZE = 100; export interface PerspectiveDatabaseConfig { conid: string; database: string; } export interface PerspectiveDataLoadProps { databaseConfig: PerspectiveDatabaseConfig; schemaName: string; pureName: string; dataColumns: string[]; orderBy: string[]; bindingColumns?: string[]; bindingValues?: any[][]; range?: RangeDefinition; topCount?: number; condition?: Condition; } export class PerspectiveDataProvider { constructor(public cache: PerspectiveCache, public loader: PerspectiveDataLoader) {} async loadData(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> { // console.log('LOAD DATA', props); if (props.bindingColumns) { return this.loadDataNested(props); } else { return this.loadDataFlat(props); } } async loadDataNested(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> { const tableCache = this.cache.getTableCache(props); const uncached = tableCache.getUncachedBindingGroups(props); if (uncached.length > 0) { const counts = await this.loader.loadGrouping({ ...props, bindingValues: uncached, }); // console.log('COUNTS', counts); for (const resetItem of uncached) { tableCache.storeGroupSize(props, resetItem, 0); } for (const countItem of counts) { const { _perspective_group_size_, ...fields } = countItem; tableCache.storeGroupSize( props, props.bindingColumns.map(col => fields[col]), _perspective_group_size_ ); } } const rows = []; // console.log('CACHE', tableCache.bindingGroups); let groupIndex = 0; for (; groupIndex < props.bindingValues.length; groupIndex++) { const groupValues = props.bindingValues[groupIndex]; const group = tableCache.getBindingGroup(groupValues); if (!group.loadedAll) { // wee need to load next data await this.loadNextGroup(props, groupIndex); } // console.log('GRP', groupValues, group); rows.push(...group.loadedRows); if (rows.length >= props.topCount) { return { rows: rows.slice(0, props.topCount), incomplete: props.topCount < rows.length || !group.loadedAll || groupIndex < props.bindingValues.length - 1, }; } } if (groupIndex >= props.bindingValues.length) { // all groups are fully loaded return { rows, incomplete: false }; } } async loadNextGroup(props: PerspectiveDataLoadProps, groupIndex: number) { const tableCache = this.cache.getTableCache(props); const planLoadingGroupIndexes: number[] = []; const planLoadingGroups: PerspectiveBindingGroup[] = []; let planLoadRowCount = 0; const loadPlanned = async () => { // console.log( // 'LOAD PLANNED', // planLoadingGroupIndexes, // planLoadingGroupIndexes.map(idx => props.bindingValues[idx]) // ); const rows = await this.loader.loadData({ ...props, bindingValues: planLoadingGroupIndexes.map(idx => props.bindingValues[idx]), }); // console.log('LOADED PLANNED', rows); // distribute rows into groups for (const row of rows) { const group = planLoadingGroups.find(x => x.matchRow(row)); if (group) { group.loadedRows.push(row); } } for (const group of planLoadingGroups) { group.loadedAll = true; } }; for (; groupIndex < props.bindingValues.length; groupIndex++) { const groupValues = props.bindingValues[groupIndex]; const group = tableCache.getBindingGroup(groupValues); if (group.loadedAll) continue; if (group.groupSize == 0) { group.loadedAll = true; continue; } if (group.groupSize >= PERSPECTIVE_PAGE_SIZE) { if (planLoadingGroupIndexes.length > 0) { await loadPlanned(); return; } const nextRows = await this.loader.loadData({ ...props, topCount: null, range: { offset: group.loadedRows.length, limit: PERSPECTIVE_PAGE_SIZE, }, bindingValues: [group.bindingValues], }); group.loadedRows = [...group.loadedRows, ...nextRows]; group.loadedAll = nextRows.length < PERSPECTIVE_PAGE_SIZE; return; } else { if (planLoadRowCount + group.groupSize > PERSPECTIVE_PAGE_SIZE) { await loadPlanned(); return; } planLoadingGroupIndexes.push(groupIndex); planLoadingGroups.push(group); planLoadRowCount += group.groupSize; } } if (planLoadingGroupIndexes.length > 0) { await loadPlanned(); } } async loadDataFlat(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> { const tableCache = this.cache.getTableCache(props); if (props.topCount <= tableCache.loadedCount) { return tableCache.getRowsResult(props); } // load missing rows tableCache.dataColumns = props.dataColumns; const nextRows = await this.loader.loadData({ ...props, topCount: null, range: { offset: tableCache.loadedCount, limit: props.topCount - tableCache.loadedCount, }, }); if (nextRows.errorMessage) { throw new Error(nextRows.errorMessage); } tableCache.loadedRows = [...tableCache.loadedRows, ...nextRows]; tableCache.loadedAll = nextRows.length < props.topCount - tableCache.loadedCount; // const rows=tableCache.getRows(props); return tableCache.getRowsResult(props); } }