dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-29 15:43:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

SYNC: Merge pull request #4 from dbgate/feature/charts

Browse Source
This commit is contained in:
Jan Prochazka
2025-06-09 09:15:21 +02:00
committed by Diflow
parent 6f69205818
commit f03cffe3f8
22 changed files with 1687 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
packages/api/src/controllers/jsldata.js
View File

@@ -10,6 +10,7 @@ const requirePluginFunction = require('../utility/requirePluginFunction');
const socket = require('../utility/socket');
const crypto = require('crypto');
const dbgateApi = require('../shell');
const { ChartProcessor } = require('dbgate-datalib');
function readFirstLine(file) {
return new Promise((resolve, reject) => {
@@ -302,4 +303,29 @@ module.exports = {
await dbgateApi.download(uri, { targetFile: getJslFileName(jslid) });
return { jslid };
},
buildChart_meta: true,
async buildChart({ jslid, definition }) {
const datastore = new JsonLinesDatastore(getJslFileName(jslid));
const processor = new ChartProcessor(definition ? [definition] : undefined);
await datastore.enumRows(row => {
processor.addRow(row);
return true;
});
processor.finalize();
return processor.charts;
},
detectChartColumns_meta: true,
async detectChartColumns({ jslid }) {
const datastore = new JsonLinesDatastore(getJslFileName(jslid));
const processor = new ChartProcessor();
processor.autoDetectCharts = false;
await datastore.enumRows(row => {
processor.addRow(row);
return true;
});
processor.finalize();
return processor.availableColumns;
},
};

9
packages/api/src/controllers/sessions.js
View File

@@ -83,6 +83,11 @@ module.exports = {
jsldata.notifyChangedStats(stats);
},
handle_charts(sesid, props) {
const { jslid, charts, resultIndex } = props;
socket.emit(`session-charts-${sesid}`, { jslid, resultIndex, charts });
},
handle_initializeFile(sesid, props) {
const { jslid } = props;
socket.emit(`session-initialize-file-${jslid}`);
@@ -141,7 +146,7 @@ module.exports = {
},
executeQuery_meta: true,
async executeQuery({ sesid, sql, autoCommit, limitRows }) {
async executeQuery({ sesid, sql, autoCommit, limitRows, frontMatter }) {
const session = this.opened.find(x => x.sesid == sesid);
if (!session) {
throw new Error('Invalid session');
@@ -149,7 +154,7 @@ module.exports = {
logger.info({ sesid, sql }, 'Processing query');
this.dispatchMessage(sesid, 'Query execution started');
session.subprocess.send({ msgtype: 'executeQuery', sql, autoCommit, limitRows });
session.subprocess.send({ msgtype: 'executeQuery', sql, autoCommit, limitRows, frontMatter });
return { state: 'ok' };
},

4
packages/api/src/proc/sessionProcess.js
View File

@@ -117,7 +117,7 @@ async function handleExecuteControlCommand({ command }) {
}
}
async function handleExecuteQuery({ sql, autoCommit, limitRows }) {
async function handleExecuteQuery({ sql, autoCommit, limitRows, frontMatter }) {
lastActivity = new Date().getTime();
await waitConnected();
@@ -146,7 +146,7 @@ async function handleExecuteQuery({ sql, autoCommit, limitRows }) {
...driver.getQuerySplitterOptions('stream'),
returnRichInfo: true,
})) {
await handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, undefined, limitRows);
await handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, undefined, limitRows, frontMatter);
// const handler = new StreamHandler(resultIndex);
// const stream = await driver.stream(systemConnection, sqlItem, handler);
// handler.stream = stream;

68
packages/api/src/utility/handleQueryStream.js
View File

@@ -5,6 +5,8 @@ const _ = require('lodash');
const { jsldir } = require('../utility/directories');
const { serializeJsTypesReplacer } = require('dbgate-tools');
const { ChartProcessor } = require('dbgate-datalib');
const { isProApp } = require('./checkLicense');
class QueryStreamTableWriter {
constructor(sesid = undefined) {
@@ -12,9 +14,12 @@ class QueryStreamTableWriter {
this.currentChangeIndex = 1;
this.initializedFile = false;
this.sesid = sesid;
if (isProApp()) {
this.chartProcessor = new ChartProcessor();
}
}
initializeFromQuery(structure, resultIndex) {
initializeFromQuery(structure, resultIndex, chartDefinition) {
this.jslid = crypto.randomUUID();
this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
fs.writeFileSync(
@@ -28,6 +33,9 @@ class QueryStreamTableWriter {
this.writeCurrentStats(false, false);
this.resultIndex = resultIndex;
this.initializedFile = true;
if (isProApp() && chartDefinition) {
this.chartProcessor = new ChartProcessor([chartDefinition]);
}
process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex, sesid: this.sesid });
}
@@ -40,6 +48,15 @@ class QueryStreamTableWriter {
row(row) {
// console.log('ACCEPT ROW', row);
this.currentStream.write(JSON.stringify(row, serializeJsTypesReplacer) + '\n');
try {
if (this.chartProcessor) {
this.chartProcessor.addRow(row);
}
} catch (e) {
console.error('Error processing chart row', e);
this.chartProcessor = null;
}
this.currentRowCount += 1;
if (!this.plannedStats) {
@@ -87,6 +104,23 @@ class QueryStreamTableWriter {
this.currentStream.end(() => {
this.writeCurrentStats(true, true);
if (afterClose) afterClose();
if (this.chartProcessor) {
try {
this.chartProcessor.finalize();
if (this.chartProcessor.charts.length > 0) {
process.send({
msgtype: 'charts',
sesid: this.sesid,
jslid: this.jslid,
charts: this.chartProcessor.charts,
resultIndex: this.resultIndex,
});
}
} catch (e) {
console.error('Error finalizing chart processor', e);
this.chartProcessor = null;
}
}
resolve();
});
} else {
@@ -97,10 +131,18 @@ class QueryStreamTableWriter {
}
class StreamHandler {
constructor(queryStreamInfoHolder, resolve, startLine, sesid = undefined, limitRows = undefined) {
constructor(
queryStreamInfoHolder,
resolve,
startLine,
sesid = undefined,
limitRows = undefined,
frontMatter = undefined
) {
this.recordset = this.recordset.bind(this);
this.startLine = startLine;
this.sesid = sesid;
this.frontMatter = frontMatter;
this.limitRows = limitRows;
this.rowsLimitOverflow = false;
this.row = this.row.bind(this);
@@ -133,7 +175,8 @@ class StreamHandler {
this.currentWriter = new QueryStreamTableWriter(this.sesid);
this.currentWriter.initializeFromQuery(
Array.isArray(columns) ? { columns } : columns,
this.queryStreamInfoHolder.resultIndex
this.queryStreamInfoHolder.resultIndex,
this.frontMatter?.[`chart-${this.queryStreamInfoHolder.resultIndex + 1}`]
);
this.queryStreamInfoHolder.resultIndex += 1;
this.rowCounter = 0;
@@ -201,10 +244,25 @@ class StreamHandler {
}
}
function handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, sesid = undefined, limitRows = undefined) {
function handleQueryStream(
dbhan,
driver,
queryStreamInfoHolder,
sqlItem,
sesid = undefined,
limitRows = undefined,
frontMatter = undefined
) {
return new Promise((resolve, reject) => {
const start = sqlItem.trimStart || sqlItem.start;
const handler = new StreamHandler(queryStreamInfoHolder, resolve, start && start.line, sesid, limitRows);
const handler = new StreamHandler(
queryStreamInfoHolder,
resolve,
start && start.line,
sesid,
limitRows,
frontMatter
);
driver.stream(dbhan, sqlItem.text, handler);
});
}

8
packages/datalib/package.json
View File

@@ -6,6 +6,7 @@
"scripts": {
"build": "tsc",
"test": "jest",
"test:charts": "jest -t \"Chart processor\"",
"test:ci": "jest --json --outputFile=result.json --testLocationInResults",
"start": "tsc --watch"
},
@@ -13,16 +14,17 @@
"lib"
],
"dependencies": {
"date-fns": "^4.1.0",
"dbgate-filterparser": "^6.0.0-alpha.1",
"dbgate-sqltree": "^6.0.0-alpha.1",
"dbgate-tools": "^6.0.0-alpha.1",
"dbgate-filterparser": "^6.0.0-alpha.1",
"uuid": "^3.4.0"
},
"devDependencies": {
"dbgate-types": "^6.0.0-alpha.1",
"@types/node": "^13.7.0",
"dbgate-types": "^6.0.0-alpha.1",
"jest": "^28.1.3",
"ts-jest": "^28.0.7",
"typescript": "^4.4.3"
}
}
}

84
packages/datalib/src/chartDefinitions.ts Normal file
View File

@@ -0,0 +1,84 @@
export type ChartTypeEnum = 'bar' | 'line' | 'pie' | 'polarArea';
export type ChartXTransformFunction =
| 'identity'
| 'date:minute'
| 'date:hour'
| 'date:day'
| 'date:month'
| 'date:year';
export type ChartYAggregateFunction = 'sum' | 'first' | 'last' | 'min' | 'max' | 'count' | 'avg';
export const ChartConstDefaults = {
sortOrder: ' asc',
windowAlign: 'end',
windowSize: 100,
parentAggregateLimit: 200,
};
export const ChartLimits = {
AUTODETECT_CHART_LIMIT: 10, // limit for auto-detecting charts, to avoid too many charts
AUTODETECT_MEASURES_LIMIT: 10, // limit for auto-detecting measures, to avoid too many measures
APPLY_LIMIT_AFTER_ROWS: 100,
MAX_DISTINCT_VALUES: 10, // max number of distinct values to keep in topDistinctValues
VALID_VALUE_RATIO_LIMIT: 0.5, // limit for valid value ratio, y defs below this will not be used in auto-detect
PIE_RATIO_LIMIT: 0.05, // limit for other values in pie chart, if the value is below this, it will be grouped into "Other"
PIE_COUNT_LIMIT: 10, // limit for number of pie chart slices, if the number of slices is above this, it will be grouped into "Other"
};
export interface ChartXFieldDefinition {
field: string;
title?: string;
transformFunction: ChartXTransformFunction;
sortOrder?: 'natural' | 'ascKeys' | 'descKeys' | 'ascValues' | 'descValues';
windowAlign?: 'start' | 'end';
windowSize?: number;
parentAggregateLimit?: number;
}
export interface ChartYFieldDefinition {
field: string;
title?: string;
aggregateFunction: ChartYAggregateFunction;
}
export interface ChartDefinition {
chartType: ChartTypeEnum;
title?: string;
pieRatioLimit?: number; // limit for pie chart, if the value is below this, it will be grouped into "Other"
pieCountLimit?: number; // limit for number of pie chart slices, if the number of slices is above this, it will be grouped into "Other"
xdef: ChartXFieldDefinition;
ydefs: ChartYFieldDefinition[];
}
export interface ChartDateParsed {
year: number;
month?: number;
day?: number;
hour?: number;
minute?: number;
second?: number;
fraction?: string;
}
export interface ChartAvailableColumn {
field: string;
}
export interface ProcessedChart {
minX?: string;
maxX?: string;
rowsAdded: number;
buckets: { [key: string]: any }; // key is the bucket key, value is aggregated data
bucketKeysOrdered: string[];
bucketKeyDateParsed: { [key: string]: ChartDateParsed }; // key is the bucket key, value is parsed date
isGivenDefinition: boolean; // true if the chart was created with a given definition, false if it was created from raw data
invalidXRows: number;
invalidYRows: { [key: string]: number }; // key is the y field, value is the count of invalid rows
validYRows: { [key: string]: number }; // key is the field, value is the count of valid rows
topDistinctValues: { [key: string]: Set<any> }; // key is the field, value is the set of distinct values
availableColumns: ChartAvailableColumn[];
definition: ChartDefinition;
}

374
packages/datalib/src/chartProcessor.ts Normal file
View File

@@ -0,0 +1,374 @@
import {
ChartAvailableColumn,
ChartDateParsed,
ChartDefinition,
ChartLimits,
ProcessedChart,
} from './chartDefinitions';
import _sortBy from 'lodash/sortBy';
import _sum from 'lodash/sum';
import {
aggregateChartNumericValuesFromSource,
autoAggregateCompactTimelineChart,
computeChartBucketCardinality,
computeChartBucketKey,
fillChartTimelineBuckets,
tryParseChartDate,
} from './chartTools';
import { getChartScore, getChartYFieldScore } from './chartScoring';
export class ChartProcessor {
chartsProcessing: ProcessedChart[] = [];
charts: ProcessedChart[] = [];
availableColumnsDict: { [field: string]: ChartAvailableColumn } = {};
availableColumns: ChartAvailableColumn[] = [];
autoDetectCharts = false;
rowsAdded = 0;
constructor(public givenDefinitions: ChartDefinition[] = []) {
for (const definition of givenDefinitions) {
this.chartsProcessing.push({
definition,
rowsAdded: 0,
bucketKeysOrdered: [],
buckets: {},
bucketKeyDateParsed: {},
isGivenDefinition: true,
invalidXRows: 0,
invalidYRows: {},
availableColumns: [],
validYRows: {},
topDistinctValues: {},
});
}
this.autoDetectCharts = this.givenDefinitions.length == 0;
}
// findOrCreateChart(definition: ChartDefinition, isGivenDefinition: boolean): ProcessedChart {
// const signatureItems = [
// definition.chartType,
// definition.xdef.field,
// definition.xdef.transformFunction,
// definition.ydefs.map(y => y.field).join(','),
// ];
// const signature = signatureItems.join('::');
// if (this.chartsBySignature[signature]) {
// return this.chartsBySignature[signature];
// }
// const chart: ProcessedChart = {
// definition,
// rowsAdded: 0,
// bucketKeysOrdered: [],
// buckets: {},
// bucketKeyDateParsed: {},
// isGivenDefinition,
// };
// this.chartsBySignature[signature] = chart;
// return chart;
// }
addRow(row: any) {
const dateColumns: { [key: string]: ChartDateParsed } = {};
const numericColumns: { [key: string]: number } = {};
const numericColumnsForAutodetect: { [key: string]: number } = {};
const stringColumns: { [key: string]: string } = {};
for (const [key, value] of Object.entries(row)) {
const number: number = typeof value == 'string' ? Number(value) : typeof value == 'number' ? value : NaN;
this.availableColumnsDict[key] = {
field: key,
};
const keyLower = key.toLowerCase();
const keyIsId = keyLower.endsWith('_id') || keyLower == 'id' || key.endsWith('Id');
const parsedDate = tryParseChartDate(value);
if (parsedDate) {
dateColumns[key] = parsedDate;
continue;
}
if (!isNaN(number) && isFinite(number)) {
numericColumns[key] = number;
if (!keyIsId) {
numericColumnsForAutodetect[key] = number; // for auto-detecting charts
}
continue;
}
if (typeof value === 'string' && isNaN(number) && value.length < 100) {
stringColumns[key] = value;
}
}
// const sortedNumericColumnns = Object.keys(numericColumns).sort();
if (this.autoDetectCharts) {
// create charts from data, if there are no given definitions
for (const datecol in dateColumns) {
let usedChart = this.chartsProcessing.find(
chart =>
!chart.isGivenDefinition &&
chart.definition.xdef.field === datecol &&
chart.definition.xdef.transformFunction?.startsWith('date:')
);
if (
!usedChart &&
(this.rowsAdded < ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
this.chartsProcessing.length < ChartLimits.AUTODETECT_CHART_LIMIT)
) {
usedChart = {
definition: {
chartType: 'line',
xdef: {
field: datecol,
transformFunction: 'date:day',
},
ydefs: [],
},
rowsAdded: 0,
bucketKeysOrdered: [],
buckets: {},
bucketKeyDateParsed: {},
isGivenDefinition: false,
invalidXRows: 0,
invalidYRows: {},
availableColumns: [],
validYRows: {},
topDistinctValues: {},
};
this.chartsProcessing.push(usedChart);
}
for (const [key, value] of Object.entries(row)) {
if (value == null) continue;
if (key == datecol) continue; // skip date column itself
let existingYDef = usedChart.definition.ydefs.find(y => y.field === key);
if (
!existingYDef &&
(this.rowsAdded < ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
usedChart.definition.ydefs.length < ChartLimits.AUTODETECT_MEASURES_LIMIT)
) {
existingYDef = {
field: key,
aggregateFunction: 'sum',
};
usedChart.definition.ydefs.push(existingYDef);
}
}
}
}
// apply on all charts with this date column
for (const chart of this.chartsProcessing) {
this.applyRawData(
chart,
row,
dateColumns[chart.definition.xdef.field],
chart.isGivenDefinition ? numericColumns : numericColumnsForAutodetect,
stringColumns
);
}
for (let i = 0; i < this.chartsProcessing.length; i++) {
this.chartsProcessing[i] = autoAggregateCompactTimelineChart(this.chartsProcessing[i]);
}
this.rowsAdded += 1;
if (this.rowsAdded == ChartLimits.APPLY_LIMIT_AFTER_ROWS) {
this.applyLimitsOnCharts();
}
}
applyLimitsOnCharts() {
const autodetectProcessingCharts = this.chartsProcessing.filter(chart => !chart.isGivenDefinition);
if (autodetectProcessingCharts.length > ChartLimits.AUTODETECT_CHART_LIMIT) {
const newAutodetectProcessingCharts = _sortBy(
this.chartsProcessing.slice(0, ChartLimits.AUTODETECT_CHART_LIMIT),
chart => -getChartScore(chart)
);
for (const chart of autodetectProcessingCharts) {
chart.definition.ydefs = _sortBy(chart.definition.ydefs, yfield => -getChartYFieldScore(chart, yfield)).slice(
0,
ChartLimits.AUTODETECT_MEASURES_LIMIT
);
}
this.chartsProcessing = [
...this.chartsProcessing.filter(chart => chart.isGivenDefinition),
...newAutodetectProcessingCharts,
];
}
}
addRows(...rows: any[]) {
for (const row of rows) {
this.addRow(row);
}
}
finalize() {
this.applyLimitsOnCharts();
this.availableColumns = Object.values(this.availableColumnsDict);
for (const chart of this.chartsProcessing) {
let addedChart: ProcessedChart = chart;
if (chart.rowsAdded == 0) {
continue; // skip empty charts
}
const sortOrder = chart.definition.xdef.sortOrder ?? 'ascKeys';
if (sortOrder != 'natural') {
if (sortOrder == 'ascKeys' || sortOrder == 'descKeys') {
if (chart.definition.xdef.transformFunction.startsWith('date:')) {
addedChart = autoAggregateCompactTimelineChart(addedChart);
fillChartTimelineBuckets(addedChart);
}
addedChart.bucketKeysOrdered = _sortBy(Object.keys(addedChart.buckets));
if (sortOrder == 'descKeys') {
addedChart.bucketKeysOrdered.reverse();
}
}
if (sortOrder == 'ascValues' || sortOrder == 'descValues') {
addedChart.bucketKeysOrdered = _sortBy(Object.keys(addedChart.buckets), key =>
computeChartBucketCardinality(addedChart.buckets[key])
);
if (sortOrder == 'descValues') {
addedChart.bucketKeysOrdered.reverse();
}
}
}
if (!addedChart.isGivenDefinition) {
addedChart = {
...addedChart,
definition: {
...addedChart.definition,
ydefs: addedChart.definition.ydefs.filter(
y =>
!addedChart.invalidYRows[y.field] &&
addedChart.validYRows[y.field] / addedChart.rowsAdded >= ChartLimits.VALID_VALUE_RATIO_LIMIT
),
},
};
}
if (addedChart) {
addedChart.availableColumns = this.availableColumns;
this.charts.push(addedChart);
}
this.groupPieOtherBuckets(addedChart);
}
this.charts = [
...this.charts.filter(x => x.isGivenDefinition),
..._sortBy(
this.charts.filter(x => !x.isGivenDefinition),
chart => -getChartScore(chart)
),
];
}
groupPieOtherBuckets(chart: ProcessedChart) {
if (chart.definition.chartType !== 'pie') {
return; // only for pie charts
}
const ratioLimit = chart.definition.pieRatioLimit ?? ChartLimits.PIE_RATIO_LIMIT;
const countLimit = chart.definition.pieCountLimit ?? ChartLimits.PIE_COUNT_LIMIT;
if (ratioLimit == 0 && countLimit == 0) {
return; // no grouping if limit is 0
}
const otherBucket: any = {};
let newBuckets: any = {};
const cardSum = _sum(Object.values(chart.buckets).map(bucket => computeChartBucketCardinality(bucket)));
if (cardSum == 0) {
return; // no buckets to process
}
for (const [bucketKey, bucket] of Object.entries(chart.buckets)) {
if (computeChartBucketCardinality(bucket) / cardSum < ratioLimit) {
for (const field in bucket) {
otherBucket[field] = (otherBucket[field] ?? 0) + bucket[field];
}
} else {
newBuckets[bucketKey] = bucket;
}
}
if (Object.keys(newBuckets).length > countLimit) {
const sortedBucketKeys = _sortBy(
Object.entries(newBuckets),
([, bucket]) => -computeChartBucketCardinality(bucket)
).map(([key]) => key);
const newBuckets2 = {};
sortedBucketKeys.forEach((key, index) => {
if (index < countLimit) {
newBuckets2[key] = newBuckets[key];
} else {
for (const field in newBuckets[key]) {
otherBucket[field] = (otherBucket[field] ?? 0) + newBuckets[key][field];
}
}
});
newBuckets = newBuckets2;
}
if (Object.keys(otherBucket).length > 0) {
newBuckets['Other'] = otherBucket;
}
chart.buckets = newBuckets;
chart.bucketKeysOrdered = [...chart.bucketKeysOrdered, 'Other'].filter(key => key in newBuckets);
}
applyRawData(
chart: ProcessedChart,
row: any,
dateParsed: ChartDateParsed,
numericColumns: { [key: string]: number },
stringColumns: { [key: string]: string }
) {
if (chart.definition.xdef == null) {
return;
}
if (row[chart.definition.xdef.field] == null) {
return;
}
if (dateParsed == null && chart.definition.xdef.transformFunction.startsWith('date:')) {
chart.invalidXRows += 1;
return; // skip if date is invalid
}
const [bucketKey, bucketKeyParsed] = computeChartBucketKey(dateParsed, chart, row);
if (!bucketKey) {
return; // skip if no bucket key
}
if (bucketKeyParsed) {
chart.bucketKeyDateParsed[bucketKey] = bucketKeyParsed;
}
if (chart.minX == null || bucketKey < chart.minX) {
chart.minX = bucketKey;
}
if (chart.maxX == null || bucketKey > chart.maxX) {
chart.maxX = bucketKey;
}
if (!chart.buckets[bucketKey]) {
chart.buckets[bucketKey] = {};
if (chart.definition.xdef.sortOrder == 'natural') {
chart.bucketKeysOrdered.push(bucketKey);
}
}
aggregateChartNumericValuesFromSource(chart, bucketKey, numericColumns, row);
chart.rowsAdded += 1;
}
}

23
packages/datalib/src/chartScoring.ts Normal file
View File

@@ -0,0 +1,23 @@
import _sortBy from 'lodash/sortBy';
import _sum from 'lodash/sum';
import { ChartLimits, ChartYFieldDefinition, ProcessedChart } from './chartDefinitions';
export function getChartScore(chart: ProcessedChart): number {
let res = 0;
res += chart.rowsAdded * 5;
const ydefScores = chart.definition.ydefs.map(yField => getChartYFieldScore(chart, yField));
const sorted = _sortBy(ydefScores).reverse();
res += _sum(sorted.slice(0, ChartLimits.AUTODETECT_MEASURES_LIMIT));
return res;
}
export function getChartYFieldScore(chart: ProcessedChart, yField: ChartYFieldDefinition): number {
let res = 0;
res += chart.validYRows[yField.field] * 5; // score for valid Y rows
res += (chart.topDistinctValues[yField.field]?.size ?? 0) * 20; // score for distinct values in Y field
res += chart.rowsAdded * 2; // base score for rows added
res -= (chart.invalidYRows[yField.field] ?? 0) * 50; // penalty for invalid Y rows
return res;
}

542
packages/datalib/src/chartTools.ts Normal file
View File

@@ -0,0 +1,542 @@
import _toPairs from 'lodash/toPairs';
import _sumBy from 'lodash/sumBy';
import {
ChartConstDefaults,
ChartDateParsed,
ChartLimits,
ChartXTransformFunction,
ProcessedChart,
} from './chartDefinitions';
import { addMinutes, addHours, addDays, addMonths, addYears } from 'date-fns';
export function getChartDebugPrint(chart: ProcessedChart) {
let res = '';
res += `Chart: ${chart.definition.chartType} (${chart.definition.xdef.transformFunction})\n`;
for (const key of chart.bucketKeysOrdered) {
res += `${key}: ${_toPairs(chart.buckets[key])
.map(([k, v]) => `${k}=${v}`)
.join(', ')}\n`;
}
return res;
}
export function tryParseChartDate(dateInput: any): ChartDateParsed | null {
if (dateInput instanceof Date) {
return {
year: dateInput.getFullYear(),
month: dateInput.getMonth() + 1,
day: dateInput.getDate(),
hour: dateInput.getHours(),
minute: dateInput.getMinutes(),
second: dateInput.getSeconds(),
fraction: undefined, // Date object does not have fraction
};
}
if (typeof dateInput !== 'string') return null;
const m = dateInput.match(
/^(\d{4})-(\d{2})-(\d{2})(?:[ T](\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|[+-]\d{2}:\d{2})?)?$/
);
if (!m) return null;
const [_notUsed, year, month, day, hour, minute, second, fraction] = m;
return {
year: parseInt(year, 10),
month: parseInt(month, 10),
day: parseInt(day, 10),
hour: parseInt(hour, 10) || 0,
minute: parseInt(minute, 10) || 0,
second: parseInt(second, 10) || 0,
fraction: fraction || undefined,
};
}
function pad2Digits(number) {
return ('00' + number).slice(-2);
}
export function stringifyChartDate(value: ChartDateParsed, transform: ChartXTransformFunction): string {
switch (transform) {
case 'date:year':
return `${value.year}`;
case 'date:month':
return `${value.year}-${pad2Digits(value.month)}`;
case 'date:day':
return `${value.year}-${pad2Digits(value.month)}-${pad2Digits(value.day)}`;
case 'date:hour':
return `${value.year}-${pad2Digits(value.month)}-${pad2Digits(value.day)} ${pad2Digits(value.hour)}`;
case 'date:minute':
return `${value.year}-${pad2Digits(value.month)}-${pad2Digits(value.day)} ${pad2Digits(value.hour)}:${pad2Digits(
value.minute
)}`;
default:
return '';
}
}
export function incrementChartDate(value: ChartDateParsed, transform: ChartXTransformFunction): ChartDateParsed {
const dateRepresentation = new Date(
value.year,
(value.month ?? 1) - 1,
value.day ?? 1,
value.hour ?? 0,
value.minute ?? 0
);
let newDateRepresentation: Date;
switch (transform) {
case 'date:year':
newDateRepresentation = addYears(dateRepresentation, 1);
break;
case 'date:month':
newDateRepresentation = addMonths(dateRepresentation, 1);
break;
case 'date:day':
newDateRepresentation = addDays(dateRepresentation, 1);
break;
case 'date:hour':
newDateRepresentation = addHours(dateRepresentation, 1);
break;
case 'date:minute':
newDateRepresentation = addMinutes(dateRepresentation, 1);
break;
}
switch (transform) {
case 'date:year':
return { year: newDateRepresentation.getFullYear() };
case 'date:month':
return {
year: newDateRepresentation.getFullYear(),
month: newDateRepresentation.getMonth() + 1,
};
case 'date:day':
return {
year: newDateRepresentation.getFullYear(),
month: newDateRepresentation.getMonth() + 1,
day: newDateRepresentation.getDate(),
};
case 'date:hour':
return {
year: newDateRepresentation.getFullYear(),
month: newDateRepresentation.getMonth() + 1,
day: newDateRepresentation.getDate(),
hour: newDateRepresentation.getHours(),
};
case 'date:minute':
return {
year: newDateRepresentation.getFullYear(),
month: newDateRepresentation.getMonth() + 1,
day: newDateRepresentation.getDate(),
hour: newDateRepresentation.getHours(),
minute: newDateRepresentation.getMinutes(),
};
}
}
export function computeChartBucketKey(
dateParsed: ChartDateParsed,
chart: ProcessedChart,
row: any
): [string, ChartDateParsed] {
switch (chart.definition.xdef.transformFunction) {
case 'date:year':
return [dateParsed ? `${dateParsed.year}` : null, { year: dateParsed.year }];
case 'date:month':
return [
dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}` : null,
{
year: dateParsed.year,
month: dateParsed.month,
},
];
case 'date:day':
return [
dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)}` : null,
{
year: dateParsed.year,
month: dateParsed.month,
day: dateParsed.day,
},
];
case 'date:hour':
return [
dateParsed
? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits(
dateParsed.hour
)}`
: null,
{
year: dateParsed.year,
month: dateParsed.month,
day: dateParsed.day,
hour: dateParsed.hour,
},
];
case 'date:minute':
return [
dateParsed
? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits(
dateParsed.hour
)}:${pad2Digits(dateParsed.minute)}`
: null,
{
year: dateParsed.year,
month: dateParsed.month,
day: dateParsed.day,
hour: dateParsed.hour,
minute: dateParsed.minute,
},
];
case 'identity':
default:
return [row[chart.definition.xdef.field], null];
}
}
export function computeDateBucketDistance(
begin: ChartDateParsed,
end: ChartDateParsed,
transform: ChartXTransformFunction
): number {
switch (transform) {
case 'date:year':
return end.year - begin.year;
case 'date:month':
return (end.year - begin.year) * 12 + (end.month - begin.month);
case 'date:day':
return (
(end.year - begin.year) * 365 +
(end.month - begin.month) * 30 + // rough approximation
(end.day - begin.day)
);
case 'date:hour':
return (
(end.year - begin.year) * 365 * 24 +
(end.month - begin.month) * 30 * 24 + // rough approximation
(end.day - begin.day) * 24 +
(end.hour - begin.hour)
);
case 'date:minute':
return (
(end.year - begin.year) * 365 * 24 * 60 +
(end.month - begin.month) * 30 * 24 * 60 + // rough approximation
(end.day - begin.day) * 24 * 60 +
(end.hour - begin.hour) * 60 +
(end.minute - begin.minute)
);
case 'identity':
default:
return NaN;
}
}
export function compareChartDatesParsed(
a: ChartDateParsed,
b: ChartDateParsed,
transform: ChartXTransformFunction
): number {
switch (transform) {
case 'date:year':
return a.year - b.year;
case 'date:month':
return a.year === b.year ? a.month - b.month : a.year - b.year;
case 'date:day':
return a.year === b.year && a.month === b.month
? a.day - b.day
: a.year === b.year
? a.month - b.month
: a.year - b.year;
case 'date:hour':
return a.year === b.year && a.month === b.month && a.day === b.day
? a.hour - b.hour
: a.year === b.year && a.month === b.month
? a.day - b.day
: a.year === b.year
? a.month - b.month
: a.year - b.year;
case 'date:minute':
return a.year === b.year && a.month === b.month && a.day === b.day && a.hour === b.hour
? a.minute - b.minute
: a.year === b.year && a.month === b.month && a.day === b.day
? a.hour - b.hour
: a.year === b.year && a.month === b.month
? a.day - b.day
: a.year === b.year
? a.month - b.month
: a.year - b.year;
}
}
function getParentDateBucketKey(bucketKey: string, transform: ChartXTransformFunction): string | null {
switch (transform) {
case 'date:year':
return null; // no parent for year
case 'date:month':
return bucketKey.slice(0, 4);
case 'date:day':
return bucketKey.slice(0, 7);
case 'date:hour':
return bucketKey.slice(0, 10);
case 'date:minute':
return bucketKey.slice(0, 13);
}
}
function getParentDateBucketTransform(transform: ChartXTransformFunction): ChartXTransformFunction | null {
switch (transform) {
case 'date:year':
return null; // no parent for year
case 'date:month':
return 'date:year';
case 'date:day':
return 'date:month';
case 'date:hour':
return 'date:day';
case 'date:minute':
return 'date:hour';
default:
return null;
}
}
function getParentKeyParsed(date: ChartDateParsed, transform: ChartXTransformFunction): ChartDateParsed | null {
switch (transform) {
case 'date:year':
return null; // no parent for year
case 'date:month':
return { year: date.year };
case 'date:day':
return { year: date.year, month: date.month };
case 'date:hour':
return { year: date.year, month: date.month, day: date.day };
case 'date:minute':
return { year: date.year, month: date.month, day: date.day, hour: date.hour };
default:
return null;
}
}
function createParentChartAggregation(chart: ProcessedChart): ProcessedChart | null {
if (chart.isGivenDefinition) {
// if the chart is created with a given definition, we cannot create a parent aggregation
return null;
}
const parentTransform = getParentDateBucketTransform(chart.definition.xdef.transformFunction);
if (!parentTransform) {
return null;
}
const res: ProcessedChart = {
definition: {
...chart.definition,
xdef: {
...chart.definition.xdef,
transformFunction: parentTransform,
},
},
rowsAdded: chart.rowsAdded,
bucketKeysOrdered: [],
buckets: {},
bucketKeyDateParsed: {},
isGivenDefinition: false,
invalidXRows: chart.invalidXRows,
invalidYRows: { ...chart.invalidYRows }, // copy invalid Y rows
validYRows: { ...chart.validYRows }, // copy valid Y rows
topDistinctValues: { ...chart.topDistinctValues }, // copy top distinct values
availableColumns: chart.availableColumns,
};
for (const [bucketKey, bucketValues] of Object.entries(chart.buckets)) {
const parentKey = getParentDateBucketKey(bucketKey, chart.definition.xdef.transformFunction);
if (!parentKey) {
// skip if the bucket is already a parent
continue;
}
res.bucketKeyDateParsed[parentKey] = getParentKeyParsed(
chart.bucketKeyDateParsed[bucketKey],
chart.definition.xdef.transformFunction
);
aggregateChartNumericValuesFromChild(res, parentKey, bucketValues);
}
const bucketKeys = Object.keys(res.buckets).sort();
res.minX = bucketKeys.length > 0 ? bucketKeys[0] : null;
res.maxX = bucketKeys.length > 0 ? bucketKeys[bucketKeys.length - 1] : null;
return res;
}
export function autoAggregateCompactTimelineChart(chart: ProcessedChart) {
while (true) {
const fromParsed = chart.bucketKeyDateParsed[chart.minX];
const toParsed = chart.bucketKeyDateParsed[chart.maxX];
if (!fromParsed || !toParsed) {
return chart; // cannot fill timeline buckets without valid date range
}
const transform = chart.definition.xdef.transformFunction;
if (!transform.startsWith('date:')) {
return chart; // cannot aggregate non-date charts
}
const dateDistance = computeDateBucketDistance(fromParsed, toParsed, transform);
if (dateDistance < (chart.definition.xdef.parentAggregateLimit ?? ChartConstDefaults.parentAggregateLimit)) {
return chart; // no need to aggregate further, the distance is less than the limit
}
const parentChart = createParentChartAggregation(chart);
if (!parentChart) {
return chart; // cannot create parent aggregation
}
chart = parentChart;
}
}
export function aggregateChartNumericValuesFromSource(
chart: ProcessedChart,
bucketKey: string,
numericColumns: { [key: string]: number },
row: any
) {
for (const ydef of chart.definition.ydefs) {
if (numericColumns[ydef.field] == null) {
if (row[ydef.field]) {
chart.invalidYRows[ydef.field] = (chart.invalidYRows[ydef.field] || 0) + 1; // increment invalid row count if the field is not numeric
}
continue;
}
chart.validYRows[ydef.field] = (chart.validYRows[ydef.field] || 0) + 1; // increment valid row count
let distinctValues = chart.topDistinctValues[ydef.field];
if (!distinctValues) {
distinctValues = new Set();
chart.topDistinctValues[ydef.field] = distinctValues;
}
if (distinctValues.size < ChartLimits.MAX_DISTINCT_VALUES) {
chart.topDistinctValues[ydef.field].add(numericColumns[ydef.field]);
}
switch (ydef.aggregateFunction) {
case 'sum':
chart.buckets[bucketKey][ydef.field] =
(chart.buckets[bucketKey][ydef.field] || 0) + (numericColumns[ydef.field] || 0);
break;
case 'first':
if (chart.buckets[bucketKey][ydef.field] === undefined) {
chart.buckets[bucketKey][ydef.field] = numericColumns[ydef.field];
}
break;
case 'last':
chart.buckets[bucketKey][ydef.field] = numericColumns[ydef.field];
break;
case 'min':
if (chart.buckets[bucketKey][ydef.field] === undefined) {
chart.buckets[bucketKey][ydef.field] = numericColumns[ydef.field];
} else {
chart.buckets[bucketKey][ydef.field] = Math.min(
chart.buckets[bucketKey][ydef.field],
numericColumns[ydef.field]
);
}
break;
case 'max':
if (chart.buckets[bucketKey][ydef.field] === undefined) {
chart.buckets[bucketKey][ydef.field] = numericColumns[ydef.field];
} else {
chart.buckets[bucketKey][ydef.field] = Math.max(
chart.buckets[bucketKey][ydef.field],
numericColumns[ydef.field]
);
}
break;
case 'count':
chart.buckets[bucketKey][ydef.field] = (chart.buckets[bucketKey][ydef.field] || 0) + 1;
break;
case 'avg':
if (chart.buckets[bucketKey][ydef.field] === undefined) {
chart.buckets[bucketKey][ydef.field] = [numericColumns[ydef.field], 1]; // [sum, count]
} else {
chart.buckets[bucketKey][ydef.field][0] += numericColumns[ydef.field];
chart.buckets[bucketKey][ydef.field][1] += 1;
}
break;
}
}
}
export function aggregateChartNumericValuesFromChild(
chart: ProcessedChart,
bucketKey: string,
childBucketValues: { [key: string]: any }
) {
for (const ydef of chart.definition.ydefs) {
if (childBucketValues[ydef.field] == undefined) {
continue; // skip if the field is not present in the child bucket
}
if (!chart.buckets[bucketKey]) {
chart.buckets[bucketKey] = {};
}
switch (ydef.aggregateFunction) {
case 'sum':
case 'count':
chart.buckets[bucketKey][ydef.field] =
(chart.buckets[bucketKey][ydef.field] || 0) + (childBucketValues[ydef.field] || 0);
break;
case 'min':
if (chart.buckets[bucketKey][ydef.field] === undefined) {
chart.buckets[bucketKey][ydef.field] = childBucketValues[ydef.field];
} else {
chart.buckets[bucketKey][ydef.field] = Math.min(
chart.buckets[bucketKey][ydef.field],
childBucketValues[ydef.field]
);
}
break;
case 'max':
if (chart.buckets[bucketKey][ydef.field] === undefined) {
chart.buckets[bucketKey][ydef.field] = childBucketValues[ydef.field];
} else {
chart.buckets[bucketKey][ydef.field] = Math.max(
chart.buckets[bucketKey][ydef.field],
childBucketValues[ydef.field]
);
}
break;
case 'avg':
if (chart.buckets[bucketKey][ydef.field] === undefined) {
chart.buckets[bucketKey][ydef.field] = childBucketValues[ydef.field];
} else {
chart.buckets[bucketKey][ydef.field][0] += childBucketValues[ydef.field][0];
chart.buckets[bucketKey][ydef.field][1] += childBucketValues[ydef.field][1];
}
break;
case 'first':
case 'last':
throw new Error(`Cannot aggregate ${ydef.aggregateFunction} for ${ydef.field} in child bucket`);
}
}
}
export function fillChartTimelineBuckets(chart: ProcessedChart) {
const fromParsed = chart.bucketKeyDateParsed[chart.minX];
const toParsed = chart.bucketKeyDateParsed[chart.maxX];
if (!fromParsed || !toParsed) {
return; // cannot fill timeline buckets without valid date range
}
const transform = chart.definition.xdef.transformFunction;
let currentParsed = fromParsed;
while (compareChartDatesParsed(currentParsed, toParsed, transform) <= 0) {
const bucketKey = stringifyChartDate(currentParsed, transform);
if (!chart.buckets[bucketKey]) {
chart.buckets[bucketKey] = {};
chart.bucketKeyDateParsed[bucketKey] = currentParsed;
}
currentParsed = incrementChartDate(currentParsed, transform);
}
}
export function computeChartBucketCardinality(bucket: { [key: string]: any }): number {
return _sumBy(Object.keys(bucket), field => bucket[field]);
}

2
packages/datalib/src/index.ts
View File

@@ -23,3 +23,5 @@ export * from './FreeTableGridDisplay';
export * from './FreeTableModel';
export * from './CustomGridDisplay';
export * from './ScriptDrivedDeployer';
export * from './chartDefinitions';
export * from './chartProcessor';

376
packages/datalib/src/tests/chartProcessor.test.ts Normal file
View File

@@ -0,0 +1,376 @@
import exp from 'constants';
import { ChartProcessor } from '../chartProcessor';
import { getChartDebugPrint } from '../chartTools';
const DS1 = [
{
timestamp: '2023-10-01T12:00:00Z',
value: 42.5,
category: 'B',
related_id: 12,
},
{
timestamp: '2023-10-02T10:05:00Z',
value: 12,
category: 'A',
related_id: 13,
},
{
timestamp: '2023-10-03T07:10:00Z',
value: 57,
category: 'A',
related_id: 5,
},
{
timestamp: '2024-08-03T07:10:00Z',
value: 33,
category: 'B',
related_id: 22,
},
];
const DS2 = [
{
ts1: '2023-10-01T12:00:00Z',
ts2: '2024-10-01T12:00:00Z',
dummy1: 1,
dummy2: 1,
dummy3: 1,
dummy4: 1,
dummy5: 1,
dummy6: 1,
dummy7: 1,
dummy8: 1,
dummy9: 1,
dummy10: 1,
price1: '11',
price2: '22',
},
{
ts1: '2023-10-02T10:05:00Z',
ts2: '2024-10-02T10:05:00Z',
price1: '12',
price2: '23',
},
{
ts1: '2023-10-03T07:10:00Z',
ts2: '2024-10-03T07:10:00Z',
price1: '13',
price2: '24',
},
{
ts1: '2023-11-04T12:00:00Z',
ts2: '2024-11-04T12:00:00Z',
price1: 1,
price2: 2,
},
];
const DS3 = [
{
timestamp: '2023-10-01T12:00:00Z',
value: 42.5,
bitval: true,
},
{
timestamp: '2023-10-02T10:05:00Z',
value: 12,
bitval: false,
},
{
timestamp: '2023-10-03T07:10:00Z',
value: 57,
bitval: null,
},
];
const DS4 = [
{
object_id: 710293590,
ObjectName: 'Journal',
Total_Reserved_kb: '68696',
RowsCount: '405452',
},
{
object_id: 182291709,
ObjectName: 'Employee',
Total_Reserved_kb: '732008',
RowsCount: '1980067',
},
{
object_id: 23432525,
ObjectName: 'User',
Total_Reserved_kb: '325352',
RowsCount: '2233',
},
{
object_id: 4985159,
ObjectName: 'Project',
Total_Reserved_kb: '293523',
RowsCount: '1122',
},
];
describe('Chart processor', () => {
test('Simple by day test, autodetected', () => {
const processor = new ChartProcessor();
processor.addRows(...DS1.slice(0, 3));
processor.finalize();
expect(processor.charts.length).toEqual(1);
const chart = processor.charts[0];
expect(chart.definition.xdef.transformFunction).toEqual('date:day');
expect(chart.definition.ydefs).toEqual([
expect.objectContaining({
field: 'value',
}),
]);
expect(chart.bucketKeysOrdered).toEqual(['2023-10-01', '2023-10-02', '2023-10-03']);
});
test('By month grouped, autedetected', () => {
const processor = new ChartProcessor();
processor.addRows(...DS1.slice(0, 4));
processor.finalize();
expect(processor.charts.length).toEqual(1);
const chart = processor.charts[0];
expect(chart.definition.xdef.transformFunction).toEqual('date:month');
expect(chart.bucketKeysOrdered).toEqual([
'2023-10',
'2023-11',
'2023-12',
'2024-01',
'2024-02',
'2024-03',
'2024-04',
'2024-05',
'2024-06',
'2024-07',
'2024-08',
]);
});
test('Detect columns', () => {
const processor = new ChartProcessor();
processor.autoDetectCharts = false;
processor.addRows(...DS1);
processor.finalize();
expect(processor.charts.length).toEqual(0);
expect(processor.availableColumns).toEqual([
expect.objectContaining({
field: 'timestamp',
}),
expect.objectContaining({
field: 'value',
}),
expect.objectContaining({
field: 'category',
}),
expect.objectContaining({
field: 'related_id',
}),
]);
});
test('Explicit definition', () => {
const processor = new ChartProcessor([
{
chartType: 'pie',
xdef: {
field: 'category',
transformFunction: 'identity',
sortOrder: 'natural',
},
ydefs: [
{
field: 'related_id',
aggregateFunction: 'sum',
},
],
},
]);
processor.addRows(...DS1);
processor.finalize();
expect(processor.charts.length).toEqual(1);
const chart = processor.charts[0];
expect(chart.definition.xdef.transformFunction).toEqual('identity');
expect(chart.bucketKeysOrdered).toEqual(['B', 'A']);
expect(chart.buckets).toEqual({
B: { related_id: 34 },
A: { related_id: 18 },
});
});
test('Two data sets with different date columns', () => {
const processor = new ChartProcessor();
processor.addRows(...DS2);
processor.finalize();
expect(processor.charts.length).toEqual(2);
expect(processor.charts[0].definition).toEqual(
expect.objectContaining({
xdef: expect.objectContaining({
field: 'ts1',
transformFunction: 'date:day',
}),
ydefs: [
expect.objectContaining({
field: 'price1',
aggregateFunction: 'sum',
}),
expect.objectContaining({
field: 'price2',
aggregateFunction: 'sum',
}),
],
})
);
expect(processor.charts[1].definition).toEqual(
expect.objectContaining({
xdef: expect.objectContaining({
field: 'ts2',
transformFunction: 'date:day',
}),
ydefs: [
expect.objectContaining({
field: 'price1',
aggregateFunction: 'sum',
}),
expect.objectContaining({
field: 'price2',
aggregateFunction: 'sum',
}),
],
})
);
});
test('Exclude boolean fields in autodetected', () => {
const processor = new ChartProcessor();
processor.addRows(...DS3);
processor.finalize();
expect(processor.charts.length).toEqual(1);
const chart = processor.charts[0];
expect(chart.definition.xdef.transformFunction).toEqual('date:day');
expect(chart.definition.ydefs).toEqual([
expect.objectContaining({
field: 'value',
}),
]);
});
test('Added field manual from GUI', () => {
const processor = new ChartProcessor([
{
chartType: 'bar',
xdef: {
field: 'object_id',
transformFunction: 'identity',
},
ydefs: [
{
field: 'object_id',
aggregateFunction: 'sum',
},
],
},
]);
processor.addRows(...DS4);
processor.finalize();
expect(processor.charts.length).toEqual(1);
const chart = processor.charts[0];
expect(chart.definition.xdef.transformFunction).toEqual('identity');
expect(chart.definition.ydefs).toEqual([
expect.objectContaining({
field: 'object_id',
aggregateFunction: 'sum',
}),
]);
});
const PieMainTestData = [
['natural', ['Journal', 'Employee', 'User', 'Project']],
['ascKeys', ['Employee', 'Journal', 'Project', 'User']],
['descKeys', ['User', 'Project', 'Journal', 'Employee']],
['ascValues', ['Project', 'User', 'Journal', 'Employee']],
['descValues', ['Employee', 'Journal', 'User', 'Project']],
];
test.each(PieMainTestData)('Pie chart - used space for DB objects (%s)', (sortOrder, expectedOrder) => {
const processor = new ChartProcessor([
{
chartType: 'bar',
xdef: {
field: 'ObjectName',
transformFunction: 'identity',
sortOrder: sortOrder as any,
},
ydefs: [
{
field: 'RowsCount',
aggregateFunction: 'sum',
},
],
},
]);
processor.addRows(...DS4);
processor.finalize();
expect(processor.charts.length).toEqual(1);
const chart = processor.charts[0];
expect(chart.bucketKeysOrdered).toEqual(expectedOrder);
expect(chart.buckets).toEqual({
Employee: { RowsCount: 1980067 },
Journal: { RowsCount: 405452 },
Project: { RowsCount: 1122 },
User: { RowsCount: 2233 },
});
});
const PieOtherTestData = [
[
'ratio',
0.1,
5,
['Employee', 'Journal', 'Other'],
{
Employee: { RowsCount: 1980067 },
Journal: { RowsCount: 405452 },
Other: { RowsCount: 3355 },
},
],
[
'count',
0,
1,
['Employee', 'Other'],
{
Employee: { RowsCount: 1980067 },
Other: { RowsCount: 408807 },
},
],
];
test.each(PieOtherTestData)(
'Pie limit test - %s',
(_description, pieRatioLimit, pieCountLimit, expectedOrder, expectedBuckets) => {
const processor = new ChartProcessor([
{
chartType: 'pie',
pieRatioLimit: pieRatioLimit as number,
pieCountLimit: pieCountLimit as number,
xdef: {
field: 'ObjectName',
transformFunction: 'identity',
},
ydefs: [
{
field: 'RowsCount',
aggregateFunction: 'sum',
},
],
},
]);
processor.addRows(...DS4);
processor.finalize();
expect(processor.charts.length).toEqual(1);
const chart = processor.charts[0];
expect(chart.bucketKeysOrdered).toEqual(expectedOrder);
expect(chart.buckets).toEqual(expectedBuckets);
}
);
});

38
packages/tools/src/stringTools.ts
View File

@@ -5,7 +5,10 @@ import _isNumber from 'lodash/isNumber';
import _isPlainObject from 'lodash/isPlainObject';
import _pad from 'lodash/pad';
import _cloneDeepWith from 'lodash/cloneDeepWith';
import _isEmpty from 'lodash/isEmpty';
import _omitBy from 'lodash/omitBy';
import { DataEditorTypesBehaviour } from 'dbgate-types';
import isPlainObject from 'lodash/isPlainObject';
export type EditorDataType =
| 'null'
@@ -633,3 +636,38 @@ export function parseNumberSafe(value) {
}
return parseFloat(value);
}
const frontMatterRe = /^--\ >>>[ \t]*\n(.*)\n-- <<<[ \t]*\n/s;
export function getSqlFrontMatter(text: string, yamlModule) {
const match = text.match(frontMatterRe);
if (!match) return null;
const yamlContentMapped = match[1].replace(/^--[ ]?/gm, '');
return yamlModule.load(yamlContentMapped);
}
export function removeSqlFrontMatter(text: string) {
return text.replace(frontMatterRe, '');
}
export function setSqlFrontMatter(text: string, data: { [key: string]: any }, yamlModule) {
const textClean = removeSqlFrontMatter(text);
if (!isPlainObject(data)) {
return textClean;
}
const dataClean = _omitBy(data, v => v === undefined);
if (_isEmpty(dataClean)) {
return textClean;
}
const yamlContent = yamlModule.dump(dataClean);
const yamlContentMapped = yamlContent
.trimRight()
.split('\n')
.map(line => '-- ' + line)
.join('\n');
const frontMatterContent = `-- >>>\n${yamlContentMapped}\n-- <<<\n`;
return frontMatterContent + textClean;
}

2
packages/tools/tsconfig.json
View File

@@ -1,6 +1,6 @@
{
"compilerOptions": {
"target": "ES2015",
"target": "ES2018",
"module": "commonjs",
"declaration": true,
"skipLibCheck": true,

35
packages/web/src/appobj/DatabaseObjectAppObject.svelte
View File

@@ -185,10 +185,6 @@
isImport: true,
requiresWriteAccess: true,
},
hasPermission('dbops/charts') && {
label: 'Open active chart',
isActiveChart: true,
},
];
case 'views':
return [
@@ -245,10 +241,6 @@
isExport: true,
functionName: 'tableReader',
},
{
label: 'Open active chart',
isActiveChart: true,
},
];
case 'matviews':
return [
@@ -299,10 +291,6 @@
isExport: true,
functionName: 'tableReader',
},
{
label: 'Open active chart',
isActiveChart: true,
},
];
case 'queries':
return [
@@ -472,28 +460,7 @@
return driver;
};
if (menu.isActiveChart) {
const driver = await getDriver();
const dmp = driver.createDumper();
dmp.put('^select * from %f', data);
openNewTab(
{
title: data.pureName,
icon: 'img chart',
tabComponent: 'ChartTab',
props: {
conid: data.conid,
database: data.database,
},
},
{
editor: {
config: { chartType: 'bar' },
sql: dmp.s,
},
}
);
} else if (menu.isQueryDesigner) {
if (menu.isQueryDesigner) {
openNewTab(
{
title: 'Query #',

11
packages/web/src/appobj/SavedFileAppObject.svelte
View File

@@ -41,16 +41,6 @@
label: 'Markdown file',
};
const charts: FileTypeHandler = {
icon: 'img chart',
format: 'json',
tabComponent: 'ChartTab',
folder: 'charts',
currentConnection: true,
extension: 'json',
label: 'Chart file',
};
const query: FileTypeHandler = {
icon: 'img query-design',
format: 'json',
@@ -139,7 +129,6 @@
sql,
shell,
markdown,
charts,
query,
sqlite,
diagrams,

31
packages/web/src/datagrid/DataGridCore.svelte
View File

@@ -261,13 +261,6 @@
testEnabled: () => getCurrentDataGrid() != null,
onClick: () => getCurrentDataGrid().openFreeTable(),
});
registerCommand({
id: 'dataGrid.openChartFromSelection',
category: 'Data grid',
name: 'Open chart from selection',
testEnabled: () => getCurrentDataGrid() != null,
onClick: () => getCurrentDataGrid().openChartFromSelection(),
});
registerCommand({
id: 'dataGrid.newJson',
category: 'Data grid',
@@ -469,6 +462,7 @@
export let hideGridLeftColumn = false;
export let overlayDefinition = null;
export let onGetSelectionMenu = null;
export let onOpenChart = null;
export const activator = createActivator('DataGridCore', false);
@@ -715,23 +709,6 @@
openJsonLinesData(getSelectedFreeDataRows());
}
export function openChartFromSelection() {
openNewTab(
{
title: 'Chart #',
icon: 'img chart',
tabComponent: 'ChartTab',
props: {},
},
{
editor: {
data: getSelectedFreeData(),
config: { chartType: 'bar' },
},
}
);
}
export function viewJsonDocumentEnabled() {
return isDynamicStructure && _.uniq(selectedCells.map(x => x[0])).length == 1;
}
@@ -1869,9 +1846,13 @@
// ],
// },
isProApp() && { command: 'dataGrid.sendToDataDeploy' },
isProApp() &&
onOpenChart && {
text: 'Open chart',
onClick: () => onOpenChart(),
},
{ command: 'dataGrid.generateSqlFromData' },
{ command: 'dataGrid.openFreeTable' },
{ command: 'dataGrid.openChartFromSelection' },
{ command: 'dataGrid.openSelectionInMap', hideDisabled: true },
{ placeTag: 'chart' }
);

30
packages/web/src/datagrid/SqlDataGridCore.svelte
View File

@@ -1,14 +1,6 @@
<script context="module" lang="ts">
const getCurrentEditor = () => getActiveComponent('SqlDataGridCore');
registerCommand({
id: 'sqlDataGrid.openActiveChart',
category: 'Data grid',
name: 'Open active chart',
testEnabled: () => getCurrentEditor() != null && hasPermission('dbops/charts'),
onClick: () => getCurrentEditor().openActiveChart(),
});
registerCommand({
id: 'sqlDataGrid.openQuery',
category: 'Data grid',
@@ -190,28 +182,6 @@
openQuery(display.getPageQueryText(0, getIntSettingsValue('dataGrid.pageSize', 100, 5, 1000)));
}
export function openActiveChart() {
openNewTab(
{
title: 'Chart #',
icon: 'img chart',
tabComponent: 'ChartTab',
props: {
conid,
database,
},
},
{
editor: {
config: { chartType: 'bar' },
sql: display.getExportQuery(select => {
select.orderBy = null;
}),
},
}
);
}
const quickExportHandler = fmt => async () => {
const coninfo = await getConnectionInfo({ conid });
exportQuickExportFile(

2
packages/web/src/elements/HorizontalSplitter.svelte
View File

@@ -39,7 +39,7 @@
$: size = computeSplitterSize(initialValue, clientWidth, customRatio, initialSizeRight);
$: if (onChangeSize) onChangeSize(size);
$: if (onChangeSize) onChangeSize(size, clientWidth - size);
</script>
<div class="container" bind:clientWidth>

14
packages/web/src/elements/TabControl.svelte
View File

@@ -18,6 +18,7 @@
export let flex1 = true;
export let contentTestId = undefined;
export let inlineTabs = false;
export let onUserChange = null;
export function setValue(index) {
value = index;
@@ -30,8 +31,16 @@
<div class="main" class:flex1>
<div class="tabs" class:inlineTabs>
{#each _.compact(tabs) as tab, index}
<div class="tab-item" class:selected={value == index} on:click={() => (value = index)} data-testid={tab.testid}>
<span class="ml-2">
<div
class="tab-item"
class:selected={value == index}
on:click={() => {
value = index;
onUserChange?.(index);
}}
data-testid={tab.testid}
>
<span class="ml-2 noselect">
{tab.label}
</span>
</div>
@@ -139,5 +148,4 @@
.container.isInline:not(.tabVisible) {
display: none;
}
</style>

1
packages/web/src/icons/FontIcon.svelte
View File

@@ -71,6 +71,7 @@
'icon trigger': 'mdi mdi-lightning-bolt',
'icon scheduler-event': 'mdi mdi-calendar-blank',
'icon arrow-link': 'mdi mdi-arrow-top-right-thick',
'icon reset': 'mdi mdi-cancel',
'icon window-restore': 'mdi mdi-window-restore',
'icon window-maximize': 'mdi mdi-window-maximize',

67
packages/web/src/query/ResultTabs.svelte
View File

@@ -1,5 +1,5 @@
<script lang="ts">
import _ from 'lodash';
import _, { result } from 'lodash';
import { onMount, tick } from 'svelte';
@@ -9,6 +9,7 @@
import { apiOff, apiOn } from '../utility/api';
import useEffect from '../utility/useEffect';
import AllResultsTab from './AllResultsTab.svelte';
import JslChart from '../charts/JslChart.svelte';
export let tabs = [];
export let sessionId;
@@ -16,6 +17,8 @@
export let driver;
export let resultCount;
export let onSetFrontMatterField;
export let onGetFrontMatter;
onMount(() => {
allResultsInOneTab = $allResultsInOneTabDefault;
@@ -23,6 +26,7 @@
let allResultsInOneTab = null;
let resultInfos = [];
let charts = [];
let domTabs;
$: resultCount = resultInfos.length;
@@ -35,6 +39,23 @@
if (!currentTab?.isResult) domTabs.setValue(_.findIndex(allTabs, x => x.isResult));
};
const handleCharts = async props => {
charts = [
...charts,
{
jslid: props.jslid,
charts: props.charts,
resultIndex: props.resultIndex,
},
];
const selectedChart = onGetFrontMatter?.()?.['selected-chart'];
await tick();
if (selectedChart && props.resultIndex == selectedChart - 1) {
domTabs.setValue(_.findIndex(allTabs, x => x.isChart && x.resultIndex === props.resultIndex));
}
// console.log('Charts received for jslid:', props.jslid, 'Charts:', props.charts);
};
$: oneTab = allResultsInOneTab ?? $allResultsInOneTabDefault;
$: allTabs = [
@@ -55,13 +76,27 @@
label: `Result ${index + 1}`,
isResult: true,
component: JslDataGrid,
props: { jslid: info.jslid, driver },
props: { jslid: info.jslid, driver, onOpenChart: () => handleOpenChart(info.resultIndex) },
}))),
...charts.map((info, index) => ({
label: `Chart ${info.resultIndex + 1}`,
isChart: true,
resultIndex: info.resultIndex,
component: JslChart,
props: {
jslid: info.jslid,
initialCharts: info.charts,
onEditDefinition: definition => {
onSetFrontMatterField?.(`chart-${info.resultIndex + 1}`, definition ?? undefined);
},
},
})),
];
$: {
if (executeNumber >= 0) {
resultInfos = [];
charts = [];
if (domTabs) domTabs.setValue(0);
}
}
@@ -72,8 +107,10 @@
function onSession(sid) {
if (sid) {
apiOn(`session-recordset-${sid}`, handleResultSet);
apiOn(`session-charts-${sid}`, handleCharts);
return () => {
apiOff(`session-recordset-${sid}`, handleResultSet);
apiOff(`session-charts-${sid}`, handleCharts);
};
}
return () => {};
@@ -84,6 +121,25 @@
allResultsInOneTab = value;
$allResultsInOneTabDefault = value;
}
async function handleOpenChart(resultIndex) {
const chartTab = _.find(allTabs, x => x.isChart && x.resultIndex === resultIndex);
if (chartTab) {
domTabs.setValue(_.findIndex(allTabs, x => x.isChart && x.resultIndex === resultIndex));
} else {
charts = [
...charts,
{
jslid: resultInfos[resultIndex].jslid,
charts: [],
resultIndex,
},
];
await tick();
domTabs.setValue(_.findIndex(allTabs, x => x.isChart && x.resultIndex === resultIndex));
}
onSetFrontMatterField?.('selected-chart', resultIndex + 1);
}
</script>
<TabControl
@@ -94,6 +150,13 @@
? { text: 'Every result in single tab', onClick: () => setOneTabValue(false) }
: { text: 'All results in one tab', onClick: () => setOneTabValue(true) },
]}
onUserChange={value => {
if (allTabs[value].isChart) {
onSetFrontMatterField?.(`selected-chart`, allTabs[value].resultIndex + 1);
} else {
onSetFrontMatterField?.(`selected-chart`, undefined);
}
}}
>
<slot name="0" slot="0" />
<slot name="1" slot="1" />

62
packages/web/src/tabs/QueryTab.svelte
View File

@@ -1,6 +1,7 @@
<script lang="ts" context="module">
import registerCommand from '../commands/registerCommand';
import { copyTextToClipboard } from '../utility/clipboard';
import yaml from 'js-yaml';
const getCurrentEditor = () => getActiveComponent('QueryTab');
@@ -60,6 +61,13 @@
getCurrentEditor() != null && !getCurrentEditor()?.isBusy() && getCurrentEditor()?.hasConnection(),
onClick: () => getCurrentEditor().executeCurrent(),
});
registerCommand({
id: 'query.toggleAutoExecute',
category: 'Query',
name: 'Toggle auto execute',
testEnabled: () => getCurrentEditor() != null,
onClick: () => getCurrentEditor().toggleAutoExecute(),
});
registerCommand({
id: 'query.beginTransaction',
category: 'Query',
@@ -126,7 +134,7 @@
import InsertJoinModal from '../modals/InsertJoinModal.svelte';
import useTimerLabel from '../utility/useTimerLabel';
import createActivator, { getActiveComponent } from '../utility/createActivator';
import { findEngineDriver, safeJsonParse } from 'dbgate-tools';
import { findEngineDriver, getSqlFrontMatter, safeJsonParse, setSqlFrontMatter } from 'dbgate-tools';
import AceEditor from '../query/AceEditor.svelte';
import StatusBarTabItem from '../widgets/StatusBarTabItem.svelte';
import { showSnackbarError } from '../utility/snackbar';
@@ -147,6 +155,7 @@
import ToolStripButton from '../buttons/ToolStripButton.svelte';
import { getIntSettingsValue } from '../settings/settingsTools';
import RowsLimitModal from '../modals/RowsLimitModal.svelte';
import _ from 'lodash';
export let tabid;
export let conid;
@@ -199,6 +208,7 @@
let domAiAssistant;
let isInTransaction = false;
let isAutocommit = false;
let splitterInitialValue = undefined;
const queryRowsLimitLocalStorageKey = `tabdata_limitRows_${tabid}`;
function getInitialRowsLimit() {
@@ -350,6 +360,7 @@
executeStartLine = startLine;
executeNumber++;
visibleResultTabs = true;
const frontMatter = getSqlFrontMatter($editorValue, yaml);
busy = true;
timerLabel.start();
@@ -381,6 +392,7 @@
sql,
autoCommit: driver?.implicitTransactions && isAutocommit,
limitRows: queryRowsLimit ? queryRowsLimit : undefined,
frontMatter,
});
}
await apiCall('query-history/write', {
@@ -550,12 +562,47 @@
initialArgs && initialArgs.scriptTemplate
? () => applyScriptTemplate(initialArgs.scriptTemplate, $extensions, $$props)
: null,
onInitialData: value => {
const frontMatter = getSqlFrontMatter(value, yaml);
if (frontMatter?.autoExecute) {
executeCore(value, 0);
}
if (frontMatter?.splitterInitialValue) {
splitterInitialValue = frontMatter.splitterInitialValue;
}
},
});
function handleChangeErrors(errors) {
errorMessages = errors;
}
function handleSetFrontMatterField(field, value) {
const text = $editorValue;
setEditorData(
setSqlFrontMatter(
text,
{
...getSqlFrontMatter(text, yaml),
[field]: value,
},
yaml
)
);
}
export function toggleAutoExecute() {
const frontMatter = getSqlFrontMatter($editorValue, yaml);
setEditorData(
setSqlFrontMatter(
$editorValue,
{ ...frontMatter, autoExecute: frontMatter?.autoExecute ? undefined : true },
yaml
)
);
}
async function handleKeyDown(event) {
if (isProApp()) {
if (event.code == 'Space' && event.shiftKey && event.ctrlKey && !isAiAssistantVisible) {
@@ -584,6 +631,7 @@
{ command: 'query.execute' },
{ command: 'query.executeCurrent' },
{ command: 'query.kill' },
{ command: 'query.toggleAutoExecute' },
{ divider: true },
{ command: 'query.toggleComment' },
{ command: 'query.formatCode' },
@@ -625,7 +673,7 @@
<ToolStripContainer bind:this={domToolStrip}>
<HorizontalSplitter isSplitter={isAiAssistantVisible} initialSizeRight={300}>
<svelte:fragment slot="1">
<VerticalSplitter isSplitter={visibleResultTabs}>
<VerticalSplitter isSplitter={visibleResultTabs} initialValue={splitterInitialValue}>
<svelte:fragment slot="1">
{#if driver?.databaseEngineTypes?.includes('sql')}
<SqlEditor
@@ -678,7 +726,15 @@
{/if}
</svelte:fragment>
<svelte:fragment slot="2">
<ResultTabs tabs={[{ label: 'Messages', slot: 0 }]} {sessionId} {executeNumber} bind:resultCount {driver}>
<ResultTabs
tabs={[{ label: 'Messages', slot: 0 }]}
{sessionId}
{executeNumber}
bind:resultCount
{driver}
onSetFrontMatterField={handleSetFrontMatterField}
onGetFrontMatter={() => getSqlFrontMatter($editorValue, yaml)}
>
<svelte:fragment slot="0">
<SocketMessageView
eventName={sessionId ? `session-info-${sessionId}` : null}