diff --git a/packages/datalib/src/chartDefinitions.ts b/packages/datalib/src/chartDefinitions.ts index 7452b65df..9c1a0bd31 100644 --- a/packages/datalib/src/chartDefinitions.ts +++ b/packages/datalib/src/chartDefinitions.ts @@ -25,6 +25,7 @@ export const ChartLimits = { PIE_RATIO_LIMIT: 0.05, // limit for other values in pie chart, if the value is below this, it will be grouped into "Other" PIE_COUNT_LIMIT: 10, // limit for number of pie chart slices, if the number of slices is above this, it will be grouped into "Other" CHART_FILL_LIMIT: 10000, // limit for filled charts (time intervals), to avoid too many points + CHART_GROUP_LIMIT: 32, // limit for number of groups in a chart }; export interface ChartXFieldDefinition { @@ -52,6 +53,8 @@ export interface ChartDefinition { xdef: ChartXFieldDefinition; ydefs: ChartYFieldDefinition[]; + groupingField?: string; + groupTransformFunction?: ChartXTransformFunction; useDataLabels?: boolean; dataLabelFormatter?: ChartDataLabelFormatter; @@ -77,11 +80,14 @@ export interface ProcessedChart { rowsAdded: number; buckets: { [key: string]: any }; // key is the bucket key, value is aggregated data bucketKeysOrdered: string[]; + bucketKeysSet: Set; bucketKeyDateParsed: { [key: string]: ChartDateParsed }; // key is the bucket key, value is parsed date isGivenDefinition: boolean; // true if the chart was created with a given definition, false if it was created from raw data invalidXRows: number; invalidYRows: { [key: string]: number }; // key is the y field, value is the count of invalid rows validYRows: { [key: string]: number }; // key is the field, value is the count of valid rows + groups: string[]; + groupSet: Set; topDistinctValues: { [key: string]: Set }; // key is the field, value is the set of distinct values availableColumns: ChartAvailableColumn[]; diff --git a/packages/datalib/src/chartProcessor.ts b/packages/datalib/src/chartProcessor.ts index ea0df257d..270681875 100644 --- a/packages/datalib/src/chartProcessor.ts +++ b/packages/datalib/src/chartProcessor.ts @@ -13,6 +13,7 @@ import { computeChartBucketCardinality, computeChartBucketKey, fillChartTimelineBuckets, + runTransformFunction, tryParseChartDate, } from './chartTools'; import { getChartScore, getChartYFieldScore } from './chartScoring'; @@ -40,6 +41,9 @@ export class ChartProcessor { availableColumns: [], validYRows: {}, topDistinctValues: {}, + groups: [], + groupSet: new Set(), + bucketKeysSet: new Set(), }); } this.autoDetectCharts = this.givenDefinitions.length == 0; @@ -132,6 +136,7 @@ export class ChartProcessor { rowsAdded: 0, bucketKeysOrdered: [], buckets: {}, + groups: [], bucketKeyDateParsed: {}, isGivenDefinition: false, invalidXRows: 0, @@ -139,6 +144,8 @@ export class ChartProcessor { availableColumns: [], validYRows: {}, topDistinctValues: {}, + groupSet: new Set(), + bucketKeysSet: new Set(), }; this.chartsProcessing.push(usedChart); } @@ -247,14 +254,14 @@ export class ChartProcessor { continue; } - addedChart.bucketKeysOrdered = _sortBy(Object.keys(addedChart.buckets)); + addedChart.bucketKeysOrdered = _sortBy([...addedChart.bucketKeysSet]); if (sortOrder == 'descKeys') { addedChart.bucketKeysOrdered.reverse(); } } if (sortOrder == 'ascValues' || sortOrder == 'descValues') { - addedChart.bucketKeysOrdered = _sortBy(Object.keys(addedChart.buckets), key => + addedChart.bucketKeysOrdered = _sortBy([...addedChart.bucketKeysSet], key => computeChartBucketCardinality(addedChart.buckets[key]) ); if (sortOrder == 'descValues') { @@ -290,6 +297,10 @@ export class ChartProcessor { } this.groupPieOtherBuckets(addedChart); + + addedChart.groups = [...addedChart.groupSet]; + addedChart.bucketKeysSet = undefined; + addedChart.groupSet = undefined; } this.charts = [ @@ -373,6 +384,15 @@ export class ChartProcessor { } const [bucketKey, bucketKeyParsed] = computeChartBucketKey(dateParsed, chart, row); + const bucketGroup = chart.definition.groupingField + ? runTransformFunction(row[chart.definition.groupingField], chart.definition.groupTransformFunction) + : null; + if (bucketGroup) { + chart.groupSet.add(bucketGroup); + } + if (chart.groupSet.size > ChartLimits.CHART_GROUP_LIMIT) { + chart.errorMessage = `Chart has too many groups, limit is ${ChartLimits.CHART_GROUP_LIMIT}.`; + } if (!bucketKey) { return; // skip if no bucket key @@ -389,14 +409,19 @@ export class ChartProcessor { chart.maxX = bucketKey; } - if (!chart.buckets[bucketKey]) { - chart.buckets[bucketKey] = {}; + const groupedBucketKey = chart.definition.groupingField ? `${bucketGroup ?? ''}::${bucketKey}` : bucketKey; + if (!chart.buckets[groupedBucketKey]) { + chart.buckets[groupedBucketKey] = {}; + } + + if (!chart.bucketKeysSet.has(bucketKey)) { + chart.bucketKeysSet.add(bucketKey); if (chart.definition.xdef.sortOrder == 'natural') { chart.bucketKeysOrdered.push(bucketKey); } } - aggregateChartNumericValuesFromSource(chart, bucketKey, numericColumns, row); + aggregateChartNumericValuesFromSource(chart, groupedBucketKey, numericColumns, row); chart.rowsAdded += 1; } } diff --git a/packages/datalib/src/chartTools.ts b/packages/datalib/src/chartTools.ts index 970ea6528..7e8d9f908 100644 --- a/packages/datalib/src/chartTools.ts +++ b/packages/datalib/src/chartTools.ts @@ -133,6 +133,33 @@ export function incrementChartDate(value: ChartDateParsed, transform: ChartXTran } } +export function runTransformFunction(value: string, transformFunction: ChartXTransformFunction): string { + const dateParsed = tryParseChartDate(value); + switch (transformFunction) { + case 'date:year': + return dateParsed ? `${dateParsed.year}` : null; + case 'date:month': + return dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}` : null; + case 'date:day': + return dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)}` : null; + case 'date:hour': + return dateParsed + ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits( + dateParsed.hour + )}` + : null; + case 'date:minute': + return dateParsed + ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits( + dateParsed.hour + )}:${pad2Digits(dateParsed.minute)}` + : null; + case 'identity': + default: + return value; + } +} + export function computeChartBucketKey( dateParsed: ChartDateParsed, chart: ProcessedChart, @@ -268,7 +295,19 @@ export function compareChartDatesParsed( } } -function getParentDateBucketKey(bucketKey: string, transform: ChartXTransformFunction): string | null { +function getParentDateBucketKey( + bucketKey: string, + transform: ChartXTransformFunction, + isGrouped: boolean +): string | null { + if (isGrouped) { + const [group, key] = bucketKey.split('::', 2); + if (!key) { + return null; // no parent for grouped bucket + } + return `${group}::${getParentDateBucketKey(key, transform, false)}`; + } + switch (transform) { case 'date:year': return null; // no parent for year @@ -345,10 +384,21 @@ function createParentChartAggregation(chart: ProcessedChart): ProcessedChart | n validYRows: { ...chart.validYRows }, // copy valid Y rows topDistinctValues: { ...chart.topDistinctValues }, // copy top distinct values availableColumns: chart.availableColumns, + groups: [...chart.groups], // copy groups + groupSet: new Set(chart.groups), // create a set from the groups + bucketKeysSet: new Set(), // initialize empty set for bucket keys }; + for (const bucketKey of chart.bucketKeysSet) { + res.bucketKeysSet.add(getParentDateBucketKey(bucketKey, chart.definition.xdef.transformFunction, false)); + } + for (const [bucketKey, bucketValues] of Object.entries(chart.buckets)) { - const parentKey = getParentDateBucketKey(bucketKey, chart.definition.xdef.transformFunction); + const parentKey = getParentDateBucketKey( + bucketKey, + chart.definition.xdef.transformFunction, + !!chart.definition.groupingField + ); if (!parentKey) { // skip if the bucket is already a parent continue; @@ -532,8 +582,11 @@ export function fillChartTimelineBuckets(chart: ProcessedChart) { const bucketKey = stringifyChartDate(currentParsed, transform); if (!chart.buckets[bucketKey]) { chart.buckets[bucketKey] = {}; + } + if (!chart.bucketKeyDateParsed[bucketKey]) { chart.bucketKeyDateParsed[bucketKey] = currentParsed; } + chart.bucketKeysSet.add(bucketKey); currentParsed = incrementChartDate(currentParsed, transform); count++; if (count > ChartLimits.CHART_FILL_LIMIT) { @@ -544,5 +597,5 @@ export function fillChartTimelineBuckets(chart: ProcessedChart) { } export function computeChartBucketCardinality(bucket: { [key: string]: any }): number { - return _sumBy(Object.keys(bucket), field => bucket[field]); + return _sumBy(Object.keys(bucket ?? {}), field => bucket[field]); }