SYNC: charts - grouping field support

packages/datalib/src/chartDefinitions.ts

@@ -25,6 +25,7 @@ export const ChartLimits = {
   PIE_RATIO_LIMIT: 0.05, // limit for other values in pie chart, if the value is below this, it will be grouped into "Other"
   PIE_COUNT_LIMIT: 10, // limit for number of pie chart slices, if the number of slices is above this, it will be grouped into "Other"
   CHART_FILL_LIMIT: 10000, // limit for filled charts (time intervals), to avoid too many points
+  CHART_GROUP_LIMIT: 32, // limit for number of groups in a chart
 };
 
 export interface ChartXFieldDefinition {
@@ -52,6 +53,8 @@ export interface ChartDefinition {
 
   xdef: ChartXFieldDefinition;
   ydefs: ChartYFieldDefinition[];
+  groupingField?: string;
+  groupTransformFunction?: ChartXTransformFunction;
 
   useDataLabels?: boolean;
   dataLabelFormatter?: ChartDataLabelFormatter;
@@ -77,11 +80,14 @@ export interface ProcessedChart {
   rowsAdded: number;
   buckets: { [key: string]: any }; // key is the bucket key, value is aggregated data
   bucketKeysOrdered: string[];
+  bucketKeysSet: Set<string>;
   bucketKeyDateParsed: { [key: string]: ChartDateParsed }; // key is the bucket key, value is parsed date
   isGivenDefinition: boolean; // true if the chart was created with a given definition, false if it was created from raw data
   invalidXRows: number;
   invalidYRows: { [key: string]: number }; // key is the y field, value is the count of invalid rows
   validYRows: { [key: string]: number }; // key is the field, value is the count of valid rows
+  groups: string[];
+  groupSet: Set<string>;
 
   topDistinctValues: { [key: string]: Set<any> }; // key is the field, value is the set of distinct values
   availableColumns: ChartAvailableColumn[];

packages/datalib/src/chartProcessor.ts

@@ -13,6 +13,7 @@ import {
   computeChartBucketCardinality,
   computeChartBucketKey,
   fillChartTimelineBuckets,
+  runTransformFunction,
   tryParseChartDate,
 } from './chartTools';
 import { getChartScore, getChartYFieldScore } from './chartScoring';
@@ -40,6 +41,9 @@ export class ChartProcessor {
         availableColumns: [],
         validYRows: {},
         topDistinctValues: {},
+        groups: [],
+        groupSet: new Set<string>(),
+        bucketKeysSet: new Set<string>(),
       });
     }
     this.autoDetectCharts = this.givenDefinitions.length == 0;
@@ -132,6 +136,7 @@ export class ChartProcessor {
             rowsAdded: 0,
             bucketKeysOrdered: [],
             buckets: {},
+            groups: [],
             bucketKeyDateParsed: {},
             isGivenDefinition: false,
             invalidXRows: 0,
@@ -139,6 +144,8 @@ export class ChartProcessor {
             availableColumns: [],
             validYRows: {},
             topDistinctValues: {},
+            groupSet: new Set<string>(),
+            bucketKeysSet: new Set<string>(),
           };
           this.chartsProcessing.push(usedChart);
         }
@@ -247,14 +254,14 @@ export class ChartProcessor {
             continue;
           }
 
-          addedChart.bucketKeysOrdered = _sortBy(Object.keys(addedChart.buckets));
+          addedChart.bucketKeysOrdered = _sortBy([...addedChart.bucketKeysSet]);
           if (sortOrder == 'descKeys') {
             addedChart.bucketKeysOrdered.reverse();
           }
         }
 
         if (sortOrder == 'ascValues' || sortOrder == 'descValues') {
-          addedChart.bucketKeysOrdered = _sortBy(Object.keys(addedChart.buckets), key =>
+          addedChart.bucketKeysOrdered = _sortBy([...addedChart.bucketKeysSet], key =>
             computeChartBucketCardinality(addedChart.buckets[key])
           );
           if (sortOrder == 'descValues') {
@@ -290,6 +297,10 @@ export class ChartProcessor {
       }
 
       this.groupPieOtherBuckets(addedChart);
+
+      addedChart.groups = [...addedChart.groupSet];
+      addedChart.bucketKeysSet = undefined;
+      addedChart.groupSet = undefined;
     }
 
     this.charts = [
@@ -373,6 +384,15 @@ export class ChartProcessor {
     }
 
     const [bucketKey, bucketKeyParsed] = computeChartBucketKey(dateParsed, chart, row);
+    const bucketGroup = chart.definition.groupingField
+      ? runTransformFunction(row[chart.definition.groupingField], chart.definition.groupTransformFunction)
+      : null;
+    if (bucketGroup) {
+      chart.groupSet.add(bucketGroup);
+    }
+    if (chart.groupSet.size > ChartLimits.CHART_GROUP_LIMIT) {
+      chart.errorMessage = `Chart has too many groups, limit is ${ChartLimits.CHART_GROUP_LIMIT}.`;
+    }
 
     if (!bucketKey) {
       return; // skip if no bucket key
@@ -389,14 +409,19 @@ export class ChartProcessor {
       chart.maxX = bucketKey;
     }
 
-    if (!chart.buckets[bucketKey]) {
-      chart.buckets[bucketKey] = {};
+    const groupedBucketKey = chart.definition.groupingField ? `${bucketGroup ?? ''}::${bucketKey}` : bucketKey;
+    if (!chart.buckets[groupedBucketKey]) {
+      chart.buckets[groupedBucketKey] = {};
+    }
+
+    if (!chart.bucketKeysSet.has(bucketKey)) {
+      chart.bucketKeysSet.add(bucketKey);
       if (chart.definition.xdef.sortOrder == 'natural') {
         chart.bucketKeysOrdered.push(bucketKey);
       }
     }
 
-    aggregateChartNumericValuesFromSource(chart, bucketKey, numericColumns, row);
+    aggregateChartNumericValuesFromSource(chart, groupedBucketKey, numericColumns, row);
     chart.rowsAdded += 1;
   }
 }

packages/datalib/src/chartTools.ts

@@ -133,6 +133,33 @@ export function incrementChartDate(value: ChartDateParsed, transform: ChartXTran
   }
 }
 
+export function runTransformFunction(value: string, transformFunction: ChartXTransformFunction): string {
+  const dateParsed = tryParseChartDate(value);
+  switch (transformFunction) {
+    case 'date:year':
+      return dateParsed ? `${dateParsed.year}` : null;
+    case 'date:month':
+      return dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}` : null;
+    case 'date:day':
+      return dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)}` : null;
+    case 'date:hour':
+      return dateParsed
+        ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits(
+            dateParsed.hour
+          )}`
+        : null;
+    case 'date:minute':
+      return dateParsed
+        ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits(
+            dateParsed.hour
+          )}:${pad2Digits(dateParsed.minute)}`
+        : null;
+    case 'identity':
+    default:
+      return value;
+  }
+}
+
 export function computeChartBucketKey(
   dateParsed: ChartDateParsed,
   chart: ProcessedChart,
@@ -268,7 +295,19 @@ export function compareChartDatesParsed(
   }
 }
 
-function getParentDateBucketKey(bucketKey: string, transform: ChartXTransformFunction): string | null {
+function getParentDateBucketKey(
+  bucketKey: string,
+  transform: ChartXTransformFunction,
+  isGrouped: boolean
+): string | null {
+  if (isGrouped) {
+    const [group, key] = bucketKey.split('::', 2);
+    if (!key) {
+      return null; // no parent for grouped bucket
+    }
+    return `${group}::${getParentDateBucketKey(key, transform, false)}`;
+  }
+
   switch (transform) {
     case 'date:year':
       return null; // no parent for year
@@ -345,10 +384,21 @@ function createParentChartAggregation(chart: ProcessedChart): ProcessedChart | n
     validYRows: { ...chart.validYRows }, // copy valid Y rows
     topDistinctValues: { ...chart.topDistinctValues }, // copy top distinct values
     availableColumns: chart.availableColumns,
+    groups: [...chart.groups], // copy groups
+    groupSet: new Set(chart.groups), // create a set from the groups
+    bucketKeysSet: new Set<string>(), // initialize empty set for bucket keys
   };
 
+  for (const bucketKey of chart.bucketKeysSet) {
+    res.bucketKeysSet.add(getParentDateBucketKey(bucketKey, chart.definition.xdef.transformFunction, false));
+  }
+
   for (const [bucketKey, bucketValues] of Object.entries(chart.buckets)) {
-    const parentKey = getParentDateBucketKey(bucketKey, chart.definition.xdef.transformFunction);
+    const parentKey = getParentDateBucketKey(
+      bucketKey,
+      chart.definition.xdef.transformFunction,
+      !!chart.definition.groupingField
+    );
     if (!parentKey) {
       // skip if the bucket is already a parent
       continue;
@@ -532,8 +582,11 @@ export function fillChartTimelineBuckets(chart: ProcessedChart) {
     const bucketKey = stringifyChartDate(currentParsed, transform);
     if (!chart.buckets[bucketKey]) {
       chart.buckets[bucketKey] = {};
+    }
+    if (!chart.bucketKeyDateParsed[bucketKey]) {
       chart.bucketKeyDateParsed[bucketKey] = currentParsed;
     }
+    chart.bucketKeysSet.add(bucketKey);
     currentParsed = incrementChartDate(currentParsed, transform);
     count++;
     if (count > ChartLimits.CHART_FILL_LIMIT) {
@@ -544,5 +597,5 @@ export function fillChartTimelineBuckets(chart: ProcessedChart) {
 }
 
 export function computeChartBucketCardinality(bucket: { [key: string]: any }): number {
-  return _sumBy(Object.keys(bucket), field => bucket[field]);
+  return _sumBy(Object.keys(bucket ?? {}), field => bucket[field]);
 }