SYNC: charts UX, error handling, bucket count limit

packages/datalib/src/chartDefinitions.ts

@@ -24,6 +24,7 @@ export const ChartLimits = {
   VALID_VALUE_RATIO_LIMIT: 0.5, // limit for valid value ratio, y defs below this will not be used in auto-detect
   PIE_RATIO_LIMIT: 0.05, // limit for other values in pie chart, if the value is below this, it will be grouped into "Other"
   PIE_COUNT_LIMIT: 10, // limit for number of pie chart slices, if the number of slices is above this, it will be grouped into "Other"
+  CHART_FILL_LIMIT: 10000, // limit for filled charts (time intervals), to avoid too many points
 };
 
 export interface ChartXFieldDefinition {
@@ -83,6 +84,7 @@ export interface ProcessedChart {
 
   topDistinctValues: { [key: string]: Set<any> }; // key is the field, value is the set of distinct values
   availableColumns: ChartAvailableColumn[];
+  errorMessage?: string; // error message if there was an error processing the chart
 
   definition: ChartDefinition;
 }

packages/datalib/src/chartProcessor.ts

@@ -24,6 +24,7 @@ export class ChartProcessor {
   availableColumns: ChartAvailableColumn[] = [];
   autoDetectCharts = false;
   rowsAdded = 0;
+  errorMessage?: string;
 
   constructor(public givenDefinitions: ChartDefinition[] = []) {
     for (const definition of givenDefinitions) {
@@ -163,6 +164,10 @@ export class ChartProcessor {
 
     // apply on all charts with this date column
     for (const chart of this.chartsProcessing) {
+      if (chart.errorMessage) {
+        continue; // skip charts with errors
+      }
+
       this.applyRawData(
         chart,
         row,
@@ -170,9 +175,16 @@ export class ChartProcessor {
         chart.isGivenDefinition ? numericColumns : numericColumnsForAutodetect,
         stringColumns
       );
+
+      if (Object.keys(chart.buckets).length > ChartLimits.CHART_FILL_LIMIT) {
+        chart.errorMessage = `Chart has too many buckets, limit is ${ChartLimits.CHART_FILL_LIMIT}.`;
+      }
     }
 
     for (let i = 0; i < this.chartsProcessing.length; i++) {
+      if (this.chartsProcessing[i].errorMessage) {
+        continue; // skip charts with errors
+      }
       this.chartsProcessing[i] = autoAggregateCompactTimelineChart(this.chartsProcessing[i]);
     }
 
@@ -214,6 +226,10 @@ export class ChartProcessor {
     this.applyLimitsOnCharts();
     this.availableColumns = Object.values(this.availableColumnsDict);
     for (const chart of this.chartsProcessing) {
+      if (chart.errorMessage) {
+        this.charts.push(chart);
+        continue;
+      }
       let addedChart: ProcessedChart = chart;
       if (chart.rowsAdded == 0) {
         continue; // skip empty charts
@@ -221,11 +237,16 @@ export class ChartProcessor {
       const sortOrder = chart.definition.xdef.sortOrder ?? 'ascKeys';
       if (sortOrder != 'natural') {
         if (sortOrder == 'ascKeys' || sortOrder == 'descKeys') {
-          if (chart.definition.xdef.transformFunction.startsWith('date:')) {
+          if (chart.definition.chartType == 'line' && chart.definition.xdef.transformFunction.startsWith('date:')) {
             addedChart = autoAggregateCompactTimelineChart(addedChart);
             fillChartTimelineBuckets(addedChart);
           }
 
+          if (addedChart.errorMessage) {
+            this.charts.push(addedChart);
+            continue;
+          }
+
           addedChart.bucketKeysOrdered = _sortBy(Object.keys(addedChart.buckets));
           if (sortOrder == 'descKeys') {
             addedChart.bucketKeysOrdered.reverse();

packages/datalib/src/chartTools.ts

@@ -527,6 +527,7 @@ export function fillChartTimelineBuckets(chart: ProcessedChart) {
   const transform = chart.definition.xdef.transformFunction;
 
   let currentParsed = fromParsed;
+  let count = 0;
   while (compareChartDatesParsed(currentParsed, toParsed, transform) <= 0) {
     const bucketKey = stringifyChartDate(currentParsed, transform);
     if (!chart.buckets[bucketKey]) {
@@ -534,6 +535,11 @@ export function fillChartTimelineBuckets(chart: ProcessedChart) {
       chart.bucketKeyDateParsed[bucketKey] = currentParsed;
     }
     currentParsed = incrementChartDate(currentParsed, transform);
+    count++;
+    if (count > ChartLimits.CHART_FILL_LIMIT) {
+      chart.errorMessage = `Too many buckets to fill in chart, limit is ${ChartLimits.CHART_FILL_LIMIT}`;
+      return;
+    }
   }
 }