SYNC: Merge pull request #4 from dbgate/feature/charts

packages/api/src/controllers/jsldata.js

@@ -10,6 +10,7 @@ const requirePluginFunction = require('../utility/requirePluginFunction');
 const socket = require('../utility/socket');
 const crypto = require('crypto');
 const dbgateApi = require('../shell');
+const { ChartProcessor } = require('dbgate-datalib');
 
 function readFirstLine(file) {
   return new Promise((resolve, reject) => {
@@ -302,4 +303,29 @@ module.exports = {
     await dbgateApi.download(uri, { targetFile: getJslFileName(jslid) });
     return { jslid };
   },
+
+  buildChart_meta: true,
+  async buildChart({ jslid, definition }) {
+    const datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    const processor = new ChartProcessor(definition ? [definition] : undefined);
+    await datastore.enumRows(row => {
+      processor.addRow(row);
+      return true;
+    });
+    processor.finalize();
+    return processor.charts;
+  },
+
+  detectChartColumns_meta: true,
+  async detectChartColumns({ jslid }) {
+    const datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    const processor = new ChartProcessor();
+    processor.autoDetectCharts = false;
+    await datastore.enumRows(row => {
+      processor.addRow(row);
+      return true;
+    });
+    processor.finalize();
+    return processor.availableColumns;
+  },
 };

packages/api/src/controllers/sessions.js

@@ -83,6 +83,11 @@ module.exports = {
     jsldata.notifyChangedStats(stats);
   },
 
+  handle_charts(sesid, props) {
+    const { jslid, charts, resultIndex } = props;
+    socket.emit(`session-charts-${sesid}`, { jslid, resultIndex, charts });
+  },
+
   handle_initializeFile(sesid, props) {
     const { jslid } = props;
     socket.emit(`session-initialize-file-${jslid}`);
@@ -141,7 +146,7 @@ module.exports = {
   },
 
   executeQuery_meta: true,
-  async executeQuery({ sesid, sql, autoCommit, limitRows }) {
+  async executeQuery({ sesid, sql, autoCommit, limitRows, frontMatter }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
       throw new Error('Invalid session');
@@ -149,7 +154,7 @@ module.exports = {
 
     logger.info({ sesid, sql }, 'Processing query');
     this.dispatchMessage(sesid, 'Query execution started');
-    session.subprocess.send({ msgtype: 'executeQuery', sql, autoCommit, limitRows });
+    session.subprocess.send({ msgtype: 'executeQuery', sql, autoCommit, limitRows, frontMatter });
 
     return { state: 'ok' };
   },

packages/api/src/proc/sessionProcess.js

@@ -117,7 +117,7 @@ async function handleExecuteControlCommand({ command }) {
   }
 }
 
-async function handleExecuteQuery({ sql, autoCommit, limitRows }) {
+async function handleExecuteQuery({ sql, autoCommit, limitRows, frontMatter }) {
   lastActivity = new Date().getTime();
 
   await waitConnected();
@@ -146,7 +146,7 @@ async function handleExecuteQuery({ sql, autoCommit, limitRows }) {
       ...driver.getQuerySplitterOptions('stream'),
       returnRichInfo: true,
     })) {
-      await handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, undefined, limitRows);
+      await handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, undefined, limitRows, frontMatter);
       // const handler = new StreamHandler(resultIndex);
       // const stream = await driver.stream(systemConnection, sqlItem, handler);
       // handler.stream = stream;

packages/api/src/utility/handleQueryStream.js

@@ -5,6 +5,8 @@ const _ = require('lodash');
 
 const { jsldir } = require('../utility/directories');
 const { serializeJsTypesReplacer } = require('dbgate-tools');
+const { ChartProcessor } = require('dbgate-datalib');
+const { isProApp } = require('./checkLicense');
 
 class QueryStreamTableWriter {
   constructor(sesid = undefined) {
@@ -12,9 +14,12 @@ class QueryStreamTableWriter {
     this.currentChangeIndex = 1;
     this.initializedFile = false;
     this.sesid = sesid;
+    if (isProApp()) {
+      this.chartProcessor = new ChartProcessor();
+    }
   }
 
-  initializeFromQuery(structure, resultIndex) {
+  initializeFromQuery(structure, resultIndex, chartDefinition) {
     this.jslid = crypto.randomUUID();
     this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     fs.writeFileSync(
@@ -28,6 +33,9 @@ class QueryStreamTableWriter {
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
     this.initializedFile = true;
+    if (isProApp() && chartDefinition) {
+      this.chartProcessor = new ChartProcessor([chartDefinition]);
+    }
     process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex, sesid: this.sesid });
   }
 
@@ -40,6 +48,15 @@ class QueryStreamTableWriter {
   row(row) {
     // console.log('ACCEPT ROW', row);
     this.currentStream.write(JSON.stringify(row, serializeJsTypesReplacer) + '\n');
+    try {
+      if (this.chartProcessor) {
+        this.chartProcessor.addRow(row);
+      }
+    } catch (e) {
+      console.error('Error processing chart row', e);
+      this.chartProcessor = null;
+    }
+
     this.currentRowCount += 1;
 
     if (!this.plannedStats) {
@@ -87,6 +104,23 @@ class QueryStreamTableWriter {
         this.currentStream.end(() => {
           this.writeCurrentStats(true, true);
           if (afterClose) afterClose();
+          if (this.chartProcessor) {
+            try {
+              this.chartProcessor.finalize();
+              if (this.chartProcessor.charts.length > 0) {
+                process.send({
+                  msgtype: 'charts',
+                  sesid: this.sesid,
+                  jslid: this.jslid,
+                  charts: this.chartProcessor.charts,
+                  resultIndex: this.resultIndex,
+                });
+              }
+            } catch (e) {
+              console.error('Error finalizing chart processor', e);
+              this.chartProcessor = null;
+            }
+          }
           resolve();
         });
       } else {
@@ -97,10 +131,18 @@ class QueryStreamTableWriter {
 }
 
 class StreamHandler {
-  constructor(queryStreamInfoHolder, resolve, startLine, sesid = undefined, limitRows = undefined) {
+  constructor(
+    queryStreamInfoHolder,
+    resolve,
+    startLine,
+    sesid = undefined,
+    limitRows = undefined,
+    frontMatter = undefined
+  ) {
     this.recordset = this.recordset.bind(this);
     this.startLine = startLine;
     this.sesid = sesid;
+    this.frontMatter = frontMatter;
     this.limitRows = limitRows;
     this.rowsLimitOverflow = false;
     this.row = this.row.bind(this);
@@ -133,7 +175,8 @@ class StreamHandler {
     this.currentWriter = new QueryStreamTableWriter(this.sesid);
     this.currentWriter.initializeFromQuery(
       Array.isArray(columns) ? { columns } : columns,
-      this.queryStreamInfoHolder.resultIndex
+      this.queryStreamInfoHolder.resultIndex,
+      this.frontMatter?.[`chart-${this.queryStreamInfoHolder.resultIndex + 1}`]
     );
     this.queryStreamInfoHolder.resultIndex += 1;
     this.rowCounter = 0;
@@ -201,10 +244,25 @@ class StreamHandler {
   }
 }
 
-function handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, sesid = undefined, limitRows = undefined) {
+function handleQueryStream(
+  dbhan,
+  driver,
+  queryStreamInfoHolder,
+  sqlItem,
+  sesid = undefined,
+  limitRows = undefined,
+  frontMatter = undefined
+) {
   return new Promise((resolve, reject) => {
     const start = sqlItem.trimStart || sqlItem.start;
-    const handler = new StreamHandler(queryStreamInfoHolder, resolve, start && start.line, sesid, limitRows);
+    const handler = new StreamHandler(
+      queryStreamInfoHolder,
+      resolve,
+      start && start.line,
+      sesid,
+      limitRows,
+      frontMatter
+    );
     driver.stream(dbhan, sqlItem.text, handler);
   });
 }

packages/datalib/package.json

@@ -6,6 +6,7 @@
   "scripts": {
     "build": "tsc",
     "test": "jest",
+    "test:charts": "jest -t \"Chart processor\"",
     "test:ci": "jest --json --outputFile=result.json --testLocationInResults",
     "start": "tsc --watch"
   },
@@ -13,16 +14,17 @@
     "lib"
   ],
   "dependencies": {
+    "date-fns": "^4.1.0",
+    "dbgate-filterparser": "^6.0.0-alpha.1",
     "dbgate-sqltree": "^6.0.0-alpha.1",
     "dbgate-tools": "^6.0.0-alpha.1",
-    "dbgate-filterparser": "^6.0.0-alpha.1",
     "uuid": "^3.4.0"
   },
   "devDependencies": {
-    "dbgate-types": "^6.0.0-alpha.1",
     "@types/node": "^13.7.0",
+    "dbgate-types": "^6.0.0-alpha.1",
     "jest": "^28.1.3",
     "ts-jest": "^28.0.7",
     "typescript": "^4.4.3"
   }
-}
+}

packages/datalib/src/chartDefinitions.ts

@@ -0,0 +1,84 @@
+export type ChartTypeEnum = 'bar' | 'line' | 'pie' | 'polarArea';
+export type ChartXTransformFunction =
+  | 'identity'
+  | 'date:minute'
+  | 'date:hour'
+  | 'date:day'
+  | 'date:month'
+  | 'date:year';
+export type ChartYAggregateFunction = 'sum' | 'first' | 'last' | 'min' | 'max' | 'count' | 'avg';
+
+export const ChartConstDefaults = {
+  sortOrder: ' asc',
+  windowAlign: 'end',
+  windowSize: 100,
+  parentAggregateLimit: 200,
+};
+
+export const ChartLimits = {
+  AUTODETECT_CHART_LIMIT: 10, // limit for auto-detecting charts, to avoid too many charts
+  AUTODETECT_MEASURES_LIMIT: 10, // limit for auto-detecting measures, to avoid too many measures
+  APPLY_LIMIT_AFTER_ROWS: 100,
+  MAX_DISTINCT_VALUES: 10, // max number of distinct values to keep in topDistinctValues
+  VALID_VALUE_RATIO_LIMIT: 0.5, // limit for valid value ratio, y defs below this will not be used in auto-detect
+  PIE_RATIO_LIMIT: 0.05, // limit for other values in pie chart, if the value is below this, it will be grouped into "Other"
+  PIE_COUNT_LIMIT: 10, // limit for number of pie chart slices, if the number of slices is above this, it will be grouped into "Other"
+};
+
+export interface ChartXFieldDefinition {
+  field: string;
+  title?: string;
+  transformFunction: ChartXTransformFunction;
+  sortOrder?: 'natural' | 'ascKeys' | 'descKeys' | 'ascValues' | 'descValues';
+  windowAlign?: 'start' | 'end';
+  windowSize?: number;
+  parentAggregateLimit?: number;
+}
+
+export interface ChartYFieldDefinition {
+  field: string;
+  title?: string;
+  aggregateFunction: ChartYAggregateFunction;
+}
+
+export interface ChartDefinition {
+  chartType: ChartTypeEnum;
+  title?: string;
+  pieRatioLimit?: number; // limit for pie chart, if the value is below this, it will be grouped into "Other"
+  pieCountLimit?: number; // limit for number of pie chart slices, if the number of slices is above this, it will be grouped into "Other"
+
+  xdef: ChartXFieldDefinition;
+  ydefs: ChartYFieldDefinition[];
+}
+
+export interface ChartDateParsed {
+  year: number;
+  month?: number;
+  day?: number;
+  hour?: number;
+  minute?: number;
+  second?: number;
+  fraction?: string;
+}
+
+export interface ChartAvailableColumn {
+  field: string;
+}
+
+export interface ProcessedChart {
+  minX?: string;
+  maxX?: string;
+  rowsAdded: number;
+  buckets: { [key: string]: any }; // key is the bucket key, value is aggregated data
+  bucketKeysOrdered: string[];
+  bucketKeyDateParsed: { [key: string]: ChartDateParsed }; // key is the bucket key, value is parsed date
+  isGivenDefinition: boolean; // true if the chart was created with a given definition, false if it was created from raw data
+  invalidXRows: number;
+  invalidYRows: { [key: string]: number }; // key is the y field, value is the count of invalid rows
+  validYRows: { [key: string]: number }; // key is the field, value is the count of valid rows
+
+  topDistinctValues: { [key: string]: Set<any> }; // key is the field, value is the set of distinct values
+  availableColumns: ChartAvailableColumn[];
+
+  definition: ChartDefinition;
+}

packages/datalib/src/chartProcessor.ts

@@ -0,0 +1,374 @@
+import {
+  ChartAvailableColumn,
+  ChartDateParsed,
+  ChartDefinition,
+  ChartLimits,
+  ProcessedChart,
+} from './chartDefinitions';
+import _sortBy from 'lodash/sortBy';
+import _sum from 'lodash/sum';
+import {
+  aggregateChartNumericValuesFromSource,
+  autoAggregateCompactTimelineChart,
+  computeChartBucketCardinality,
+  computeChartBucketKey,
+  fillChartTimelineBuckets,
+  tryParseChartDate,
+} from './chartTools';
+import { getChartScore, getChartYFieldScore } from './chartScoring';
+
+export class ChartProcessor {
+  chartsProcessing: ProcessedChart[] = [];
+  charts: ProcessedChart[] = [];
+  availableColumnsDict: { [field: string]: ChartAvailableColumn } = {};
+  availableColumns: ChartAvailableColumn[] = [];
+  autoDetectCharts = false;
+  rowsAdded = 0;
+
+  constructor(public givenDefinitions: ChartDefinition[] = []) {
+    for (const definition of givenDefinitions) {
+      this.chartsProcessing.push({
+        definition,
+        rowsAdded: 0,
+        bucketKeysOrdered: [],
+        buckets: {},
+        bucketKeyDateParsed: {},
+        isGivenDefinition: true,
+        invalidXRows: 0,
+        invalidYRows: {},
+        availableColumns: [],
+        validYRows: {},
+        topDistinctValues: {},
+      });
+    }
+    this.autoDetectCharts = this.givenDefinitions.length == 0;
+  }
+
+  // findOrCreateChart(definition: ChartDefinition, isGivenDefinition: boolean): ProcessedChart {
+  //   const signatureItems = [
+  //     definition.chartType,
+  //     definition.xdef.field,
+  //     definition.xdef.transformFunction,
+  //     definition.ydefs.map(y => y.field).join(','),
+  //   ];
+  //   const signature = signatureItems.join('::');
+
+  //   if (this.chartsBySignature[signature]) {
+  //     return this.chartsBySignature[signature];
+  //   }
+  //   const chart: ProcessedChart = {
+  //     definition,
+  //     rowsAdded: 0,
+  //     bucketKeysOrdered: [],
+  //     buckets: {},
+  //     bucketKeyDateParsed: {},
+  //     isGivenDefinition,
+  //   };
+  //   this.chartsBySignature[signature] = chart;
+  //   return chart;
+  // }
+
+  addRow(row: any) {
+    const dateColumns: { [key: string]: ChartDateParsed } = {};
+    const numericColumns: { [key: string]: number } = {};
+    const numericColumnsForAutodetect: { [key: string]: number } = {};
+    const stringColumns: { [key: string]: string } = {};
+
+    for (const [key, value] of Object.entries(row)) {
+      const number: number = typeof value == 'string' ? Number(value) : typeof value == 'number' ? value : NaN;
+      this.availableColumnsDict[key] = {
+        field: key,
+      };
+
+      const keyLower = key.toLowerCase();
+      const keyIsId = keyLower.endsWith('_id') || keyLower == 'id' || key.endsWith('Id');
+
+      const parsedDate = tryParseChartDate(value);
+      if (parsedDate) {
+        dateColumns[key] = parsedDate;
+        continue;
+      }
+
+      if (!isNaN(number) && isFinite(number)) {
+        numericColumns[key] = number;
+        if (!keyIsId) {
+          numericColumnsForAutodetect[key] = number; // for auto-detecting charts
+        }
+        continue;
+      }
+
+      if (typeof value === 'string' && isNaN(number) && value.length < 100) {
+        stringColumns[key] = value;
+      }
+    }
+
+    // const sortedNumericColumnns = Object.keys(numericColumns).sort();
+
+    if (this.autoDetectCharts) {
+      // create charts from data, if there are no given definitions
+      for (const datecol in dateColumns) {
+        let usedChart = this.chartsProcessing.find(
+          chart =>
+            !chart.isGivenDefinition &&
+            chart.definition.xdef.field === datecol &&
+            chart.definition.xdef.transformFunction?.startsWith('date:')
+        );
+
+        if (
+          !usedChart &&
+          (this.rowsAdded < ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
+            this.chartsProcessing.length < ChartLimits.AUTODETECT_CHART_LIMIT)
+        ) {
+          usedChart = {
+            definition: {
+              chartType: 'line',
+              xdef: {
+                field: datecol,
+                transformFunction: 'date:day',
+              },
+              ydefs: [],
+            },
+            rowsAdded: 0,
+            bucketKeysOrdered: [],
+            buckets: {},
+            bucketKeyDateParsed: {},
+            isGivenDefinition: false,
+            invalidXRows: 0,
+            invalidYRows: {},
+            availableColumns: [],
+            validYRows: {},
+            topDistinctValues: {},
+          };
+          this.chartsProcessing.push(usedChart);
+        }
+
+        for (const [key, value] of Object.entries(row)) {
+          if (value == null) continue;
+          if (key == datecol) continue; // skip date column itself
+          let existingYDef = usedChart.definition.ydefs.find(y => y.field === key);
+          if (
+            !existingYDef &&
+            (this.rowsAdded < ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
+              usedChart.definition.ydefs.length < ChartLimits.AUTODETECT_MEASURES_LIMIT)
+          ) {
+            existingYDef = {
+              field: key,
+              aggregateFunction: 'sum',
+            };
+            usedChart.definition.ydefs.push(existingYDef);
+          }
+        }
+      }
+    }
+
+    // apply on all charts with this date column
+    for (const chart of this.chartsProcessing) {
+      this.applyRawData(
+        chart,
+        row,
+        dateColumns[chart.definition.xdef.field],
+        chart.isGivenDefinition ? numericColumns : numericColumnsForAutodetect,
+        stringColumns
+      );
+    }
+
+    for (let i = 0; i < this.chartsProcessing.length; i++) {
+      this.chartsProcessing[i] = autoAggregateCompactTimelineChart(this.chartsProcessing[i]);
+    }
+
+    this.rowsAdded += 1;
+    if (this.rowsAdded == ChartLimits.APPLY_LIMIT_AFTER_ROWS) {
+      this.applyLimitsOnCharts();
+    }
+  }
+
+  applyLimitsOnCharts() {
+    const autodetectProcessingCharts = this.chartsProcessing.filter(chart => !chart.isGivenDefinition);
+    if (autodetectProcessingCharts.length > ChartLimits.AUTODETECT_CHART_LIMIT) {
+      const newAutodetectProcessingCharts = _sortBy(
+        this.chartsProcessing.slice(0, ChartLimits.AUTODETECT_CHART_LIMIT),
+        chart => -getChartScore(chart)
+      );
+
+      for (const chart of autodetectProcessingCharts) {
+        chart.definition.ydefs = _sortBy(chart.definition.ydefs, yfield => -getChartYFieldScore(chart, yfield)).slice(
+          0,
+          ChartLimits.AUTODETECT_MEASURES_LIMIT
+        );
+      }
+
+      this.chartsProcessing = [
+        ...this.chartsProcessing.filter(chart => chart.isGivenDefinition),
+        ...newAutodetectProcessingCharts,
+      ];
+    }
+  }
+
+  addRows(...rows: any[]) {
+    for (const row of rows) {
+      this.addRow(row);
+    }
+  }
+
+  finalize() {
+    this.applyLimitsOnCharts();
+    this.availableColumns = Object.values(this.availableColumnsDict);
+    for (const chart of this.chartsProcessing) {
+      let addedChart: ProcessedChart = chart;
+      if (chart.rowsAdded == 0) {
+        continue; // skip empty charts
+      }
+      const sortOrder = chart.definition.xdef.sortOrder ?? 'ascKeys';
+      if (sortOrder != 'natural') {
+        if (sortOrder == 'ascKeys' || sortOrder == 'descKeys') {
+          if (chart.definition.xdef.transformFunction.startsWith('date:')) {
+            addedChart = autoAggregateCompactTimelineChart(addedChart);
+            fillChartTimelineBuckets(addedChart);
+          }
+
+          addedChart.bucketKeysOrdered = _sortBy(Object.keys(addedChart.buckets));
+          if (sortOrder == 'descKeys') {
+            addedChart.bucketKeysOrdered.reverse();
+          }
+        }
+
+        if (sortOrder == 'ascValues' || sortOrder == 'descValues') {
+          addedChart.bucketKeysOrdered = _sortBy(Object.keys(addedChart.buckets), key =>
+            computeChartBucketCardinality(addedChart.buckets[key])
+          );
+          if (sortOrder == 'descValues') {
+            addedChart.bucketKeysOrdered.reverse();
+          }
+        }
+      }
+
+      if (!addedChart.isGivenDefinition) {
+        addedChart = {
+          ...addedChart,
+          definition: {
+            ...addedChart.definition,
+            ydefs: addedChart.definition.ydefs.filter(
+              y =>
+                !addedChart.invalidYRows[y.field] &&
+                addedChart.validYRows[y.field] / addedChart.rowsAdded >= ChartLimits.VALID_VALUE_RATIO_LIMIT
+            ),
+          },
+        };
+      }
+
+      if (addedChart) {
+        addedChart.availableColumns = this.availableColumns;
+        this.charts.push(addedChart);
+      }
+
+      this.groupPieOtherBuckets(addedChart);
+    }
+
+    this.charts = [
+      ...this.charts.filter(x => x.isGivenDefinition),
+      ..._sortBy(
+        this.charts.filter(x => !x.isGivenDefinition),
+        chart => -getChartScore(chart)
+      ),
+    ];
+  }
+  groupPieOtherBuckets(chart: ProcessedChart) {
+    if (chart.definition.chartType !== 'pie') {
+      return; // only for pie charts
+    }
+    const ratioLimit = chart.definition.pieRatioLimit ?? ChartLimits.PIE_RATIO_LIMIT;
+    const countLimit = chart.definition.pieCountLimit ?? ChartLimits.PIE_COUNT_LIMIT;
+    if (ratioLimit == 0 && countLimit == 0) {
+      return; // no grouping if limit is 0
+    }
+    const otherBucket: any = {};
+    let newBuckets: any = {};
+    const cardSum = _sum(Object.values(chart.buckets).map(bucket => computeChartBucketCardinality(bucket)));
+
+    if (cardSum == 0) {
+      return; // no buckets to process
+    }
+
+    for (const [bucketKey, bucket] of Object.entries(chart.buckets)) {
+      if (computeChartBucketCardinality(bucket) / cardSum < ratioLimit) {
+        for (const field in bucket) {
+          otherBucket[field] = (otherBucket[field] ?? 0) + bucket[field];
+        }
+      } else {
+        newBuckets[bucketKey] = bucket;
+      }
+    }
+
+    if (Object.keys(newBuckets).length > countLimit) {
+      const sortedBucketKeys = _sortBy(
+        Object.entries(newBuckets),
+        ([, bucket]) => -computeChartBucketCardinality(bucket)
+      ).map(([key]) => key);
+      const newBuckets2 = {};
+      sortedBucketKeys.forEach((key, index) => {
+        if (index < countLimit) {
+          newBuckets2[key] = newBuckets[key];
+        } else {
+          for (const field in newBuckets[key]) {
+            otherBucket[field] = (otherBucket[field] ?? 0) + newBuckets[key][field];
+          }
+        }
+      });
+      newBuckets = newBuckets2;
+    }
+
+    if (Object.keys(otherBucket).length > 0) {
+      newBuckets['Other'] = otherBucket;
+    }
+    chart.buckets = newBuckets;
+    chart.bucketKeysOrdered = [...chart.bucketKeysOrdered, 'Other'].filter(key => key in newBuckets);
+  }
+
+  applyRawData(
+    chart: ProcessedChart,
+    row: any,
+    dateParsed: ChartDateParsed,
+    numericColumns: { [key: string]: number },
+    stringColumns: { [key: string]: string }
+  ) {
+    if (chart.definition.xdef == null) {
+      return;
+    }
+
+    if (row[chart.definition.xdef.field] == null) {
+      return;
+    }
+
+    if (dateParsed == null && chart.definition.xdef.transformFunction.startsWith('date:')) {
+      chart.invalidXRows += 1;
+      return; // skip if date is invalid
+    }
+
+    const [bucketKey, bucketKeyParsed] = computeChartBucketKey(dateParsed, chart, row);
+
+    if (!bucketKey) {
+      return; // skip if no bucket key
+    }
+
+    if (bucketKeyParsed) {
+      chart.bucketKeyDateParsed[bucketKey] = bucketKeyParsed;
+    }
+
+    if (chart.minX == null || bucketKey < chart.minX) {
+      chart.minX = bucketKey;
+    }
+    if (chart.maxX == null || bucketKey > chart.maxX) {
+      chart.maxX = bucketKey;
+    }
+
+    if (!chart.buckets[bucketKey]) {
+      chart.buckets[bucketKey] = {};
+      if (chart.definition.xdef.sortOrder == 'natural') {
+        chart.bucketKeysOrdered.push(bucketKey);
+      }
+    }
+
+    aggregateChartNumericValuesFromSource(chart, bucketKey, numericColumns, row);
+    chart.rowsAdded += 1;
+  }
+}

packages/datalib/src/chartScoring.ts

@@ -0,0 +1,23 @@
+import _sortBy from 'lodash/sortBy';
+import _sum from 'lodash/sum';
+import { ChartLimits, ChartYFieldDefinition, ProcessedChart } from './chartDefinitions';
+
+export function getChartScore(chart: ProcessedChart): number {
+  let res = 0;
+  res += chart.rowsAdded * 5;
+
+  const ydefScores = chart.definition.ydefs.map(yField => getChartYFieldScore(chart, yField));
+  const sorted = _sortBy(ydefScores).reverse();
+  res += _sum(sorted.slice(0, ChartLimits.AUTODETECT_MEASURES_LIMIT));
+  return res;
+}
+
+export function getChartYFieldScore(chart: ProcessedChart, yField: ChartYFieldDefinition): number {
+  let res = 0;
+  res += chart.validYRows[yField.field] * 5; // score for valid Y rows
+  res += (chart.topDistinctValues[yField.field]?.size ?? 0) * 20; // score for distinct values in Y field
+  res += chart.rowsAdded * 2; // base score for rows added
+  res -= (chart.invalidYRows[yField.field] ?? 0) * 50; // penalty for invalid Y rows
+
+  return res;
+}

packages/datalib/src/chartTools.ts

@@ -0,0 +1,542 @@
+import _toPairs from 'lodash/toPairs';
+import _sumBy from 'lodash/sumBy';
+import {
+  ChartConstDefaults,
+  ChartDateParsed,
+  ChartLimits,
+  ChartXTransformFunction,
+  ProcessedChart,
+} from './chartDefinitions';
+import { addMinutes, addHours, addDays, addMonths, addYears } from 'date-fns';
+
+export function getChartDebugPrint(chart: ProcessedChart) {
+  let res = '';
+  res += `Chart: ${chart.definition.chartType} (${chart.definition.xdef.transformFunction})\n`;
+  for (const key of chart.bucketKeysOrdered) {
+    res += `${key}: ${_toPairs(chart.buckets[key])
+      .map(([k, v]) => `${k}=${v}`)
+      .join(', ')}\n`;
+  }
+  return res;
+}
+
+export function tryParseChartDate(dateInput: any): ChartDateParsed | null {
+  if (dateInput instanceof Date) {
+    return {
+      year: dateInput.getFullYear(),
+      month: dateInput.getMonth() + 1,
+      day: dateInput.getDate(),
+      hour: dateInput.getHours(),
+      minute: dateInput.getMinutes(),
+      second: dateInput.getSeconds(),
+      fraction: undefined, // Date object does not have fraction
+    };
+  }
+
+  if (typeof dateInput !== 'string') return null;
+  const m = dateInput.match(
+    /^(\d{4})-(\d{2})-(\d{2})(?:[ T](\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|[+-]\d{2}:\d{2})?)?$/
+  );
+  if (!m) return null;
+
+  const [_notUsed, year, month, day, hour, minute, second, fraction] = m;
+
+  return {
+    year: parseInt(year, 10),
+    month: parseInt(month, 10),
+    day: parseInt(day, 10),
+    hour: parseInt(hour, 10) || 0,
+    minute: parseInt(minute, 10) || 0,
+    second: parseInt(second, 10) || 0,
+    fraction: fraction || undefined,
+  };
+}
+
+function pad2Digits(number) {
+  return ('00' + number).slice(-2);
+}
+
+export function stringifyChartDate(value: ChartDateParsed, transform: ChartXTransformFunction): string {
+  switch (transform) {
+    case 'date:year':
+      return `${value.year}`;
+    case 'date:month':
+      return `${value.year}-${pad2Digits(value.month)}`;
+    case 'date:day':
+      return `${value.year}-${pad2Digits(value.month)}-${pad2Digits(value.day)}`;
+    case 'date:hour':
+      return `${value.year}-${pad2Digits(value.month)}-${pad2Digits(value.day)} ${pad2Digits(value.hour)}`;
+    case 'date:minute':
+      return `${value.year}-${pad2Digits(value.month)}-${pad2Digits(value.day)} ${pad2Digits(value.hour)}:${pad2Digits(
+        value.minute
+      )}`;
+    default:
+      return '';
+  }
+}
+
+export function incrementChartDate(value: ChartDateParsed, transform: ChartXTransformFunction): ChartDateParsed {
+  const dateRepresentation = new Date(
+    value.year,
+    (value.month ?? 1) - 1,
+    value.day ?? 1,
+    value.hour ?? 0,
+    value.minute ?? 0
+  );
+  let newDateRepresentation: Date;
+  switch (transform) {
+    case 'date:year':
+      newDateRepresentation = addYears(dateRepresentation, 1);
+      break;
+    case 'date:month':
+      newDateRepresentation = addMonths(dateRepresentation, 1);
+      break;
+    case 'date:day':
+      newDateRepresentation = addDays(dateRepresentation, 1);
+      break;
+    case 'date:hour':
+      newDateRepresentation = addHours(dateRepresentation, 1);
+      break;
+    case 'date:minute':
+      newDateRepresentation = addMinutes(dateRepresentation, 1);
+      break;
+  }
+  switch (transform) {
+    case 'date:year':
+      return { year: newDateRepresentation.getFullYear() };
+    case 'date:month':
+      return {
+        year: newDateRepresentation.getFullYear(),
+        month: newDateRepresentation.getMonth() + 1,
+      };
+    case 'date:day':
+      return {
+        year: newDateRepresentation.getFullYear(),
+        month: newDateRepresentation.getMonth() + 1,
+        day: newDateRepresentation.getDate(),
+      };
+    case 'date:hour':
+      return {
+        year: newDateRepresentation.getFullYear(),
+        month: newDateRepresentation.getMonth() + 1,
+        day: newDateRepresentation.getDate(),
+        hour: newDateRepresentation.getHours(),
+      };
+    case 'date:minute':
+      return {
+        year: newDateRepresentation.getFullYear(),
+        month: newDateRepresentation.getMonth() + 1,
+        day: newDateRepresentation.getDate(),
+        hour: newDateRepresentation.getHours(),
+        minute: newDateRepresentation.getMinutes(),
+      };
+  }
+}
+
+export function computeChartBucketKey(
+  dateParsed: ChartDateParsed,
+  chart: ProcessedChart,
+  row: any
+): [string, ChartDateParsed] {
+  switch (chart.definition.xdef.transformFunction) {
+    case 'date:year':
+      return [dateParsed ? `${dateParsed.year}` : null, { year: dateParsed.year }];
+    case 'date:month':
+      return [
+        dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}` : null,
+        {
+          year: dateParsed.year,
+          month: dateParsed.month,
+        },
+      ];
+    case 'date:day':
+      return [
+        dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)}` : null,
+        {
+          year: dateParsed.year,
+          month: dateParsed.month,
+          day: dateParsed.day,
+        },
+      ];
+    case 'date:hour':
+      return [
+        dateParsed
+          ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits(
+              dateParsed.hour
+            )}`
+          : null,
+        {
+          year: dateParsed.year,
+          month: dateParsed.month,
+          day: dateParsed.day,
+          hour: dateParsed.hour,
+        },
+      ];
+    case 'date:minute':
+      return [
+        dateParsed
+          ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits(
+              dateParsed.hour
+            )}:${pad2Digits(dateParsed.minute)}`
+          : null,
+        {
+          year: dateParsed.year,
+          month: dateParsed.month,
+          day: dateParsed.day,
+          hour: dateParsed.hour,
+          minute: dateParsed.minute,
+        },
+      ];
+    case 'identity':
+    default:
+      return [row[chart.definition.xdef.field], null];
+  }
+}
+
+export function computeDateBucketDistance(
+  begin: ChartDateParsed,
+  end: ChartDateParsed,
+  transform: ChartXTransformFunction
+): number {
+  switch (transform) {
+    case 'date:year':
+      return end.year - begin.year;
+    case 'date:month':
+      return (end.year - begin.year) * 12 + (end.month - begin.month);
+    case 'date:day':
+      return (
+        (end.year - begin.year) * 365 +
+        (end.month - begin.month) * 30 + // rough approximation
+        (end.day - begin.day)
+      );
+    case 'date:hour':
+      return (
+        (end.year - begin.year) * 365 * 24 +
+        (end.month - begin.month) * 30 * 24 + // rough approximation
+        (end.day - begin.day) * 24 +
+        (end.hour - begin.hour)
+      );
+    case 'date:minute':
+      return (
+        (end.year - begin.year) * 365 * 24 * 60 +
+        (end.month - begin.month) * 30 * 24 * 60 + // rough approximation
+        (end.day - begin.day) * 24 * 60 +
+        (end.hour - begin.hour) * 60 +
+        (end.minute - begin.minute)
+      );
+    case 'identity':
+    default:
+      return NaN;
+  }
+}
+
+export function compareChartDatesParsed(
+  a: ChartDateParsed,
+  b: ChartDateParsed,
+  transform: ChartXTransformFunction
+): number {
+  switch (transform) {
+    case 'date:year':
+      return a.year - b.year;
+    case 'date:month':
+      return a.year === b.year ? a.month - b.month : a.year - b.year;
+    case 'date:day':
+      return a.year === b.year && a.month === b.month
+        ? a.day - b.day
+        : a.year === b.year
+        ? a.month - b.month
+        : a.year - b.year;
+    case 'date:hour':
+      return a.year === b.year && a.month === b.month && a.day === b.day
+        ? a.hour - b.hour
+        : a.year === b.year && a.month === b.month
+        ? a.day - b.day
+        : a.year === b.year
+        ? a.month - b.month
+        : a.year - b.year;
+
+    case 'date:minute':
+      return a.year === b.year && a.month === b.month && a.day === b.day && a.hour === b.hour
+        ? a.minute - b.minute
+        : a.year === b.year && a.month === b.month && a.day === b.day
+        ? a.hour - b.hour
+        : a.year === b.year && a.month === b.month
+        ? a.day - b.day
+        : a.year === b.year
+        ? a.month - b.month
+        : a.year - b.year;
+  }
+}
+
+function getParentDateBucketKey(bucketKey: string, transform: ChartXTransformFunction): string | null {
+  switch (transform) {
+    case 'date:year':
+      return null; // no parent for year
+    case 'date:month':
+      return bucketKey.slice(0, 4);
+    case 'date:day':
+      return bucketKey.slice(0, 7);
+    case 'date:hour':
+      return bucketKey.slice(0, 10);
+    case 'date:minute':
+      return bucketKey.slice(0, 13);
+  }
+}
+
+function getParentDateBucketTransform(transform: ChartXTransformFunction): ChartXTransformFunction | null {
+  switch (transform) {
+    case 'date:year':
+      return null; // no parent for year
+    case 'date:month':
+      return 'date:year';
+    case 'date:day':
+      return 'date:month';
+    case 'date:hour':
+      return 'date:day';
+    case 'date:minute':
+      return 'date:hour';
+    default:
+      return null;
+  }
+}
+
+function getParentKeyParsed(date: ChartDateParsed, transform: ChartXTransformFunction): ChartDateParsed | null {
+  switch (transform) {
+    case 'date:year':
+      return null; // no parent for year
+    case 'date:month':
+      return { year: date.year };
+    case 'date:day':
+      return { year: date.year, month: date.month };
+    case 'date:hour':
+      return { year: date.year, month: date.month, day: date.day };
+    case 'date:minute':
+      return { year: date.year, month: date.month, day: date.day, hour: date.hour };
+    default:
+      return null;
+  }
+}
+
+function createParentChartAggregation(chart: ProcessedChart): ProcessedChart | null {
+  if (chart.isGivenDefinition) {
+    // if the chart is created with a given definition, we cannot create a parent aggregation
+    return null;
+  }
+  const parentTransform = getParentDateBucketTransform(chart.definition.xdef.transformFunction);
+  if (!parentTransform) {
+    return null;
+  }
+
+  const res: ProcessedChart = {
+    definition: {
+      ...chart.definition,
+      xdef: {
+        ...chart.definition.xdef,
+        transformFunction: parentTransform,
+      },
+    },
+    rowsAdded: chart.rowsAdded,
+    bucketKeysOrdered: [],
+    buckets: {},
+    bucketKeyDateParsed: {},
+    isGivenDefinition: false,
+    invalidXRows: chart.invalidXRows,
+    invalidYRows: { ...chart.invalidYRows }, // copy invalid Y rows
+    validYRows: { ...chart.validYRows }, // copy valid Y rows
+    topDistinctValues: { ...chart.topDistinctValues }, // copy top distinct values
+    availableColumns: chart.availableColumns,
+  };
+
+  for (const [bucketKey, bucketValues] of Object.entries(chart.buckets)) {
+    const parentKey = getParentDateBucketKey(bucketKey, chart.definition.xdef.transformFunction);
+    if (!parentKey) {
+      // skip if the bucket is already a parent
+      continue;
+    }
+    res.bucketKeyDateParsed[parentKey] = getParentKeyParsed(
+      chart.bucketKeyDateParsed[bucketKey],
+      chart.definition.xdef.transformFunction
+    );
+    aggregateChartNumericValuesFromChild(res, parentKey, bucketValues);
+  }
+
+  const bucketKeys = Object.keys(res.buckets).sort();
+  res.minX = bucketKeys.length > 0 ? bucketKeys[0] : null;
+  res.maxX = bucketKeys.length > 0 ? bucketKeys[bucketKeys.length - 1] : null;
+
+  return res;
+}
+
+export function autoAggregateCompactTimelineChart(chart: ProcessedChart) {
+  while (true) {
+    const fromParsed = chart.bucketKeyDateParsed[chart.minX];
+    const toParsed = chart.bucketKeyDateParsed[chart.maxX];
+
+    if (!fromParsed || !toParsed) {
+      return chart; // cannot fill timeline buckets without valid date range
+    }
+    const transform = chart.definition.xdef.transformFunction;
+    if (!transform.startsWith('date:')) {
+      return chart; // cannot aggregate non-date charts
+    }
+    const dateDistance = computeDateBucketDistance(fromParsed, toParsed, transform);
+    if (dateDistance < (chart.definition.xdef.parentAggregateLimit ?? ChartConstDefaults.parentAggregateLimit)) {
+      return chart; // no need to aggregate further, the distance is less than the limit
+    }
+
+    const parentChart = createParentChartAggregation(chart);
+    if (!parentChart) {
+      return chart; // cannot create parent aggregation
+    }
+
+    chart = parentChart;
+  }
+}
+
+export function aggregateChartNumericValuesFromSource(
+  chart: ProcessedChart,
+  bucketKey: string,
+  numericColumns: { [key: string]: number },
+  row: any
+) {
+  for (const ydef of chart.definition.ydefs) {
+    if (numericColumns[ydef.field] == null) {
+      if (row[ydef.field]) {
+        chart.invalidYRows[ydef.field] = (chart.invalidYRows[ydef.field] || 0) + 1; // increment invalid row count if the field is not numeric
+      }
+      continue;
+    }
+    chart.validYRows[ydef.field] = (chart.validYRows[ydef.field] || 0) + 1; // increment valid row count
+
+    let distinctValues = chart.topDistinctValues[ydef.field];
+    if (!distinctValues) {
+      distinctValues = new Set();
+      chart.topDistinctValues[ydef.field] = distinctValues;
+    }
+    if (distinctValues.size < ChartLimits.MAX_DISTINCT_VALUES) {
+      chart.topDistinctValues[ydef.field].add(numericColumns[ydef.field]);
+    }
+
+    switch (ydef.aggregateFunction) {
+      case 'sum':
+        chart.buckets[bucketKey][ydef.field] =
+          (chart.buckets[bucketKey][ydef.field] || 0) + (numericColumns[ydef.field] || 0);
+        break;
+      case 'first':
+        if (chart.buckets[bucketKey][ydef.field] === undefined) {
+          chart.buckets[bucketKey][ydef.field] = numericColumns[ydef.field];
+        }
+        break;
+      case 'last':
+        chart.buckets[bucketKey][ydef.field] = numericColumns[ydef.field];
+        break;
+      case 'min':
+        if (chart.buckets[bucketKey][ydef.field] === undefined) {
+          chart.buckets[bucketKey][ydef.field] = numericColumns[ydef.field];
+        } else {
+          chart.buckets[bucketKey][ydef.field] = Math.min(
+            chart.buckets[bucketKey][ydef.field],
+            numericColumns[ydef.field]
+          );
+        }
+        break;
+      case 'max':
+        if (chart.buckets[bucketKey][ydef.field] === undefined) {
+          chart.buckets[bucketKey][ydef.field] = numericColumns[ydef.field];
+        } else {
+          chart.buckets[bucketKey][ydef.field] = Math.max(
+            chart.buckets[bucketKey][ydef.field],
+            numericColumns[ydef.field]
+          );
+        }
+        break;
+      case 'count':
+        chart.buckets[bucketKey][ydef.field] = (chart.buckets[bucketKey][ydef.field] || 0) + 1;
+        break;
+      case 'avg':
+        if (chart.buckets[bucketKey][ydef.field] === undefined) {
+          chart.buckets[bucketKey][ydef.field] = [numericColumns[ydef.field], 1]; // [sum, count]
+        } else {
+          chart.buckets[bucketKey][ydef.field][0] += numericColumns[ydef.field];
+          chart.buckets[bucketKey][ydef.field][1] += 1;
+        }
+        break;
+    }
+  }
+}
+
+export function aggregateChartNumericValuesFromChild(
+  chart: ProcessedChart,
+  bucketKey: string,
+  childBucketValues: { [key: string]: any }
+) {
+  for (const ydef of chart.definition.ydefs) {
+    if (childBucketValues[ydef.field] == undefined) {
+      continue; // skip if the field is not present in the child bucket
+    }
+    if (!chart.buckets[bucketKey]) {
+      chart.buckets[bucketKey] = {};
+    }
+    switch (ydef.aggregateFunction) {
+      case 'sum':
+      case 'count':
+        chart.buckets[bucketKey][ydef.field] =
+          (chart.buckets[bucketKey][ydef.field] || 0) + (childBucketValues[ydef.field] || 0);
+        break;
+      case 'min':
+        if (chart.buckets[bucketKey][ydef.field] === undefined) {
+          chart.buckets[bucketKey][ydef.field] = childBucketValues[ydef.field];
+        } else {
+          chart.buckets[bucketKey][ydef.field] = Math.min(
+            chart.buckets[bucketKey][ydef.field],
+            childBucketValues[ydef.field]
+          );
+        }
+        break;
+      case 'max':
+        if (chart.buckets[bucketKey][ydef.field] === undefined) {
+          chart.buckets[bucketKey][ydef.field] = childBucketValues[ydef.field];
+        } else {
+          chart.buckets[bucketKey][ydef.field] = Math.max(
+            chart.buckets[bucketKey][ydef.field],
+            childBucketValues[ydef.field]
+          );
+        }
+        break;
+      case 'avg':
+        if (chart.buckets[bucketKey][ydef.field] === undefined) {
+          chart.buckets[bucketKey][ydef.field] = childBucketValues[ydef.field];
+        } else {
+          chart.buckets[bucketKey][ydef.field][0] += childBucketValues[ydef.field][0];
+          chart.buckets[bucketKey][ydef.field][1] += childBucketValues[ydef.field][1];
+        }
+        break;
+      case 'first':
+      case 'last':
+        throw new Error(`Cannot aggregate ${ydef.aggregateFunction} for ${ydef.field} in child bucket`);
+    }
+  }
+}
+
+export function fillChartTimelineBuckets(chart: ProcessedChart) {
+  const fromParsed = chart.bucketKeyDateParsed[chart.minX];
+  const toParsed = chart.bucketKeyDateParsed[chart.maxX];
+  if (!fromParsed || !toParsed) {
+    return; // cannot fill timeline buckets without valid date range
+  }
+  const transform = chart.definition.xdef.transformFunction;
+
+  let currentParsed = fromParsed;
+  while (compareChartDatesParsed(currentParsed, toParsed, transform) <= 0) {
+    const bucketKey = stringifyChartDate(currentParsed, transform);
+    if (!chart.buckets[bucketKey]) {
+      chart.buckets[bucketKey] = {};
+      chart.bucketKeyDateParsed[bucketKey] = currentParsed;
+    }
+    currentParsed = incrementChartDate(currentParsed, transform);
+  }
+}
+
+export function computeChartBucketCardinality(bucket: { [key: string]: any }): number {
+  return _sumBy(Object.keys(bucket), field => bucket[field]);
+}

packages/datalib/src/index.ts

@@ -23,3 +23,5 @@ export * from './FreeTableGridDisplay';
 export * from './FreeTableModel';
 export * from './CustomGridDisplay';
 export * from './ScriptDrivedDeployer';
+export * from './chartDefinitions';
+export * from './chartProcessor';

packages/datalib/src/tests/chartProcessor.test.ts

@@ -0,0 +1,376 @@
+import exp from 'constants';
+import { ChartProcessor } from '../chartProcessor';
+import { getChartDebugPrint } from '../chartTools';
+
+const DS1 = [
+  {
+    timestamp: '2023-10-01T12:00:00Z',
+    value: 42.5,
+    category: 'B',
+    related_id: 12,
+  },
+  {
+    timestamp: '2023-10-02T10:05:00Z',
+    value: 12,
+    category: 'A',
+    related_id: 13,
+  },
+  {
+    timestamp: '2023-10-03T07:10:00Z',
+    value: 57,
+    category: 'A',
+    related_id: 5,
+  },
+  {
+    timestamp: '2024-08-03T07:10:00Z',
+    value: 33,
+    category: 'B',
+    related_id: 22,
+  },
+];
+
+const DS2 = [
+  {
+    ts1: '2023-10-01T12:00:00Z',
+    ts2: '2024-10-01T12:00:00Z',
+    dummy1: 1,
+    dummy2: 1,
+    dummy3: 1,
+    dummy4: 1,
+    dummy5: 1,
+    dummy6: 1,
+    dummy7: 1,
+    dummy8: 1,
+    dummy9: 1,
+    dummy10: 1,
+    price1: '11',
+    price2: '22',
+  },
+  {
+    ts1: '2023-10-02T10:05:00Z',
+    ts2: '2024-10-02T10:05:00Z',
+    price1: '12',
+    price2: '23',
+  },
+  {
+    ts1: '2023-10-03T07:10:00Z',
+    ts2: '2024-10-03T07:10:00Z',
+    price1: '13',
+    price2: '24',
+  },
+  {
+    ts1: '2023-11-04T12:00:00Z',
+    ts2: '2024-11-04T12:00:00Z',
+    price1: 1,
+    price2: 2,
+  },
+];
+
+const DS3 = [
+  {
+    timestamp: '2023-10-01T12:00:00Z',
+    value: 42.5,
+    bitval: true,
+  },
+  {
+    timestamp: '2023-10-02T10:05:00Z',
+    value: 12,
+    bitval: false,
+  },
+  {
+    timestamp: '2023-10-03T07:10:00Z',
+    value: 57,
+    bitval: null,
+  },
+];
+
+const DS4 = [
+  {
+    object_id: 710293590,
+    ObjectName: 'Journal',
+    Total_Reserved_kb: '68696',
+    RowsCount: '405452',
+  },
+  {
+    object_id: 182291709,
+    ObjectName: 'Employee',
+    Total_Reserved_kb: '732008',
+    RowsCount: '1980067',
+  },
+  {
+    object_id: 23432525,
+    ObjectName: 'User',
+    Total_Reserved_kb: '325352',
+    RowsCount: '2233',
+  },
+  {
+    object_id: 4985159,
+    ObjectName: 'Project',
+    Total_Reserved_kb: '293523',
+    RowsCount: '1122',
+  },
+];
+
+describe('Chart processor', () => {
+  test('Simple by day test, autodetected', () => {
+    const processor = new ChartProcessor();
+    processor.addRows(...DS1.slice(0, 3));
+    processor.finalize();
+    expect(processor.charts.length).toEqual(1);
+    const chart = processor.charts[0];
+    expect(chart.definition.xdef.transformFunction).toEqual('date:day');
+    expect(chart.definition.ydefs).toEqual([
+      expect.objectContaining({
+        field: 'value',
+      }),
+    ]);
+    expect(chart.bucketKeysOrdered).toEqual(['2023-10-01', '2023-10-02', '2023-10-03']);
+  });
+  test('By month grouped, autedetected', () => {
+    const processor = new ChartProcessor();
+    processor.addRows(...DS1.slice(0, 4));
+    processor.finalize();
+    expect(processor.charts.length).toEqual(1);
+    const chart = processor.charts[0];
+    expect(chart.definition.xdef.transformFunction).toEqual('date:month');
+    expect(chart.bucketKeysOrdered).toEqual([
+      '2023-10',
+      '2023-11',
+      '2023-12',
+      '2024-01',
+      '2024-02',
+      '2024-03',
+      '2024-04',
+      '2024-05',
+      '2024-06',
+      '2024-07',
+      '2024-08',
+    ]);
+  });
+  test('Detect columns', () => {
+    const processor = new ChartProcessor();
+    processor.autoDetectCharts = false;
+    processor.addRows(...DS1);
+    processor.finalize();
+    expect(processor.charts.length).toEqual(0);
+    expect(processor.availableColumns).toEqual([
+      expect.objectContaining({
+        field: 'timestamp',
+      }),
+      expect.objectContaining({
+        field: 'value',
+      }),
+      expect.objectContaining({
+        field: 'category',
+      }),
+      expect.objectContaining({
+        field: 'related_id',
+      }),
+    ]);
+  });
+  test('Explicit definition', () => {
+    const processor = new ChartProcessor([
+      {
+        chartType: 'pie',
+        xdef: {
+          field: 'category',
+          transformFunction: 'identity',
+          sortOrder: 'natural',
+        },
+        ydefs: [
+          {
+            field: 'related_id',
+            aggregateFunction: 'sum',
+          },
+        ],
+      },
+    ]);
+    processor.addRows(...DS1);
+    processor.finalize();
+    expect(processor.charts.length).toEqual(1);
+    const chart = processor.charts[0];
+    expect(chart.definition.xdef.transformFunction).toEqual('identity');
+    expect(chart.bucketKeysOrdered).toEqual(['B', 'A']);
+    expect(chart.buckets).toEqual({
+      B: { related_id: 34 },
+      A: { related_id: 18 },
+    });
+  });
+
+  test('Two data sets with different date columns', () => {
+    const processor = new ChartProcessor();
+    processor.addRows(...DS2);
+    processor.finalize();
+    expect(processor.charts.length).toEqual(2);
+    expect(processor.charts[0].definition).toEqual(
+      expect.objectContaining({
+        xdef: expect.objectContaining({
+          field: 'ts1',
+          transformFunction: 'date:day',
+        }),
+        ydefs: [
+          expect.objectContaining({
+            field: 'price1',
+            aggregateFunction: 'sum',
+          }),
+          expect.objectContaining({
+            field: 'price2',
+            aggregateFunction: 'sum',
+          }),
+        ],
+      })
+    );
+    expect(processor.charts[1].definition).toEqual(
+      expect.objectContaining({
+        xdef: expect.objectContaining({
+          field: 'ts2',
+          transformFunction: 'date:day',
+        }),
+        ydefs: [
+          expect.objectContaining({
+            field: 'price1',
+            aggregateFunction: 'sum',
+          }),
+          expect.objectContaining({
+            field: 'price2',
+            aggregateFunction: 'sum',
+          }),
+        ],
+      })
+    );
+  });
+
+  test('Exclude boolean fields in autodetected', () => {
+    const processor = new ChartProcessor();
+    processor.addRows(...DS3);
+    processor.finalize();
+    expect(processor.charts.length).toEqual(1);
+    const chart = processor.charts[0];
+    expect(chart.definition.xdef.transformFunction).toEqual('date:day');
+    expect(chart.definition.ydefs).toEqual([
+      expect.objectContaining({
+        field: 'value',
+      }),
+    ]);
+  });
+
+  test('Added field manual from GUI', () => {
+    const processor = new ChartProcessor([
+      {
+        chartType: 'bar',
+        xdef: {
+          field: 'object_id',
+          transformFunction: 'identity',
+        },
+        ydefs: [
+          {
+            field: 'object_id',
+            aggregateFunction: 'sum',
+          },
+        ],
+      },
+    ]);
+    processor.addRows(...DS4);
+    processor.finalize();
+    expect(processor.charts.length).toEqual(1);
+    const chart = processor.charts[0];
+    expect(chart.definition.xdef.transformFunction).toEqual('identity');
+    expect(chart.definition.ydefs).toEqual([
+      expect.objectContaining({
+        field: 'object_id',
+        aggregateFunction: 'sum',
+      }),
+    ]);
+  });
+
+  const PieMainTestData = [
+    ['natural', ['Journal', 'Employee', 'User', 'Project']],
+    ['ascKeys', ['Employee', 'Journal', 'Project', 'User']],
+    ['descKeys', ['User', 'Project', 'Journal', 'Employee']],
+    ['ascValues', ['Project', 'User', 'Journal', 'Employee']],
+    ['descValues', ['Employee', 'Journal', 'User', 'Project']],
+  ];
+
+  test.each(PieMainTestData)('Pie chart - used space for DB objects (%s)', (sortOrder, expectedOrder) => {
+    const processor = new ChartProcessor([
+      {
+        chartType: 'bar',
+        xdef: {
+          field: 'ObjectName',
+          transformFunction: 'identity',
+          sortOrder: sortOrder as any,
+        },
+        ydefs: [
+          {
+            field: 'RowsCount',
+            aggregateFunction: 'sum',
+          },
+        ],
+      },
+    ]);
+    processor.addRows(...DS4);
+    processor.finalize();
+    expect(processor.charts.length).toEqual(1);
+    const chart = processor.charts[0];
+    expect(chart.bucketKeysOrdered).toEqual(expectedOrder);
+    expect(chart.buckets).toEqual({
+      Employee: { RowsCount: 1980067 },
+      Journal: { RowsCount: 405452 },
+      Project: { RowsCount: 1122 },
+      User: { RowsCount: 2233 },
+    });
+  });
+
+  const PieOtherTestData = [
+    [
+      'ratio',
+      0.1,
+      5,
+      ['Employee', 'Journal', 'Other'],
+      {
+        Employee: { RowsCount: 1980067 },
+        Journal: { RowsCount: 405452 },
+        Other: { RowsCount: 3355 },
+      },
+    ],
+    [
+      'count',
+      0,
+      1,
+      ['Employee', 'Other'],
+      {
+        Employee: { RowsCount: 1980067 },
+        Other: { RowsCount: 408807 },
+      },
+    ],
+  ];
+
+  test.each(PieOtherTestData)(
+    'Pie limit test - %s',
+    (_description, pieRatioLimit, pieCountLimit, expectedOrder, expectedBuckets) => {
+      const processor = new ChartProcessor([
+        {
+          chartType: 'pie',
+          pieRatioLimit: pieRatioLimit as number,
+          pieCountLimit: pieCountLimit as number,
+          xdef: {
+            field: 'ObjectName',
+            transformFunction: 'identity',
+          },
+          ydefs: [
+            {
+              field: 'RowsCount',
+              aggregateFunction: 'sum',
+            },
+          ],
+        },
+      ]);
+      processor.addRows(...DS4);
+      processor.finalize();
+      expect(processor.charts.length).toEqual(1);
+      const chart = processor.charts[0];
+      expect(chart.bucketKeysOrdered).toEqual(expectedOrder);
+      expect(chart.buckets).toEqual(expectedBuckets);
+    }
+  );
+});

packages/tools/src/stringTools.ts

@@ -5,7 +5,10 @@ import _isNumber from 'lodash/isNumber';
 import _isPlainObject from 'lodash/isPlainObject';
 import _pad from 'lodash/pad';
 import _cloneDeepWith from 'lodash/cloneDeepWith';
+import _isEmpty from 'lodash/isEmpty';
+import _omitBy from 'lodash/omitBy';
 import { DataEditorTypesBehaviour } from 'dbgate-types';
+import isPlainObject from 'lodash/isPlainObject';
 
 export type EditorDataType =
   | 'null'
@@ -633,3 +636,38 @@ export function parseNumberSafe(value) {
   }
   return parseFloat(value);
 }
+
+const frontMatterRe = /^--\ >>>[ \t]*\n(.*)\n-- <<<[ \t]*\n/s;
+
+export function getSqlFrontMatter(text: string, yamlModule) {
+  const match = text.match(frontMatterRe);
+  if (!match) return null;
+  const yamlContentMapped = match[1].replace(/^--[ ]?/gm, '');
+  return yamlModule.load(yamlContentMapped);
+}
+
+export function removeSqlFrontMatter(text: string) {
+  return text.replace(frontMatterRe, '');
+}
+
+export function setSqlFrontMatter(text: string, data: { [key: string]: any }, yamlModule) {
+  const textClean = removeSqlFrontMatter(text);
+
+  if (!isPlainObject(data)) {
+    return textClean;
+  }
+
+  const dataClean = _omitBy(data, v => v === undefined);
+
+  if (_isEmpty(dataClean)) {
+    return textClean;
+  }
+  const yamlContent = yamlModule.dump(dataClean);
+  const yamlContentMapped = yamlContent
+    .trimRight()
+    .split('\n')
+    .map(line => '-- ' + line)
+    .join('\n');
+  const frontMatterContent = `-- >>>\n${yamlContentMapped}\n-- <<<\n`;
+  return frontMatterContent + textClean;
+}

packages/tools/tsconfig.json

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "target": "ES2015",
+    "target": "ES2018",
     "module": "commonjs",
     "declaration": true,
     "skipLibCheck": true,

packages/web/src/appobj/DatabaseObjectAppObject.svelte

@@ -185,10 +185,6 @@
             isImport: true,
             requiresWriteAccess: true,
           },
-          hasPermission('dbops/charts') && {
-            label: 'Open active chart',
-            isActiveChart: true,
-          },
         ];
       case 'views':
         return [
@@ -245,10 +241,6 @@
             isExport: true,
             functionName: 'tableReader',
           },
-          {
-            label: 'Open active chart',
-            isActiveChart: true,
-          },
         ];
       case 'matviews':
         return [
@@ -299,10 +291,6 @@
             isExport: true,
             functionName: 'tableReader',
           },
-          {
-            label: 'Open active chart',
-            isActiveChart: true,
-          },
         ];
       case 'queries':
         return [
@@ -472,28 +460,7 @@
       return driver;
     };
 
-    if (menu.isActiveChart) {
+    if (menu.isQueryDesigner) {
-      const driver = await getDriver();
-      const dmp = driver.createDumper();
-      dmp.put('^select * from %f', data);
-      openNewTab(
-        {
-          title: data.pureName,
-          icon: 'img chart',
-          tabComponent: 'ChartTab',
-          props: {
-            conid: data.conid,
-            database: data.database,
-          },
-        },
-        {
-          editor: {
-            config: { chartType: 'bar' },
-            sql: dmp.s,
-          },
-        }
-      );
-    } else if (menu.isQueryDesigner) {
       openNewTab(
         {
           title: 'Query #',

packages/web/src/appobj/SavedFileAppObject.svelte

@@ -41,16 +41,6 @@
     label: 'Markdown file',
   };
 
-  const charts: FileTypeHandler = {
-    icon: 'img chart',
-    format: 'json',
-    tabComponent: 'ChartTab',
-    folder: 'charts',
-    currentConnection: true,
-    extension: 'json',
-    label: 'Chart file',
-  };
-
   const query: FileTypeHandler = {
     icon: 'img query-design',
     format: 'json',
@@ -139,7 +129,6 @@
     sql,
     shell,
     markdown,
-    charts,
     query,
     sqlite,
     diagrams,

packages/web/src/datagrid/DataGridCore.svelte

@@ -261,13 +261,6 @@
     testEnabled: () => getCurrentDataGrid() != null,
     onClick: () => getCurrentDataGrid().openFreeTable(),
   });
-  registerCommand({
-    id: 'dataGrid.openChartFromSelection',
-    category: 'Data grid',
-    name: 'Open chart from selection',
-    testEnabled: () => getCurrentDataGrid() != null,
-    onClick: () => getCurrentDataGrid().openChartFromSelection(),
-  });
   registerCommand({
     id: 'dataGrid.newJson',
     category: 'Data grid',
@@ -469,6 +462,7 @@
   export let hideGridLeftColumn = false;
   export let overlayDefinition = null;
   export let onGetSelectionMenu = null;
+  export let onOpenChart = null;
 
   export const activator = createActivator('DataGridCore', false);
 
@@ -715,23 +709,6 @@
     openJsonLinesData(getSelectedFreeDataRows());
   }
 
-  export function openChartFromSelection() {
-    openNewTab(
-      {
-        title: 'Chart #',
-        icon: 'img chart',
-        tabComponent: 'ChartTab',
-        props: {},
-      },
-      {
-        editor: {
-          data: getSelectedFreeData(),
-          config: { chartType: 'bar' },
-        },
-      }
-    );
-  }
-
   export function viewJsonDocumentEnabled() {
     return isDynamicStructure && _.uniq(selectedCells.map(x => x[0])).length == 1;
   }
@@ -1869,9 +1846,13 @@
     //   ],
     // },
     isProApp() && { command: 'dataGrid.sendToDataDeploy' },
+    isProApp() &&
+      onOpenChart && {
+        text: 'Open chart',
+        onClick: () => onOpenChart(),
+      },
     { command: 'dataGrid.generateSqlFromData' },
     { command: 'dataGrid.openFreeTable' },
-    { command: 'dataGrid.openChartFromSelection' },
     { command: 'dataGrid.openSelectionInMap', hideDisabled: true },
     { placeTag: 'chart' }
   );

packages/web/src/datagrid/SqlDataGridCore.svelte

@@ -1,14 +1,6 @@
 <script context="module" lang="ts">
   const getCurrentEditor = () => getActiveComponent('SqlDataGridCore');
 
-  registerCommand({
-    id: 'sqlDataGrid.openActiveChart',
-    category: 'Data grid',
-    name: 'Open active chart',
-    testEnabled: () => getCurrentEditor() != null && hasPermission('dbops/charts'),
-    onClick: () => getCurrentEditor().openActiveChart(),
-  });
-
   registerCommand({
     id: 'sqlDataGrid.openQuery',
     category: 'Data grid',
@@ -190,28 +182,6 @@
     openQuery(display.getPageQueryText(0, getIntSettingsValue('dataGrid.pageSize', 100, 5, 1000)));
   }
 
-  export function openActiveChart() {
-    openNewTab(
-      {
-        title: 'Chart #',
-        icon: 'img chart',
-        tabComponent: 'ChartTab',
-        props: {
-          conid,
-          database,
-        },
-      },
-      {
-        editor: {
-          config: { chartType: 'bar' },
-          sql: display.getExportQuery(select => {
-            select.orderBy = null;
-          }),
-        },
-      }
-    );
-  }
-
   const quickExportHandler = fmt => async () => {
     const coninfo = await getConnectionInfo({ conid });
     exportQuickExportFile(

packages/web/src/elements/HorizontalSplitter.svelte

@@ -39,7 +39,7 @@
 
   $: size = computeSplitterSize(initialValue, clientWidth, customRatio, initialSizeRight);
 
-  $: if (onChangeSize) onChangeSize(size);
+  $: if (onChangeSize) onChangeSize(size, clientWidth - size);
 </script>
 
 <div class="container" bind:clientWidth>

packages/web/src/elements/TabControl.svelte

@@ -18,6 +18,7 @@
   export let flex1 = true;
   export let contentTestId = undefined;
   export let inlineTabs = false;
+  export let onUserChange = null;
 
   export function setValue(index) {
     value = index;
@@ -30,8 +31,16 @@
 <div class="main" class:flex1>
   <div class="tabs" class:inlineTabs>
     {#each _.compact(tabs) as tab, index}
-      <div class="tab-item" class:selected={value == index} on:click={() => (value = index)} data-testid={tab.testid}>
+      <div
-        <span class="ml-2">
+        class="tab-item"
+        class:selected={value == index}
+        on:click={() => {
+          value = index;
+          onUserChange?.(index);
+        }}
+        data-testid={tab.testid}
+      >
+        <span class="ml-2 noselect">
           {tab.label}
         </span>
       </div>
@@ -139,5 +148,4 @@
   .container.isInline:not(.tabVisible) {
     display: none;
   }
-
 </style>

packages/web/src/icons/FontIcon.svelte

@@ -71,6 +71,7 @@
     'icon trigger': 'mdi mdi-lightning-bolt',
     'icon scheduler-event': 'mdi mdi-calendar-blank',
     'icon arrow-link': 'mdi mdi-arrow-top-right-thick',
+    'icon reset': 'mdi mdi-cancel',
 
     'icon window-restore': 'mdi mdi-window-restore',
     'icon window-maximize': 'mdi mdi-window-maximize',

packages/web/src/query/ResultTabs.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import _ from 'lodash';
+  import _, { result } from 'lodash';
 
   import { onMount, tick } from 'svelte';
 
@@ -9,6 +9,7 @@
   import { apiOff, apiOn } from '../utility/api';
   import useEffect from '../utility/useEffect';
   import AllResultsTab from './AllResultsTab.svelte';
+  import JslChart from '../charts/JslChart.svelte';
 
   export let tabs = [];
   export let sessionId;
@@ -16,6 +17,8 @@
   export let driver;
 
   export let resultCount;
+  export let onSetFrontMatterField;
+  export let onGetFrontMatter;
 
   onMount(() => {
     allResultsInOneTab = $allResultsInOneTabDefault;
@@ -23,6 +26,7 @@
 
   let allResultsInOneTab = null;
   let resultInfos = [];
+  let charts = [];
   let domTabs;
 
   $: resultCount = resultInfos.length;
@@ -35,6 +39,23 @@
     if (!currentTab?.isResult) domTabs.setValue(_.findIndex(allTabs, x => x.isResult));
   };
 
+  const handleCharts = async props => {
+    charts = [
+      ...charts,
+      {
+        jslid: props.jslid,
+        charts: props.charts,
+        resultIndex: props.resultIndex,
+      },
+    ];
+    const selectedChart = onGetFrontMatter?.()?.['selected-chart'];
+    await tick();
+    if (selectedChart && props.resultIndex == selectedChart - 1) {
+      domTabs.setValue(_.findIndex(allTabs, x => x.isChart && x.resultIndex === props.resultIndex));
+    }
+    // console.log('Charts received for jslid:', props.jslid, 'Charts:', props.charts);
+  };
+
   $: oneTab = allResultsInOneTab ?? $allResultsInOneTabDefault;
 
   $: allTabs = [
@@ -55,13 +76,27 @@
           label: `Result ${index + 1}`,
           isResult: true,
           component: JslDataGrid,
-          props: { jslid: info.jslid, driver },
+          props: { jslid: info.jslid, driver, onOpenChart: () => handleOpenChart(info.resultIndex) },
         }))),
+    ...charts.map((info, index) => ({
+      label: `Chart ${info.resultIndex + 1}`,
+      isChart: true,
+      resultIndex: info.resultIndex,
+      component: JslChart,
+      props: {
+        jslid: info.jslid,
+        initialCharts: info.charts,
+        onEditDefinition: definition => {
+          onSetFrontMatterField?.(`chart-${info.resultIndex + 1}`, definition ?? undefined);
+        },
+      },
+    })),
   ];
 
   $: {
     if (executeNumber >= 0) {
       resultInfos = [];
+      charts = [];
       if (domTabs) domTabs.setValue(0);
     }
   }
@@ -72,8 +107,10 @@
   function onSession(sid) {
     if (sid) {
       apiOn(`session-recordset-${sid}`, handleResultSet);
+      apiOn(`session-charts-${sid}`, handleCharts);
       return () => {
         apiOff(`session-recordset-${sid}`, handleResultSet);
+        apiOff(`session-charts-${sid}`, handleCharts);
       };
     }
     return () => {};
@@ -84,6 +121,25 @@
     allResultsInOneTab = value;
     $allResultsInOneTabDefault = value;
   }
+
+  async function handleOpenChart(resultIndex) {
+    const chartTab = _.find(allTabs, x => x.isChart && x.resultIndex === resultIndex);
+    if (chartTab) {
+      domTabs.setValue(_.findIndex(allTabs, x => x.isChart && x.resultIndex === resultIndex));
+    } else {
+      charts = [
+        ...charts,
+        {
+          jslid: resultInfos[resultIndex].jslid,
+          charts: [],
+          resultIndex,
+        },
+      ];
+      await tick();
+      domTabs.setValue(_.findIndex(allTabs, x => x.isChart && x.resultIndex === resultIndex));
+    }
+    onSetFrontMatterField?.('selected-chart', resultIndex + 1);
+  }
 </script>
 
 <TabControl
@@ -94,6 +150,13 @@
       ? { text: 'Every result in single tab', onClick: () => setOneTabValue(false) }
       : { text: 'All results in one tab', onClick: () => setOneTabValue(true) },
   ]}
+  onUserChange={value => {
+    if (allTabs[value].isChart) {
+      onSetFrontMatterField?.(`selected-chart`, allTabs[value].resultIndex + 1);
+    } else {
+      onSetFrontMatterField?.(`selected-chart`, undefined);
+    }
+  }}
 >
   <slot name="0" slot="0" />
   <slot name="1" slot="1" />

packages/web/src/tabs/QueryTab.svelte

@@ -1,6 +1,7 @@
 <script lang="ts" context="module">
   import registerCommand from '../commands/registerCommand';
   import { copyTextToClipboard } from '../utility/clipboard';
+  import yaml from 'js-yaml';
 
   const getCurrentEditor = () => getActiveComponent('QueryTab');
 
@@ -60,6 +61,13 @@
       getCurrentEditor() != null && !getCurrentEditor()?.isBusy() && getCurrentEditor()?.hasConnection(),
     onClick: () => getCurrentEditor().executeCurrent(),
   });
+  registerCommand({
+    id: 'query.toggleAutoExecute',
+    category: 'Query',
+    name: 'Toggle auto execute',
+    testEnabled: () => getCurrentEditor() != null,
+    onClick: () => getCurrentEditor().toggleAutoExecute(),
+  });
   registerCommand({
     id: 'query.beginTransaction',
     category: 'Query',
@@ -126,7 +134,7 @@
   import InsertJoinModal from '../modals/InsertJoinModal.svelte';
   import useTimerLabel from '../utility/useTimerLabel';
   import createActivator, { getActiveComponent } from '../utility/createActivator';
-  import { findEngineDriver, safeJsonParse } from 'dbgate-tools';
+  import { findEngineDriver, getSqlFrontMatter, safeJsonParse, setSqlFrontMatter } from 'dbgate-tools';
   import AceEditor from '../query/AceEditor.svelte';
   import StatusBarTabItem from '../widgets/StatusBarTabItem.svelte';
   import { showSnackbarError } from '../utility/snackbar';
@@ -147,6 +155,7 @@
   import ToolStripButton from '../buttons/ToolStripButton.svelte';
   import { getIntSettingsValue } from '../settings/settingsTools';
   import RowsLimitModal from '../modals/RowsLimitModal.svelte';
+  import _ from 'lodash';
 
   export let tabid;
   export let conid;
@@ -199,6 +208,7 @@
   let domAiAssistant;
   let isInTransaction = false;
   let isAutocommit = false;
+  let splitterInitialValue = undefined;
 
   const queryRowsLimitLocalStorageKey = `tabdata_limitRows_${tabid}`;
   function getInitialRowsLimit() {
@@ -350,6 +360,7 @@
     executeStartLine = startLine;
     executeNumber++;
     visibleResultTabs = true;
+    const frontMatter = getSqlFrontMatter($editorValue, yaml);
 
     busy = true;
     timerLabel.start();
@@ -381,6 +392,7 @@
         sql,
         autoCommit: driver?.implicitTransactions && isAutocommit,
         limitRows: queryRowsLimit ? queryRowsLimit : undefined,
+        frontMatter,
       });
     }
     await apiCall('query-history/write', {
@@ -550,12 +562,47 @@
       initialArgs && initialArgs.scriptTemplate
         ? () => applyScriptTemplate(initialArgs.scriptTemplate, $extensions, $$props)
         : null,
+
+    onInitialData: value => {
+      const frontMatter = getSqlFrontMatter(value, yaml);
+      if (frontMatter?.autoExecute) {
+        executeCore(value, 0);
+      }
+      if (frontMatter?.splitterInitialValue) {
+        splitterInitialValue = frontMatter.splitterInitialValue;
+      }
+    },
   });
 
   function handleChangeErrors(errors) {
     errorMessages = errors;
   }
 
+  function handleSetFrontMatterField(field, value) {
+    const text = $editorValue;
+    setEditorData(
+      setSqlFrontMatter(
+        text,
+        {
+          ...getSqlFrontMatter(text, yaml),
+          [field]: value,
+        },
+        yaml
+      )
+    );
+  }
+
+  export function toggleAutoExecute() {
+    const frontMatter = getSqlFrontMatter($editorValue, yaml);
+    setEditorData(
+      setSqlFrontMatter(
+        $editorValue,
+        { ...frontMatter, autoExecute: frontMatter?.autoExecute ? undefined : true },
+        yaml
+      )
+    );
+  }
+
   async function handleKeyDown(event) {
     if (isProApp()) {
       if (event.code == 'Space' && event.shiftKey && event.ctrlKey && !isAiAssistantVisible) {
@@ -584,6 +631,7 @@
       { command: 'query.execute' },
       { command: 'query.executeCurrent' },
       { command: 'query.kill' },
+      { command: 'query.toggleAutoExecute' },
       { divider: true },
       { command: 'query.toggleComment' },
       { command: 'query.formatCode' },
@@ -625,7 +673,7 @@
 <ToolStripContainer bind:this={domToolStrip}>
   <HorizontalSplitter isSplitter={isAiAssistantVisible} initialSizeRight={300}>
     <svelte:fragment slot="1">
-      <VerticalSplitter isSplitter={visibleResultTabs}>
+      <VerticalSplitter isSplitter={visibleResultTabs} initialValue={splitterInitialValue}>
         <svelte:fragment slot="1">
           {#if driver?.databaseEngineTypes?.includes('sql')}
             <SqlEditor
@@ -678,7 +726,15 @@
           {/if}
         </svelte:fragment>
         <svelte:fragment slot="2">
-          <ResultTabs tabs={[{ label: 'Messages', slot: 0 }]} {sessionId} {executeNumber} bind:resultCount {driver}>
+          <ResultTabs
+            tabs={[{ label: 'Messages', slot: 0 }]}
+            {sessionId}
+            {executeNumber}
+            bind:resultCount
+            {driver}
+            onSetFrontMatterField={handleSetFrontMatterField}
+            onGetFrontMatter={() => getSqlFrontMatter($editorValue, yaml)}
+          >
             <svelte:fragment slot="0">
               <SocketMessageView
                 eventName={sessionId ? `session-info-${sessionId}` : null}