running row macros

2026-04-25 12:06:00 +00:00 · 2023-02-24 19:04:22 +01:00
parent a519c78301
commit 7c4a47c4c6
5 changed files with 280 additions and 324 deletions
--- a/packages/api/src/shell/modifyJsonLinesReader.js
+++ b/packages/api/src/shell/modifyJsonLinesReader.js
@@ -58,10 +58,14 @@ class ParseStream extends stream.Transform {

        const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex);
        if (update) {
-          obj = {
-            ...obj,
-            ...update.fields,
-          };
+          if (update.document) {
+            obj = update.document;
+          } else {
+            obj = {
+              ...obj,
+              ...update.fields,
+            };
+          }
        }

        if (obj) {
--- a/packages/datalib/src/MacroDefinition.ts
+++ b/packages/datalib/src/MacroDefinition.ts
@@ -11,7 +11,7 @@ export interface MacroDefinition {
  name: string;
  group: string;
  description?: string;
-  type: 'transformValue';
+  type: 'transformValue' | 'transformRow';
  code: string;
  args?: MacroArgument[];
 }
--- a/packages/datalib/src/runMacro.ts
+++ b/packages/datalib/src/runMacro.ts
@@ -4,7 +4,7 @@ import uuidv1 from 'uuid/v1';
 import uuidv4 from 'uuid/v4';
 import moment from 'moment';
 import { MacroDefinition, MacroSelectedCell } from './MacroDefinition';
-import { ChangeSet, setChangeSetValue } from './ChangeSet';
+import { ChangeSet, setChangeSetValue, setChangeSetRowData } from './ChangeSet';
 import { GridDisplay } from './GridDisplay';

 const getMacroFunction = {
@@ -13,13 +13,8 @@ const getMacroFunction = {
    ${code}
 }
 `,
-  transformRows: code => `
-(rows, args, modules, selectedCells, cols, columns) => {
-  ${code}
-}
-`,
-  transformData: code => `
-(rows, args, modules, selectedCells, cols, columns) => {
+  transformRow: code => `
+(row, args, modules, rowIndex, columns) => {
  ${code}
 }
 `,
@@ -32,159 +27,159 @@ const modules = {
  moment,
 };

-function runTramsformValue(
-  func,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-) {
-  const selectedRows = _.groupBy(selectedCells, 'row');
-  const rows = data.rows.map((row, rowIndex) => {
-    const selectedRow = selectedRows[rowIndex];
-    if (selectedRow) {
-      const modifiedFields = [];
-      let res = null;
-      for (const cell of selectedRow) {
-        const { column } = cell;
-        const oldValue = row[column];
-        let newValue = oldValue;
-        try {
-          newValue = func(oldValue, macroArgs, modules, rowIndex, row, column);
-        } catch (err) {
-          errors.push(`Error processing column ${column} on row ${rowIndex}: ${err.message}`);
-        }
-        if (newValue != oldValue) {
-          if (res == null) {
-            res = { ...row };
-          }
-          res[column] = newValue;
-          if (preview) modifiedFields.push(column);
-        }
-      }
-      if (res) {
-        if (modifiedFields.length > 0) {
-          return {
-            ...res,
-            __modifiedFields: new Set(modifiedFields),
-          };
-        }
-        return res;
-      }
-      return row;
-    } else {
-      return row;
-    }
-  });
+// function runTramsformValue(
+//   func,
+//   macroArgs: {},
+//   data: FreeTableModel,
+//   preview: boolean,
+//   selectedCells: MacroSelectedCell[],
+//   errors: string[] = []
+// ) {
+//   const selectedRows = _.groupBy(selectedCells, 'row');
+//   const rows = data.rows.map((row, rowIndex) => {
+//     const selectedRow = selectedRows[rowIndex];
+//     if (selectedRow) {
+//       const modifiedFields = [];
+//       let res = null;
+//       for (const cell of selectedRow) {
+//         const { column } = cell;
+//         const oldValue = row[column];
+//         let newValue = oldValue;
+//         try {
+//           newValue = func(oldValue, macroArgs, modules, rowIndex, row, column);
+//         } catch (err) {
+//           errors.push(`Error processing column ${column} on row ${rowIndex}: ${err.message}`);
+//         }
+//         if (newValue != oldValue) {
+//           if (res == null) {
+//             res = { ...row };
+//           }
+//           res[column] = newValue;
+//           if (preview) modifiedFields.push(column);
+//         }
+//       }
+//       if (res) {
+//         if (modifiedFields.length > 0) {
+//           return {
+//             ...res,
+//             __modifiedFields: new Set(modifiedFields),
+//           };
+//         }
+//         return res;
+//       }
+//       return row;
+//     } else {
+//       return row;
+//     }
+//   });

-  return {
-    structure: data.structure,
-    rows,
-  };
-}
+//   return {
+//     structure: data.structure,
+//     rows,
+//   };
+// }

-function removePreviewRowFlags(rows) {
-  rows = rows.filter(row => row.__rowStatus != 'deleted');
-  rows = rows.map(row => {
-    if (row.__rowStatus || row.__modifiedFields || row.__insertedFields || row.__deletedFields)
-      return _.omit(row, ['__rowStatus', '__modifiedFields', '__insertedFields', '__deletedFields']);
-    return row;
-  });
-  return rows;
-}
+// function removePreviewRowFlags(rows) {
+//   rows = rows.filter(row => row.__rowStatus != 'deleted');
+//   rows = rows.map(row => {
+//     if (row.__rowStatus || row.__modifiedFields || row.__insertedFields || row.__deletedFields)
+//       return _.omit(row, ['__rowStatus', '__modifiedFields', '__insertedFields', '__deletedFields']);
+//     return row;
+//   });
+//   return rows;
+// }

-function runTramsformRows(
-  func,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-) {
-  let rows = data.rows;
-  try {
-    rows = func(
-      data.rows,
-      macroArgs,
-      modules,
-      selectedCells,
-      data.structure.columns.map(x => x.columnName),
-      data.structure.columns
-    );
-    if (!preview) {
-      rows = removePreviewRowFlags(rows);
-    }
-  } catch (err) {
-    errors.push(`Error processing rows: ${err.message}`);
-  }
-  return {
-    structure: data.structure,
-    rows,
-  };
-}
+// function runTramsformRows(
+//   func,
+//   macroArgs: {},
+//   data: FreeTableModel,
+//   preview: boolean,
+//   selectedCells: MacroSelectedCell[],
+//   errors: string[] = []
+// ) {
+//   let rows = data.rows;
+//   try {
+//     rows = func(
+//       data.rows,
+//       macroArgs,
+//       modules,
+//       selectedCells,
+//       data.structure.columns.map(x => x.columnName),
+//       data.structure.columns
+//     );
+//     if (!preview) {
+//       rows = removePreviewRowFlags(rows);
+//     }
+//   } catch (err) {
+//     errors.push(`Error processing rows: ${err.message}`);
+//   }
+//   return {
+//     structure: data.structure,
+//     rows,
+//   };
+// }

-function runTramsformData(
-  func,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-) {
-  try {
-    let { rows, columns, cols } = func(
-      data.rows,
-      macroArgs,
-      modules,
-      selectedCells,
-      data.structure.columns.map(x => x.columnName),
-      data.structure.columns
-    );
-    if (cols && !columns) {
-      columns = cols.map(columnName => ({ columnName }));
-    }
-    columns = _.uniqBy(columns, 'columnName');
-    if (!preview) {
-      rows = removePreviewRowFlags(rows);
-    }
-    return {
-      structure: { columns },
-      rows,
-    };
-  } catch (err) {
-    errors.push(`Error processing data: ${err.message}`);
-  }
-  return data;
-}
+// function runTramsformData(
+//   func,
+//   macroArgs: {},
+//   data: FreeTableModel,
+//   preview: boolean,
+//   selectedCells: MacroSelectedCell[],
+//   errors: string[] = []
+// ) {
+//   try {
+//     let { rows, columns, cols } = func(
+//       data.rows,
+//       macroArgs,
+//       modules,
+//       selectedCells,
+//       data.structure.columns.map(x => x.columnName),
+//       data.structure.columns
+//     );
+//     if (cols && !columns) {
+//       columns = cols.map(columnName => ({ columnName }));
+//     }
+//     columns = _.uniqBy(columns, 'columnName');
+//     if (!preview) {
+//       rows = removePreviewRowFlags(rows);
+//     }
+//     return {
+//       structure: { columns },
+//       rows,
+//     };
+//   } catch (err) {
+//     errors.push(`Error processing data: ${err.message}`);
+//   }
+//   return data;
+// }

-export function runMacro(
-  macro: MacroDefinition,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-): FreeTableModel {
-  let func;
-  try {
-    func = eval(getMacroFunction[macro.type](macro.code));
-  } catch (err) {
-    errors.push(`Error compiling macro ${macro.name}: ${err.message}`);
-    return data;
-  }
-  if (macro.type == 'transformValue') {
-    return runTramsformValue(func, macroArgs, data, preview, selectedCells, errors);
-  }
-  if (macro.type == 'transformRows') {
-    return runTramsformRows(func, macroArgs, data, preview, selectedCells, errors);
-  }
-  if (macro.type == 'transformData') {
-    // @ts-ignore
-    return runTramsformData(func, macroArgs, data, preview, selectedCells, errors);
-  }
-  return data;
-}
+// export function runMacro(
+//   macro: MacroDefinition,
+//   macroArgs: {},
+//   data: FreeTableModel,
+//   preview: boolean,
+//   selectedCells: MacroSelectedCell[],
+//   errors: string[] = []
+// ): FreeTableModel {
+//   let func;
+//   try {
+//     func = eval(getMacroFunction[macro.type](macro.code));
+//   } catch (err) {
+//     errors.push(`Error compiling macro ${macro.name}: ${err.message}`);
+//     return data;
+//   }
+//   if (macro.type == 'transformValue') {
+//     return runTramsformValue(func, macroArgs, data, preview, selectedCells, errors);
+//   }
+//   if (macro.type == 'transformRows') {
+//     return runTramsformRows(func, macroArgs, data, preview, selectedCells, errors);
+//   }
+//   if (macro.type == 'transformData') {
+//     // @ts-ignore
+//     return runTramsformData(func, macroArgs, data, preview, selectedCells, errors);
+//   }
+//   return data;
+// }

 export function compileMacroFunction(macro: MacroDefinition, errors = []) {
  if (!macro) return null;
@@ -198,7 +193,7 @@ export function compileMacroFunction(macro: MacroDefinition, errors = []) {
  }
 }

-export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, column, errors = []) {
+export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex: number, row, column: string, errors = []) {
  if (!compiledFunc) return value;
  try {
    const res = compiledFunc(value, macroArgs, modules, rowIndex, row, column);
@@ -209,6 +204,17 @@ export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, c
  }
 }

+export function runMacroOnRow(compiledFunc, macroArgs, rowIndex: number, row: any, columns: string[], errors = []) {
+  if (!compiledFunc) return row;
+  try {
+    const res = compiledFunc(row, macroArgs, modules, rowIndex, columns);
+    return res;
+  } catch (err) {
+    errors.push(`Error processing row ${rowIndex}: ${err.message}`);
+    return row;
+  }
+}
+
 export function runMacroOnChangeSet(
  macro: MacroDefinition,
  macroArgs: {},
@@ -221,26 +227,39 @@ export function runMacroOnChangeSet(
  const compiledMacroFunc = compileMacroFunction(macro, errors);
  if (!compiledMacroFunc) return null;

-  let res = changeSet;
-  for (const cell of selectedCells) {
-    const definition = display.getChangeSetField(
-      cell.rowData,
-      cell.column,
-      undefined,
-      useRowIndexInsteaOfCondition ? cell.row : undefined,
-      useRowIndexInsteaOfCondition
-    );
-    const macroResult = runMacroOnValue(
-      compiledMacroFunc,
-      macroArgs,
-      cell.value,
-      cell.row,
-      cell.rowData,
-      cell.column,
-      errors
-    );
-    res = setChangeSetValue(res, definition, macroResult);
+  if (macro.type == 'transformValue') {
+    let res = changeSet;
+    for (const cell of selectedCells) {
+      const definition = display.getChangeSetField(
+        cell.rowData,
+        cell.column,
+        undefined,
+        useRowIndexInsteaOfCondition ? cell.row : undefined,
+        useRowIndexInsteaOfCondition
+      );
+      const macroResult = runMacroOnValue(
+        compiledMacroFunc,
+        macroArgs,
+        cell.value,
+        cell.row,
+        cell.rowData,
+        cell.column,
+        errors
+      );
+      res = setChangeSetValue(res, definition, macroResult);
+    }
+    return res;
+  }
+  if (macro.type == 'transformRow') {
+    let res = changeSet;
+    const rowIndexes = _.uniq(selectedCells.map(x => x.row));
+    for (const index of rowIndexes) {
+      const rowData = selectedCells.find(x => x.row == index)?.rowData;
+      const columns = _.uniq(selectedCells.map(x => x.column));
+      const definition = display.getChangeSetRow(rowData, null, index, true);
+      const newRow = runMacroOnRow(compiledMacroFunc, macroArgs, index, rowData, columns);
+      res = setChangeSetRowData(res, definition, newRow);
+    }
+    return res;
  }
-
-  return res;
 }
--- a/packages/web/src/datagrid/ChangeSetGrider.ts
+++ b/packages/web/src/datagrid/ChangeSetGrider.ts
@@ -1,4 +1,4 @@
-import type { ChangeSet, MacroDefinition, MacroSelectedCell } from 'dbgate-datalib';
+import { ChangeSet, MacroDefinition, MacroSelectedCell, runMacroOnRow } from 'dbgate-datalib';

 import {
  changeSetContainsChanges,
@@ -17,6 +17,7 @@ import {
 } from 'dbgate-datalib';
 import Grider from './Grider';
 import type { GriderRowStatus } from './Grider';
+import _ from 'lodash';

 function getRowFromItem(row, matchedChangeSetItem) {
  return matchedChangeSetItem.document
@@ -38,7 +39,7 @@ export default class ChangeSetGrider extends Grider {
  private rowStatusCache;
  private rowDefinitionsCache;
  private batchChangeSet: ChangeSet;
-  private _errors = null;
+  private _errors = [];
  private compiledMacroFunc;

  constructor(
@@ -89,7 +90,7 @@ export default class ChangeSetGrider extends Grider {
      this.useRowIndexInsteaOfCondition
    );
    const [matchedField, matchedChangeSetItem] = findExistingChangeSetItem(this.changeSet, rowDefinition);
-    const rowUpdated = matchedChangeSetItem
+    let rowUpdated = matchedChangeSetItem
      ? getRowFromItem(row, matchedChangeSetItem)
      : this.compiledMacroFunc
      ? { ...row }
@@ -105,18 +106,32 @@ export default class ChangeSetGrider extends Grider {
    };

    if (this.compiledMacroFunc) {
-      for (const cell of this.selectedCells) {
-        if (cell.row != index) continue;
-        const newValue = runMacroOnValue(
-          this.compiledMacroFunc,
-          this.macroArgs,
-          rowUpdated[cell.column],
-          index,
-          rowUpdated,
-          cell.column,
-          this._errors
-        );
-        rowUpdated[cell.column] = newValue;
+      if (this.macro?.type == 'transformValue') {
+        for (const cell of this.selectedCells) {
+          if (cell.row != index) continue;
+          const newValue = runMacroOnValue(
+            this.compiledMacroFunc,
+            this.macroArgs,
+            rowUpdated[cell.column],
+            index,
+            rowUpdated,
+            cell.column,
+            this._errors
+          );
+          rowUpdated[cell.column] = newValue;
+        }
+      }
+      if (this.macro?.type == 'transformRow') {
+        if (this.selectedCells.find(x => x.row == index)) {
+          rowUpdated = runMacroOnRow(
+            this.compiledMacroFunc,
+            this.macroArgs,
+            index,
+            rowUpdated,
+            _.uniq(this.selectedCells.map(x => x.column)),
+            this._errors
+          );
+        }
      }
    }

--- a/packages/web/src/freetable/macros.js
+++ b/packages/web/src/freetable/macros.js
@@ -203,65 +203,17 @@ return !!value;
    ],
    code: `return modules.moment().format(args.format)`,
  },
-  {
-    title: 'Duplicate rows',
-    name: 'duplicateRows',
-    group: 'Tools',
-    description: 'Duplicate selected rows',
-    type: 'transformRows',
-    code: `
-const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row));
-const selectedRows = modules.lodash.groupBy(selectedCells, 'row');
-const maxIndex = modules.lodash.max(selectedRowIndexes);
-return [
-  ...rows.slice(0, maxIndex + 1),
-  ...selectedRowIndexes.map(index => ({
-    ...modules.lodash.pick(rows[index], selectedRows[index].map(x => x.column)),
-    __rowStatus: 'inserted',
-  })),
-  ...rows.slice(maxIndex + 1),
-]
-    `,
-  },
-  {
-    title: 'Delete empty rows',
-    name: 'deleteEmptyRows',
-    group: 'Tools',
-    description: 'Delete empty rows - rows with all values null or empty string',
-    type: 'transformRows',
-    code: `
-return rows.map(row => {
-  if (cols.find(col => row[col])) return row;
-  return {
-    ...row,
-    __rowStatus: 'deleted',
-  };
-})
-`,
-  },
  {
    title: 'Duplicate columns',
    name: 'duplicateColumns',
    group: 'Tools',
    description: 'Duplicate selected columns',
-    type: 'transformData',
+    type: 'transformRow',
    code: `
-const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column));
-const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row));
-const addedColumnNames = selectedColumnNames.map(col => (args.prefix || '') + col + (args.postfix || ''));
-const resultRows = rows.map((row, rowIndex) => ({
-  ...row,
-  ...(selectedRowIndexes.includes(rowIndex) ? modules.lodash.fromPairs(selectedColumnNames.map(col => [(args.prefix || '') + col + (args.postfix || ''), row[col]])) : {}),
-  __insertedFields: addedColumnNames,
-}));
-const resultCols = [
-  ...cols,
-  ...addedColumnNames,
-];
-return {
-  rows: resultRows,
-  cols: resultCols,
-}
+      return {
+        ...row,
+        ...modules.lodash.fromPairs(columns.map(col=>[(args.prefix || '') + col + (args.postfix || ''), row[col]]))
+      }
    `,
    args: [
      {
@@ -282,42 +234,27 @@ return {
    name: 'splitColumns',
    group: 'Tools',
    description: 'Split selected columns',
-    type: 'transformData',
+    type: 'transformRow',
    code: `
-const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column));
-const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row));
-
-const addedColumnNames = new Set();
-
-const resultRows = modules.lodash.cloneDeep(rows);
-resultRows.forEach((row, rowIndex) => {
-  for(const cell of selectedCells) {
-    if (cell.row == rowIndex && modules.lodash.isString(cell.value)) {
-      const splitted = cell.value.split(args.delimiter);
-      splitted.forEach((value, valueIndex) => {
-        const name = cell.column + '_' + (valueIndex + 1).toString();
-        row[name] = value;
-        addedColumnNames.add(name);
-      });
-    }
-  }
-});
-
-const resultCols = [
-  ...cols,
-  ...addedColumnNames,
-];
-return {
-  rows: resultRows,
-  cols: resultCols,
-}
+       const res = {...row};
+       for(const col of columns) {
+          const value = row[col];
+          if (modules.lodash.isString(value)) {
+            const splitted = value.split(args.delimiter);      
+            splitted.forEach((splitValue, valueIndex) => {
+              const name = col + '_' + (valueIndex + 1).toString();
+              res[name] = splitValue;
+            });
+          }
+       }
+       return res;
    `,
    args: [
      {
        type: 'text',
        label: 'Delimiter',
        name: 'delimiter',
-        default: ','
+        default: ',',
      },
    ],
  },
@@ -342,53 +279,34 @@ return {
    name: 'extractDateFields',
    group: 'Tools',
    description: 'Extract yaear, month, day and other date/time fields from selection and adds it as new columns',
-    type: 'transformData',
+    type: 'transformRow',
    code: `
-const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column));
-const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row));
-const addedColumnNames = modules.lodash.compact([args.year, args.month, args.day, args.hour, args.minute, args.second]);
-const selectedRows = modules.lodash.groupBy(selectedCells, 'row');
-const resultRows = rows.map((row, rowIndex) => {
-  if (!selectedRowIndexes.includes(rowIndex)) return {
-    ...row,
-    __insertedFields: addedColumnNames,
-  };
-  let mom = null;
-  for(const cell of selectedRows[rowIndex]) {
-    const m = modules.moment(row[cell.column]);
-    if (m.isValid()) {
-      mom = m;
-      break;
-    }
-  }
-  if (!mom) return {
-    ...row,
-    __insertedFields: addedColumnNames,
-  };
+      let mom = null;
+      for(const col of columns) {
+        const m = modules.moment(row[col]);
+        if (m.isValid()) {
+          mom = m;
+          break;
+        }
+      }

-  const fields = {
-    [args.year]: mom.year(),
-    [args.month]: mom.month() + 1,
-    [args.day]: mom.day(),
-    [args.hour]: mom.hour(),
-    [args.minute]: mom.minute(),
-    [args.second]: mom.second(),
-  };
+      if (!mom) return row;

-  return {
-    ...row,
-    ...modules.lodash.pick(fields, addedColumnNames),
-    __insertedFields: addedColumnNames,
-  }
-});
-const resultCols = [
-  ...cols,
-  ...addedColumnNames,
-];
-return {
-  rows: resultRows,
-  cols: resultCols,
-}
+      const addedColumnNames = modules.lodash.compact([args.year, args.month, args.day, args.hour, args.minute, args.second]);
+
+      const fields = {
+        [args.year]: mom.year(),
+        [args.month]: mom.month() + 1,
+        [args.day]: mom.day(),
+        [args.hour]: mom.hour(),
+        [args.minute]: mom.minute(),
+        [args.second]: mom.second(),
+      };
+    
+      return {
+        ...row,
+        ...modules.lodash.pick(fields, addedColumnNames),
+      };
    `,
    args: [
      {