dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-29 15:43:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

running row macros

Browse Source
This commit is contained in:
Jan Prochazka
2023-02-24 19:04:22 +01:00
parent a519c78301
commit 7c4a47c4c6
5 changed files with 280 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/api/src/shell/modifyJsonLinesReader.js
View File

@@ -58,11 +58,15 @@ class ParseStream extends stream.Transform {
const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex); const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex);
if (update) { if (update) {
if (update.document) {
obj = update.document;
} else {
obj = { obj = {
...obj, ...obj,
...update.fields, ...update.fields,
}; };
} }
}
if (obj) { if (obj) {
if (this.changeSet.dataUpdateCommands) { if (this.changeSet.dataUpdateCommands) {

2
packages/datalib/src/MacroDefinition.ts
View File

@@ -11,7 +11,7 @@ export interface MacroDefinition {
name: string; name: string;
group: string; group: string;
description?: string; description?: string;
type: 'transformValue'; type: 'transformValue' | 'transformRow';
code: string; code: string;
args?: MacroArgument[]; args?: MacroArgument[];
} }

335
packages/datalib/src/runMacro.ts
View File

@@ -4,7 +4,7 @@ import uuidv1 from 'uuid/v1';
import uuidv4 from 'uuid/v4'; import uuidv4 from 'uuid/v4';
import moment from 'moment'; import moment from 'moment';
import { MacroDefinition, MacroSelectedCell } from './MacroDefinition'; import { MacroDefinition, MacroSelectedCell } from './MacroDefinition';
import { ChangeSet, setChangeSetValue } from './ChangeSet'; import { ChangeSet, setChangeSetValue, setChangeSetRowData } from './ChangeSet';
import { GridDisplay } from './GridDisplay'; import { GridDisplay } from './GridDisplay';
const getMacroFunction = { const getMacroFunction = {
@@ -13,13 +13,8 @@ const getMacroFunction = {
${code} ${code}
} }
`, `,
transformRows: code => ` transformRow: code => `
(rows, args, modules, selectedCells, cols, columns) => { (row, args, modules, rowIndex, columns) => {
${code}
}
`,
transformData: code => `
(rows, args, modules, selectedCells, cols, columns) => {
${code} ${code}
} }
`, `,
@@ -32,159 +27,159 @@ const modules = {
moment, moment,
}; };
function runTramsformValue( // function runTramsformValue(
func, // func,
macroArgs: {}, // macroArgs: {},
data: FreeTableModel, // data: FreeTableModel,
preview: boolean, // preview: boolean,
selectedCells: MacroSelectedCell[], // selectedCells: MacroSelectedCell[],
errors: string[] = [] // errors: string[] = []
) { // ) {
const selectedRows = _.groupBy(selectedCells, 'row'); // const selectedRows = _.groupBy(selectedCells, 'row');
const rows = data.rows.map((row, rowIndex) => { // const rows = data.rows.map((row, rowIndex) => {
const selectedRow = selectedRows[rowIndex]; // const selectedRow = selectedRows[rowIndex];
if (selectedRow) { // if (selectedRow) {
const modifiedFields = []; // const modifiedFields = [];
let res = null; // let res = null;
for (const cell of selectedRow) { // for (const cell of selectedRow) {
const { column } = cell; // const { column } = cell;
const oldValue = row[column]; // const oldValue = row[column];
let newValue = oldValue; // let newValue = oldValue;
try { // try {
newValue = func(oldValue, macroArgs, modules, rowIndex, row, column); // newValue = func(oldValue, macroArgs, modules, rowIndex, row, column);
} catch (err) { // } catch (err) {
errors.push(`Error processing column ${column} on row ${rowIndex}: ${err.message}`); // errors.push(`Error processing column ${column} on row ${rowIndex}: ${err.message}`);
} // }
if (newValue != oldValue) { // if (newValue != oldValue) {
if (res == null) { // if (res == null) {
res = { ...row }; // res = { ...row };
} // }
res[column] = newValue; // res[column] = newValue;
if (preview) modifiedFields.push(column); // if (preview) modifiedFields.push(column);
} // }
} // }
if (res) { // if (res) {
if (modifiedFields.length > 0) { // if (modifiedFields.length > 0) {
return { // return {
...res, // ...res,
__modifiedFields: new Set(modifiedFields), // __modifiedFields: new Set(modifiedFields),
}; // };
} // }
return res; // return res;
} // }
return row; // return row;
} else { // } else {
return row; // return row;
} // }
}); // });
return { // return {
structure: data.structure, // structure: data.structure,
rows, // rows,
}; // };
} // }
function removePreviewRowFlags(rows) { // function removePreviewRowFlags(rows) {
rows = rows.filter(row => row.__rowStatus != 'deleted'); // rows = rows.filter(row => row.__rowStatus != 'deleted');
rows = rows.map(row => { // rows = rows.map(row => {
if (row.__rowStatus || row.__modifiedFields || row.__insertedFields || row.__deletedFields) // if (row.__rowStatus || row.__modifiedFields || row.__insertedFields || row.__deletedFields)
return _.omit(row, ['__rowStatus', '__modifiedFields', '__insertedFields', '__deletedFields']); // return _.omit(row, ['__rowStatus', '__modifiedFields', '__insertedFields', '__deletedFields']);
return row; // return row;
}); // });
return rows; // return rows;
} // }
function runTramsformRows( // function runTramsformRows(
func, // func,
macroArgs: {}, // macroArgs: {},
data: FreeTableModel, // data: FreeTableModel,
preview: boolean, // preview: boolean,
selectedCells: MacroSelectedCell[], // selectedCells: MacroSelectedCell[],
errors: string[] = [] // errors: string[] = []
) { // ) {
let rows = data.rows; // let rows = data.rows;
try { // try {
rows = func( // rows = func(
data.rows, // data.rows,
macroArgs, // macroArgs,
modules, // modules,
selectedCells, // selectedCells,
data.structure.columns.map(x => x.columnName), // data.structure.columns.map(x => x.columnName),
data.structure.columns // data.structure.columns
); // );
if (!preview) { // if (!preview) {
rows = removePreviewRowFlags(rows); // rows = removePreviewRowFlags(rows);
} // }
} catch (err) { // } catch (err) {
errors.push(`Error processing rows: ${err.message}`); // errors.push(`Error processing rows: ${err.message}`);
} // }
return { // return {
structure: data.structure, // structure: data.structure,
rows, // rows,
}; // };
} // }
function runTramsformData( // function runTramsformData(
func, // func,
macroArgs: {}, // macroArgs: {},
data: FreeTableModel, // data: FreeTableModel,
preview: boolean, // preview: boolean,
selectedCells: MacroSelectedCell[], // selectedCells: MacroSelectedCell[],
errors: string[] = [] // errors: string[] = []
) { // ) {
try { // try {
let { rows, columns, cols } = func( // let { rows, columns, cols } = func(
data.rows, // data.rows,
macroArgs, // macroArgs,
modules, // modules,
selectedCells, // selectedCells,
data.structure.columns.map(x => x.columnName), // data.structure.columns.map(x => x.columnName),
data.structure.columns // data.structure.columns
); // );
if (cols && !columns) { // if (cols && !columns) {
columns = cols.map(columnName => ({ columnName })); // columns = cols.map(columnName => ({ columnName }));
} // }
columns = _.uniqBy(columns, 'columnName'); // columns = _.uniqBy(columns, 'columnName');
if (!preview) { // if (!preview) {
rows = removePreviewRowFlags(rows); // rows = removePreviewRowFlags(rows);
} // }
return { // return {
structure: { columns }, // structure: { columns },
rows, // rows,
}; // };
} catch (err) { // } catch (err) {
errors.push(`Error processing data: ${err.message}`); // errors.push(`Error processing data: ${err.message}`);
} // }
return data; // return data;
} // }
export function runMacro( // export function runMacro(
macro: MacroDefinition, // macro: MacroDefinition,
macroArgs: {}, // macroArgs: {},
data: FreeTableModel, // data: FreeTableModel,
preview: boolean, // preview: boolean,
selectedCells: MacroSelectedCell[], // selectedCells: MacroSelectedCell[],
errors: string[] = [] // errors: string[] = []
): FreeTableModel { // ): FreeTableModel {
let func; // let func;
try { // try {
func = eval(getMacroFunction[macro.type](macro.code)); // func = eval(getMacroFunction[macro.type](macro.code));
} catch (err) { // } catch (err) {
errors.push(`Error compiling macro ${macro.name}: ${err.message}`); // errors.push(`Error compiling macro ${macro.name}: ${err.message}`);
return data; // return data;
} // }
if (macro.type == 'transformValue') { // if (macro.type == 'transformValue') {
return runTramsformValue(func, macroArgs, data, preview, selectedCells, errors); // return runTramsformValue(func, macroArgs, data, preview, selectedCells, errors);
} // }
if (macro.type == 'transformRows') { // if (macro.type == 'transformRows') {
return runTramsformRows(func, macroArgs, data, preview, selectedCells, errors); // return runTramsformRows(func, macroArgs, data, preview, selectedCells, errors);
} // }
if (macro.type == 'transformData') { // if (macro.type == 'transformData') {
// @ts-ignore // // @ts-ignore
return runTramsformData(func, macroArgs, data, preview, selectedCells, errors); // return runTramsformData(func, macroArgs, data, preview, selectedCells, errors);
} // }
return data; // return data;
} // }
export function compileMacroFunction(macro: MacroDefinition, errors = []) { export function compileMacroFunction(macro: MacroDefinition, errors = []) {
if (!macro) return null; if (!macro) return null;
@@ -198,7 +193,7 @@ export function compileMacroFunction(macro: MacroDefinition, errors = []) {
} }
} }
export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, column, errors = []) { export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex: number, row, column: string, errors = []) {
if (!compiledFunc) return value; if (!compiledFunc) return value;
try { try {
const res = compiledFunc(value, macroArgs, modules, rowIndex, row, column); const res = compiledFunc(value, macroArgs, modules, rowIndex, row, column);
@@ -209,6 +204,17 @@ export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, c
} }
} }
export function runMacroOnRow(compiledFunc, macroArgs, rowIndex: number, row: any, columns: string[], errors = []) {
if (!compiledFunc) return row;
try {
const res = compiledFunc(row, macroArgs, modules, rowIndex, columns);
return res;
} catch (err) {
errors.push(`Error processing row ${rowIndex}: ${err.message}`);
return row;
}
}
export function runMacroOnChangeSet( export function runMacroOnChangeSet(
macro: MacroDefinition, macro: MacroDefinition,
macroArgs: {}, macroArgs: {},
@@ -221,6 +227,7 @@ export function runMacroOnChangeSet(
const compiledMacroFunc = compileMacroFunction(macro, errors); const compiledMacroFunc = compileMacroFunction(macro, errors);
if (!compiledMacroFunc) return null; if (!compiledMacroFunc) return null;
if (macro.type == 'transformValue') {
let res = changeSet; let res = changeSet;
for (const cell of selectedCells) { for (const cell of selectedCells) {
const definition = display.getChangeSetField( const definition = display.getChangeSetField(
@@ -241,6 +248,18 @@ export function runMacroOnChangeSet(
); );
res = setChangeSetValue(res, definition, macroResult); res = setChangeSetValue(res, definition, macroResult);
} }
return res; return res;
} }
if (macro.type == 'transformRow') {
let res = changeSet;
const rowIndexes = _.uniq(selectedCells.map(x => x.row));
for (const index of rowIndexes) {
const rowData = selectedCells.find(x => x.row == index)?.rowData;
const columns = _.uniq(selectedCells.map(x => x.column));
const definition = display.getChangeSetRow(rowData, null, index, true);
const newRow = runMacroOnRow(compiledMacroFunc, macroArgs, index, rowData, columns);
res = setChangeSetRowData(res, definition, newRow);
}
return res;
}
}

21
packages/web/src/datagrid/ChangeSetGrider.ts
View File

@@ -1,4 +1,4 @@
import type { ChangeSet, MacroDefinition, MacroSelectedCell } from 'dbgate-datalib'; import { ChangeSet, MacroDefinition, MacroSelectedCell, runMacroOnRow } from 'dbgate-datalib';
import { import {
changeSetContainsChanges, changeSetContainsChanges,
@@ -17,6 +17,7 @@ import {
} from 'dbgate-datalib'; } from 'dbgate-datalib';
import Grider from './Grider'; import Grider from './Grider';
import type { GriderRowStatus } from './Grider'; import type { GriderRowStatus } from './Grider';
import _ from 'lodash';
function getRowFromItem(row, matchedChangeSetItem) { function getRowFromItem(row, matchedChangeSetItem) {
return matchedChangeSetItem.document return matchedChangeSetItem.document
@@ -38,7 +39,7 @@ export default class ChangeSetGrider extends Grider {
private rowStatusCache; private rowStatusCache;
private rowDefinitionsCache; private rowDefinitionsCache;
private batchChangeSet: ChangeSet; private batchChangeSet: ChangeSet;
private _errors = null; private _errors = [];
private compiledMacroFunc; private compiledMacroFunc;
constructor( constructor(
@@ -89,7 +90,7 @@ export default class ChangeSetGrider extends Grider {
this.useRowIndexInsteaOfCondition this.useRowIndexInsteaOfCondition
); );
const [matchedField, matchedChangeSetItem] = findExistingChangeSetItem(this.changeSet, rowDefinition); const [matchedField, matchedChangeSetItem] = findExistingChangeSetItem(this.changeSet, rowDefinition);
const rowUpdated = matchedChangeSetItem let rowUpdated = matchedChangeSetItem
? getRowFromItem(row, matchedChangeSetItem) ? getRowFromItem(row, matchedChangeSetItem)
: this.compiledMacroFunc : this.compiledMacroFunc
? { ...row } ? { ...row }
@@ -105,6 +106,7 @@ export default class ChangeSetGrider extends Grider {
}; };
if (this.compiledMacroFunc) { if (this.compiledMacroFunc) {
if (this.macro?.type == 'transformValue') {
for (const cell of this.selectedCells) { for (const cell of this.selectedCells) {
if (cell.row != index) continue; if (cell.row != index) continue;
const newValue = runMacroOnValue( const newValue = runMacroOnValue(
@@ -119,6 +121,19 @@ export default class ChangeSetGrider extends Grider {
rowUpdated[cell.column] = newValue; rowUpdated[cell.column] = newValue;
} }
} }
if (this.macro?.type == 'transformRow') {
if (this.selectedCells.find(x => x.row == index)) {
rowUpdated = runMacroOnRow(
this.compiledMacroFunc,
this.macroArgs,
index,
rowUpdated,
_.uniq(this.selectedCells.map(x => x.column)),
this._errors
);
}
}
}
this.rowDataCache[index] = rowUpdated; this.rowDataCache[index] = rowUpdated;
this.rowStatusCache[index] = rowStatus; this.rowStatusCache[index] = rowStatus;

126
packages/web/src/freetable/macros.js
View File

@@ -203,64 +203,16 @@ return !!value;
], ],
code: `return modules.moment().format(args.format)`, code: `return modules.moment().format(args.format)`,
}, },
{
title: 'Duplicate rows',
name: 'duplicateRows',
group: 'Tools',
description: 'Duplicate selected rows',
type: 'transformRows',
code: `
const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row));
const selectedRows = modules.lodash.groupBy(selectedCells, 'row');
const maxIndex = modules.lodash.max(selectedRowIndexes);
return [
...rows.slice(0, maxIndex + 1),
...selectedRowIndexes.map(index => ({
...modules.lodash.pick(rows[index], selectedRows[index].map(x => x.column)),
__rowStatus: 'inserted',
})),
...rows.slice(maxIndex + 1),
]
`,
},
{
title: 'Delete empty rows',
name: 'deleteEmptyRows',
group: 'Tools',
description: 'Delete empty rows - rows with all values null or empty string',
type: 'transformRows',
code: `
return rows.map(row => {
if (cols.find(col => row[col])) return row;
return {
...row,
__rowStatus: 'deleted',
};
})
`,
},
{ {
title: 'Duplicate columns', title: 'Duplicate columns',
name: 'duplicateColumns', name: 'duplicateColumns',
group: 'Tools', group: 'Tools',
description: 'Duplicate selected columns', description: 'Duplicate selected columns',
type: 'transformData', type: 'transformRow',
code: ` code: `
const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column));
const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row));
const addedColumnNames = selectedColumnNames.map(col => (args.prefix || '') + col + (args.postfix || ''));
const resultRows = rows.map((row, rowIndex) => ({
...row,
...(selectedRowIndexes.includes(rowIndex) ? modules.lodash.fromPairs(selectedColumnNames.map(col => [(args.prefix || '') + col + (args.postfix || ''), row[col]])) : {}),
__insertedFields: addedColumnNames,
}));
const resultCols = [
...cols,
...addedColumnNames,
];
return { return {
rows: resultRows, ...row,
cols: resultCols, ...modules.lodash.fromPairs(columns.map(col=>[(args.prefix || '') + col + (args.postfix || ''), row[col]]))
} }
`, `,
args: [ args: [
@@ -282,42 +234,27 @@ return {
name: 'splitColumns', name: 'splitColumns',
group: 'Tools', group: 'Tools',
description: 'Split selected columns', description: 'Split selected columns',
type: 'transformData', type: 'transformRow',
code: ` code: `
const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column)); const res = {...row};
const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row)); for(const col of columns) {
const value = row[col];
const addedColumnNames = new Set(); if (modules.lodash.isString(value)) {
const splitted = value.split(args.delimiter);
const resultRows = modules.lodash.cloneDeep(rows); splitted.forEach((splitValue, valueIndex) => {
resultRows.forEach((row, rowIndex) => { const name = col + '_' + (valueIndex + 1).toString();
for(const cell of selectedCells) { res[name] = splitValue;
if (cell.row == rowIndex && modules.lodash.isString(cell.value)) {
const splitted = cell.value.split(args.delimiter);
splitted.forEach((value, valueIndex) => {
const name = cell.column + '_' + (valueIndex + 1).toString();
row[name] = value;
addedColumnNames.add(name);
}); });
} }
} }
}); return res;
const resultCols = [
...cols,
...addedColumnNames,
];
return {
rows: resultRows,
cols: resultCols,
}
`, `,
args: [ args: [
{ {
type: 'text', type: 'text',
label: 'Delimiter', label: 'Delimiter',
name: 'delimiter', name: 'delimiter',
default: ',' default: ',',
}, },
], ],
}, },
@@ -342,29 +279,20 @@ return {
name: 'extractDateFields', name: 'extractDateFields',
group: 'Tools', group: 'Tools',
description: 'Extract yaear, month, day and other date/time fields from selection and adds it as new columns', description: 'Extract yaear, month, day and other date/time fields from selection and adds it as new columns',
type: 'transformData', type: 'transformRow',
code: ` code: `
const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column));
const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row));
const addedColumnNames = modules.lodash.compact([args.year, args.month, args.day, args.hour, args.minute, args.second]);
const selectedRows = modules.lodash.groupBy(selectedCells, 'row');
const resultRows = rows.map((row, rowIndex) => {
if (!selectedRowIndexes.includes(rowIndex)) return {
...row,
__insertedFields: addedColumnNames,
};
let mom = null; let mom = null;
for(const cell of selectedRows[rowIndex]) { for(const col of columns) {
const m = modules.moment(row[cell.column]); const m = modules.moment(row[col]);
if (m.isValid()) { if (m.isValid()) {
mom = m; mom = m;
break; break;
} }
} }
if (!mom) return {
...row, if (!mom) return row;
__insertedFields: addedColumnNames,
}; const addedColumnNames = modules.lodash.compact([args.year, args.month, args.day, args.hour, args.minute, args.second]);
const fields = { const fields = {
[args.year]: mom.year(), [args.year]: mom.year(),
@@ -378,17 +306,7 @@ const resultRows = rows.map((row, rowIndex) => {
return { return {
...row, ...row,
...modules.lodash.pick(fields, addedColumnNames), ...modules.lodash.pick(fields, addedColumnNames),
__insertedFields: addedColumnNames, };
}
});
const resultCols = [
...cols,
...addedColumnNames,
];
return {
rows: resultRows,
cols: resultCols,
}
`, `,
args: [ args: [
{ {