diff --git a/packages/api/src/shell/modifyJsonLinesReader.js b/packages/api/src/shell/modifyJsonLinesReader.js index bb5c6b1ed..acffb32f9 100644 --- a/packages/api/src/shell/modifyJsonLinesReader.js +++ b/packages/api/src/shell/modifyJsonLinesReader.js @@ -58,10 +58,14 @@ class ParseStream extends stream.Transform { const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex); if (update) { - obj = { - ...obj, - ...update.fields, - }; + if (update.document) { + obj = update.document; + } else { + obj = { + ...obj, + ...update.fields, + }; + } } if (obj) { diff --git a/packages/datalib/src/MacroDefinition.ts b/packages/datalib/src/MacroDefinition.ts index e7fa0b5b1..cf1839631 100644 --- a/packages/datalib/src/MacroDefinition.ts +++ b/packages/datalib/src/MacroDefinition.ts @@ -11,7 +11,7 @@ export interface MacroDefinition { name: string; group: string; description?: string; - type: 'transformValue'; + type: 'transformValue' | 'transformRow'; code: string; args?: MacroArgument[]; } diff --git a/packages/datalib/src/runMacro.ts b/packages/datalib/src/runMacro.ts index ebd7c2983..bd545ef3a 100644 --- a/packages/datalib/src/runMacro.ts +++ b/packages/datalib/src/runMacro.ts @@ -4,7 +4,7 @@ import uuidv1 from 'uuid/v1'; import uuidv4 from 'uuid/v4'; import moment from 'moment'; import { MacroDefinition, MacroSelectedCell } from './MacroDefinition'; -import { ChangeSet, setChangeSetValue } from './ChangeSet'; +import { ChangeSet, setChangeSetValue, setChangeSetRowData } from './ChangeSet'; import { GridDisplay } from './GridDisplay'; const getMacroFunction = { @@ -13,13 +13,8 @@ const getMacroFunction = { ${code} } `, - transformRows: code => ` -(rows, args, modules, selectedCells, cols, columns) => { - ${code} -} -`, - transformData: code => ` -(rows, args, modules, selectedCells, cols, columns) => { + transformRow: code => ` +(row, args, modules, rowIndex, columns) => { ${code} } `, @@ -32,159 +27,159 @@ const modules = { moment, }; -function runTramsformValue( - func, - macroArgs: {}, - data: FreeTableModel, - preview: boolean, - selectedCells: MacroSelectedCell[], - errors: string[] = [] -) { - const selectedRows = _.groupBy(selectedCells, 'row'); - const rows = data.rows.map((row, rowIndex) => { - const selectedRow = selectedRows[rowIndex]; - if (selectedRow) { - const modifiedFields = []; - let res = null; - for (const cell of selectedRow) { - const { column } = cell; - const oldValue = row[column]; - let newValue = oldValue; - try { - newValue = func(oldValue, macroArgs, modules, rowIndex, row, column); - } catch (err) { - errors.push(`Error processing column ${column} on row ${rowIndex}: ${err.message}`); - } - if (newValue != oldValue) { - if (res == null) { - res = { ...row }; - } - res[column] = newValue; - if (preview) modifiedFields.push(column); - } - } - if (res) { - if (modifiedFields.length > 0) { - return { - ...res, - __modifiedFields: new Set(modifiedFields), - }; - } - return res; - } - return row; - } else { - return row; - } - }); +// function runTramsformValue( +// func, +// macroArgs: {}, +// data: FreeTableModel, +// preview: boolean, +// selectedCells: MacroSelectedCell[], +// errors: string[] = [] +// ) { +// const selectedRows = _.groupBy(selectedCells, 'row'); +// const rows = data.rows.map((row, rowIndex) => { +// const selectedRow = selectedRows[rowIndex]; +// if (selectedRow) { +// const modifiedFields = []; +// let res = null; +// for (const cell of selectedRow) { +// const { column } = cell; +// const oldValue = row[column]; +// let newValue = oldValue; +// try { +// newValue = func(oldValue, macroArgs, modules, rowIndex, row, column); +// } catch (err) { +// errors.push(`Error processing column ${column} on row ${rowIndex}: ${err.message}`); +// } +// if (newValue != oldValue) { +// if (res == null) { +// res = { ...row }; +// } +// res[column] = newValue; +// if (preview) modifiedFields.push(column); +// } +// } +// if (res) { +// if (modifiedFields.length > 0) { +// return { +// ...res, +// __modifiedFields: new Set(modifiedFields), +// }; +// } +// return res; +// } +// return row; +// } else { +// return row; +// } +// }); - return { - structure: data.structure, - rows, - }; -} +// return { +// structure: data.structure, +// rows, +// }; +// } -function removePreviewRowFlags(rows) { - rows = rows.filter(row => row.__rowStatus != 'deleted'); - rows = rows.map(row => { - if (row.__rowStatus || row.__modifiedFields || row.__insertedFields || row.__deletedFields) - return _.omit(row, ['__rowStatus', '__modifiedFields', '__insertedFields', '__deletedFields']); - return row; - }); - return rows; -} +// function removePreviewRowFlags(rows) { +// rows = rows.filter(row => row.__rowStatus != 'deleted'); +// rows = rows.map(row => { +// if (row.__rowStatus || row.__modifiedFields || row.__insertedFields || row.__deletedFields) +// return _.omit(row, ['__rowStatus', '__modifiedFields', '__insertedFields', '__deletedFields']); +// return row; +// }); +// return rows; +// } -function runTramsformRows( - func, - macroArgs: {}, - data: FreeTableModel, - preview: boolean, - selectedCells: MacroSelectedCell[], - errors: string[] = [] -) { - let rows = data.rows; - try { - rows = func( - data.rows, - macroArgs, - modules, - selectedCells, - data.structure.columns.map(x => x.columnName), - data.structure.columns - ); - if (!preview) { - rows = removePreviewRowFlags(rows); - } - } catch (err) { - errors.push(`Error processing rows: ${err.message}`); - } - return { - structure: data.structure, - rows, - }; -} +// function runTramsformRows( +// func, +// macroArgs: {}, +// data: FreeTableModel, +// preview: boolean, +// selectedCells: MacroSelectedCell[], +// errors: string[] = [] +// ) { +// let rows = data.rows; +// try { +// rows = func( +// data.rows, +// macroArgs, +// modules, +// selectedCells, +// data.structure.columns.map(x => x.columnName), +// data.structure.columns +// ); +// if (!preview) { +// rows = removePreviewRowFlags(rows); +// } +// } catch (err) { +// errors.push(`Error processing rows: ${err.message}`); +// } +// return { +// structure: data.structure, +// rows, +// }; +// } -function runTramsformData( - func, - macroArgs: {}, - data: FreeTableModel, - preview: boolean, - selectedCells: MacroSelectedCell[], - errors: string[] = [] -) { - try { - let { rows, columns, cols } = func( - data.rows, - macroArgs, - modules, - selectedCells, - data.structure.columns.map(x => x.columnName), - data.structure.columns - ); - if (cols && !columns) { - columns = cols.map(columnName => ({ columnName })); - } - columns = _.uniqBy(columns, 'columnName'); - if (!preview) { - rows = removePreviewRowFlags(rows); - } - return { - structure: { columns }, - rows, - }; - } catch (err) { - errors.push(`Error processing data: ${err.message}`); - } - return data; -} +// function runTramsformData( +// func, +// macroArgs: {}, +// data: FreeTableModel, +// preview: boolean, +// selectedCells: MacroSelectedCell[], +// errors: string[] = [] +// ) { +// try { +// let { rows, columns, cols } = func( +// data.rows, +// macroArgs, +// modules, +// selectedCells, +// data.structure.columns.map(x => x.columnName), +// data.structure.columns +// ); +// if (cols && !columns) { +// columns = cols.map(columnName => ({ columnName })); +// } +// columns = _.uniqBy(columns, 'columnName'); +// if (!preview) { +// rows = removePreviewRowFlags(rows); +// } +// return { +// structure: { columns }, +// rows, +// }; +// } catch (err) { +// errors.push(`Error processing data: ${err.message}`); +// } +// return data; +// } -export function runMacro( - macro: MacroDefinition, - macroArgs: {}, - data: FreeTableModel, - preview: boolean, - selectedCells: MacroSelectedCell[], - errors: string[] = [] -): FreeTableModel { - let func; - try { - func = eval(getMacroFunction[macro.type](macro.code)); - } catch (err) { - errors.push(`Error compiling macro ${macro.name}: ${err.message}`); - return data; - } - if (macro.type == 'transformValue') { - return runTramsformValue(func, macroArgs, data, preview, selectedCells, errors); - } - if (macro.type == 'transformRows') { - return runTramsformRows(func, macroArgs, data, preview, selectedCells, errors); - } - if (macro.type == 'transformData') { - // @ts-ignore - return runTramsformData(func, macroArgs, data, preview, selectedCells, errors); - } - return data; -} +// export function runMacro( +// macro: MacroDefinition, +// macroArgs: {}, +// data: FreeTableModel, +// preview: boolean, +// selectedCells: MacroSelectedCell[], +// errors: string[] = [] +// ): FreeTableModel { +// let func; +// try { +// func = eval(getMacroFunction[macro.type](macro.code)); +// } catch (err) { +// errors.push(`Error compiling macro ${macro.name}: ${err.message}`); +// return data; +// } +// if (macro.type == 'transformValue') { +// return runTramsformValue(func, macroArgs, data, preview, selectedCells, errors); +// } +// if (macro.type == 'transformRows') { +// return runTramsformRows(func, macroArgs, data, preview, selectedCells, errors); +// } +// if (macro.type == 'transformData') { +// // @ts-ignore +// return runTramsformData(func, macroArgs, data, preview, selectedCells, errors); +// } +// return data; +// } export function compileMacroFunction(macro: MacroDefinition, errors = []) { if (!macro) return null; @@ -198,7 +193,7 @@ export function compileMacroFunction(macro: MacroDefinition, errors = []) { } } -export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, column, errors = []) { +export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex: number, row, column: string, errors = []) { if (!compiledFunc) return value; try { const res = compiledFunc(value, macroArgs, modules, rowIndex, row, column); @@ -209,6 +204,17 @@ export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, c } } +export function runMacroOnRow(compiledFunc, macroArgs, rowIndex: number, row: any, columns: string[], errors = []) { + if (!compiledFunc) return row; + try { + const res = compiledFunc(row, macroArgs, modules, rowIndex, columns); + return res; + } catch (err) { + errors.push(`Error processing row ${rowIndex}: ${err.message}`); + return row; + } +} + export function runMacroOnChangeSet( macro: MacroDefinition, macroArgs: {}, @@ -221,26 +227,39 @@ export function runMacroOnChangeSet( const compiledMacroFunc = compileMacroFunction(macro, errors); if (!compiledMacroFunc) return null; - let res = changeSet; - for (const cell of selectedCells) { - const definition = display.getChangeSetField( - cell.rowData, - cell.column, - undefined, - useRowIndexInsteaOfCondition ? cell.row : undefined, - useRowIndexInsteaOfCondition - ); - const macroResult = runMacroOnValue( - compiledMacroFunc, - macroArgs, - cell.value, - cell.row, - cell.rowData, - cell.column, - errors - ); - res = setChangeSetValue(res, definition, macroResult); + if (macro.type == 'transformValue') { + let res = changeSet; + for (const cell of selectedCells) { + const definition = display.getChangeSetField( + cell.rowData, + cell.column, + undefined, + useRowIndexInsteaOfCondition ? cell.row : undefined, + useRowIndexInsteaOfCondition + ); + const macroResult = runMacroOnValue( + compiledMacroFunc, + macroArgs, + cell.value, + cell.row, + cell.rowData, + cell.column, + errors + ); + res = setChangeSetValue(res, definition, macroResult); + } + return res; + } + if (macro.type == 'transformRow') { + let res = changeSet; + const rowIndexes = _.uniq(selectedCells.map(x => x.row)); + for (const index of rowIndexes) { + const rowData = selectedCells.find(x => x.row == index)?.rowData; + const columns = _.uniq(selectedCells.map(x => x.column)); + const definition = display.getChangeSetRow(rowData, null, index, true); + const newRow = runMacroOnRow(compiledMacroFunc, macroArgs, index, rowData, columns); + res = setChangeSetRowData(res, definition, newRow); + } + return res; } - - return res; } diff --git a/packages/web/src/datagrid/ChangeSetGrider.ts b/packages/web/src/datagrid/ChangeSetGrider.ts index 5ba218b56..6d7d1abc9 100644 --- a/packages/web/src/datagrid/ChangeSetGrider.ts +++ b/packages/web/src/datagrid/ChangeSetGrider.ts @@ -1,4 +1,4 @@ -import type { ChangeSet, MacroDefinition, MacroSelectedCell } from 'dbgate-datalib'; +import { ChangeSet, MacroDefinition, MacroSelectedCell, runMacroOnRow } from 'dbgate-datalib'; import { changeSetContainsChanges, @@ -17,6 +17,7 @@ import { } from 'dbgate-datalib'; import Grider from './Grider'; import type { GriderRowStatus } from './Grider'; +import _ from 'lodash'; function getRowFromItem(row, matchedChangeSetItem) { return matchedChangeSetItem.document @@ -38,7 +39,7 @@ export default class ChangeSetGrider extends Grider { private rowStatusCache; private rowDefinitionsCache; private batchChangeSet: ChangeSet; - private _errors = null; + private _errors = []; private compiledMacroFunc; constructor( @@ -89,7 +90,7 @@ export default class ChangeSetGrider extends Grider { this.useRowIndexInsteaOfCondition ); const [matchedField, matchedChangeSetItem] = findExistingChangeSetItem(this.changeSet, rowDefinition); - const rowUpdated = matchedChangeSetItem + let rowUpdated = matchedChangeSetItem ? getRowFromItem(row, matchedChangeSetItem) : this.compiledMacroFunc ? { ...row } @@ -105,18 +106,32 @@ export default class ChangeSetGrider extends Grider { }; if (this.compiledMacroFunc) { - for (const cell of this.selectedCells) { - if (cell.row != index) continue; - const newValue = runMacroOnValue( - this.compiledMacroFunc, - this.macroArgs, - rowUpdated[cell.column], - index, - rowUpdated, - cell.column, - this._errors - ); - rowUpdated[cell.column] = newValue; + if (this.macro?.type == 'transformValue') { + for (const cell of this.selectedCells) { + if (cell.row != index) continue; + const newValue = runMacroOnValue( + this.compiledMacroFunc, + this.macroArgs, + rowUpdated[cell.column], + index, + rowUpdated, + cell.column, + this._errors + ); + rowUpdated[cell.column] = newValue; + } + } + if (this.macro?.type == 'transformRow') { + if (this.selectedCells.find(x => x.row == index)) { + rowUpdated = runMacroOnRow( + this.compiledMacroFunc, + this.macroArgs, + index, + rowUpdated, + _.uniq(this.selectedCells.map(x => x.column)), + this._errors + ); + } } } diff --git a/packages/web/src/freetable/macros.js b/packages/web/src/freetable/macros.js index 0cee83a8c..f131e3b6c 100644 --- a/packages/web/src/freetable/macros.js +++ b/packages/web/src/freetable/macros.js @@ -203,65 +203,17 @@ return !!value; ], code: `return modules.moment().format(args.format)`, }, - { - title: 'Duplicate rows', - name: 'duplicateRows', - group: 'Tools', - description: 'Duplicate selected rows', - type: 'transformRows', - code: ` -const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row)); -const selectedRows = modules.lodash.groupBy(selectedCells, 'row'); -const maxIndex = modules.lodash.max(selectedRowIndexes); -return [ - ...rows.slice(0, maxIndex + 1), - ...selectedRowIndexes.map(index => ({ - ...modules.lodash.pick(rows[index], selectedRows[index].map(x => x.column)), - __rowStatus: 'inserted', - })), - ...rows.slice(maxIndex + 1), -] - `, - }, - { - title: 'Delete empty rows', - name: 'deleteEmptyRows', - group: 'Tools', - description: 'Delete empty rows - rows with all values null or empty string', - type: 'transformRows', - code: ` -return rows.map(row => { - if (cols.find(col => row[col])) return row; - return { - ...row, - __rowStatus: 'deleted', - }; -}) -`, - }, { title: 'Duplicate columns', name: 'duplicateColumns', group: 'Tools', description: 'Duplicate selected columns', - type: 'transformData', + type: 'transformRow', code: ` -const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column)); -const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row)); -const addedColumnNames = selectedColumnNames.map(col => (args.prefix || '') + col + (args.postfix || '')); -const resultRows = rows.map((row, rowIndex) => ({ - ...row, - ...(selectedRowIndexes.includes(rowIndex) ? modules.lodash.fromPairs(selectedColumnNames.map(col => [(args.prefix || '') + col + (args.postfix || ''), row[col]])) : {}), - __insertedFields: addedColumnNames, -})); -const resultCols = [ - ...cols, - ...addedColumnNames, -]; -return { - rows: resultRows, - cols: resultCols, -} + return { + ...row, + ...modules.lodash.fromPairs(columns.map(col=>[(args.prefix || '') + col + (args.postfix || ''), row[col]])) + } `, args: [ { @@ -282,42 +234,27 @@ return { name: 'splitColumns', group: 'Tools', description: 'Split selected columns', - type: 'transformData', + type: 'transformRow', code: ` -const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column)); -const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row)); - -const addedColumnNames = new Set(); - -const resultRows = modules.lodash.cloneDeep(rows); -resultRows.forEach((row, rowIndex) => { - for(const cell of selectedCells) { - if (cell.row == rowIndex && modules.lodash.isString(cell.value)) { - const splitted = cell.value.split(args.delimiter); - splitted.forEach((value, valueIndex) => { - const name = cell.column + '_' + (valueIndex + 1).toString(); - row[name] = value; - addedColumnNames.add(name); - }); - } - } -}); - -const resultCols = [ - ...cols, - ...addedColumnNames, -]; -return { - rows: resultRows, - cols: resultCols, -} + const res = {...row}; + for(const col of columns) { + const value = row[col]; + if (modules.lodash.isString(value)) { + const splitted = value.split(args.delimiter); + splitted.forEach((splitValue, valueIndex) => { + const name = col + '_' + (valueIndex + 1).toString(); + res[name] = splitValue; + }); + } + } + return res; `, args: [ { type: 'text', label: 'Delimiter', name: 'delimiter', - default: ',' + default: ',', }, ], }, @@ -342,53 +279,34 @@ return { name: 'extractDateFields', group: 'Tools', description: 'Extract yaear, month, day and other date/time fields from selection and adds it as new columns', - type: 'transformData', + type: 'transformRow', code: ` -const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column)); -const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row)); -const addedColumnNames = modules.lodash.compact([args.year, args.month, args.day, args.hour, args.minute, args.second]); -const selectedRows = modules.lodash.groupBy(selectedCells, 'row'); -const resultRows = rows.map((row, rowIndex) => { - if (!selectedRowIndexes.includes(rowIndex)) return { - ...row, - __insertedFields: addedColumnNames, - }; - let mom = null; - for(const cell of selectedRows[rowIndex]) { - const m = modules.moment(row[cell.column]); - if (m.isValid()) { - mom = m; - break; - } - } - if (!mom) return { - ...row, - __insertedFields: addedColumnNames, - }; + let mom = null; + for(const col of columns) { + const m = modules.moment(row[col]); + if (m.isValid()) { + mom = m; + break; + } + } - const fields = { - [args.year]: mom.year(), - [args.month]: mom.month() + 1, - [args.day]: mom.day(), - [args.hour]: mom.hour(), - [args.minute]: mom.minute(), - [args.second]: mom.second(), - }; + if (!mom) return row; - return { - ...row, - ...modules.lodash.pick(fields, addedColumnNames), - __insertedFields: addedColumnNames, - } -}); -const resultCols = [ - ...cols, - ...addedColumnNames, -]; -return { - rows: resultRows, - cols: resultCols, -} + const addedColumnNames = modules.lodash.compact([args.year, args.month, args.day, args.hour, args.minute, args.second]); + + const fields = { + [args.year]: mom.year(), + [args.month]: mom.month() + 1, + [args.day]: mom.day(), + [args.hour]: mom.hour(), + [args.minute]: mom.minute(), + [args.second]: mom.second(), + }; + + return { + ...row, + ...modules.lodash.pick(fields, addedColumnNames), + }; `, args: [ {