diff --git a/packages/api/src/controllers/archive.js b/packages/api/src/controllers/archive.js index 8beb16330..e520871e5 100644 --- a/packages/api/src/controllers/archive.js +++ b/packages/api/src/controllers/archive.js @@ -108,12 +108,33 @@ module.exports = { return true; }, - saveChangeSet_meta: true, - async saveChangeSet({ folder, file, changeSet }) { + modifyFile_meta: true, + async modifyFile({ folder, file, changeSet, mergedRows, mergeKey, mergeMode }) { await jsldata.closeDataStore(`archive://${folder}/${file}`); const changedFilePath = path.join(resolveArchiveFolder(folder), `${file}.jsonl`); + + if (!fs.existsSync(changedFilePath)) { + if (!mergedRows) { + return false; + } + const fileStream = fs.createWriteStream(changedFilePath); + for (const row of mergedRows) { + await fileStream.write(JSON.stringify(row) + '\n'); + } + await fileStream.close(); + + socket.emitChanged(`archive-files-changed`, { folder }); + return true; + } + const tmpchangedFilePath = path.join(resolveArchiveFolder(folder), `${file}-${uuidv1()}.jsonl`); - const reader = await dbgateApi.changeSetOverJsonLinesReader({ fileName: changedFilePath, changeSet }); + const reader = await dbgateApi.modifyJsonLinesReader({ + fileName: changedFilePath, + changeSet, + mergedRows, + mergeKey, + mergeMode, + }); const writer = await dbgateApi.jsonLinesWriter({ fileName: tmpchangedFilePath }); await dbgateApi.copyStream(reader, writer); await fs.unlink(changedFilePath); diff --git a/packages/api/src/shell/changeSetOverJsonLinesReader.js b/packages/api/src/shell/changeSetOverJsonLinesReader.js deleted file mode 100644 index 7113a3d53..000000000 --- a/packages/api/src/shell/changeSetOverJsonLinesReader.js +++ /dev/null @@ -1,69 +0,0 @@ -const fs = require('fs'); -const stream = require('stream'); -const byline = require('byline'); -const { getLogger } = require('dbgate-tools'); -const logger = getLogger('changeSetOverJsonLinesReader'); - -class ParseStream extends stream.Transform { - constructor({ limitRows, changeSet }) { - super({ objectMode: true }); - this.limitRows = limitRows; - this.changeSet = changeSet; - this.currentRowIndex = 0; - } - _transform(chunk, encoding, done) { - let obj = JSON.parse(chunk); - if (obj.__isStreamHeader) { - this.push(obj); - done(); - return; - } - - if (!this.limitRows || this.currentRowIndex < this.limitRows) { - if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) { - obj = null; - } - - const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex); - if (update) { - obj = { - ...obj, - ...update.fields, - }; - } - - if (obj) { - this.push(obj); - } - this.currentRowIndex += 1; - } - done(); - } - - _flush(done) { - for (const insert of this.changeSet.inserts) { - this.push({ - ...insert.document, - ...insert.fields, - }); - } - done(); - } -} - -async function changeSetOverJsonLinesReader({ - fileName, - encoding = 'utf-8', - limitRows = undefined, - changeSet = { inserts: [], updates: [], deletes: [] }, -}) { - logger.info(`Reading file ${fileName} with change set`); - - const fileStream = fs.createReadStream(fileName, encoding); - const liner = byline(fileStream); - const parser = new ParseStream({ limitRows, changeSet }); - liner.pipe(parser); - return parser; -} - -module.exports = changeSetOverJsonLinesReader; diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index d03bc83b4..31fd6ac1a 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -25,7 +25,7 @@ const dumpDatabase = require('./dumpDatabase'); const importDatabase = require('./importDatabase'); const loadDatabase = require('./loadDatabase'); const generateModelSql = require('./generateModelSql'); -const changeSetOverJsonLinesReader = require('./changeSetOverJsonLinesReader'); +const modifyJsonLinesReader = require('./modifyJsonLinesReader'); const dbgateApi = { queryReader, @@ -54,7 +54,7 @@ const dbgateApi = { importDatabase, loadDatabase, generateModelSql, - changeSetOverJsonLinesReader, + modifyJsonLinesReader, }; requirePlugin.initializeDbgateApi(dbgateApi); diff --git a/packages/api/src/shell/modifyJsonLinesReader.js b/packages/api/src/shell/modifyJsonLinesReader.js new file mode 100644 index 000000000..5f1e73444 --- /dev/null +++ b/packages/api/src/shell/modifyJsonLinesReader.js @@ -0,0 +1,117 @@ +const fs = require('fs'); +const _ = require('lodash'); +const stream = require('stream'); +const byline = require('byline'); +const { getLogger } = require('dbgate-tools'); +const logger = getLogger('modifyJsonLinesReader'); +const stableStringify = require('json-stable-stringify'); + +class ParseStream extends stream.Transform { + constructor({ limitRows, changeSet, mergedRows, mergeKey, mergeMode }) { + super({ objectMode: true }); + this.limitRows = limitRows; + this.changeSet = changeSet; + this.currentRowIndex = 0; + if (mergeMode == 'merge') { + if (mergedRows && mergeKey) { + this.mergedRowsDict = {}; + for (const row of mergedRows) { + const key = stableStringify(_.pick(row, mergeKey)); + this.mergedRowsDict[key] = row; + } + } + } + this.mergedRowsArray = mergedRows; + this.mergeKey = mergeKey; + this.mergeMode = mergeMode; + } + _transform(chunk, encoding, done) { + let obj = JSON.parse(chunk); + if (obj.__isStreamHeader) { + this.push(obj); + done(); + return; + } + + if (this.changeSet) { + if (!this.limitRows || this.currentRowIndex < this.limitRows) { + if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) { + obj = null; + } + + const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex); + if (update) { + obj = { + ...obj, + ...update.fields, + }; + } + + if (obj) { + this.push(obj); + } + this.currentRowIndex += 1; + } + } else if (this.mergedRowsArray && this.mergeKey && this.mergeMode) { + if (this.mergeMode == 'merge') { + const key = stableStringify(_.pick(obj, this.mergeKey)); + if (this.mergedRowsDict[key]) { + this.push({ ...obj, ...this.mergedRowsDict[key] }); + delete this.mergedRowsDict[key]; + } else { + this.push(obj); + } + } else if (this.mergeMode == 'append') { + this.push(obj); + } + } else { + this.push(obj); + } + done(); + } + + _flush(done) { + if (this.changeSet) { + for (const insert of this.changeSet.inserts) { + this.push({ + ...insert.document, + ...insert.fields, + }); + } + } else if (this.mergedRowsArray && this.mergeKey) { + if (this.mergeMode == 'merge') { + for (const row of this.mergedRowsArray) { + const key = stableStringify(_.pick(row, this.mergeKey)); + if (this.mergedRowsDict[key]) { + this.push(row); + } + } + } else { + for (const row of this.mergedRowsArray) { + this.push(row); + } + } + } + done(); + } +} + +async function modifyJsonLinesReader({ + fileName, + encoding = 'utf-8', + limitRows = undefined, + changeSet = null, + mergedRows = null, + mergeKey = null, + mergeMode = 'merge', +}) { + logger.info(`Reading file ${fileName} with change set`); + + const fileStream = fs.createReadStream(fileName, encoding); + const liner = byline(fileStream); + const parser = new ParseStream({ limitRows, changeSet, mergedRows, mergeKey, mergeMode }); + liner.pipe(parser); + return parser; +} + +module.exports = modifyJsonLinesReader; diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index 6eba775a5..bd016e309 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -267,6 +267,54 @@ onClick: () => getCurrentDataGrid().editCellValue(), }); + registerCommand({ + id: 'dataGrid.mergeSelectedCellsIntoMirror', + category: 'Data grid', + name: 'Merge selected cells', + testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true), + onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'merge', fullRows: false }), + }); + + registerCommand({ + id: 'dataGrid.mergeSelectedRowsIntoMirror', + category: 'Data grid', + name: 'Merge selected rows', + testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true), + onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'merge', fullRows: true }), + }); + + registerCommand({ + id: 'dataGrid.appendSelectedCellsIntoMirror', + category: 'Data grid', + name: 'Append selected cells', + testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true), + onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'append', fullRows: false }), + }); + + registerCommand({ + id: 'dataGrid.appendSelectedRowsIntoMirror', + category: 'Data grid', + name: 'Append selected rows', + testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true), + onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'append', fullRows: true }), + }); + + registerCommand({ + id: 'dataGrid.replaceSelectedCellsIntoMirror', + category: 'Data grid', + name: 'Replace with selected cells', + testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true), + onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'replace', fullRows: false }), + }); + + registerCommand({ + id: 'dataGrid.replaceSelectedRowsIntoMirror', + category: 'Data grid', + name: 'Replace with selected rows', + testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true), + onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'replace', fullRows: true }), + }); + function getSelectedCellsInfo(selectedCells, grider, realColumnUniqueNames, selectedRowData) { if (selectedCells.length > 1 && selectedCells.every(x => _.isNumber(x[0]) && _.isNumber(x[1]))) { let sum = _.sumBy(selectedCells, cell => { @@ -295,7 +343,7 @@ import { GridDisplay } from 'dbgate-datalib'; import { driverBase, parseCellValue } from 'dbgate-tools'; import { getContext, onDestroy } from 'svelte'; - import _ from 'lodash'; + import _, { map } from 'lodash'; import registerCommand from '../commands/registerCommand'; import ColumnHeaderControl from './ColumnHeaderControl.svelte'; import DataGridRow from './DataGridRow.svelte'; @@ -319,7 +367,7 @@ import DataFilterControl from './DataFilterControl.svelte'; import createReducer from '../utility/createReducer'; import keycodes from '../utility/keycodes'; - import { copyRowsFormat, selectedCellsCallback } from '../stores'; + import { copyRowsFormat, currentArchive, selectedCellsCallback } from '../stores'; import { copyRowsFormatDefs, copyRowsToClipboard, @@ -349,6 +397,7 @@ import ErrorMessageModal from '../modals/ErrorMessageModal.svelte'; import EditCellDataModal, { shouldOpenMultilineDialog } from '../modals/EditCellDataModal.svelte'; import { getDatabaseInfo, useDatabaseStatus } from '../utility/metadataLoaders'; + import { showSnackbarSuccess } from '../utility/snackbar'; export let onLoadNextData = undefined; export let grider = undefined; @@ -913,6 +962,34 @@ }); } + export function mirrorWriteEnabled(requireKey) { + return requireKey ? !!display.baseTable?.primaryKey || !!display.baseCollection : !!display.baseTableOrSimilar; + } + + export async function mergeSelectionIntoMirror({ fullRows, mergeMode = 'merge' }) { + const file = display.baseTableOrSimilar?.pureName; + const mergeKey = display.baseCollection ? ['_id'] : display.baseTable?.primaryKey.columns.map(x => x.columnName); + + const cells = cellsToRegularCells(selectedCells); + const rowIndexes = _.sortBy(_.uniq(cells.map(x => x[0]))); + const colIndexes = _.sortBy(_.uniq(cells.map(x => x[1]))); + const rows = rowIndexes.map(rowIndex => grider.getRowData(rowIndex)); + // @ts-ignore + const columns = colIndexes.map(col => realColumnUniqueNames[col]); + const mergedRows = fullRows ? rows : rows.map(x => _.pick(x, _.uniq([...columns, ...mergeKey]))); + + const res = await apiCall('archive/modify-file', { + folder: $currentArchive, + file, + mergedRows, + mergeKey, + mergeMode, + }); + if (res) { + showSnackbarSuccess(`Merged ${mergedRows.length} rows into ${file} in archive ${$currentArchive}`); + } + } + $: autofillMarkerCell = selectedCells && selectedCells.length > 0 && _.uniq(selectedCells.map(x => x[0])).length == 1 ? [_.max(selectedCells.map(x => x[0])), _.max(selectedCells.map(x => x[1]))] @@ -1622,6 +1699,17 @@ // { command: 'dataGrid.copyJsonDocument', hideDisabled: true }, { divider: true }, { placeTag: 'export' }, + { + label: 'Archive mirror', + submenu: [ + { command: 'dataGrid.mergeSelectedCellsIntoMirror' }, + { command: 'dataGrid.mergeSelectedRowsIntoMirror' }, + { command: 'dataGrid.appendSelectedCellsIntoMirror' }, + { command: 'dataGrid.appendSelectedRowsIntoMirror' }, + { command: 'dataGrid.replaceSelectedCellsIntoMirror' }, + { command: 'dataGrid.replaceSelectedRowsIntoMirror' }, + ], + }, { command: 'dataGrid.generateSqlFromData' }, { command: 'dataGrid.openFreeTable' }, { command: 'dataGrid.openChartFromSelection' }, diff --git a/packages/web/src/tabs/ArchiveFileTab.svelte b/packages/web/src/tabs/ArchiveFileTab.svelte index 5f43193bc..868404c26 100644 --- a/packages/web/src/tabs/ArchiveFileTab.svelte +++ b/packages/web/src/tabs/ArchiveFileTab.svelte @@ -68,7 +68,7 @@ } export async function save() { - await apiCall('archive/save-change-set', { + await apiCall('archive/modify-file', { folder: archiveFolder, file: archiveFile, changeSet: $changeSetStore.value, @@ -88,6 +88,7 @@ jslid={jslid || `archive://${archiveFolder}/${archiveFile}`} supportsReload changeSetState={$changeSetStore} + focusOnVisible {changeSetStore} {dispatchChangeSet} />