more mirroe archive commands

packages/api/src/controllers/archive.js

@@ -108,12 +108,33 @@ module.exports = {
     return true;
   },
 
-  saveChangeSet_meta: true,
+  modifyFile_meta: true,
-  async saveChangeSet({ folder, file, changeSet }) {
+  async modifyFile({ folder, file, changeSet, mergedRows, mergeKey, mergeMode }) {
     await jsldata.closeDataStore(`archive://${folder}/${file}`);
     const changedFilePath = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
+
+    if (!fs.existsSync(changedFilePath)) {
+      if (!mergedRows) {
+        return false;
+      }
+      const fileStream = fs.createWriteStream(changedFilePath);
+      for (const row of mergedRows) {
+        await fileStream.write(JSON.stringify(row) + '\n');
+      }
+      await fileStream.close();
+
+      socket.emitChanged(`archive-files-changed`, { folder });
+      return true;
+    }
+
     const tmpchangedFilePath = path.join(resolveArchiveFolder(folder), `${file}-${uuidv1()}.jsonl`);
-    const reader = await dbgateApi.changeSetOverJsonLinesReader({ fileName: changedFilePath, changeSet });
+    const reader = await dbgateApi.modifyJsonLinesReader({
+      fileName: changedFilePath,
+      changeSet,
+      mergedRows,
+      mergeKey,
+      mergeMode,
+    });
     const writer = await dbgateApi.jsonLinesWriter({ fileName: tmpchangedFilePath });
     await dbgateApi.copyStream(reader, writer);
     await fs.unlink(changedFilePath);

packages/api/src/shell/changeSetOverJsonLinesReader.js

@@ -1,69 +0,0 @@
-const fs = require('fs');
-const stream = require('stream');
-const byline = require('byline');
-const { getLogger } = require('dbgate-tools');
-const logger = getLogger('changeSetOverJsonLinesReader');
-
-class ParseStream extends stream.Transform {
-  constructor({ limitRows, changeSet }) {
-    super({ objectMode: true });
-    this.limitRows = limitRows;
-    this.changeSet = changeSet;
-    this.currentRowIndex = 0;
-  }
-  _transform(chunk, encoding, done) {
-    let obj = JSON.parse(chunk);
-    if (obj.__isStreamHeader) {
-      this.push(obj);
-      done();
-      return;
-    }
-
-    if (!this.limitRows || this.currentRowIndex < this.limitRows) {
-      if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) {
-        obj = null;
-      }
-
-      const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex);
-      if (update) {
-        obj = {
-          ...obj,
-          ...update.fields,
-        };
-      }
-
-      if (obj) {
-        this.push(obj);
-      }
-      this.currentRowIndex += 1;
-    }
-    done();
-  }
-
-  _flush(done) {
-    for (const insert of this.changeSet.inserts) {
-      this.push({
-        ...insert.document,
-        ...insert.fields,
-      });
-    }
-    done();
-  }
-}
-
-async function changeSetOverJsonLinesReader({
-  fileName,
-  encoding = 'utf-8',
-  limitRows = undefined,
-  changeSet = { inserts: [], updates: [], deletes: [] },
-}) {
-  logger.info(`Reading file ${fileName} with change set`);
-
-  const fileStream = fs.createReadStream(fileName, encoding);
-  const liner = byline(fileStream);
-  const parser = new ParseStream({ limitRows, changeSet });
-  liner.pipe(parser);
-  return parser;
-}
-
-module.exports = changeSetOverJsonLinesReader;

packages/api/src/shell/index.js

@@ -25,7 +25,7 @@ const dumpDatabase = require('./dumpDatabase');
 const importDatabase = require('./importDatabase');
 const loadDatabase = require('./loadDatabase');
 const generateModelSql = require('./generateModelSql');
-const changeSetOverJsonLinesReader = require('./changeSetOverJsonLinesReader');
+const modifyJsonLinesReader = require('./modifyJsonLinesReader');
 
 const dbgateApi = {
   queryReader,
@@ -54,7 +54,7 @@ const dbgateApi = {
   importDatabase,
   loadDatabase,
   generateModelSql,
-  changeSetOverJsonLinesReader,
+  modifyJsonLinesReader,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

packages/api/src/shell/modifyJsonLinesReader.js

@@ -0,0 +1,117 @@
+const fs = require('fs');
+const _ = require('lodash');
+const stream = require('stream');
+const byline = require('byline');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('modifyJsonLinesReader');
+const stableStringify = require('json-stable-stringify');
+
+class ParseStream extends stream.Transform {
+  constructor({ limitRows, changeSet, mergedRows, mergeKey, mergeMode }) {
+    super({ objectMode: true });
+    this.limitRows = limitRows;
+    this.changeSet = changeSet;
+    this.currentRowIndex = 0;
+    if (mergeMode == 'merge') {
+      if (mergedRows && mergeKey) {
+        this.mergedRowsDict = {};
+        for (const row of mergedRows) {
+          const key = stableStringify(_.pick(row, mergeKey));
+          this.mergedRowsDict[key] = row;
+        }
+      }
+    }
+    this.mergedRowsArray = mergedRows;
+    this.mergeKey = mergeKey;
+    this.mergeMode = mergeMode;
+  }
+  _transform(chunk, encoding, done) {
+    let obj = JSON.parse(chunk);
+    if (obj.__isStreamHeader) {
+      this.push(obj);
+      done();
+      return;
+    }
+
+    if (this.changeSet) {
+      if (!this.limitRows || this.currentRowIndex < this.limitRows) {
+        if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) {
+          obj = null;
+        }
+
+        const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex);
+        if (update) {
+          obj = {
+            ...obj,
+            ...update.fields,
+          };
+        }
+
+        if (obj) {
+          this.push(obj);
+        }
+        this.currentRowIndex += 1;
+      }
+    } else if (this.mergedRowsArray && this.mergeKey && this.mergeMode) {
+      if (this.mergeMode == 'merge') {
+        const key = stableStringify(_.pick(obj, this.mergeKey));
+        if (this.mergedRowsDict[key]) {
+          this.push({ ...obj, ...this.mergedRowsDict[key] });
+          delete this.mergedRowsDict[key];
+        } else {
+          this.push(obj);
+        }
+      } else if (this.mergeMode == 'append') {
+        this.push(obj);
+      }
+    } else {
+      this.push(obj);
+    }
+    done();
+  }
+
+  _flush(done) {
+    if (this.changeSet) {
+      for (const insert of this.changeSet.inserts) {
+        this.push({
+          ...insert.document,
+          ...insert.fields,
+        });
+      }
+    } else if (this.mergedRowsArray && this.mergeKey) {
+      if (this.mergeMode == 'merge') {
+        for (const row of this.mergedRowsArray) {
+          const key = stableStringify(_.pick(row, this.mergeKey));
+          if (this.mergedRowsDict[key]) {
+            this.push(row);
+          }
+        }
+      } else {
+        for (const row of this.mergedRowsArray) {
+          this.push(row);
+        }
+      }
+    }
+    done();
+  }
+}
+
+async function modifyJsonLinesReader({
+  fileName,
+  encoding = 'utf-8',
+  limitRows = undefined,
+  changeSet = null,
+  mergedRows = null,
+  mergeKey = null,
+  mergeMode = 'merge',
+}) {
+  logger.info(`Reading file ${fileName} with change set`);
+
+  const fileStream = fs.createReadStream(fileName, encoding);
+  const liner = byline(fileStream);
+  const parser = new ParseStream({ limitRows, changeSet, mergedRows, mergeKey, mergeMode });
+  liner.pipe(parser);
+  return parser;
+}
+
+module.exports = modifyJsonLinesReader;

packages/web/src/datagrid/DataGridCore.svelte

@@ -267,6 +267,54 @@
     onClick: () => getCurrentDataGrid().editCellValue(),
   });
 
+  registerCommand({
+    id: 'dataGrid.mergeSelectedCellsIntoMirror',
+    category: 'Data grid',
+    name: 'Merge selected cells',
+    testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true),
+    onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'merge', fullRows: false }),
+  });
+
+  registerCommand({
+    id: 'dataGrid.mergeSelectedRowsIntoMirror',
+    category: 'Data grid',
+    name: 'Merge selected rows',
+    testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true),
+    onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'merge', fullRows: true }),
+  });
+
+  registerCommand({
+    id: 'dataGrid.appendSelectedCellsIntoMirror',
+    category: 'Data grid',
+    name: 'Append selected cells',
+    testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true),
+    onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'append', fullRows: false }),
+  });
+
+  registerCommand({
+    id: 'dataGrid.appendSelectedRowsIntoMirror',
+    category: 'Data grid',
+    name: 'Append selected rows',
+    testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true),
+    onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'append', fullRows: true }),
+  });
+
+  registerCommand({
+    id: 'dataGrid.replaceSelectedCellsIntoMirror',
+    category: 'Data grid',
+    name: 'Replace with selected cells',
+    testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true),
+    onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'replace', fullRows: false }),
+  });
+
+  registerCommand({
+    id: 'dataGrid.replaceSelectedRowsIntoMirror',
+    category: 'Data grid',
+    name: 'Replace with selected rows',
+    testEnabled: () => getCurrentDataGrid()?.mirrorWriteEnabled(true),
+    onClick: () => getCurrentDataGrid().mergeSelectionIntoMirror({ mergeMode: 'replace', fullRows: true }),
+  });
+
   function getSelectedCellsInfo(selectedCells, grider, realColumnUniqueNames, selectedRowData) {
     if (selectedCells.length > 1 && selectedCells.every(x => _.isNumber(x[0]) && _.isNumber(x[1]))) {
       let sum = _.sumBy(selectedCells, cell => {
@@ -295,7 +343,7 @@
   import { GridDisplay } from 'dbgate-datalib';
   import { driverBase, parseCellValue } from 'dbgate-tools';
   import { getContext, onDestroy } from 'svelte';
-  import _ from 'lodash';
+  import _, { map } from 'lodash';
   import registerCommand from '../commands/registerCommand';
   import ColumnHeaderControl from './ColumnHeaderControl.svelte';
   import DataGridRow from './DataGridRow.svelte';
@@ -319,7 +367,7 @@
   import DataFilterControl from './DataFilterControl.svelte';
   import createReducer from '../utility/createReducer';
   import keycodes from '../utility/keycodes';
-  import { copyRowsFormat, selectedCellsCallback } from '../stores';
+  import { copyRowsFormat, currentArchive, selectedCellsCallback } from '../stores';
   import {
     copyRowsFormatDefs,
     copyRowsToClipboard,
@@ -349,6 +397,7 @@
   import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
   import EditCellDataModal, { shouldOpenMultilineDialog } from '../modals/EditCellDataModal.svelte';
   import { getDatabaseInfo, useDatabaseStatus } from '../utility/metadataLoaders';
+  import { showSnackbarSuccess } from '../utility/snackbar';
 
   export let onLoadNextData = undefined;
   export let grider = undefined;
@@ -913,6 +962,34 @@
     });
   }
 
+  export function mirrorWriteEnabled(requireKey) {
+    return requireKey ? !!display.baseTable?.primaryKey || !!display.baseCollection : !!display.baseTableOrSimilar;
+  }
+
+  export async function mergeSelectionIntoMirror({ fullRows, mergeMode = 'merge' }) {
+    const file = display.baseTableOrSimilar?.pureName;
+    const mergeKey = display.baseCollection ? ['_id'] : display.baseTable?.primaryKey.columns.map(x => x.columnName);
+
+    const cells = cellsToRegularCells(selectedCells);
+    const rowIndexes = _.sortBy(_.uniq(cells.map(x => x[0])));
+    const colIndexes = _.sortBy(_.uniq(cells.map(x => x[1])));
+    const rows = rowIndexes.map(rowIndex => grider.getRowData(rowIndex));
+    // @ts-ignore
+    const columns = colIndexes.map(col => realColumnUniqueNames[col]);
+    const mergedRows = fullRows ? rows : rows.map(x => _.pick(x, _.uniq([...columns, ...mergeKey])));
+
+    const res = await apiCall('archive/modify-file', {
+      folder: $currentArchive,
+      file,
+      mergedRows,
+      mergeKey,
+      mergeMode,
+    });
+    if (res) {
+      showSnackbarSuccess(`Merged ${mergedRows.length} rows into ${file} in archive ${$currentArchive}`);
+    }
+  }
+
   $: autofillMarkerCell =
     selectedCells && selectedCells.length > 0 && _.uniq(selectedCells.map(x => x[0])).length == 1
       ? [_.max(selectedCells.map(x => x[0])), _.max(selectedCells.map(x => x[1]))]
@@ -1622,6 +1699,17 @@
     // { command: 'dataGrid.copyJsonDocument', hideDisabled: true },
     { divider: true },
     { placeTag: 'export' },
+    {
+      label: 'Archive mirror',
+      submenu: [
+        { command: 'dataGrid.mergeSelectedCellsIntoMirror' },
+        { command: 'dataGrid.mergeSelectedRowsIntoMirror' },
+        { command: 'dataGrid.appendSelectedCellsIntoMirror' },
+        { command: 'dataGrid.appendSelectedRowsIntoMirror' },
+        { command: 'dataGrid.replaceSelectedCellsIntoMirror' },
+        { command: 'dataGrid.replaceSelectedRowsIntoMirror' },
+      ],
+    },
     { command: 'dataGrid.generateSqlFromData' },
     { command: 'dataGrid.openFreeTable' },
     { command: 'dataGrid.openChartFromSelection' },

packages/web/src/tabs/ArchiveFileTab.svelte

@@ -68,7 +68,7 @@
   }
 
   export async function save() {
-    await apiCall('archive/save-change-set', {
+    await apiCall('archive/modify-file', {
       folder: archiveFolder,
       file: archiveFile,
       changeSet: $changeSetStore.value,
@@ -88,6 +88,7 @@
     jslid={jslid || `archive://${archiveFolder}/${archiveFile}`}
     supportsReload
     changeSetState={$changeSetStore}
+    focusOnVisible
     {changeSetStore}
     {dispatchChangeSet}
   />