save archive algorithm

packages/api/src/controllers/archive.js

@@ -7,6 +7,8 @@ const { saveFreeTableData } = require('../utility/freeTableStorage');
 const loadFilesRecursive = require('../utility/loadFilesRecursive');
 const getJslFileName = require('../utility/getJslFileName');
 const { getLogger } = require('dbgate-tools');
+const uuidv1 = require('uuid/v1');
+const dbgateApi = require('../shell');
 
 const logger = getLogger('archive');
 
@@ -79,17 +81,20 @@ module.exports = {
   refreshFiles_meta: true,
   async refreshFiles({ folder }) {
     socket.emitChanged('archive-files-changed', { folder });
+    return true;
   },
 
   refreshFolders_meta: true,
   async refreshFolders() {
     socket.emitChanged(`archive-folders-changed`);
+    return true;
   },
 
   deleteFile_meta: true,
   async deleteFile({ folder, file, fileType }) {
     await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
     socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
   },
 
   renameFile_meta: true,
@@ -99,6 +104,19 @@ module.exports = {
       path.join(resolveArchiveFolder(folder), `${newFile}.${fileType}`)
     );
     socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
+  saveChangeSet_meta: true,
+  async saveChangeSet({ folder, file, changeSet }) {
+    const changedFilePath = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
+    const tmpchangedFilePath = path.join(resolveArchiveFolder(folder), `${file}-${uuidv1()}.jsonl`);
+    const reader = await dbgateApi.changeSetOverJsonLinesReader({ fileName: changedFilePath, changeSet });
+    const writer = await dbgateApi.jsonLinesWriter({ fileName: tmpchangedFilePath });
+    await dbgateApi.copyStream(reader, writer);
+    await fs.unlink(changedFilePath);
+    await fs.rename(path.join(tmpchangedFilePath), path.join(changedFilePath));
+    return true;
   },
 
   renameFolder_meta: true,
@@ -106,6 +124,7 @@ module.exports = {
     const uniqueName = await this.getNewArchiveFolder({ database: newFolder });
     await fs.rename(path.join(archivedir(), folder), path.join(archivedir(), uniqueName));
     socket.emitChanged(`archive-folders-changed`);
+    return true;
   },
 
   deleteFolder_meta: true,
@@ -117,6 +136,7 @@ module.exports = {
       await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
     }
     socket.emitChanged(`archive-folders-changed`);
+    return true;
   },
 
   saveFreeTable_meta: true,

packages/api/src/shell/changeSetOverJsonLinesReader.js

@@ -0,0 +1,69 @@
+const fs = require('fs');
+const stream = require('stream');
+const byline = require('byline');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('changeSetOverJsonLinesReader');
+
+class ParseStream extends stream.Transform {
+  constructor({ limitRows, changeSet }) {
+    super({ objectMode: true });
+    this.limitRows = limitRows;
+    this.changeSet = changeSet;
+    this.currentRowIndex = 0;
+  }
+  _transform(chunk, encoding, done) {
+    let obj = JSON.parse(chunk);
+    if (obj.__isStreamHeader) {
+      this.push(obj);
+      done();
+      return;
+    }
+
+    if (!this.limitRows || this.currentRowIndex < this.limitRows) {
+      if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) {
+        obj = null;
+      }
+
+      const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex);
+      if (update) {
+        obj = {
+          ...obj,
+          ...update.fields,
+        };
+      }
+
+      if (obj) {
+        this.push(obj);
+      }
+      this.currentRowIndex += 1;
+    }
+    done();
+  }
+
+  _flush(done) {
+    for (const insert of this.changeSet.inserts) {
+      this.push({
+        ...insert.document,
+        ...insert.fields,
+      });
+    }
+    done();
+  }
+}
+
+async function changeSetOverJsonLinesReader({
+  fileName,
+  encoding = 'utf-8',
+  limitRows = undefined,
+  changeSet = { inserts: [], updates: [], deletes: [] },
+}) {
+  logger.info(`Reading file ${fileName} with change set`);
+
+  const fileStream = fs.createReadStream(fileName, encoding);
+  const liner = byline(fileStream);
+  const parser = new ParseStream({ limitRows, changeSet });
+  liner.pipe(parser);
+  return parser;
+}
+
+module.exports = changeSetOverJsonLinesReader;

packages/api/src/shell/index.js

@@ -25,6 +25,7 @@ const dumpDatabase = require('./dumpDatabase');
 const importDatabase = require('./importDatabase');
 const loadDatabase = require('./loadDatabase');
 const generateModelSql = require('./generateModelSql');
+const changeSetOverJsonLinesReader = require('./changeSetOverJsonLinesReader');
 
 const dbgateApi = {
   queryReader,
@@ -53,6 +54,7 @@ const dbgateApi = {
   importDatabase,
   loadDatabase,
   generateModelSql,
+  changeSetOverJsonLinesReader,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

packages/web/src/tabs/ArchiveFileTab.svelte

@@ -1,5 +1,19 @@
 <script lang="ts" context="module">
   export const matchingProps = ['archiveFile', 'archiveFolder', 'jslid'];
+
+  const getCurrentEditor = () => getActiveComponent('ArchiveFileTab');
+
+  registerCommand({
+    id: 'archiveFile.save',
+    group: 'save',
+    category: 'Archive file',
+    name: 'Save',
+    toolbar: true,
+    isRelatedToTab: true,
+    icon: 'icon save',
+    testEnabled: () => getCurrentEditor()?.canSave(),
+    onClick: () => getCurrentEditor().save(),
+  });
 </script>
 
 <script lang="ts">
@@ -10,12 +24,18 @@
   import ToolStripContainer from '../buttons/ToolStripContainer.svelte';
   import ToolStripExportButton, { createQuickExportHandlerRef } from '../buttons/ToolStripExportButton.svelte';
   import invalidateCommands from '../commands/invalidateCommands';
+  import registerCommand from '../commands/registerCommand';
+  import runCommand from '../commands/runCommand';
 
   import JslDataGrid from '../datagrid/JslDataGrid.svelte';
   import useEditorData from '../query/useEditorData';
+  import { apiCall } from '../utility/api';
   import { markTabSaved, markTabUnsaved } from '../utility/common';
+  import createActivator, { getActiveComponent } from '../utility/createActivator';
   import createUndoReducer from '../utility/createUndoReducer';
 
+  export const activator = createActivator('ArchiveFileTab', true);
+
   export let archiveFolder = undefined;
   export let archiveFile = undefined;
   export let jslid = undefined;
@@ -45,6 +65,20 @@
       markTabSaved(tabid);
     }
   }
+
+  export async function save() {
+    await apiCall('archive/save-change-set', {
+      folder: archiveFolder,
+      file: archiveFile,
+      changeSet: $changeSetStore.value,
+    });
+    dispatchChangeSet({ type: 'reset', value: createChangeSet() });
+    runCommand('dataGrid.refresh');
+  }
+
+  export function canSave() {
+    return changeSetContainsChanges($changeSetStore?.value);
+  }
 </script>
 
 <ToolStripContainer>
@@ -58,5 +92,6 @@
   <svelte:fragment slot="toolstrip">
     <ToolStripCommandButton command="dataGrid.refresh" />
     <ToolStripExportButton command="jslTableGrid.export" {quickExportHandlerRef} />
+    <ToolStripCommandButton command="archiveFile.save" />
   </svelte:fragment>
 </ToolStripContainer>