save archive algorithm

packages/api/src/controllers/archive.js

@@ -7,6 +7,8 @@ const { saveFreeTableData } = require('../utility/freeTableStorage');
 const loadFilesRecursive = require('../utility/loadFilesRecursive');
 const getJslFileName = require('../utility/getJslFileName');
 const { getLogger } = require('dbgate-tools');
+const uuidv1 = require('uuid/v1');
+const dbgateApi = require('../shell');
 
 const logger = getLogger('archive');
 
@@ -79,17 +81,20 @@ module.exports = {
   refreshFiles_meta: true,
   async refreshFiles({ folder }) {
     socket.emitChanged('archive-files-changed', { folder });
+    return true;
   },
 
   refreshFolders_meta: true,
   async refreshFolders() {
     socket.emitChanged(`archive-folders-changed`);
+    return true;
   },
 
   deleteFile_meta: true,
   async deleteFile({ folder, file, fileType }) {
     await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
     socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
   },
 
   renameFile_meta: true,
@@ -99,6 +104,19 @@ module.exports = {
       path.join(resolveArchiveFolder(folder), `${newFile}.${fileType}`)
     );
     socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
+  saveChangeSet_meta: true,
+  async saveChangeSet({ folder, file, changeSet }) {
+    const changedFilePath = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
+    const tmpchangedFilePath = path.join(resolveArchiveFolder(folder), `${file}-${uuidv1()}.jsonl`);
+    const reader = await dbgateApi.changeSetOverJsonLinesReader({ fileName: changedFilePath, changeSet });
+    const writer = await dbgateApi.jsonLinesWriter({ fileName: tmpchangedFilePath });
+    await dbgateApi.copyStream(reader, writer);
+    await fs.unlink(changedFilePath);
+    await fs.rename(path.join(tmpchangedFilePath), path.join(changedFilePath));
+    return true;
   },
 
   renameFolder_meta: true,
@@ -106,6 +124,7 @@ module.exports = {
     const uniqueName = await this.getNewArchiveFolder({ database: newFolder });
     await fs.rename(path.join(archivedir(), folder), path.join(archivedir(), uniqueName));
     socket.emitChanged(`archive-folders-changed`);
+    return true;
   },
 
   deleteFolder_meta: true,
@@ -117,6 +136,7 @@ module.exports = {
       await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
     }
     socket.emitChanged(`archive-folders-changed`);
+    return true;
   },
 
   saveFreeTable_meta: true,

packages/api/src/shell/changeSetOverJsonLinesReader.js

@@ -0,0 +1,69 @@
+const fs = require('fs');
+const stream = require('stream');
+const byline = require('byline');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('changeSetOverJsonLinesReader');
+
+class ParseStream extends stream.Transform {
+  constructor({ limitRows, changeSet }) {
+    super({ objectMode: true });
+    this.limitRows = limitRows;
+    this.changeSet = changeSet;
+    this.currentRowIndex = 0;
+  }
+  _transform(chunk, encoding, done) {
+    let obj = JSON.parse(chunk);
+    if (obj.__isStreamHeader) {
+      this.push(obj);
+      done();
+      return;
+    }
+
+    if (!this.limitRows || this.currentRowIndex < this.limitRows) {
+      if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) {
+        obj = null;
+      }
+
+      const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex);
+      if (update) {
+        obj = {
+          ...obj,
+          ...update.fields,
+        };
+      }
+
+      if (obj) {
+        this.push(obj);
+      }
+      this.currentRowIndex += 1;
+    }
+    done();
+  }
+
+  _flush(done) {
+    for (const insert of this.changeSet.inserts) {
+      this.push({
+        ...insert.document,
+        ...insert.fields,
+      });
+    }
+    done();
+  }
+}
+
+async function changeSetOverJsonLinesReader({
+  fileName,
+  encoding = 'utf-8',
+  limitRows = undefined,
+  changeSet = { inserts: [], updates: [], deletes: [] },
+}) {
+  logger.info(`Reading file ${fileName} with change set`);
+
+  const fileStream = fs.createReadStream(fileName, encoding);
+  const liner = byline(fileStream);
+  const parser = new ParseStream({ limitRows, changeSet });
+  liner.pipe(parser);
+  return parser;
+}
+
+module.exports = changeSetOverJsonLinesReader;

packages/api/src/shell/index.js

@@ -25,6 +25,7 @@ const dumpDatabase = require('./dumpDatabase');
 const importDatabase = require('./importDatabase');
 const loadDatabase = require('./loadDatabase');
 const generateModelSql = require('./generateModelSql');
+const changeSetOverJsonLinesReader = require('./changeSetOverJsonLinesReader');
 
 const dbgateApi = {
   queryReader,
@@ -53,6 +54,7 @@ const dbgateApi = {
   importDatabase,
   loadDatabase,
   generateModelSql,
+  changeSetOverJsonLinesReader,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);