diff --git a/packages/api/src/controllers/archive.js b/packages/api/src/controllers/archive.js index fbde95256..4e61d5fed 100644 --- a/packages/api/src/controllers/archive.js +++ b/packages/api/src/controllers/archive.js @@ -7,6 +7,8 @@ const { saveFreeTableData } = require('../utility/freeTableStorage'); const loadFilesRecursive = require('../utility/loadFilesRecursive'); const getJslFileName = require('../utility/getJslFileName'); const { getLogger } = require('dbgate-tools'); +const uuidv1 = require('uuid/v1'); +const dbgateApi = require('../shell'); const logger = getLogger('archive'); @@ -79,17 +81,20 @@ module.exports = { refreshFiles_meta: true, async refreshFiles({ folder }) { socket.emitChanged('archive-files-changed', { folder }); + return true; }, refreshFolders_meta: true, async refreshFolders() { socket.emitChanged(`archive-folders-changed`); + return true; }, deleteFile_meta: true, async deleteFile({ folder, file, fileType }) { await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`)); socket.emitChanged(`archive-files-changed`, { folder }); + return true; }, renameFile_meta: true, @@ -99,6 +104,19 @@ module.exports = { path.join(resolveArchiveFolder(folder), `${newFile}.${fileType}`) ); socket.emitChanged(`archive-files-changed`, { folder }); + return true; + }, + + saveChangeSet_meta: true, + async saveChangeSet({ folder, file, changeSet }) { + const changedFilePath = path.join(resolveArchiveFolder(folder), `${file}.jsonl`); + const tmpchangedFilePath = path.join(resolveArchiveFolder(folder), `${file}-${uuidv1()}.jsonl`); + const reader = await dbgateApi.changeSetOverJsonLinesReader({ fileName: changedFilePath, changeSet }); + const writer = await dbgateApi.jsonLinesWriter({ fileName: tmpchangedFilePath }); + await dbgateApi.copyStream(reader, writer); + await fs.unlink(changedFilePath); + await fs.rename(path.join(tmpchangedFilePath), path.join(changedFilePath)); + return true; }, renameFolder_meta: true, @@ -106,6 +124,7 @@ module.exports = { const uniqueName = await this.getNewArchiveFolder({ database: newFolder }); await fs.rename(path.join(archivedir(), folder), path.join(archivedir(), uniqueName)); socket.emitChanged(`archive-folders-changed`); + return true; }, deleteFolder_meta: true, @@ -117,6 +136,7 @@ module.exports = { await fs.rmdir(path.join(archivedir(), folder), { recursive: true }); } socket.emitChanged(`archive-folders-changed`); + return true; }, saveFreeTable_meta: true, diff --git a/packages/api/src/shell/changeSetOverJsonLinesReader.js b/packages/api/src/shell/changeSetOverJsonLinesReader.js new file mode 100644 index 000000000..7113a3d53 --- /dev/null +++ b/packages/api/src/shell/changeSetOverJsonLinesReader.js @@ -0,0 +1,69 @@ +const fs = require('fs'); +const stream = require('stream'); +const byline = require('byline'); +const { getLogger } = require('dbgate-tools'); +const logger = getLogger('changeSetOverJsonLinesReader'); + +class ParseStream extends stream.Transform { + constructor({ limitRows, changeSet }) { + super({ objectMode: true }); + this.limitRows = limitRows; + this.changeSet = changeSet; + this.currentRowIndex = 0; + } + _transform(chunk, encoding, done) { + let obj = JSON.parse(chunk); + if (obj.__isStreamHeader) { + this.push(obj); + done(); + return; + } + + if (!this.limitRows || this.currentRowIndex < this.limitRows) { + if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) { + obj = null; + } + + const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex); + if (update) { + obj = { + ...obj, + ...update.fields, + }; + } + + if (obj) { + this.push(obj); + } + this.currentRowIndex += 1; + } + done(); + } + + _flush(done) { + for (const insert of this.changeSet.inserts) { + this.push({ + ...insert.document, + ...insert.fields, + }); + } + done(); + } +} + +async function changeSetOverJsonLinesReader({ + fileName, + encoding = 'utf-8', + limitRows = undefined, + changeSet = { inserts: [], updates: [], deletes: [] }, +}) { + logger.info(`Reading file ${fileName} with change set`); + + const fileStream = fs.createReadStream(fileName, encoding); + const liner = byline(fileStream); + const parser = new ParseStream({ limitRows, changeSet }); + liner.pipe(parser); + return parser; +} + +module.exports = changeSetOverJsonLinesReader; diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index 420a4f65f..d03bc83b4 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -25,6 +25,7 @@ const dumpDatabase = require('./dumpDatabase'); const importDatabase = require('./importDatabase'); const loadDatabase = require('./loadDatabase'); const generateModelSql = require('./generateModelSql'); +const changeSetOverJsonLinesReader = require('./changeSetOverJsonLinesReader'); const dbgateApi = { queryReader, @@ -53,6 +54,7 @@ const dbgateApi = { importDatabase, loadDatabase, generateModelSql, + changeSetOverJsonLinesReader, }; requirePlugin.initializeDbgateApi(dbgateApi); diff --git a/packages/web/src/tabs/ArchiveFileTab.svelte b/packages/web/src/tabs/ArchiveFileTab.svelte index 2c25cff32..1e1be23ad 100644 --- a/packages/web/src/tabs/ArchiveFileTab.svelte +++ b/packages/web/src/tabs/ArchiveFileTab.svelte @@ -1,5 +1,19 @@ @@ -58,5 +92,6 @@ +