data duplicator works in simple case

packages/api/src/shell/dataDuplicator.js

@@ -0,0 +1,38 @@
+const stream = require('stream');
+const path = require('path');
+const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const logger = getLogger('dataDuplicator');
+const { DataDuplicator } = require('dbgate-datalib');
+const copyStream = require('./copyStream');
+const jsonLinesReader = require('./jsonLinesReader');
+const { resolveArchiveFolder } = require('../utility/directories');
+
+async function dataDuplicator({ connection, archive, items, analysedStructure = null }) {
+  const driver = requireEngineDriver(connection);
+  const pool = await connectUtility(driver, connection, 'write');
+  logger.info(`Connected.`);
+
+  if (!analysedStructure) {
+    analysedStructure = await driver.analyseFull(pool);
+  }
+
+  const dupl = new DataDuplicator(
+    pool,
+    driver,
+    analysedStructure,
+    items.map(item => ({
+      name: item.name,
+      operation: item.operation,
+      matchColumns: item.matchColumns,
+      openStream: () => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) }),
+    })),
+    stream,
+    copyStream
+  );
+
+  await dupl.run();
+}
+
+module.exports = dataDuplicator;

packages/api/src/shell/index.js

@@ -26,6 +26,7 @@ const importDatabase = require('./importDatabase');
 const loadDatabase = require('./loadDatabase');
 const generateModelSql = require('./generateModelSql');
 const modifyJsonLinesReader = require('./modifyJsonLinesReader');
+const dataDuplicator = require('./dataDuplicator');
 
 const dbgateApi = {
   queryReader,
@@ -55,6 +56,7 @@ const dbgateApi = {
   loadDatabase,
   generateModelSql,
   modifyJsonLinesReader,
+  dataDuplicator,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

packages/api/src/shell/jsonLinesReader.js

@@ -35,7 +35,11 @@ class ParseStream extends stream.Transform {
 async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
   logger.info(`Reading file ${fileName}`);
 
-  const fileStream = fs.createReadStream(fileName, encoding);
+  const fileStream = fs.createReadStream(
+    fileName,
+    // @ts-ignore
+    encoding
+  );
   const liner = byline(fileStream);
   const parser = new ParseStream({ limitRows });
   liner.pipe(parser);

packages/api/src/shell/modifyJsonLinesReader.js

@@ -107,7 +107,11 @@ async function modifyJsonLinesReader({
 }) {
   logger.info(`Reading file ${fileName} with change set`);
 
-  const fileStream = fs.createReadStream(fileName, encoding);
+  const fileStream = fs.createReadStream(
+    fileName,
+    // @ts-ignore
+    encoding
+  );
   const liner = byline(fileStream);
   const parser = new ParseStream({ limitRows, changeSet, mergedRows, mergeKey, mergeMode });
   liner.pipe(parser);