data duplicator test

integration-tests/__tests__/data-duplicator.spec.js

@@ -0,0 +1,73 @@
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator');
+const fakeObjectReader = require('dbgate-api/src/shell/fakeObjectReader');
+
+const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val varchar(50) null)';
+const t2Sql =
+  'CREATE TABLE t2 (id int not null primary key, val varchar(50) null, valfk int, foreign key (valfk) references t2(id))';
+
+describe('Data duplicator', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Insert simple data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+
+      const t1 = await fakeObjectReader({
+        dynamicData: [
+          { id: 1, val: 'v1' },
+          { id: 2, val: 'v2' },
+          { id: 3, val: 'v3' },
+        ],
+      });
+      const t2 = await fakeObjectReader({
+        dynamicData: [
+          { id: 1, val: 'v1', valfk: 1 },
+          { id: 2, val: 'v2', valfk: 2 },
+          { id: 3, val: 'v3', valfk: 3 },
+        ],
+      });
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't1',
+            operation: 'copy',
+            openStream: () => t1,
+          },
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: () => t2,
+          },
+        ],
+      });
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't1',
+            operation: 'copy',
+            openStream: () => t1,
+          },
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: () => t2,
+          },
+        ],
+      });
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('6');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+});

integration-tests/engines.js

@@ -134,9 +134,9 @@ const engines = [
 const filterLocal = [
   // filter local testing
   '-MySQL',
-  '-MariaDB',
+  'MariaDB',
   '-PostgreSQL',
-  'SQL Server',
+  '-SQL Server',
   '-SQLite',
   '-CockroachDB',
 ];

packages/api/src/shell/dataDuplicator.js

@@ -9,9 +9,18 @@ const copyStream = require('./copyStream');
 const jsonLinesReader = require('./jsonLinesReader');
 const { resolveArchiveFolder } = require('../utility/directories');
 
-async function dataDuplicator({ connection, archive, items, options, analysedStructure = null }) {
+async function dataDuplicator({
-  const driver = requireEngineDriver(connection);
+  connection,
-  const pool = await connectUtility(driver, connection, 'write');
+  archive,
+  items,
+  options,
+  analysedStructure = null,
+  driver,
+  systemConnection,
+}) {
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
+
   logger.info(`Connected.`);
 
   if (!analysedStructure) {

packages/api/src/shell/fakeObjectReader.js

@@ -1,18 +1,26 @@
 const stream = require('stream');
 
-async function fakeObjectReader({ delay = 0 } = {}) {
+async function fakeObjectReader({ delay = 0, dynamicData = null } = {}) {
   const pass = new stream.PassThrough({
     objectMode: true,
   });
   function doWrite() {
-    pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
+    if (dynamicData) {
-    pass.write({ id: 1, country: 'Czechia' });
+      pass.write({ __isStreamHeader: true, __isDynamicStructure: true });
-    pass.write({ id: 2, country: 'Austria' });
+      for (const item of dynamicData) {
-    pass.write({ country: 'Germany', id: 3 });
+        pass.write(item);
-    pass.write({ country: 'Romania', id: 4 });
+      }
-    pass.write({ country: 'Great Britain', id: 5 });
+      pass.end();
-    pass.write({ country: 'Bosna, Hecegovina', id: 6 });
+    } else {
-    pass.end();
+      pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
+      pass.write({ id: 1, country: 'Czechia' });
+      pass.write({ id: 2, country: 'Austria' });
+      pass.write({ country: 'Germany', id: 3 });
+      pass.write({ country: 'Romania', id: 4 });
+      pass.write({ country: 'Great Britain', id: 5 });
+      pass.write({ country: 'Bosna, Hecegovina', id: 6 });
+      pass.end();
+    }
   }
 
   if (delay) {

packages/datalib/src/DataDuplicator.ts

@@ -13,8 +13,8 @@ export interface DataDuplicatorItem {
 }
 
 export interface DataDuplicatorOptions {
-  rollbackAfterFinish: boolean;
+  rollbackAfterFinish?: boolean;
-  skipRowsWithUnresolvedRefs: boolean;
+  skipRowsWithUnresolvedRefs?: boolean;
 }
 
 class DuplicatorReference {
@@ -198,7 +198,7 @@ export class DataDuplicator {
     public items: DataDuplicatorItem[],
     public stream,
     public copyStream: (input, output) => Promise<void>,
-    public options: DataDuplicatorOptions
+    public options: DataDuplicatorOptions = {}
   ) {
     this.itemHolders = items.map(x => new DuplicatorItemHolder(x, this));
     this.itemHolders.forEach(x => x.initializeReferences());