From b6c5f26eb47a2706c7424f10d2dbcbb7bda8aabf Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 17 Feb 2023 09:15:13 +0100 Subject: [PATCH] data duplicator test --- .../__tests__/data-duplicator.spec.js | 73 +++++++++++++++++++ integration-tests/engines.js | 4 +- packages/api/src/shell/dataDuplicator.js | 15 +++- packages/api/src/shell/fakeObjectReader.js | 26 ++++--- packages/datalib/src/DataDuplicator.ts | 6 +- 5 files changed, 107 insertions(+), 17 deletions(-) create mode 100644 integration-tests/__tests__/data-duplicator.spec.js diff --git a/integration-tests/__tests__/data-duplicator.spec.js b/integration-tests/__tests__/data-duplicator.spec.js new file mode 100644 index 000000000..08992645a --- /dev/null +++ b/integration-tests/__tests__/data-duplicator.spec.js @@ -0,0 +1,73 @@ +const engines = require('../engines'); +const { testWrapper } = require('../tools'); +const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator'); +const fakeObjectReader = require('dbgate-api/src/shell/fakeObjectReader'); + +const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val varchar(50) null)'; +const t2Sql = + 'CREATE TABLE t2 (id int not null primary key, val varchar(50) null, valfk int, foreign key (valfk) references t2(id))'; + +describe('Data duplicator', () => { + test.each(engines.map(engine => [engine.label, engine]))( + 'Insert simple data - %s', + testWrapper(async (conn, driver, engine) => { + await driver.query(conn, t1Sql); + await driver.query(conn, t2Sql); + + const t1 = await fakeObjectReader({ + dynamicData: [ + { id: 1, val: 'v1' }, + { id: 2, val: 'v2' }, + { id: 3, val: 'v3' }, + ], + }); + const t2 = await fakeObjectReader({ + dynamicData: [ + { id: 1, val: 'v1', valfk: 1 }, + { id: 2, val: 'v2', valfk: 2 }, + { id: 3, val: 'v3', valfk: 3 }, + ], + }); + + await dataDuplicator({ + systemConnection: conn, + driver, + items: [ + { + name: 't1', + operation: 'copy', + openStream: () => t1, + }, + { + name: 't2', + operation: 'copy', + openStream: () => t2, + }, + ], + }); + + await dataDuplicator({ + systemConnection: conn, + driver, + items: [ + { + name: 't1', + operation: 'copy', + openStream: () => t1, + }, + { + name: 't2', + operation: 'copy', + openStream: () => t2, + }, + ], + }); + + const res1 = await driver.query(conn, `select count(*) as cnt from t1`); + expect(res1.rows[0].cnt.toString()).toEqual('6'); + + const res2 = await driver.query(conn, `select count(*) as cnt from t2`); + expect(res2.rows[0].cnt.toString()).toEqual('6'); + }) + ); +}); diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 8f10950d5..41e0345aa 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -134,9 +134,9 @@ const engines = [ const filterLocal = [ // filter local testing '-MySQL', - '-MariaDB', + 'MariaDB', '-PostgreSQL', - 'SQL Server', + '-SQL Server', '-SQLite', '-CockroachDB', ]; diff --git a/packages/api/src/shell/dataDuplicator.js b/packages/api/src/shell/dataDuplicator.js index dedc37d01..702d3a7b8 100644 --- a/packages/api/src/shell/dataDuplicator.js +++ b/packages/api/src/shell/dataDuplicator.js @@ -9,9 +9,18 @@ const copyStream = require('./copyStream'); const jsonLinesReader = require('./jsonLinesReader'); const { resolveArchiveFolder } = require('../utility/directories'); -async function dataDuplicator({ connection, archive, items, options, analysedStructure = null }) { - const driver = requireEngineDriver(connection); - const pool = await connectUtility(driver, connection, 'write'); +async function dataDuplicator({ + connection, + archive, + items, + options, + analysedStructure = null, + driver, + systemConnection, +}) { + if (!driver) driver = requireEngineDriver(connection); + const pool = systemConnection || (await connectUtility(driver, connection, 'write')); + logger.info(`Connected.`); if (!analysedStructure) { diff --git a/packages/api/src/shell/fakeObjectReader.js b/packages/api/src/shell/fakeObjectReader.js index 18e2d70a4..d1bdb593e 100644 --- a/packages/api/src/shell/fakeObjectReader.js +++ b/packages/api/src/shell/fakeObjectReader.js @@ -1,18 +1,26 @@ const stream = require('stream'); -async function fakeObjectReader({ delay = 0 } = {}) { +async function fakeObjectReader({ delay = 0, dynamicData = null } = {}) { const pass = new stream.PassThrough({ objectMode: true, }); function doWrite() { - pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true }); - pass.write({ id: 1, country: 'Czechia' }); - pass.write({ id: 2, country: 'Austria' }); - pass.write({ country: 'Germany', id: 3 }); - pass.write({ country: 'Romania', id: 4 }); - pass.write({ country: 'Great Britain', id: 5 }); - pass.write({ country: 'Bosna, Hecegovina', id: 6 }); - pass.end(); + if (dynamicData) { + pass.write({ __isStreamHeader: true, __isDynamicStructure: true }); + for (const item of dynamicData) { + pass.write(item); + } + pass.end(); + } else { + pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true }); + pass.write({ id: 1, country: 'Czechia' }); + pass.write({ id: 2, country: 'Austria' }); + pass.write({ country: 'Germany', id: 3 }); + pass.write({ country: 'Romania', id: 4 }); + pass.write({ country: 'Great Britain', id: 5 }); + pass.write({ country: 'Bosna, Hecegovina', id: 6 }); + pass.end(); + } } if (delay) { diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index 7a13e73ea..49a51b6de 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -13,8 +13,8 @@ export interface DataDuplicatorItem { } export interface DataDuplicatorOptions { - rollbackAfterFinish: boolean; - skipRowsWithUnresolvedRefs: boolean; + rollbackAfterFinish?: boolean; + skipRowsWithUnresolvedRefs?: boolean; } class DuplicatorReference { @@ -198,7 +198,7 @@ export class DataDuplicator { public items: DataDuplicatorItem[], public stream, public copyStream: (input, output) => Promise, - public options: DataDuplicatorOptions + public options: DataDuplicatorOptions = {} ) { this.itemHolders = items.map(x => new DuplicatorItemHolder(x, this)); this.itemHolders.forEach(x => x.initializeReferences());