From 8bea9b83129e39f24a2854b7f2ec0d499506e0cf Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 24 Feb 2025 16:14:45 +0100 Subject: [PATCH] import data fixes --- e2e-tests/data/my-guitar-shop/orders.jsonl | 20 +++++++++---------- .../__tests__/db-import-export.spec.js | 2 +- integration-tests/engines.js | 4 ++-- packages/api/src/shell/importDbFromFolder.js | 2 ++ packages/api/src/shell/tableWriter.js | 1 + .../tools/src/createBulkInsertStreamBase.ts | 2 +- packages/types/engines.d.ts | 1 + 7 files changed, 18 insertions(+), 14 deletions(-) diff --git a/e2e-tests/data/my-guitar-shop/orders.jsonl b/e2e-tests/data/my-guitar-shop/orders.jsonl index fd5b56830..26618b362 100644 --- a/e2e-tests/data/my-guitar-shop/orders.jsonl +++ b/e2e-tests/data/my-guitar-shop/orders.jsonl @@ -1,10 +1,10 @@ -{"__isStreamHeader":true,"pureName":"orders","tableRowCount":"9","tableEngine":"InnoDB","objectComment":"","modifyDate":"2025-02-03 02:56:32","objectId":"orders","contentHash":"2025-02-03 02:56:32","columns":[{"notNull":true,"autoIncrement":true,"columnName":"order_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"customer_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"order_date","columnComment":"","dataType":"datetime","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"ship_amount","columnComment":"","dataType":"decimal(10,2)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"tax_amount","columnComment":"","dataType":"decimal(10,2)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":false,"autoIncrement":false,"columnName":"ship_date","columnComment":"","dataType":"datetime","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"ship_address_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_type","columnComment":"","dataType":"varchar(50)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_number","columnComment":"","dataType":"char(16)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_expires","columnComment":"","dataType":"char(7)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"billing_address_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false}],"primaryKey":{"constraintName":"PRIMARY","pureName":"orders","constraintType":"primaryKey","columns":[{"columnName":"order_id"}]},"foreignKeys":[{"constraintName":"orders_fk_customers","constraintType":"foreignKey","pureName":"orders","refTableName":"customers","updateAction":"NO ACTION","deleteAction":"NO ACTION","columns":[{"columnName":"customer_id","refColumnName":"customer_id"}]}],"indexes":[{"constraintName":"orders_fk_customers","indexType":"BTREE","isUnique":false,"columns":[{"columnName":"customer_id","isDescending":0}]}],"uniques":[],"engine":"mysql@dbgate-plugin-mysql"} -{"order_id":1,"customer_id":1,"order_date":"2018-03-28 09:40:28","ship_amount":"5.00","tax_amount":"32.32","ship_date":"2018-03-30 15:32:51","ship_address_id":1,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2020","billing_address_id":2} -{"order_id":2,"customer_id":2,"order_date":"2018-03-28 11:23:20","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-03-29 12:52:14","ship_address_id":3,"card_type":"Visa","card_number":"4012888888881881","card_expires":"08/2019","billing_address_id":3} -{"order_id":3,"customer_id":1,"order_date":"2018-03-29 09:44:58","ship_amount":"10.00","tax_amount":"89.92","ship_date":"2018-03-31 09:11:41","ship_address_id":1,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2017","billing_address_id":2} -{"order_id":4,"customer_id":3,"order_date":"2018-03-30 15:22:31","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-04-03 16:32:21","ship_address_id":4,"card_type":"American Express","card_number":"378282246310005","card_expires":"04/2016","billing_address_id":4} -{"order_id":5,"customer_id":4,"order_date":"2018-03-31 05:43:11","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-04-02 14:21:12","ship_address_id":5,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2019","billing_address_id":6} -{"order_id":6,"customer_id":5,"order_date":"2018-03-31 18:37:22","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":7,"card_type":"Discover","card_number":"6011111111111117","card_expires":"04/2019","billing_address_id":7} -{"order_id":7,"customer_id":6,"order_date":"2018-04-01 23:11:12","ship_amount":"15.00","tax_amount":"0.00","ship_date":"2018-04-03 10:21:35","ship_address_id":8,"card_type":"MasterCard","card_number":"5555555555554444","card_expires":"04/2019","billing_address_id":8} -{"order_id":8,"customer_id":7,"order_date":"2018-04-02 11:26:38","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":9,"card_type":"Visa","card_number":"4012888888881881","card_expires":"04/2019","billing_address_id":10} -{"order_id":9,"customer_id":4,"order_date":"2018-04-03 12:22:31","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":5,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2019","billing_address_id":6} +{"__isStreamHeader":true,"pureName":"orders","tableRowCount":"9","tableEngine":"InnoDB","objectComment":"","modifyDate":"2025-02-03T02:56:32","objectId":"orders","contentHash":"2025-02-03 02:56:32","columns":[{"notNull":true,"autoIncrement":true,"columnName":"order_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"customer_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"order_date","columnComment":"","dataType":"datetime","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"ship_amount","columnComment":"","dataType":"decimal(10,2)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"tax_amount","columnComment":"","dataType":"decimal(10,2)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":false,"autoIncrement":false,"columnName":"ship_date","columnComment":"","dataType":"datetime","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"ship_address_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_type","columnComment":"","dataType":"varchar(50)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_number","columnComment":"","dataType":"char(16)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_expires","columnComment":"","dataType":"char(7)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"billing_address_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false}],"primaryKey":{"constraintName":"PRIMARY","pureName":"orders","constraintType":"primaryKey","columns":[{"columnName":"order_id"}]},"foreignKeys":[{"constraintName":"orders_fk_customers","constraintType":"foreignKey","pureName":"orders","refTableName":"customers","updateAction":"NO ACTION","deleteAction":"NO ACTION","columns":[{"columnName":"customer_id","refColumnName":"customer_id"}]}],"indexes":[{"constraintName":"orders_fk_customers","indexType":"BTREE","isUnique":false,"columns":[{"columnName":"customer_id","isDescending":0}]}],"uniques":[],"engine":"mysql@dbgate-plugin-mysql"} +{"order_id":1,"customer_id":1,"order_date":"2018-03-28T09:40:28","ship_amount":"5.00","tax_amount":"32.32","ship_date":"2018-03-30T15:32:51","ship_address_id":1,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2020","billing_address_id":2} +{"order_id":2,"customer_id":2,"order_date":"2018-03-28T11:23:20","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-03-29T12:52:14","ship_address_id":3,"card_type":"Visa","card_number":"4012888888881881","card_expires":"08/2019","billing_address_id":3} +{"order_id":3,"customer_id":1,"order_date":"2018-03-29T09:44:58","ship_amount":"10.00","tax_amount":"89.92","ship_date":"2018-03-31T09:11:41","ship_address_id":1,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2017","billing_address_id":2} +{"order_id":4,"customer_id":3,"order_date":"2018-03-30T15:22:31","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-04-03T16:32:21","ship_address_id":4,"card_type":"American Express","card_number":"378282246310005","card_expires":"04/2016","billing_address_id":4} +{"order_id":5,"customer_id":4,"order_date":"2018-03-31T05:43:11","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-04-02T14:21:12","ship_address_id":5,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2019","billing_address_id":6} +{"order_id":6,"customer_id":5,"order_date":"2018-03-31T18:37:22","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":7,"card_type":"Discover","card_number":"6011111111111117","card_expires":"04/2019","billing_address_id":7} +{"order_id":7,"customer_id":6,"order_date":"2018-04-01T23:11:12","ship_amount":"15.00","tax_amount":"0.00","ship_date":"2018-04-03T10:21:35","ship_address_id":8,"card_type":"MasterCard","card_number":"5555555555554444","card_expires":"04/2019","billing_address_id":8} +{"order_id":8,"customer_id":7,"order_date":"2018-04-02T11:26:38","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":9,"card_type":"Visa","card_number":"4012888888881881","card_expires":"04/2019","billing_address_id":10} +{"order_id":9,"customer_id":4,"order_date":"2018-04-03T12:22:31","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":5,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2019","billing_address_id":6} diff --git a/integration-tests/__tests__/db-import-export.spec.js b/integration-tests/__tests__/db-import-export.spec.js index 636c1440b..2a0d6b449 100644 --- a/integration-tests/__tests__/db-import-export.spec.js +++ b/integration-tests/__tests__/db-import-export.spec.js @@ -1,5 +1,6 @@ const engines = require('../engines'); const stream = require('stream'); +const path = require('path'); const { testWrapper } = require('../tools'); const tableWriter = require('dbgate-api/src/shell/tableWriter'); const tableReader = require('dbgate-api/src/shell/tableReader'); @@ -108,7 +109,6 @@ describe('DB Import/export', () => { expect(res2.rows[0].cnt.toString()).toEqual('6'); }) ); - const enginesWithDumpFile = engines.filter(x => x.dumpFile); const hasEnginesWithDumpFile = enginesWithDumpFile.length > 0; diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 0489c8bca..4ac2003c7 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -650,7 +650,7 @@ const enginesOnCi = [ const enginesOnLocal = [ // all engines, which would be run on local test - cassandraEngine, + // cassandraEngine, // mysqlEngine, // mariaDbEngine, // postgreSqlEngine, @@ -658,7 +658,7 @@ const enginesOnLocal = [ // sqliteEngine, // cockroachDbEngine, // clickhouseEngine, - // oracleEngine, + oracleEngine, ]; /** @type {import('dbgate-types').TestEngineInfo[] & Record} */ diff --git a/packages/api/src/shell/importDbFromFolder.js b/packages/api/src/shell/importDbFromFolder.js index 46edcc9fb..25053db27 100644 --- a/packages/api/src/shell/importDbFromFolder.js +++ b/packages/api/src/shell/importDbFromFolder.js @@ -59,6 +59,8 @@ async function importDbFromFolder({ connection, systemConnection, driver, folder const dst = await tableWriter({ systemConnection: dbhan, pureName: table.pureName, + driver, + targetTableStructure: table, }); await copyStream(src, dst); } diff --git a/packages/api/src/shell/tableWriter.js b/packages/api/src/shell/tableWriter.js index 727bc6d4a..7b53f0cd1 100644 --- a/packages/api/src/shell/tableWriter.js +++ b/packages/api/src/shell/tableWriter.js @@ -15,6 +15,7 @@ const logger = getLogger('tableWriter'); * @param {boolean} options.truncate - truncate table before insert * @param {boolean} options.createIfNotExists - create table if not exists * @param {boolean} options.commitAfterInsert - commit transaction after insert + * @param {any} options.targetTableStructure - target table structure (don't analyse if given) * @returns {Promise} - writer object */ async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) { diff --git a/packages/tools/src/createBulkInsertStreamBase.ts b/packages/tools/src/createBulkInsertStreamBase.ts index 534ad9fe2..2fa683cbf 100644 --- a/packages/tools/src/createBulkInsertStreamBase.ts +++ b/packages/tools/src/createBulkInsertStreamBase.ts @@ -32,7 +32,7 @@ export function createBulkInsertStreamBase(driver: EngineDriver, stream, dbhan, }; writable.checkStructure = async () => { - let structure = await driver.analyseSingleTable(dbhan, name); + let structure = options.targetTableStructure ?? (await driver.analyseSingleTable(dbhan, name)); if (structure) { writable.structure = structure; } diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index dae1f2878..61fde1e07 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -40,6 +40,7 @@ export interface WriteTableOptions { truncate?: boolean; createIfNotExists?: boolean; commitAfterInsert?: boolean; + targetTableStructure?: TableInfo; } export interface EngineAuthType {