dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-17 23:45:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

import data fixes

Browse Source
This commit is contained in:
Jan Prochazka
2025-02-24 16:14:45 +01:00
parent 805a063fa1
commit 8bea9b8312
7 changed files with 18 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
e2e-tests/data/my-guitar-shop/orders.jsonl
View File

@@ -1,10 +1,10 @@
{"__isStreamHeader":true,"pureName":"orders","tableRowCount":"9","tableEngine":"InnoDB","objectComment":"","modifyDate":"2025-02-03 02:56:32","objectId":"orders","contentHash":"2025-02-03 02:56:32","columns":[{"notNull":true,"autoIncrement":true,"columnName":"order_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"customer_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"order_date","columnComment":"","dataType":"datetime","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"ship_amount","columnComment":"","dataType":"decimal(10,2)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"tax_amount","columnComment":"","dataType":"decimal(10,2)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":false,"autoIncrement":false,"columnName":"ship_date","columnComment":"","dataType":"datetime","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"ship_address_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_type","columnComment":"","dataType":"varchar(50)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_number","columnComment":"","dataType":"char(16)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_expires","columnComment":"","dataType":"char(7)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"billing_address_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false}],"primaryKey":{"constraintName":"PRIMARY","pureName":"orders","constraintType":"primaryKey","columns":[{"columnName":"order_id"}]},"foreignKeys":[{"constraintName":"orders_fk_customers","constraintType":"foreignKey","pureName":"orders","refTableName":"customers","updateAction":"NO ACTION","deleteAction":"NO ACTION","columns":[{"columnName":"customer_id","refColumnName":"customer_id"}]}],"indexes":[{"constraintName":"orders_fk_customers","indexType":"BTREE","isUnique":false,"columns":[{"columnName":"customer_id","isDescending":0}]}],"uniques":[],"engine":"mysql@dbgate-plugin-mysql"}
{"order_id":1,"customer_id":1,"order_date":"2018-03-28 09:40:28","ship_amount":"5.00","tax_amount":"32.32","ship_date":"2018-03-30 15:32:51","ship_address_id":1,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2020","billing_address_id":2}
{"order_id":2,"customer_id":2,"order_date":"2018-03-28 11:23:20","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-03-29 12:52:14","ship_address_id":3,"card_type":"Visa","card_number":"4012888888881881","card_expires":"08/2019","billing_address_id":3}
{"order_id":3,"customer_id":1,"order_date":"2018-03-29 09:44:58","ship_amount":"10.00","tax_amount":"89.92","ship_date":"2018-03-31 09:11:41","ship_address_id":1,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2017","billing_address_id":2}
{"order_id":4,"customer_id":3,"order_date":"2018-03-30 15:22:31","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-04-03 16:32:21","ship_address_id":4,"card_type":"American Express","card_number":"378282246310005","card_expires":"04/2016","billing_address_id":4}
{"order_id":5,"customer_id":4,"order_date":"2018-03-31 05:43:11","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-04-02 14:21:12","ship_address_id":5,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2019","billing_address_id":6}
{"order_id":6,"customer_id":5,"order_date":"2018-03-31 18:37:22","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":7,"card_type":"Discover","card_number":"6011111111111117","card_expires":"04/2019","billing_address_id":7}
{"order_id":7,"customer_id":6,"order_date":"2018-04-01 23:11:12","ship_amount":"15.00","tax_amount":"0.00","ship_date":"2018-04-03 10:21:35","ship_address_id":8,"card_type":"MasterCard","card_number":"5555555555554444","card_expires":"04/2019","billing_address_id":8}
{"order_id":8,"customer_id":7,"order_date":"2018-04-02 11:26:38","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":9,"card_type":"Visa","card_number":"4012888888881881","card_expires":"04/2019","billing_address_id":10}
{"order_id":9,"customer_id":4,"order_date":"2018-04-03 12:22:31","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":5,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2019","billing_address_id":6}
{"__isStreamHeader":true,"pureName":"orders","tableRowCount":"9","tableEngine":"InnoDB","objectComment":"","modifyDate":"2025-02-03T02:56:32","objectId":"orders","contentHash":"2025-02-03 02:56:32","columns":[{"notNull":true,"autoIncrement":true,"columnName":"order_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"customer_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"order_date","columnComment":"","dataType":"datetime","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"ship_amount","columnComment":"","dataType":"decimal(10,2)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"tax_amount","columnComment":"","dataType":"decimal(10,2)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":false,"autoIncrement":false,"columnName":"ship_date","columnComment":"","dataType":"datetime","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"ship_address_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_type","columnComment":"","dataType":"varchar(50)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_number","columnComment":"","dataType":"char(16)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"card_expires","columnComment":"","dataType":"char(7)","defaultValue":null,"isUnsigned":false,"isZerofill":false},{"notNull":true,"autoIncrement":false,"columnName":"billing_address_id","columnComment":"","dataType":"int","defaultValue":null,"isUnsigned":false,"isZerofill":false}],"primaryKey":{"constraintName":"PRIMARY","pureName":"orders","constraintType":"primaryKey","columns":[{"columnName":"order_id"}]},"foreignKeys":[{"constraintName":"orders_fk_customers","constraintType":"foreignKey","pureName":"orders","refTableName":"customers","updateAction":"NO ACTION","deleteAction":"NO ACTION","columns":[{"columnName":"customer_id","refColumnName":"customer_id"}]}],"indexes":[{"constraintName":"orders_fk_customers","indexType":"BTREE","isUnique":false,"columns":[{"columnName":"customer_id","isDescending":0}]}],"uniques":[],"engine":"mysql@dbgate-plugin-mysql"}
{"order_id":1,"customer_id":1,"order_date":"2018-03-28T09:40:28","ship_amount":"5.00","tax_amount":"32.32","ship_date":"2018-03-30T15:32:51","ship_address_id":1,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2020","billing_address_id":2}
{"order_id":2,"customer_id":2,"order_date":"2018-03-28T11:23:20","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-03-29T12:52:14","ship_address_id":3,"card_type":"Visa","card_number":"4012888888881881","card_expires":"08/2019","billing_address_id":3}
{"order_id":3,"customer_id":1,"order_date":"2018-03-29T09:44:58","ship_amount":"10.00","tax_amount":"89.92","ship_date":"2018-03-31T09:11:41","ship_address_id":1,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2017","billing_address_id":2}
{"order_id":4,"customer_id":3,"order_date":"2018-03-30T15:22:31","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-04-03T16:32:21","ship_address_id":4,"card_type":"American Express","card_number":"378282246310005","card_expires":"04/2016","billing_address_id":4}
{"order_id":5,"customer_id":4,"order_date":"2018-03-31T05:43:11","ship_amount":"5.00","tax_amount":"0.00","ship_date":"2018-04-02T14:21:12","ship_address_id":5,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2019","billing_address_id":6}
{"order_id":6,"customer_id":5,"order_date":"2018-03-31T18:37:22","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":7,"card_type":"Discover","card_number":"6011111111111117","card_expires":"04/2019","billing_address_id":7}
{"order_id":7,"customer_id":6,"order_date":"2018-04-01T23:11:12","ship_amount":"15.00","tax_amount":"0.00","ship_date":"2018-04-03T10:21:35","ship_address_id":8,"card_type":"MasterCard","card_number":"5555555555554444","card_expires":"04/2019","billing_address_id":8}
{"order_id":8,"customer_id":7,"order_date":"2018-04-02T11:26:38","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":9,"card_type":"Visa","card_number":"4012888888881881","card_expires":"04/2019","billing_address_id":10}
{"order_id":9,"customer_id":4,"order_date":"2018-04-03T12:22:31","ship_amount":"5.00","tax_amount":"0.00","ship_date":null,"ship_address_id":5,"card_type":"Visa","card_number":"4111111111111111","card_expires":"04/2019","billing_address_id":6}

2
integration-tests/__tests__/db-import-export.spec.js
View File

@@ -1,5 +1,6 @@
const engines = require('../engines');
const stream = require('stream');
const path = require('path');
const { testWrapper } = require('../tools');
const tableWriter = require('dbgate-api/src/shell/tableWriter');
const tableReader = require('dbgate-api/src/shell/tableReader');
@@ -108,7 +109,6 @@ describe('DB Import/export', () => {
expect(res2.rows[0].cnt.toString()).toEqual('6');
})
);
const enginesWithDumpFile = engines.filter(x => x.dumpFile);
const hasEnginesWithDumpFile = enginesWithDumpFile.length > 0;

4
integration-tests/engines.js
View File

@@ -650,7 +650,7 @@ const enginesOnCi = [
const enginesOnLocal = [
// all engines, which would be run on local test
cassandraEngine,
// cassandraEngine,
// mysqlEngine,
// mariaDbEngine,
// postgreSqlEngine,
@@ -658,7 +658,7 @@ const enginesOnLocal = [
// sqliteEngine,
// cockroachDbEngine,
// clickhouseEngine,
// oracleEngine,
oracleEngine,
];
/** @type {import('dbgate-types').TestEngineInfo[] & Record<string, import('dbgate-types').TestEngineInfo>} */

2
packages/api/src/shell/importDbFromFolder.js
View File

@@ -59,6 +59,8 @@ async function importDbFromFolder({ connection, systemConnection, driver, folder
const dst = await tableWriter({
systemConnection: dbhan,
pureName: table.pureName,
driver,
targetTableStructure: table,
});
await copyStream(src, dst);
}

1
packages/api/src/shell/tableWriter.js
View File

@@ -15,6 +15,7 @@ const logger = getLogger('tableWriter');
* @param {boolean} options.truncate - truncate table before insert
* @param {boolean} options.createIfNotExists - create table if not exists
* @param {boolean} options.commitAfterInsert - commit transaction after insert
* @param {any} options.targetTableStructure - target table structure (don't analyse if given)
* @returns {Promise<writerType>} - writer object
*/
async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) {

2
packages/tools/src/createBulkInsertStreamBase.ts
View File

@@ -32,7 +32,7 @@ export function createBulkInsertStreamBase(driver: EngineDriver, stream, dbhan,
};
writable.checkStructure = async () => {
let structure = await driver.analyseSingleTable(dbhan, name);
let structure = options.targetTableStructure ?? (await driver.analyseSingleTable(dbhan, name));
if (structure) {
writable.structure = structure;
}

1
packages/types/engines.d.ts vendored
View File

@@ -40,6 +40,7 @@ export interface WriteTableOptions {
truncate?: boolean;
createIfNotExists?: boolean;
commitAfterInsert?: boolean;
targetTableStructure?: TableInfo;
}
export interface EngineAuthType {