From a812ff510dba1a75f40e2f3a058debd733ba4af4 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 1 Jun 2024 14:35:40 +0200 Subject: [PATCH] oracle: import data works --- .../tools/src/createBulkInsertStreamBase.ts | 48 +++++++++++++------ packages/types/dialect.d.ts | 1 + packages/types/engines.d.ts | 1 + .../src/frontend/drivers.js | 1 + .../src/backend/Analyser.js | 38 +++++++-------- .../src/backend/drivers.js | 2 +- .../src/backend/sql/columns.js | 2 +- .../src/backend/sql/foreignKeys.js | 2 +- .../src/backend/sql/indexes.js | 2 +- .../src/backend/sql/matviews.js | 2 +- .../src/backend/sql/primaryKeys.js | 2 +- .../src/backend/sql/tableList.js | 6 +-- .../src/backend/sql/uniqueNames.js | 2 +- .../src/backend/sql/views.js | 2 +- .../src/frontend/drivers.js | 1 + .../src/frontend/driver.js | 1 + 16 files changed, 68 insertions(+), 45 deletions(-) diff --git a/packages/tools/src/createBulkInsertStreamBase.ts b/packages/tools/src/createBulkInsertStreamBase.ts index 689771bb4..eb16d9ffc 100644 --- a/packages/tools/src/createBulkInsertStreamBase.ts +++ b/packages/tools/src/createBulkInsertStreamBase.ts @@ -56,24 +56,42 @@ export function createBulkInsertStreamBase(driver: EngineDriver, stream, pool, n const rows = writable.buffer; writable.buffer = []; - const dmp = driver.createDumper(); + if (driver.dialect.allowMultipleValuesInsert) { + const dmp = driver.createDumper(); + dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`); + dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col as string))); + dmp.putRaw(')\n VALUES\n'); - dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`); - dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col as string))); - dmp.putRaw(')\n VALUES\n'); + let wasRow = false; + for (const row of rows) { + if (wasRow) dmp.putRaw(',\n'); + dmp.putRaw('('); + dmp.putCollection(',', writable.columnNames, col => dmp.putValue(row[col as string])); + dmp.putRaw(')'); + wasRow = true; + } + dmp.putRaw(';'); + // require('fs').writeFileSync('/home/jena/test.sql', dmp.s); + // console.log(dmp.s); + await driver.query(pool, dmp.s, { discardResult: true }); + } else { + for (const row of rows) { + const dmp = driver.createDumper(); + dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`); + dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col as string))); + dmp.putRaw(')\n VALUES\n'); - let wasRow = false; - for (const row of rows) { - if (wasRow) dmp.putRaw(',\n'); - dmp.putRaw('('); - dmp.putCollection(',', writable.columnNames, col => dmp.putValue(row[col as string])); - dmp.putRaw(')'); - wasRow = true; + dmp.putRaw('('); + dmp.putCollection(',', writable.columnNames, col => dmp.putValue(row[col as string])); + dmp.putRaw(')'); + await driver.query(pool, dmp.s, { discardResult: true }); + } + } + if (options.commitAfterInsert) { + const dmp = driver.createDumper(); + dmp.commitTransaction(); + await driver.query(pool, dmp.s, { discardResult: true }); } - dmp.putRaw(';'); - // require('fs').writeFileSync('/home/jena/test.sql', dmp.s); - // console.log(dmp.s); - await driver.query(pool, dmp.s, { discardResult: true }); }; writable.sendIfFull = async () => { diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index 166d8298f..47642df6e 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -12,6 +12,7 @@ export interface SqlDialect { defaultSchemaName?: string; enableConstraintsPerTable?: boolean; requireStandaloneSelectForScopeIdentity?: boolean; + allowMultipleValuesInsert?: boolean; dropColumnDependencies?: string[]; changeColumnDependencies?: string[]; diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 5c2f49430..2b09547a1 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -24,6 +24,7 @@ export interface WriteTableOptions { dropIfExists?: boolean; truncate?: boolean; createIfNotExists?: boolean; + commitAfterInsert?: boolean; } export interface EngineAuthType { diff --git a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js index d773c39fc..8198e1ed4 100644 --- a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js @@ -22,6 +22,7 @@ const dialect = { enableConstraintsPerTable: false, anonymousPrimaryKey: true, explicitDropConstraint: true, + allowMultipleValuesInsert: true, quoteIdentifier(s) { return '`' + s + '`'; }, diff --git a/plugins/dbgate-plugin-oracle/src/backend/Analyser.js b/plugins/dbgate-plugin-oracle/src/backend/Analyser.js index b4eb03d8d..7006877eb 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-oracle/src/backend/Analyser.js @@ -42,8 +42,8 @@ class Analyser extends DatabaseAnalyser { } async _computeSingleObjectId() { - const { typeField, schemaName, pureName } = this.singleObjectFilter; - this.singleObjectId = `${typeField}:${schemaName || 'public'}.${pureName}`; + const { typeField, pureName } = this.singleObjectFilter; + this.singleObjectId = `${typeField}:${pureName}`; } async _runAnalysis() { @@ -78,7 +78,7 @@ class Analyser extends DatabaseAnalyser { const fkColumnsMapped = fkColumns.rows.map(x => ({ pureName: x.pure_name, - schemaName: x.schema_name, + // schemaName: x.schema_name, constraintSchema: x.constraint_schema, constraintName: x.constraint_name, columnName: x.column_name, @@ -86,11 +86,11 @@ class Analyser extends DatabaseAnalyser { updateAction: x.update_action, deleteAction: x.delete_action, refTableName: x.ref_table_name, - refSchemaName: x.ref_schema_name, + // refSchemaName: x.ref_schema_name, })); const pkColumnsMapped = pkColumns.rows.map(x => ({ pureName: x.pure_name, - schemaName: x.schema_name, + // schemaName: x.schema_name, constraintSchema: x.constraint_schema, constraintName: x.constraint_name, columnName: x.column_name, @@ -103,8 +103,8 @@ class Analyser extends DatabaseAnalyser { tables: tables.rows.map(table => { const newTable = { pureName: table.pure_name, - schemaName: table.schema_name, - objectId: `tables:${table.schema_name}.${table.pure_name}`, + // schemaName: table.schema_name, + objectId: `tables:${table.pure_name}`, contentHash: table.hash_code_columns ? `${table.hash_code_columns}-${table.hash_code_constraints}` : null, }; return { @@ -146,39 +146,39 @@ class Analyser extends DatabaseAnalyser { }; }), views: views.rows.map(view => ({ - objectId: `views:${view.schema_name}.${view.pure_name}`, + objectId: `views:${view.pure_name}`, pureName: view.pure_name, - schemaName: view.schema_name, + // schemaName: view.schema_name, contentHash: view.hash_code, - createSql: `CREATE VIEW "${view.schema_name}"."${view.pure_name}"\nAS\n${view.create_sql}`, + createSql: `CREATE VIEW "${view.pure_name}"\nAS\n${view.create_sql}`, columns: (columnsGrouped[columnGroup(view)] || []).map(col => getColumnInfo(col)), })), matviews: matviews ? matviews.rows.map(matview => ({ - objectId: `matviews:${matview.schema_name}.${matview.pure_name}`, + objectId: `matviews:${matview.pure_name}`, pureName: matview.pure_name, - schemaName: matview.schema_name, + // schemaName: matview.schema_name, contentHash: matview.hash_code, - createSql: `CREATE MATERIALIZED VIEW "${matview.schema_name}"."${matview.pure_name}"\nAS\n${matview.definition}`, + createSql: `CREATE MATERIALIZED VIEW "${matview.pure_name}"\nAS\n${matview.definition}`, columns: (columnsGrouped[columnGroup(view)] || []).map(col => getColumnInfo(col)), })) : undefined, procedures: routines.rows .filter(x => x.object_type == 'PROCEDURE') .map(proc => ({ - objectId: `procedures:${proc.schema_name}.${proc.pure_name}`, + objectId: `procedures:${proc.pure_name}`, pureName: proc.pure_name, - schemaName: proc.schema_name, - createSql: `CREATE PROCEDURE "${proc.schema_name}"."${proc.pure_name}"() LANGUAGE ${proc.language}\nAS\n$$\n${proc.definition}\n$$`, + // schemaName: proc.schema_name, + createSql: `CREATE PROCEDURE "${proc.pure_name}"() LANGUAGE ${proc.language}\nAS\n$$\n${proc.definition}\n$$`, contentHash: proc.hash_code, })), functions: routines.rows .filter(x => x.object_type == 'FUNCTION') .map(func => ({ - objectId: `functions:${func.schema_name}.${func.pure_name}`, - createSql: `CREATE FUNCTION "${func.schema_name}"."${func.pure_name}"() RETURNS ${func.data_type} LANGUAGE ${func.language}\nAS\n$$\n${func.definition}\n$$`, + objectId: `functions:${func.pure_name}`, + createSql: `CREATE FUNCTION "${func.pure_name}"() RETURNS ${func.data_type} LANGUAGE ${func.language}\nAS\n$$\n${func.definition}\n$$`, pureName: func.pure_name, - schemaName: func.schema_name, + // schemaName: func.schema_name, contentHash: func.hash_code, })), }; diff --git a/plugins/dbgate-plugin-oracle/src/backend/drivers.js b/plugins/dbgate-plugin-oracle/src/backend/drivers.js index 1d98eb5a1..02eed9dfd 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/drivers.js +++ b/plugins/dbgate-plugin-oracle/src/backend/drivers.js @@ -289,7 +289,7 @@ const drivers = driverBases.map(driverBase => ({ }, async writeTable(pool, name, options) { // @ts-ignore - return createBulkInsertStreamBase(this, stream, pool, name, options); + return createBulkInsertStreamBase(this, stream, pool, name, { ...options, commitAfterInsert: true }); }, async listDatabases(client) { const { rows } = await this.query(client, 'SELECT username as "name" from all_users order by username'); diff --git a/plugins/dbgate-plugin-oracle/src/backend/sql/columns.js b/plugins/dbgate-plugin-oracle/src/backend/sql/columns.js index f605bb3b5..a47a7960f 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/sql/columns.js +++ b/plugins/dbgate-plugin-oracle/src/backend/sql/columns.js @@ -10,6 +10,6 @@ select data_scale as "numeric_scale", data_default as "default_value" FROM all_tab_columns av - where OWNER='$owner' AND TABLE_NAME =OBJECT_ID_CONDITION + where OWNER='$owner' AND 'tables:' || TABLE_NAME =OBJECT_ID_CONDITION order by column_id `; \ No newline at end of file diff --git a/plugins/dbgate-plugin-oracle/src/backend/sql/foreignKeys.js b/plugins/dbgate-plugin-oracle/src/backend/sql/foreignKeys.js index 945ae784b..97cbeb17d 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/sql/foreignKeys.js +++ b/plugins/dbgate-plugin-oracle/src/backend/sql/foreignKeys.js @@ -19,6 +19,6 @@ and basecol.table_name = fk.table_name and refcol.owner = ref.owner and refcol.constraint_name = ref.constraint_name and refcol.table_name = ref.table_name -AND fk.constraint_name =OBJECT_ID_CONDITION +AND 'tables:' || fk.table_name =OBJECT_ID_CONDITION order by basecol.position `; diff --git a/plugins/dbgate-plugin-oracle/src/backend/sql/indexes.js b/plugins/dbgate-plugin-oracle/src/backend/sql/indexes.js index fae13e685..45577d6cf 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/sql/indexes.js +++ b/plugins/dbgate-plugin-oracle/src/backend/sql/indexes.js @@ -9,7 +9,7 @@ select i.table_name as "tableName", from all_ind_columns ic, all_indexes i where INDEX_OWNER = '$owner' AND ic.index_owner = i.owner and ic.index_name = i.index_name -and i.index_name =OBJECT_ID_CONDITION +and 'tables:' || i.table_name =OBJECT_ID_CONDITION order by i.table_owner, i.table_name, i.index_name, diff --git a/plugins/dbgate-plugin-oracle/src/backend/sql/matviews.js b/plugins/dbgate-plugin-oracle/src/backend/sql/matviews.js index dad92438b..a60e6f243 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/sql/matviews.js +++ b/plugins/dbgate-plugin-oracle/src/backend/sql/matviews.js @@ -14,6 +14,6 @@ SELECT -- owner as schema_name, '//text()' )) definition FROM all_mviews -where OWNER = '$owner' AND mview_name=OBJECT_ID_CONDITION +where OWNER = '$owner' AND 'matviews:' || mview_name=OBJECT_ID_CONDITION order by owner, mview_name `; \ No newline at end of file diff --git a/plugins/dbgate-plugin-oracle/src/backend/sql/primaryKeys.js b/plugins/dbgate-plugin-oracle/src/backend/sql/primaryKeys.js index a5ac5f739..f10ff6ff0 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/sql/primaryKeys.js +++ b/plugins/dbgate-plugin-oracle/src/backend/sql/primaryKeys.js @@ -11,7 +11,7 @@ where constraint_type = 'P' and basecol.owner = pk.owner and basecol.constraint_name = pk.constraint_name and basecol.table_name = pk.table_name -and pk.constraint_name =OBJECT_ID_CONDITION +and 'tables:' || basecol.table_name =OBJECT_ID_CONDITION and pk.owner = '$owner' order by basecol.position `; diff --git a/plugins/dbgate-plugin-oracle/src/backend/sql/tableList.js b/plugins/dbgate-plugin-oracle/src/backend/sql/tableList.js index 0a4ab93fe..47b4167e1 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/sql/tableList.js +++ b/plugins/dbgate-plugin-oracle/src/backend/sql/tableList.js @@ -1,9 +1,9 @@ module.exports = ` select - -- owner "schema_name", - table_name "pure_name" + -- owner "schema_name", + table_name "pure_name" from all_tables - where OWNER='$owner' AND TABLE_NAME =OBJECT_ID_CONDITION + where OWNER='$owner' AND 'tables:' || TABLE_NAME =OBJECT_ID_CONDITION `; diff --git a/plugins/dbgate-plugin-oracle/src/backend/sql/uniqueNames.js b/plugins/dbgate-plugin-oracle/src/backend/sql/uniqueNames.js index b2d7d9643..c752cc795 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/sql/uniqueNames.js +++ b/plugins/dbgate-plugin-oracle/src/backend/sql/uniqueNames.js @@ -2,5 +2,5 @@ module.exports = ` select constraint_name as "constraintName" from all_constraints where owner='$owner' and constraint_type = 'U' - and constraint_name =OBJECT_ID_CONDITION + and 'tables:' || table_name =OBJECT_ID_CONDITION `; diff --git a/plugins/dbgate-plugin-oracle/src/backend/sql/views.js b/plugins/dbgate-plugin-oracle/src/backend/sql/views.js index d001c8d9e..dfefd8935 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/sql/views.js +++ b/plugins/dbgate-plugin-oracle/src/backend/sql/views.js @@ -8,5 +8,5 @@ from (select from all_views av where owner = '$owner' and text_vc is not null ) avv - where "pure_name" =OBJECT_ID_CONDITION + where 'views:' || "pure_name" =OBJECT_ID_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js index fa2f2bb96..897f545fd 100644 --- a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js @@ -33,6 +33,7 @@ const dialect = { dropUnique: true, createCheck: true, dropCheck: true, + allowMultipleValuesInsert: true, dropReferencesWhenDropTable: true, requireStandaloneSelectForScopeIdentity: true, diff --git a/plugins/dbgate-plugin-sqlite/src/frontend/driver.js b/plugins/dbgate-plugin-sqlite/src/frontend/driver.js index a24bacb5a..fbf78dcf7 100644 --- a/plugins/dbgate-plugin-sqlite/src/frontend/driver.js +++ b/plugins/dbgate-plugin-sqlite/src/frontend/driver.js @@ -17,6 +17,7 @@ const dialect = { explicitDropConstraint: true, stringEscapeChar: "'", fallbackDataType: 'nvarchar', + allowMultipleValuesInsert: true, dropColumnDependencies: ['indexes', 'primaryKey', 'uniques'], quoteIdentifier(s) { return `[${s}]`;