From 014e453e5787170e9adb86c965d5cc4aa2a3d908 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 28 Nov 2025 12:13:32 +0100 Subject: [PATCH] removed ttable incremental analysis for postgres --- packages/tools/src/DatabaseAnalyser.ts | 17 --- .../src/backend/Analyser.js | 108 ++++++++---------- .../src/backend/sql/foreignKeyColumns.js | 36 ------ .../src/backend/sql/index.js | 4 - .../src/backend/sql/tableList.js | 2 +- .../src/backend/sql/tableModifications.js | 28 ----- 6 files changed, 46 insertions(+), 149 deletions(-) delete mode 100644 plugins/dbgate-plugin-postgres/src/backend/sql/foreignKeyColumns.js delete mode 100644 plugins/dbgate-plugin-postgres/src/backend/sql/tableModifications.js diff --git a/packages/tools/src/DatabaseAnalyser.ts b/packages/tools/src/DatabaseAnalyser.ts index d48cfd49b..e10e23ab8 100644 --- a/packages/tools/src/DatabaseAnalyser.ts +++ b/packages/tools/src/DatabaseAnalyser.ts @@ -299,23 +299,6 @@ export class DatabaseAnalyser { } } - objectIdConditionApplied(typeFields) { - if (!this.modifications || !typeFields || this.modifications.length == 0) { - return false; - } - if (this.modifications.some(x => typeFields.includes(x.objectTypeField) && x.action == 'all')) { - // do not filter objects - return false; - } - const filterIds = this.modifications - .filter(x => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change')) - .map(x => x.objectId); - if (filterIds.length == 0) { - return false; - } - return true; - } - async getModifications() { const snapshot = await this._getFastSnapshot(); if (!snapshot) return null; diff --git a/plugins/dbgate-plugin-postgres/src/backend/Analyser.js b/plugins/dbgate-plugin-postgres/src/backend/Analyser.js index e9361c781..fd4a9f172 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-postgres/src/backend/Analyser.js @@ -84,9 +84,7 @@ class Analyser extends DatabaseAnalyser { async _runAnalysis() { this.feedback({ analysingMessage: 'DBGM-00241 Loading tables' }); - const tables = await this.analyserQuery(this.driver.dialect.stringAgg ? 'tableModifications' : 'tableList', [ - 'tables', - ]); + const tables = await this.analyserQuery('tableList', ['tables']); this.feedback({ analysingMessage: 'DBGM-00242 Loading columns' }); const columns = await this.analyserQuery('columns', ['tables', 'views']); @@ -99,60 +97,55 @@ class Analyser extends DatabaseAnalyser { this.feedback({ analysingMessage: 'DBGM-00244 Loading foreign key constraints' }); // const fk_tableConstraints = await this.analyserQuery('fk_tableConstraints', ['tables']); - if (this.objectIdConditionApplied(['tables'])) { - this.feedback({ analysingMessage: 'DBGM-00000 Loading foreign key refs' }); - fkColumns = await this.analyserQuery('foreignKeyColumns', ['tables']); - } else { - this.feedback({ analysingMessage: 'DBGM-00245 Loading foreign key refs' }); - const foreignKeys = await this.analyserQuery('foreignKeys', ['tables']); + this.feedback({ analysingMessage: 'DBGM-00245 Loading foreign key refs' }); + const foreignKeys = await this.analyserQuery('foreignKeys', ['tables']); - this.feedback({ analysingMessage: 'DBGM-00246 Loading foreign key columns' }); - const fk_keyColumnUsage = await this.analyserQuery('fk_keyColumnUsage', ['tables']); + this.feedback({ analysingMessage: 'DBGM-00246 Loading foreign key columns' }); + const fk_keyColumnUsage = await this.analyserQuery('fk_keyColumnUsage', ['tables']); - // const cntKey = x => `${x.constraint_name}|${x.constraint_schema}`; - const fkRows = []; - // const fkConstraintDct = _.keyBy(fk_tableConstraints.rows, cntKey); - for (const fkRef of foreignKeys.rows) { - // const cntBase = fkConstraintDct[cntKey(fkRef)]; - // const cntRef = fkConstraintDct[`${fkRef.unique_constraint_name}|${fkRef.unique_constraint_schema}`]; - // if (!cntBase || !cntRef) continue; - const baseCols = _.sortBy( - fk_keyColumnUsage.rows.filter( - x => - x.table_name == fkRef.table_name && - x.constraint_name == fkRef.constraint_name && - x.table_schema == fkRef.table_schema - ), - 'ordinal_position' - ); - const refCols = _.sortBy( - fk_keyColumnUsage.rows.filter( - x => - x.table_name == fkRef.ref_table_name && - x.constraint_name == fkRef.unique_constraint_name && - x.table_schema == fkRef.ref_table_schema - ), - 'ordinal_position' - ); - if (baseCols.length != refCols.length) continue; + // const cntKey = x => `${x.constraint_name}|${x.constraint_schema}`; + const fkRows = []; + // const fkConstraintDct = _.keyBy(fk_tableConstraints.rows, cntKey); + for (const fkRef of foreignKeys.rows) { + // const cntBase = fkConstraintDct[cntKey(fkRef)]; + // const cntRef = fkConstraintDct[`${fkRef.unique_constraint_name}|${fkRef.unique_constraint_schema}`]; + // if (!cntBase || !cntRef) continue; + const baseCols = _.sortBy( + fk_keyColumnUsage.rows.filter( + x => + x.table_name == fkRef.table_name && + x.constraint_name == fkRef.constraint_name && + x.table_schema == fkRef.table_schema + ), + 'ordinal_position' + ); + const refCols = _.sortBy( + fk_keyColumnUsage.rows.filter( + x => + x.table_name == fkRef.ref_table_name && + x.constraint_name == fkRef.unique_constraint_name && + x.table_schema == fkRef.ref_table_schema + ), + 'ordinal_position' + ); + if (baseCols.length != refCols.length) continue; - for (let i = 0; i < baseCols.length; i++) { - const baseCol = baseCols[i]; - const refCol = refCols[i]; + for (let i = 0; i < baseCols.length; i++) { + const baseCol = baseCols[i]; + const refCol = refCols[i]; - fkRows.push({ - ...fkRef, - pure_name: fkRef.table_name, - schema_name: fkRef.table_schema, - ref_table_name: fkRef.ref_table_name, - ref_schema_name: fkRef.ref_table_schema, - column_name: baseCol.column_name, - ref_column_name: refCol.column_name, - }); - } + fkRows.push({ + ...fkRef, + pure_name: fkRef.table_name, + schema_name: fkRef.table_schema, + ref_table_name: fkRef.ref_table_name, + ref_schema_name: fkRef.ref_table_schema, + column_name: baseCol.column_name, + ref_column_name: refCol.column_name, + }); } - fkColumns = { rows: fkRows }; } + fkColumns = { rows: fkRows }; this.feedback({ analysingMessage: 'DBGM-00247 Loading views' }); const views = await this.analyserQuery('views', ['views']); @@ -401,9 +394,6 @@ class Analyser extends DatabaseAnalyser { } async _getFastSnapshot() { - const tableModificationsQueryData = this.driver.dialect.stringAgg - ? await this.analyserQuery('tableModifications') - : null; const viewModificationsQueryData = await this.analyserQuery('viewModifications'); const matviewModificationsQueryData = this.driver.dialect.materializedViews ? await this.analyserQuery('matviewModifications') @@ -411,15 +401,7 @@ class Analyser extends DatabaseAnalyser { const routineModificationsQueryData = await this.analyserQuery('routineModifications'); return { - tables: tableModificationsQueryData - ? tableModificationsQueryData.rows.map(x => ({ - objectId: `tables:${x.schema_name}.${x.pure_name}`, - pureName: x.pure_name, - schemaName: x.schema_name, - sizeBytes: x.size_bytes, - contentHash: `${x.hash_code_columns}-${x.hash_code_constraints}`, - })) - : null, + tables: null, views: viewModificationsQueryData.rows.map(x => ({ objectId: `views:${x.schema_name}.${x.pure_name}`, pureName: x.pure_name, diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/foreignKeyColumns.js b/plugins/dbgate-plugin-postgres/src/backend/sql/foreignKeyColumns.js deleted file mode 100644 index 1c9d5f7ed..000000000 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/foreignKeyColumns.js +++ /dev/null @@ -1,36 +0,0 @@ -module.exports = ` -SELECT - tc.table_schema as schema_name, - tc.table_name as pure_name, - tc.constraint_name, - tc_pk.table_schema AS ref_schema_name, - tc_pk.table_name AS ref_table_name, - rc.unique_constraint_name, - kcu.column_name, - kcu.ordinal_position, - ccu.column_name AS ref_column_name -FROM information_schema.table_constraints AS tc -JOIN information_schema.key_column_usage AS kcu - ON tc.constraint_name = kcu.constraint_name - AND tc.table_schema = kcu.table_schema -JOIN information_schema.referential_constraints AS rc - ON rc.constraint_name = tc.constraint_name - AND rc.constraint_schema = tc.table_schema -JOIN information_schema.table_constraints AS tc_pk - ON tc_pk.constraint_name = rc.unique_constraint_name - AND tc_pk.constraint_schema = rc.unique_constraint_schema -JOIN information_schema.key_column_usage AS ccu - ON ccu.constraint_name = rc.unique_constraint_name - AND ccu.constraint_schema = rc.unique_constraint_schema - AND ccu.ordinal_position = kcu.position_in_unique_constraint -WHERE tc.constraint_type = 'FOREIGN KEY' -AND (('tables:' || tc.table_schema || '.' || tc.table_name) =OBJECT_ID_CONDITION AND tc.table_schema =SCHEMA_NAME_CONDITION) -OR -(('tables:' || tc_pk.table_schema || '.' || tc_pk.table_name) =OBJECT_ID_CONDITION AND tc.table_schema =SCHEMA_NAME_CONDITION) -ORDER BY - tc.table_schema, - tc.table_name, - tc.constraint_name, - kcu.ordinal_position -; -`; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/index.js b/plugins/dbgate-plugin-postgres/src/backend/sql/index.js index f40108ffb..38911d0f4 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/index.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/index.js @@ -1,5 +1,4 @@ const columns = require('./columns'); -const tableModifications = require('./tableModifications'); const tableList = require('./tableList'); const viewModifications = require('./viewModifications'); const matviewModifications = require('./matviewModifications'); @@ -16,7 +15,6 @@ const geometryColumns = require('./geometryColumns'); const geographyColumns = require('./geographyColumns'); const proceduresParameters = require('./proceduresParameters'); const foreignKeys = require('./foreignKeys'); -const foreignKeyColumns = require('./foreignKeyColumns'); const triggers = require('./triggers'); const listDatabases = require('./listDatabases'); const listVariables = require('./listVariables'); @@ -26,12 +24,10 @@ const fk_keyColumnUsage = require('./fk_key_column_usage'); module.exports = { columns, - tableModifications, tableList, viewModifications, primaryKeys, fk_keyColumnUsage, - foreignKeyColumns, foreignKeys, views, routines, diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/tableList.js b/plugins/dbgate-plugin-postgres/src/backend/sql/tableList.js index 37b004655..af1261db4 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/tableList.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/tableList.js @@ -5,6 +5,6 @@ from information_schema.tables infoTables where infoTables.table_type not like '%VIEW%' and ('tables:' || infoTables.table_schema || '.' || infoTables.table_name) =OBJECT_ID_CONDITION and infoTables.table_schema <> 'pg_internal' -and infoTables.table_schema !~ '^_timescaledb_' +and infoTables.table_schema !~ '^_timescaledb_' and infoTables.table_schema =SCHEMA_NAME_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/tableModifications.js b/plugins/dbgate-plugin-postgres/src/backend/sql/tableModifications.js deleted file mode 100644 index 8d87142d0..000000000 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/tableModifications.js +++ /dev/null @@ -1,28 +0,0 @@ -module.exports = ` -select infoTables.table_schema as "schema_name", infoTables.table_name as "pure_name", - ( - select $md5Function(string_agg( - infoColumns.column_name || '|' || infoColumns.data_type || '|' || infoColumns.is_nullable::varchar(255) || '|' || coalesce(infoColumns.character_maximum_length, -1)::varchar(255) - || '|' || coalesce(infoColumns.numeric_precision, -1)::varchar(255) , - ',' order by infoColumns.ordinal_position - )) as "hash_code_columns" - from information_schema.columns infoColumns - where infoColumns.table_schema = infoTables.table_schema and infoColumns.table_name = infoTables.table_name - ), - ( - select $md5Function(string_agg( - infoConstraints.constraint_name || '|' || infoConstraints.constraint_type , - ',' order by infoConstraints.constraint_name - )) as "hash_code_constraints" - from information_schema.table_constraints infoConstraints - where infoConstraints.table_schema = infoTables.table_schema and infoConstraints.table_name = infoTables.table_name - ), - pg_relation_size('"'||infoTables.table_schema||'"."'||infoTables.table_name||'"') as "size_bytes" - -from information_schema.tables infoTables -where infoTables.table_type not like '%VIEW%' - and ('tables:' || infoTables.table_schema || '.' || infoTables.table_name) =OBJECT_ID_CONDITION -and infoTables.table_schema <> 'pg_internal' -and infoTables.table_schema !~ '^_timescaledb_' -and infoTables.table_schema =SCHEMA_NAME_CONDITION -`;