dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-30 01:03:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

postgre sql analyser - works also for redshift

Browse Source
This commit is contained in:
Jan Prochazka
2021-05-16 08:56:56 +02:00
parent 640b53e45f
commit acc49273c1
12 changed files with 152 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/tools/src/DatabaseAnalyser.ts
View File

@@ -25,6 +25,7 @@ export class DatabaseAnalyser {
async fullAnalysis() { async fullAnalysis() {
const res = await this._runAnalysis(); const res = await this._runAnalysis();
console.log('FULL ANALYSIS', res);
return res; return res;
} }

145
plugins/dbgate-plugin-postgres/src/backend/Analyser.js
View File

@@ -12,26 +12,24 @@ function normalizeTypeName(dataType) {
} }
function getColumnInfo({ function getColumnInfo({
isNullable, is_nullable,
isIdentity, column_name,
columnName, data_type,
dataType, char_max_length,
charMaxLength, numeric_precision,
numericPrecision, numeric_ccale,
numericScale, default_value,
defaultValue,
}) { }) {
const normDataType = normalizeTypeName(dataType); const normDataType = normalizeTypeName(data_type);
let fullDataType = normDataType; let fullDataType = normDataType;
if (charMaxLength && isTypeString(normDataType)) fullDataType = `${normDataType}(${charMaxLength})`; if (char_max_length && isTypeString(normDataType)) fullDataType = `${normDataType}(${char_max_length})`;
if (numericPrecision && numericScale && isTypeNumeric(normDataType)) if (numeric_precision && numeric_ccale && isTypeNumeric(normDataType))
fullDataType = `${normDataType}(${numericPrecision},${numericScale})`; fullDataType = `${normDataType}(${numeric_precision},${numeric_ccale})`;
return { return {
columnName, columnName: column_name,
dataType: fullDataType, dataType: fullDataType,
notNull: !isNullable || isNullable == 'NO' || isNullable == 'no', notNull: !is_nullable || is_nullable == 'NO' || is_nullable == 'no',
autoIncrement: !!isIdentity, defaultValue: default_value,
defaultValue,
}; };
} }
@@ -50,7 +48,10 @@ class Analyser extends DatabaseAnalyser {
} }
async _runAnalysis() { async _runAnalysis() {
const tables = await this.driver.query(this.pool, this.createQuery('tableModifications', ['tables'])); const tables = await this.driver.query(
this.pool,
this.createQuery(this.driver.dialect.stringAgg ? 'tableModifications' : 'tableList', ['tables'])
);
const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables'])); const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables']));
const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables'])); const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables']));
const fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables'])); const fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables']));
@@ -59,70 +60,110 @@ class Analyser extends DatabaseAnalyser {
// console.log('PG fkColumns', fkColumns.rows); // console.log('PG fkColumns', fkColumns.rows);
return { return {
tables: tables.rows.map(table => ({ tables: tables.rows.map(table => {
...table, const newTable = {
objectId: `tables:${table.schemaName}.${table.pureName}`, pureName: table.pure_name,
contentHash: `${table.hashCodeColumns}-${table.hashCodeConstraints}`, schemaName: table.schema_name,
columns: columns.rows objectId: `tables:${table.schema_name}.${table.pure_name}`,
.filter(col => col.pureName == table.pureName && col.schemaName == table.schemaName) contentHash: table.hash_code_columns ? `${table.hash_code_columns}-${table.hash_code_constraints}` : null,
.map(getColumnInfo), };
primaryKey: DatabaseAnalyser.extractPrimaryKeys(table, pkColumns.rows), return {
foreignKeys: DatabaseAnalyser.extractForeignKeys(table, fkColumns.rows), ...newTable,
})), columns: columns.rows
.filter(col => col.pure_name == table.pure_name && col.schema_name == table.schema_name)
.map(getColumnInfo),
primaryKey: DatabaseAnalyser.extractPrimaryKeys(
newTable,
pkColumns.rows.map(x => ({
pureName: x.pure_name,
schemaName: x.schema_name,
constraintSchema: x.constraint_schema,
constraintName: x.constraint_name,
columnName: x.column_name,
}))
),
foreignKeys: DatabaseAnalyser.extractForeignKeys(
newTable,
fkColumns.rows.map(x => ({
pureName: x.pure_name,
schemaName: x.schema_name,
constraintSchema: x.constraint_schema,
constraintName: x.constraint_name,
columnName: x.column_name,
refColumnName: x.ref_column_name,
updateAction: x.update_action,
deleteAction: x.delete_action,
refTableName: x.ref_table_name,
refSchemaName: x.ref_schema_name,
}))
),
};
}),
views: views.rows.map(view => ({ views: views.rows.map(view => ({
...view, objectId: `views:${view.schema_name}.${view.pure_name}`,
objectId: `views:${view.schemaName}.${view.pureName}`, pureName: view.pure_name,
contentHash: view.hashCode, schemaName: view.schema_name,
contentHash: view.hash_code,
columns: columns.rows columns: columns.rows
.filter(col => col.pureName == view.pureName && col.schemaName == view.schemaName) .filter(col => col.pure_name == view.pure_name && col.schema_name == view.schema_name)
.map(getColumnInfo), .map(getColumnInfo),
})), })),
procedures: routines.rows procedures: routines.rows
.filter(x => x.objectType == 'PROCEDURE') .filter(x => x.objectType == 'PROCEDURE')
.map(proc => ({ .map(proc => ({
objectId: `procedures:${proc.schemaName}.${proc.pureName}`, objectId: `procedures:${proc.schema_name}.${proc.pure_name}`,
contentHash: proc.hashCode, pureName: proc.pure_name,
...proc, schemaName: proc.schema_name,
contentHash: proc.hash_code,
})), })),
functions: routines.rows functions: routines.rows
.filter(x => x.objectType == 'FUNCTION') .filter(x => x.objectType == 'FUNCTION')
.map(func => ({ .map(func => ({
objectId: `functions:${func.schemaName}.${func.pureName}`, objectId: `functions:${func.schema_name}.${func.pure_name}`,
contentHash: func.hashCode, pureName: func.pure_name,
...func, schemaName: func.schema_name,
contentHash: func.hash_code,
})), })),
}; };
} }
async _getFastSnapshot() { async _getFastSnapshot() {
const tableModificationsQueryData = await this.driver.query(this.pool, this.createQuery('tableModifications')); const tableModificationsQueryData = this.driver.dialect.stringAgg
? await this.driver.query(this.pool, this.createQuery('tableModifications'))
: null;
const viewModificationsQueryData = await this.driver.query(this.pool, this.createQuery('viewModifications')); const viewModificationsQueryData = await this.driver.query(this.pool, this.createQuery('viewModifications'));
const routineModificationsQueryData = await this.driver.query(this.pool, this.createQuery('routineModifications')); const routineModificationsQueryData = await this.driver.query(this.pool, this.createQuery('routineModifications'));
return { return {
tables: tableModificationsQueryData.rows.map(x => ({ tables: tableModificationsQueryData
...x, ? tableModificationsQueryData.rows.map(x => ({
objectId: `tables:${x.schemaName}.${x.pureName}`, objectId: `tables:${x.schema_name}.${x.pure_name}`,
contentHash: `${x.hashCodeColumns}-${x.hashCodeConstraints}`, pureName: x.pure_name,
})), schemaName: x.schema_name,
contentHash: `${x.hash_code_columns}-${x.hash_code_constraints}`,
}))
: null,
views: viewModificationsQueryData.rows.map(x => ({ views: viewModificationsQueryData.rows.map(x => ({
...x, objectId: `views:${x.schema_name}.${x.pure_name}`,
objectId: `views:${x.schemaName}.${x.pureName}`, pureName: x.pure_name,
contentHash: x.hashCode, schemaName: x.schema_name,
contentHash: x.hash_code,
})), })),
procedures: routineModificationsQueryData.rows procedures: routineModificationsQueryData.rows
.filter(x => x.objectType == 'PROCEDURE') .filter(x => x.objectType == 'PROCEDURE')
.map(x => ({ .map(x => ({
...x, objectId: `procedures:${x.schema_name}.${x.pure_name}`,
objectId: `procedures:${x.schemaName}.${x.pureName}`, pureName: x.pure_name,
contentHash: x.hashCode, schemaName: x.schema_name,
contentHash: x.hash_code,
})), })),
functions: routineModificationsQueryData.rows functions: routineModificationsQueryData.rows
.filter(x => x.objectType == 'FUNCTION') .filter(x => x.objectType == 'FUNCTION')
.map(x => ({ .map(x => ({
...x, objectId: `functions:${x.schema_name}.${x.pure_name}`,
objectId: `functions:${x.schemaName}.${x.pureName}`, pureName: x.pure_name,
contentHash: x.hashCode, schemaName: x.schema_name,
contentHash: x.hash_code,
})), })),
}; };
} }

20
plugins/dbgate-plugin-postgres/src/backend/sql/columns.js
View File

@@ -1,19 +1,19 @@
module.exports = ` module.exports = `
select select
table_schema as "schemaName", table_schema as "schema_name",
table_name as "pureName", table_name as "pure_name",
column_name as "columnName", column_name as "column_name",
is_nullable as "isNullable", is_nullable as "is_nullable",
data_type as "dataType", data_type as "data_type",
character_maximum_length as "charMaxLength", character_maximum_length as "char_max_length",
numeric_precision as "numericPrecision", numeric_precision as "numeric_precision",
numeric_scale as "numericScale", numeric_scale as "numeric_scale",
column_default as "defaultValue" column_default as "default_value"
from information_schema.columns from information_schema.columns
where where
table_schema <> 'information_schema' table_schema <> 'information_schema'
and table_schema <> 'pg_catalog' and table_schema <> 'pg_catalog'
and table_schema !~ '^pg_toast' and table_schema !~ '^pg_toast'
and 'tables:' || table_schema || '.' || table_name =OBJECT_ID_CONDITION and ('tables:' || table_schema || '.' || table_name) =OBJECT_ID_CONDITION
order by ordinal_position order by ordinal_position
`; `;

22
plugins/dbgate-plugin-postgres/src/backend/sql/foreignKeys.js
View File

@@ -1,15 +1,15 @@
module.exports = ` module.exports = `
select select
fk.constraint_name as "constraintName", fk.constraint_name as "constraint_name",
fk.constraint_schema as "constraintSchema", fk.constraint_schema as "constraint_schema",
base.table_name as "pureName", base.table_name as "pure_name",
base.table_schema as "schemaName", base.table_schema as "schema_name",
fk.update_rule as "updateAction", fk.update_rule as "update_action",
fk.delete_rule as "deleteAction", fk.delete_rule as "delete_action",
ref.table_name as "refTableName", ref.table_name as "ref_table_name",
ref.table_schema as "refSchemaName", ref.table_schema as "ref_schema_name",
basecol.column_name as "columnName", basecol.column_name as "column_name",
refcol.column_name as "refColumnName" refcol.column_name as "ref_column_name"
from information_schema.referential_constraints fk from information_schema.referential_constraints fk
inner join information_schema.table_constraints base on fk.constraint_name = base.constraint_name and fk.constraint_schema = base.constraint_schema inner join information_schema.table_constraints base on fk.constraint_name = base.constraint_name and fk.constraint_schema = base.constraint_schema
inner join information_schema.table_constraints ref on fk.unique_constraint_name = ref.constraint_name and fk.unique_constraint_schema = ref.constraint_schema inner join information_schema.table_constraints ref on fk.unique_constraint_name = ref.constraint_name and fk.unique_constraint_schema = ref.constraint_schema
@@ -19,6 +19,6 @@ where
base.table_schema <> 'information_schema' base.table_schema <> 'information_schema'
and base.table_schema <> 'pg_catalog' and base.table_schema <> 'pg_catalog'
and base.table_schema !~ '^pg_toast' and base.table_schema !~ '^pg_toast'
and 'tables:' || base.table_schema || '.' || base.table_name =OBJECT_ID_CONDITION and ('tables:' || base.table_schema || '.' || base.table_name) =OBJECT_ID_CONDITION
order by basecol.ordinal_position order by basecol.ordinal_position
`; `;

2
plugins/dbgate-plugin-postgres/src/backend/sql/index.js
View File

@@ -1,5 +1,6 @@
const columns = require('./columns'); const columns = require('./columns');
const tableModifications = require('./tableModifications'); const tableModifications = require('./tableModifications');
const tableList = require('./tableList');
const viewModifications = require('./viewModifications'); const viewModifications = require('./viewModifications');
const primaryKeys = require('./primaryKeys'); const primaryKeys = require('./primaryKeys');
const foreignKeys = require('./foreignKeys'); const foreignKeys = require('./foreignKeys');
@@ -10,6 +11,7 @@ const routineModifications = require('./routineModifications');
module.exports = { module.exports = {
columns, columns,
tableModifications, tableModifications,
tableList,
viewModifications, viewModifications,
primaryKeys, primaryKeys,
foreignKeys, foreignKeys,

12
plugins/dbgate-plugin-postgres/src/backend/sql/primaryKeys.js
View File

@@ -1,10 +1,10 @@
module.exports = ` module.exports = `
select select
table_constraints.constraint_schema as "constraintSchema", table_constraints.constraint_schema as "constraint_schema",
table_constraints.constraint_name as "constraintName", table_constraints.constraint_name as "constraint_name",
table_constraints.table_schema as "schemaName", table_constraints.table_schema as "schema_name",
table_constraints.table_name as "pureName", table_constraints.table_name as "pure_name",
key_column_usage.column_name as "columnName" key_column_usage.column_name as "column_name"
from information_schema.table_constraints from information_schema.table_constraints
inner join information_schema.key_column_usage on table_constraints.table_name = key_column_usage.table_name and table_constraints.constraint_name = key_column_usage.constraint_name inner join information_schema.key_column_usage on table_constraints.table_name = key_column_usage.table_name and table_constraints.constraint_name = key_column_usage.constraint_name
where where
@@ -12,6 +12,6 @@ where
and table_constraints.table_schema <> 'pg_catalog' and table_constraints.table_schema <> 'pg_catalog'
and table_constraints.table_schema !~ '^pg_toast' and table_constraints.table_schema !~ '^pg_toast'
and table_constraints.constraint_type = 'PRIMARY KEY' and table_constraints.constraint_type = 'PRIMARY KEY'
and 'tables:' || table_constraints.table_schema || '.' || table_constraints.table_name =OBJECT_ID_CONDITION and ('tables:' || table_constraints.table_schema || '.' || table_constraints.table_name) =OBJECT_ID_CONDITION
order by key_column_usage.ordinal_position order by key_column_usage.ordinal_position
`; `;

8
plugins/dbgate-plugin-postgres/src/backend/sql/routineModifications.js
View File

@@ -1,9 +1,9 @@
module.exports = ` module.exports = `
select select
routine_name as "pureName", routine_name as "pure_name",
routine_schema as "schemaName", routine_schema as "schema_name",
md5(routine_definition) as "hashCode", md5(routine_definition) as "hash_code",
routine_type as "objectType" routine_type as "object_type"
from from
information_schema.routines where routine_schema != 'information_schema' and routine_schema != 'pg_catalog' information_schema.routines where routine_schema != 'information_schema' and routine_schema != 'pg_catalog'
and routine_type in ('PROCEDURE', 'FUNCTION') and routine_type in ('PROCEDURE', 'FUNCTION')

10
plugins/dbgate-plugin-postgres/src/backend/sql/tableList.js Normal file
View File

@@ -0,0 +1,10 @@
module.exports = `
select infoTables.table_schema as "schema_name", infoTables.table_name as "pure_name"
from information_schema.tables infoTables
where infoTables.table_type not like '%VIEW%'
and ('tables:' || infoTables.table_schema || '.' || infoTables.table_name) =OBJECT_ID_CONDITION
and infoTables.table_schema <> 'pg_catalog'
and infoTables.table_schema <> 'information_schema'
and infoTables.table_schema <> 'pg_internal'
and infoTables.table_schema !~ '^pg_toast'
`;

6
plugins/dbgate-plugin-postgres/src/backend/sql/tableModifications.js
View File

@@ -1,11 +1,11 @@
module.exports = ` module.exports = `
select infoTables.table_schema as "schemaName", infoTables.table_name as "pureName", select infoTables.table_schema as "schema_name", infoTables.table_name as "pure_name",
( (
select md5(string_agg( select md5(string_agg(
infoColumns.column_name || '|' || infoColumns.data_type || '|' || infoColumns.is_nullable || '|' || coalesce(infoColumns.character_maximum_length, -1) infoColumns.column_name || '|' || infoColumns.data_type || '|' || infoColumns.is_nullable || '|' || coalesce(infoColumns.character_maximum_length, -1)
|| '|' || coalesce(infoColumns.numeric_precision, -1), || '|' || coalesce(infoColumns.numeric_precision, -1),
',' order by infoColumns.ordinal_position ',' order by infoColumns.ordinal_position
)) as "hashCodeColumns" )) as "hash_code_columns"
from information_schema.columns infoColumns from information_schema.columns infoColumns
where infoColumns.table_schema = infoTables.table_schema and infoColumns.table_name = infoTables.table_name where infoColumns.table_schema = infoTables.table_schema and infoColumns.table_name = infoTables.table_name
), ),
@@ -13,7 +13,7 @@ select infoTables.table_schema as "schemaName", infoTables.table_name as "pureNa
select md5(string_agg( select md5(string_agg(
infoConstraints.constraint_name || '|' || infoConstraints.constraint_type , infoConstraints.constraint_name || '|' || infoConstraints.constraint_type ,
',' order by infoConstraints.constraint_name ',' order by infoConstraints.constraint_name
)) as "hashCodeConstraints" )) as "hash_code_constraints"
from information_schema.table_constraints infoConstraints from information_schema.table_constraints infoConstraints
where infoConstraints.table_schema = infoTables.table_schema and infoConstraints.table_name = infoTables.table_name where infoConstraints.table_schema = infoTables.table_schema and infoConstraints.table_name = infoTables.table_name
) )

6
plugins/dbgate-plugin-postgres/src/backend/sql/viewModifications.js
View File

@@ -1,8 +1,8 @@
module.exports = ` module.exports = `
select select
table_name as "pureName", table_name as "pure_name",
table_schema as "schemaName", table_schema as "schema_name",
md5(view_definition) as "hashCode" md5(view_definition) as "hash_code"
from from
information_schema.views where table_schema != 'information_schema' and table_schema != 'pg_catalog' information_schema.views where table_schema != 'information_schema' and table_schema != 'pg_catalog'
`; `;

8
plugins/dbgate-plugin-postgres/src/backend/sql/views.js
View File

@@ -1,9 +1,9 @@
module.exports = ` module.exports = `
select select
table_name as "pureName", table_name as "pure_name",
table_schema as "schemaName", table_schema as "schema_name",
view_definition as "createSql", view_definition as "create_sql",
md5(view_definition) as "hashCode" md5(view_definition) as "hash_code"
from from
information_schema.views information_schema.views
where table_schema != 'information_schema' and table_schema != 'pg_catalog' where table_schema != 'information_schema' and table_schema != 'pg_catalog'

6
plugins/dbgate-plugin-postgres/src/frontend/drivers.js
View File

@@ -12,6 +12,7 @@ const dialect = {
quoteIdentifier(s) { quoteIdentifier(s) {
return '"' + s + '"'; return '"' + s + '"';
}, },
stringAgg: true,
}; };
/** @type {import('dbgate-types').EngineDriver} */ /** @type {import('dbgate-types').EngineDriver} */
@@ -38,7 +39,10 @@ const cockroachDriver = {
const redshiftDriver = { const redshiftDriver = {
...driverBase, ...driverBase,
dumperClass: Dumper, dumperClass: Dumper,
dialect, dialect: {
...dialect,
stringAgg: false,
},
engine: 'red@dbgate-plugin-postgres', engine: 'red@dbgate-plugin-postgres',
title: 'Amazon Redshift', title: 'Amazon Redshift',
defaultPort: 5439, defaultPort: 5439,