dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-28 04:56:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

analyser refactor + optimalization

Browse Source
This commit is contained in:
Jan Prochazka
2023-01-21 10:13:08 +01:00
parent 3dfae351a6
commit f9f879272b
5 changed files with 116 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js
View File

@@ -82,33 +82,35 @@ class MsSqlAnalyser extends DatabaseAnalyser {
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
const tablesRows = await this.driver.query(this.pool, this.createQuery('tables', ['tables']));
const tablesRows = await this.analyserQuery('tables', ['tables']);
this.feedback({ analysingMessage: 'Loading columns' });
const columnsRows = await this.driver.query(this.pool, this.createQuery('columns', ['tables']));
const columnsRows = await this.analyserQuery('columns', ['tables']);
this.feedback({ analysingMessage: 'Loading primary keys' });
const pkColumnsRows = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables']));
const pkColumnsRows = await this.analyserQuery('primaryKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign keys' });
const fkColumnsRows = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables']));
const fkColumnsRows = await this.analyserQuery('foreignKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading schemas' });
const schemaRows = await this.driver.query(this.pool, this.createQuery('getSchemas'));
const schemaRows = await this.analyserQuery('getSchemas');
this.feedback({ analysingMessage: 'Loading indexes' });
const indexesRows = await this.driver.query(this.pool, this.createQuery('indexes', ['tables']));
const indexesRows = await this.analyserQuery('indexes', ['tables']);
this.feedback({ analysingMessage: 'Loading index columns' });
const indexcolsRows = await this.driver.query(this.pool, this.createQuery('indexcols', ['tables']));
const indexcolsRows = await this.analyserQuery('indexcols', ['tables']);
this.feedback({ analysingMessage: 'Loading default schema' });
const defaultSchemaRows = await this.driver.query(this.pool, 'SELECT SCHEMA_NAME() as name');
const defaultSchemaRows = await this.query(this.pool, 'SELECT SCHEMA_NAME() as name');
this.feedback({ analysingMessage: 'Loading table sizes' });
const tableSizes = await this.driver.query(this.pool, this.createQuery('tableSizes'));
const tableSizes = await this.analyserQuery('tableSizes');
const schemas = schemaRows.rows;
const tableSizesDict = _.mapValues(_.keyBy(tableSizes.rows, 'objectId'), 'tableRowCount');
this.feedback({ analysingMessage: 'Loading SQL code' });
const sqlCodeRows = await this.driver.query(
this.pool,
this.createQuery('loadSqlCode', ['views', 'procedures', 'functions', 'triggers'])
);
const sqlCodeRows = await this.analyserQuery('loadSqlCode', [
'views',
'procedures',
'functions',
'triggers',
]);
const getCreateSql = row =>
sqlCodeRows.rows
.filter(x => x.pureName == row.pureName && x.schemaName == row.schemaName)
@@ -116,14 +118,11 @@ class MsSqlAnalyser extends DatabaseAnalyser {
.join('');
this.feedback({ analysingMessage: 'Loading views' });
const viewsRows = await this.driver.query(this.pool, this.createQuery('views', ['views']));
const viewsRows = await this.analyserQuery('views', ['views']);
this.feedback({ analysingMessage: 'Loading procedures & functions' });
const programmableRows = await this.driver.query(
this.pool,
this.createQuery('programmables', ['procedures', 'functions'])
);
const programmableRows = await this.analyserQuery('programmables', ['procedures', 'functions']);
this.feedback({ analysingMessage: 'Loading view columns' });
const viewColumnRows = await this.driver.query(this.pool, this.createQuery('viewColumns', ['views']));
const viewColumnRows = await this.analyserQuery('viewColumns', ['views']);
this.feedback({ analysingMessage: 'Finalizing DB structure' });
const tables = tablesRows.rows.map(row => ({
@@ -190,8 +189,8 @@ class MsSqlAnalyser extends DatabaseAnalyser {
}
async _getFastSnapshot() {
const modificationsQueryData = await this.driver.query(this.pool, this.createQuery('modifications'));
const tableSizes = await this.driver.query(this.pool, this.createQuery('tableSizes'));
const modificationsQueryData = await this.analyserQuery('modifications');
const tableSizes = await this.analyserQuery('tableSizes');
const res = DatabaseAnalyser.createEmptyStructure();
for (const item of modificationsQueryData.rows) {

30
plugins/dbgate-plugin-mysql/src/backend/Analyser.js
View File

@@ -57,7 +57,7 @@ class Analyser extends DatabaseAnalyser {
async getViewTexts(allViewNames) {
const res = {};
const views = await this.safeQuery(this.createQuery('viewTexts', ['views']));
const views = await this.analyserQuery('viewTexts', ['views']);
for (const view of views.rows) {
res[view.pureName] = `CREATE VIEW \`${view.pureName}\` AS ${view.viewDefinition}`;
}
@@ -76,24 +76,24 @@ class Analyser extends DatabaseAnalyser {
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
const tables = await this.driver.query(this.pool, this.createQuery('tables', ['tables']));
const tables = await this.analyserQuery('tables', ['tables']);
this.feedback({ analysingMessage: 'Loading columns' });
const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables', 'views']));
const columns = await this.analyserQuery('columns', ['tables', 'views']);
this.feedback({ analysingMessage: 'Loading primary keys' });
const pkColumns = await this.safeQuery(this.createQuery('primaryKeys', ['tables']));
const pkColumns = await this.analyserQuery('primaryKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign keys' });
const fkColumns = await this.safeQuery(this.createQuery('foreignKeys', ['tables']));
const fkColumns = await this.analyserQuery('foreignKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading views' });
const views = await this.safeQuery(this.createQuery('views', ['views']));
const views = await this.analyserQuery('views', ['views']);
this.feedback({ analysingMessage: 'Loading programmables' });
const programmables = await this.safeQuery(this.createQuery('programmables', ['procedures', 'functions']));
const programmables = await this.analyserQuery('programmables', ['procedures', 'functions']);
this.feedback({ analysingMessage: 'Loading view texts' });
const viewTexts = await this.getViewTexts(views.rows.map(x => x.pureName));
this.feedback({ analysingMessage: 'Loading indexes' });
const indexes = await this.safeQuery(this.createQuery('indexes', ['tables']));
const indexes = await this.analyserQuery('indexes', ['tables']);
this.feedback({ analysingMessage: 'Loading uniques' });
const uniqueNames = await this.safeQuery(this.createQuery('uniqueNames', ['tables']));
const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']);
this.feedback({ analysingMessage: 'Finalizing DB structure' });
const res = {
@@ -169,15 +169,9 @@ class Analyser extends DatabaseAnalyser {
}
async _getFastSnapshot() {
const tableModificationsQueryData = await this.driver.query(this.pool, this.createQuery('tableModifications'));
const procedureModificationsQueryData = await this.driver.query(
this.pool,
this.createQuery('procedureModifications')
);
const functionModificationsQueryData = await this.driver.query(
this.pool,
this.createQuery('functionModifications')
);
const tableModificationsQueryData = await this.analyserQuery('tableModifications');
const procedureModificationsQueryData = await this.analyserQuery('procedureModifications');
const functionModificationsQueryData = await this.analyserQuery('functionModifications');
return {
tables: tableModificationsQueryData.rows

112
plugins/dbgate-plugin-oracle/src/backend/Analyser.js
View File

@@ -68,45 +68,40 @@ class Analyser extends DatabaseAnalyser {
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
const tables = await this.driver.query(
this.pool,
this.createQuery(this.driver.dialect.stringAgg ? 'tableList' : 'tableList', ['tables'])
);
const tables = await this.analyserQuery(this.driver.dialect.stringAgg ? 'tableList' : 'tableList', ['tables']);
this.feedback({ analysingMessage: 'Loading columns' });
const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables', 'views']));
const columns = await this.analyserQuery('columns', ['tables', 'views']);
this.feedback({ analysingMessage: 'Loading primary keys' });
const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables']));
const pkColumns = await this.analyserQuery('primaryKeys', ['tables']);
//let fkColumns = null;
this.feedback({ analysingMessage: 'Loading foreign keys' });
const fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables']));
this.feedback({ analysingMessage: 'Loading foreign keys' });
const fkColumns = await this.analyserQuery('foreignKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading views' });
const views = await this.driver.query(this.pool, this.createQuery('views', ['views']));
const views = await this.analyserQuery('views', ['views']);
let geometryColumns = { rows: [] };
let geographyColumns = { rows: [] };
this.feedback({ analysingMessage: 'Loading materialized views' });
const matviews = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviews', ['matviews']))
: null;
const matviews = this.driver.dialect.materializedViews ? await this.analyserQuery('matviews', ['matviews']) : null;
this.feedback({ analysingMessage: 'Loading materialized view columns' });
const matviewColumns = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviewColumns', ['matviews']))
? await this.analyserQuery('matviewColumns', ['matviews'])
: null;
this.feedback({ analysingMessage: 'Loading routines' });
const routines = await this.driver.query(this.pool, this.createQuery('routines', ['procedures', 'functions']));
const routines = await this.analyserQuery('routines', ['procedures', 'functions']);
this.feedback({ analysingMessage: 'Loading indexes' });
const indexes = this.driver.__analyserInternals.skipIndexes
? { rows: [] }
: await this.driver.query(this.pool, this.createQuery('indexes', ['tables']));
: await this.analyserQuery('indexes', ['tables']);
this.feedback({ analysingMessage: 'Loading index columns' });
// const indexcols = this.driver.__analyserInternals.skipIndexes
// ? { rows: [] }
// : await this.driver.query(this.pool, this.createQuery('indexcols', ['tables']));
// const indexcols = this.driver.__analyserInternals.skipIndexes
// ? { rows: [] }
// : await this.driver.query(this.pool, this.createQuery('indexcols', ['tables']));
this.feedback({ analysingMessage: 'Loading unique names' });
const uniqueNames = await this.driver.query(this.pool, this.createQuery('uniqueNames', ['tables']));
const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']);
this.feedback({ analysingMessage: 'Finalizing DB structure' });
const columnColumnsMapped = fkColumns.rows.map(x => ({
@@ -144,34 +139,35 @@ class Analyser extends DatabaseAnalyser {
.map(col => getColumnInfo(col, newTable, geometryColumns, geographyColumns)),
primaryKey: DatabaseAnalyser.extractPrimaryKeys(newTable, pkColumnsMapped),
foreignKeys: DatabaseAnalyser.extractForeignKeys(newTable, columnColumnsMapped),
indexes: _.uniqBy(
indexes.rows.filter(
idx =>
idx.tableName == table.pureName && !uniqueNames.rows.find(x => x.constraintName == idx.constraintName)
),
'constraintName'
).map(idx => ({
..._.pick(idx, ['constraintName', 'indexType']),
isUnique: idx.Unique === 'UNIQUE',
columns: indexes.rows
.filter(col => col.tableName == idx.tableName && col.constraintName == idx.constraintName)
.map(col => ({
..._.pick(col, ['columnName']),
})),
})),
uniques: _.uniqBy(
indexes.rows.filter(
idx => idx.tableName == table.pureName && uniqueNames.rows.find(x => x.constraintName == idx.constraintName)
),
'constraintName'
).map(idx => ({
..._.pick(idx, ['constraintName']),
columns: indexes.rows
.filter(col => col.tableName == idx.tableName && col.constraintName == idx.constraintName)
.map(col => ({
..._.pick(col, ['columnName']),
})),
})),
indexes: _.uniqBy(
indexes.rows.filter(
idx =>
idx.tableName == table.pureName && !uniqueNames.rows.find(x => x.constraintName == idx.constraintName)
),
'constraintName'
).map(idx => ({
..._.pick(idx, ['constraintName', 'indexType']),
isUnique: idx.Unique === 'UNIQUE',
columns: indexes.rows
.filter(col => col.tableName == idx.tableName && col.constraintName == idx.constraintName)
.map(col => ({
..._.pick(col, ['columnName']),
})),
})),
uniques: _.uniqBy(
indexes.rows.filter(
idx =>
idx.tableName == table.pureName && uniqueNames.rows.find(x => x.constraintName == idx.constraintName)
),
'constraintName'
).map(idx => ({
..._.pick(idx, ['constraintName']),
columns: indexes.rows
.filter(col => col.tableName == idx.tableName && col.constraintName == idx.constraintName)
.map(col => ({
..._.pick(col, ['columnName']),
})),
})),
};
}),
views: views.rows.map(view => ({
@@ -225,13 +221,13 @@ class Analyser extends DatabaseAnalyser {
return null;
const tableModificationsQueryData = this.driver.dialect.stringAgg
? await this.driver.query(this.pool, this.createQuery('tableModifications'))
? await this.analyserQuery('tableModifications')
: null;
const viewModificationsQueryData = await this.driver.query(this.pool, this.createQuery('viewModifications'));
const viewModificationsQueryData = await this.analyserQuery('viewModifications');
const matviewModificationsQueryData = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviewModifications'))
? await this.analyserQuery('matviewModifications')
: null;
const routineModificationsQueryData = await this.driver.query(this.pool, this.createQuery('routineModifications'));
const routineModificationsQueryData = await this.analyserQuery('routineModifications');
return {
tables: tableModificationsQueryData
@@ -243,13 +239,13 @@ class Analyser extends DatabaseAnalyser {
}))
: null,
views: viewModificationsQueryData
? viewModificationsQueryData.rows.map(x => ({
objectId: `views:${x.schema_name}.${x.pure_name}`,
pureName: x.pure_name,
schemaName: x.schema_name,
contentHash: x.hash_code,
}))
: undefined,
? viewModificationsQueryData.rows.map(x => ({
objectId: `views:${x.schema_name}.${x.pure_name}`,
pureName: x.pure_name,
schemaName: x.schema_name,
contentHash: x.hash_code,
}))
: undefined,
matviews: matviewModificationsQueryData
? matviewModificationsQueryData.rows.map(x => ({
objectId: `matviews:${x.schema_name}.${x.pure_name}`,

49
plugins/dbgate-plugin-postgres/src/backend/Analyser.js
View File

@@ -67,28 +67,23 @@ class Analyser extends DatabaseAnalyser {
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
const tables = await this.driver.query(
this.pool,
this.createQuery(this.driver.dialect.stringAgg ? 'tableModifications' : 'tableList', ['tables'])
);
const tables = await this.analyserQuery(this.driver.dialect.stringAgg ? 'tableModifications' : 'tableList', [
'tables',
]);
this.feedback({ analysingMessage: 'Loading columns' });
const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables', 'views']));
const columns = await this.analyserQuery('columns', ['tables', 'views']);
this.feedback({ analysingMessage: 'Loading primary keys' });
const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables']));
const pkColumns = await this.analyserQuery('primaryKeys', ['tables']);
let fkColumns = null;
this.feedback({ analysingMessage: 'Loading foreign key constraints' });
const fk_tableConstraints = await this.driver.query(this.pool, this.createQuery('fk_tableConstraints', ['tables']));
const fk_tableConstraints = await this.analyserQuery('fk_tableConstraints', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign key refs' });
const fk_referentialConstraints = await this.driver.query(
this.pool,
this.createQuery('fk_referentialConstraints', ['tables'])
);
const fk_referentialConstraints = await this.analyserQuery('fk_referentialConstraints', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign key columns' });
const fk_keyColumnUsage = await this.driver.query(this.pool, this.createQuery('fk_keyColumnUsage', ['tables']));
const fk_keyColumnUsage = await this.analyserQuery('fk_keyColumnUsage', ['tables']);
const cntKey = x => `${x.constraint_name}|${x.constraint_schema}`;
const fkRows = [];
@@ -129,37 +124,35 @@ class Analyser extends DatabaseAnalyser {
fkColumns = { rows: fkRows };
this.feedback({ analysingMessage: 'Loading views' });
const views = await this.driver.query(this.pool, this.createQuery('views', ['views']));
const views = await this.analyserQuery('views', ['views']);
this.feedback({ analysingMessage: 'Loading materialized views' });
const matviews = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviews', ['matviews']))
: null;
const matviews = this.driver.dialect.materializedViews ? await this.analyserQuery('matviews', ['matviews']) : null;
this.feedback({ analysingMessage: 'Loading materialized view columns' });
const matviewColumns = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviewColumns', ['matviews']))
? await this.analyserQuery('matviewColumns', ['matviews'])
: null;
this.feedback({ analysingMessage: 'Loading routines' });
const routines = await this.driver.query(this.pool, this.createQuery('routines', ['procedures', 'functions']));
const routines = await this.analyserQuery('routines', ['procedures', 'functions']);
this.feedback({ analysingMessage: 'Loading indexes' });
const indexes = this.driver.__analyserInternals.skipIndexes
? { rows: [] }
: await this.driver.query(this.pool, this.createQuery('indexes', ['tables']));
: await this.analyserQuery('indexes', ['tables']);
this.feedback({ analysingMessage: 'Loading index columns' });
const indexcols = this.driver.__analyserInternals.skipIndexes
? { rows: [] }
: await this.driver.query(this.pool, this.createQuery('indexcols', ['tables']));
: await this.analyserQuery('indexcols', ['tables']);
this.feedback({ analysingMessage: 'Loading unique names' });
const uniqueNames = await this.driver.query(this.pool, this.createQuery('uniqueNames', ['tables']));
const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']);
let geometryColumns = { rows: [] };
if (views.rows.find(x => x.pure_name == 'geometry_columns' && x.schema_name == 'public')) {
this.feedback({ analysingMessage: 'Loading geometry columns' });
geometryColumns = await this.safeQuery(this.createQuery('geometryColumns', ['tables']));
geometryColumns = await this.analyserQuery('geometryColumns', ['tables']);
}
let geographyColumns = { rows: [] };
if (views.rows.find(x => x.pure_name == 'geography_columns' && x.schema_name == 'public')) {
this.feedback({ analysingMessage: 'Loading geography columns' });
geographyColumns = await this.safeQuery(this.createQuery('geographyColumns', ['tables']));
geographyColumns = await this.analyserQuery('geographyColumns', ['tables']);
}
this.feedback({ analysingMessage: 'Finalizing DB structure' });
@@ -289,13 +282,13 @@ class Analyser extends DatabaseAnalyser {
async _getFastSnapshot() {
const tableModificationsQueryData = this.driver.dialect.stringAgg
? await this.driver.query(this.pool, this.createQuery('tableModifications'))
? await this.analyserQuery('tableModifications')
: null;
const viewModificationsQueryData = await this.driver.query(this.pool, this.createQuery('viewModifications'));
const viewModificationsQueryData = await this.analyserQuery('viewModifications');
const matviewModificationsQueryData = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviewModifications'))
? await this.analyserQuery('matviewModifications')
: null;
const routineModificationsQueryData = await this.driver.query(this.pool, this.createQuery('routineModifications'));
const routineModificationsQueryData = await this.analyserQuery('routineModifications');
return {
tables: tableModificationsQueryData