added plugins

This commit is contained in:
Jan Prochazka
2021-04-13 16:17:53 +02:00
parent 446e7c139f
commit 4d5cc119f2
115 changed files with 5519 additions and 24 deletions

View File

@@ -0,0 +1,185 @@
const fp = require('lodash/fp');
const _ = require('lodash');
const sql = require('./sql');
const { DatabaseAnalyser } = require('dbgate-tools');
const { isTypeString, isTypeNumeric } = require('dbgate-tools');
function normalizeTypeName(dataType) {
if (dataType == 'character varying') return 'varchar';
if (dataType == 'timestamp without time zone') return 'timestamp';
return dataType;
}
function getColumnInfo({
isNullable,
isIdentity,
columnName,
dataType,
charMaxLength,
numericPrecision,
numericScale,
defaultValue,
}) {
const normDataType = normalizeTypeName(dataType);
let fullDataType = normDataType;
if (charMaxLength && isTypeString(normDataType)) fullDataType = `${normDataType}(${charMaxLength})`;
if (numericPrecision && numericScale && isTypeNumeric(normDataType))
fullDataType = `${normDataType}(${numericPrecision},${numericScale})`;
return {
columnName,
dataType: fullDataType,
notNull: !isNullable || isNullable == 'NO' || isNullable == 'no',
autoIncrement: !!isIdentity,
defaultValue,
};
}
class Analyser extends DatabaseAnalyser {
constructor(pool, driver) {
super(pool, driver);
}
createQuery(resFileName, typeFields) {
let res = sql[resFileName];
if (this.singleObjectFilter) {
const { typeField, schemaName, pureName } = this.singleObjectFilter;
if (!typeFields || !typeFields.includes(typeField)) return null;
res = res.replace(/=OBJECT_ID_CONDITION/g, ` = '${typeField}:${schemaName || 'public'}.${pureName}'`);
return res;
}
if (!this.modifications || !typeFields || this.modifications.length == 0) {
res = res.replace(/=OBJECT_ID_CONDITION/g, ' is not null');
} else {
const filterNames = this.modifications
.filter(x => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change'))
.filter(x => x.newName)
.map(x => `${x.objectTypeField}:${x.newName.schemaName}.${x.newName.pureName}`);
if (filterNames.length == 0) {
res = res.replace(/=OBJECT_ID_CONDITION/g, ' IS NULL');
} else {
res = res.replace(/=OBJECT_ID_CONDITION/g, ` in (${filterNames.map(x => `'${x}'`).join(',')})`);
}
}
return res;
// let res = sql[resFileName];
// res = res.replace('=[OBJECT_ID_CONDITION]', ' is not null');
// return res;
}
async _runAnalysis() {
const tables = await this.driver.query(this.pool, this.createQuery('tableModifications', ['tables']));
const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables']));
const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables']));
const fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables']));
const views = await this.driver.query(this.pool, this.createQuery('views', ['views']));
const routines = await this.driver.query(this.pool, this.createQuery('routines', ['procedures', 'functions']));
// console.log('PG fkColumns', fkColumns.rows);
return this.mergeAnalyseResult({
tables: tables.rows.map(table => ({
...table,
objectId: `tables:${table.schemaName}.${table.pureName}`,
columns: columns.rows
.filter(col => col.pureName == table.pureName && col.schemaName == table.schemaName)
.map(getColumnInfo),
primaryKey: DatabaseAnalyser.extractPrimaryKeys(table, pkColumns.rows),
foreignKeys: DatabaseAnalyser.extractForeignKeys(table, fkColumns.rows),
})),
views: views.rows.map(view => ({
...view,
objectId: `views:${view.schemaName}.${view.pureName}`,
columns: columns.rows
.filter(col => col.pureName == view.pureName && col.schemaName == view.schemaName)
.map(getColumnInfo),
})),
procedures: routines.rows
.filter(x => x.objectType == 'PROCEDURE')
.map(proc => ({
objectId: `procedures:${proc.schemaName}.${proc.pureName}`,
...proc,
})),
functions: routines.rows
.filter(x => x.objectType == 'FUNCTION')
.map(func => ({
objectId: `functions:${func.schemaName}.${func.pureName}`,
...func,
})),
});
}
async getModifications() {
const tableModificationsQueryData = await this.driver.query(this.pool, this.createQuery('tableModifications'));
const viewModificationsQueryData = await this.driver.query(this.pool, this.createQuery('viewModifications'));
const routineModificationsQueryData = await this.driver.query(this.pool, this.createQuery('routineModifications'));
const allModifications = _.compact([
...tableModificationsQueryData.rows.map(x => ({ ...x, objectTypeField: 'tables' })),
...viewModificationsQueryData.rows.map(x => ({ ...x, objectTypeField: 'views' })),
...routineModificationsQueryData.rows
.filter(x => x.objectType == 'PROCEDURE')
.map(x => ({ ...x, objectTypeField: 'procedures' })),
...routineModificationsQueryData.rows
.filter(x => x.objectType == 'FUNCTION')
.map(x => ({ ...x, objectTypeField: 'functions' })),
]);
const modifications = allModifications.map(x => {
const { objectTypeField, hashCode, pureName, schemaName } = x;
if (!objectTypeField || !this.structure[objectTypeField]) return null;
const obj = this.structure[objectTypeField].find(x => x.pureName == pureName && x.schemaName == schemaName);
// object not modified
if (obj && obj.hashCode == hashCode) return null;
// console.log('MODIFICATION OF ', objectTypeField, schemaName, pureName);
/** @type {import('dbgate-types').DatabaseModification} */
const action = obj
? {
newName: { schemaName, pureName },
oldName: _.pick(obj, ['schemaName', 'pureName']),
action: 'change',
objectTypeField,
objectId: `${objectTypeField}:${schemaName}.${pureName}`,
}
: {
newName: { schemaName, pureName },
action: 'add',
objectTypeField,
objectId: `${objectTypeField}:${schemaName}.${pureName}`,
};
return action;
});
return [
..._.compact(modifications),
...this.getDeletedObjects([...allModifications.map(x => `${x.schemaName}.${x.pureName}`)]),
];
}
getDeletedObjectsForField(nameArray, objectTypeField) {
return this.structure[objectTypeField]
.filter(x => !nameArray.includes(`${x.schemaName}.${x.pureName}`))
.map(x => ({
oldName: _.pick(x, ['schemaName', 'pureName']),
action: 'remove',
objectTypeField,
objectId: `${objectTypeField}:${x.schemaName}.${x.pureName}`,
}));
}
getDeletedObjects(nameArray) {
return [
...this.getDeletedObjectsForField(nameArray, 'tables'),
...this.getDeletedObjectsForField(nameArray, 'views'),
...this.getDeletedObjectsForField(nameArray, 'procedures'),
...this.getDeletedObjectsForField(nameArray, 'functions'),
...this.getDeletedObjectsForField(nameArray, 'triggers'),
];
}
}
module.exports = Analyser;

View File

@@ -0,0 +1,213 @@
const _ = require('lodash');
const stream = require('stream');
const driverBase = require('../frontend/driver');
const Analyser = require('./Analyser');
const pg = require('pg');
const pgQueryStream = require('pg-query-stream');
const { createBulkInsertStreamBase, splitPostgresQuery, makeUniqueColumnNames } = require('dbgate-tools');
function extractPostgresColumns(result) {
if (!result || !result.fields) return [];
const res = result.fields.map(fld => ({
columnName: fld.name,
}));
makeUniqueColumnNames(res);
return res;
}
function zipDataRow(rowArray, columns) {
return _.zipObject(
columns.map(x => x.columnName),
rowArray
);
}
async function runStreamItem(client, sql, options) {
return new Promise((resolve, reject) => {
const query = new pgQueryStream(sql, undefined, { rowMode: 'array' });
const stream = client.query(query);
// const handleInfo = (info) => {
// const { message, lineNumber, procName } = info;
// options.info({
// message,
// line: lineNumber,
// procedure: procName,
// time: new Date(),
// severity: 'info',
// });
// };
let wasHeader = false;
const handleEnd = result => {
// console.log('RESULT', result);
resolve();
};
let columns = null;
const handleReadable = () => {
if (!wasHeader) {
columns = extractPostgresColumns(query._result);
if (columns && columns.length > 0) {
options.recordset(columns);
}
wasHeader = true;
}
for (;;) {
const row = stream.read();
if (!row) break;
options.row(zipDataRow(row, columns));
}
};
// const handleFields = (columns) => {
// // console.log('FIELDS', columns[0].name);
// options.recordset(columns);
// // options.recordset(extractColumns(columns));
// };
const handleError = error => {
console.log('ERROR', error);
const { message, lineNumber, procName } = error;
options.info({
message,
line: lineNumber,
procedure: procName,
time: new Date(),
severity: 'error',
});
resolve();
};
stream.on('error', handleError);
stream.on('readable', handleReadable);
// stream.on('result', handleRow)
// stream.on('data', handleRow)
stream.on('end', handleEnd);
});
}
/** @type {import('dbgate-types').EngineDriver} */
const driver = {
...driverBase,
analyserClass: Analyser,
async connect({ server, port, user, password, database, ssl }) {
const client = new pg.Client({
host: server,
port,
user,
password,
database: database || 'postgres',
ssl,
});
await client.connect();
return client;
},
async query(client, sql) {
if (sql == null) {
return {
rows: [],
columns: [],
};
}
const res = await client.query({ text: sql, rowMode: 'array' });
const columns = extractPostgresColumns(res);
return { rows: res.rows.map(row => zipDataRow(row, columns)), columns };
},
async stream(client, sql, options) {
const sqlSplitted = splitPostgresQuery(sql);
for (const sqlItem of sqlSplitted) {
await runStreamItem(client, sqlItem, options);
}
options.done();
// return stream;
},
// async analyseSingleObject(pool, name, typeField = 'tables') {
// const analyser = new PostgreAnalyser(pool, this);
// analyser.singleObjectFilter = { ...name, typeField };
// const res = await analyser.fullAnalysis();
// return res.tables[0];
// },
// // @ts-ignore
// analyseSingleTable(pool, name) {
// return this.analyseSingleObject(pool, name, 'tables');
// },
async getVersion(client) {
const { rows } = await this.query(client, 'SELECT version()');
const { version } = rows[0];
return { version };
},
// async analyseFull(pool) {
// const analyser = new PostgreAnalyser(pool, this);
// return analyser.fullAnalysis();
// },
// async analyseIncremental(pool, structure) {
// const analyser = new PostgreAnalyser(pool, this);
// return analyser.incrementalAnalysis(structure);
// },
async readQuery(client, sql, structure) {
const query = new pgQueryStream(sql, undefined, { rowMode: 'array' });
const queryStream = client.query(query);
let wasHeader = false;
const pass = new stream.PassThrough({
objectMode: true,
highWaterMark: 100,
});
const handleEnd = result => {
pass.end();
};
let columns = null;
const handleReadable = () => {
if (!wasHeader) {
columns = extractPostgresColumns(query._result);
pass.write({
__isStreamHeader: true,
...(structure || { columns }),
});
wasHeader = true;
}
for (;;) {
const row = queryStream.read();
if (!row) break;
pass.write(zipDataRow(row, columns));
}
};
const handleError = error => {
console.error(error);
pass.end();
};
queryStream.on('error', handleError);
queryStream.on('readable', handleReadable);
queryStream.on('end', handleEnd);
return pass;
},
// createDumper() {
// return new PostgreDumper(this);
// },
async writeTable(pool, name, options) {
// @ts-ignore
return createBulkInsertStreamBase(this, stream, pool, name, options);
},
async listDatabases(client) {
const { rows } = await this.query(client, 'SELECT datname AS name FROM pg_database WHERE datistemplate = false');
return rows;
},
};
module.exports = driver;

View File

@@ -0,0 +1,6 @@
const driver = require('./driver');
module.exports = {
packageName: 'dbgate-plugin-postgres',
driver,
};

View File

@@ -0,0 +1,19 @@
module.exports = `
select
table_schema as "schemaName",
table_name as "pureName",
column_name as "columnName",
is_nullable as "isNullable",
data_type as "dataType",
character_maximum_length as "charMaxLength",
numeric_precision as "numericPrecision",
numeric_scale as "numericScale",
column_default as "defaultValue"
from information_schema.columns
where
table_schema <> 'information_schema'
and table_schema <> 'pg_catalog'
and table_schema !~ '^pg_toast'
and 'tables:' || table_schema || '.' || table_name =OBJECT_ID_CONDITION
order by ordinal_position
`;

View File

@@ -0,0 +1,24 @@
module.exports = `
select
fk.constraint_name as "constraintName",
fk.constraint_schema as "constraintSchema",
base.table_name as "pureName",
base.table_schema as "schemaName",
fk.update_rule as "updateAction",
fk.delete_rule as "deleteAction",
ref.table_name as "refTableName",
ref.table_schema as "refSchemaName",
basecol.column_name as "columnName",
refcol.column_name as "refColumnName"
from information_schema.referential_constraints fk
inner join information_schema.table_constraints base on fk.constraint_name = base.constraint_name and fk.constraint_schema = base.constraint_schema
inner join information_schema.table_constraints ref on fk.unique_constraint_name = ref.constraint_name and fk.unique_constraint_schema = ref.constraint_schema
inner join information_schema.key_column_usage basecol on base.table_name = basecol.table_name and base.constraint_name = basecol.constraint_name
inner join information_schema.key_column_usage refcol on ref.table_name = refcol.table_name and ref.constraint_name = refcol.constraint_name and basecol.ordinal_position = refcol.ordinal_position
where
base.table_schema <> 'information_schema'
and base.table_schema <> 'pg_catalog'
and base.table_schema !~ '^pg_toast'
and 'tables:' || base.table_schema || '.' || base.table_name =OBJECT_ID_CONDITION
order by basecol.ordinal_position
`;

View File

@@ -0,0 +1,19 @@
const columns = require('./columns');
const tableModifications = require('./tableModifications');
const viewModifications = require('./viewModifications');
const primaryKeys = require('./primaryKeys');
const foreignKeys = require('./foreignKeys');
const views = require('./views');
const routines = require('./routines');
const routineModifications = require('./routineModifications');
module.exports = {
columns,
tableModifications,
viewModifications,
primaryKeys,
foreignKeys,
views,
routines,
routineModifications,
};

View File

@@ -0,0 +1,17 @@
module.exports = `
select
table_constraints.constraint_schema as "constraintSchema",
table_constraints.constraint_name as "constraintName",
table_constraints.table_schema as "schemaName",
table_constraints.table_name as "pureName",
key_column_usage.column_name as "columnName"
from information_schema.table_constraints
inner join information_schema.key_column_usage on table_constraints.table_name = key_column_usage.table_name and table_constraints.constraint_name = key_column_usage.constraint_name
where
table_constraints.table_schema <> 'information_schema'
and table_constraints.table_schema <> 'pg_catalog'
and table_constraints.table_schema !~ '^pg_toast'
and table_constraints.constraint_type = 'PRIMARY KEY'
and 'tables:' || table_constraints.table_schema || '.' || table_constraints.table_name =OBJECT_ID_CONDITION
order by key_column_usage.ordinal_position
`;

View File

@@ -0,0 +1,10 @@
module.exports = `
select
routine_name as "pureName",
routine_schema as "schemaName",
md5(routine_definition) as "hashCode",
routine_type as "objectType"
from
information_schema.routines where routine_schema != 'information_schema' and routine_schema != 'pg_catalog'
and routine_type in ('PROCEDURE', 'FUNCTION')
`;

View File

@@ -0,0 +1,15 @@
module.exports = `
select
routine_name as "pureName",
routine_schema as "schemaName",
routine_definition as "createSql",
md5(routine_definition) as "hashCode",
routine_type as "objectType"
from
information_schema.routines where routine_schema != 'information_schema' and routine_schema != 'pg_catalog'
and (
(routine_type = 'PROCEDURE' and ('procedures:' || routine_schema || '.' || routine_schema) =OBJECT_ID_CONDITION)
or
(routine_type = 'FUNCTION' and ('functions:' || routine_schema || '.' || routine_schema) =OBJECT_ID_CONDITION)
)
`;

View File

@@ -0,0 +1,52 @@
module.exports = `
with pkey as
(
select cc.conrelid, format(E'create constraint %I primary key(%s);\\n', cc.conname,
string_agg(a.attname, ', '
order by array_position(cc.conkey, a.attnum))) pkey
from pg_catalog.pg_constraint cc
join pg_catalog.pg_class c on c.oid = cc.conrelid
join pg_catalog.pg_attribute a on a.attrelid = cc.conrelid
and a.attnum = any(cc.conkey)
where cc.contype = 'p'
group by cc.conrelid, cc.conname
)
SELECT oid as "objectId", nspname as "schemaName", relname as "pureName",
md5('CREATE TABLE ' || nspname || '.' || relname || E'\\n(\\n' ||
array_to_string(
array_agg(
' ' || column_name || ' ' || type || ' '|| not_null
)
, E',\\n'
) || E'\\n);\\n' || coalesce((select pkey from pkey where pkey.conrelid = oid),'NO_PK')) as "hashCode"
from
(
SELECT
c.relname, a.attname AS column_name, c.oid,
n.nspname,
pg_catalog.format_type(a.atttypid, a.atttypmod) as type,
case
when a.attnotnull
then 'NOT NULL'
else 'NULL'
END as not_null
FROM pg_class c,
pg_namespace n,
pg_attribute a,
pg_type t
WHERE c.relkind = 'r'
AND a.attnum > 0
AND a.attrelid = c.oid
AND a.atttypid = t.oid
AND n.oid = c.relnamespace
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
ORDER BY a.attnum
) as tabledefinition
where ('tables:' || nspname || '.' || relname) =OBJECT_ID_CONDITION
group by relname, nspname, oid
`;

View File

@@ -0,0 +1,8 @@
module.exports = `
select
table_name as "pureName",
table_schema as "schemaName",
md5(view_definition) as "hashCode"
from
information_schema.views where table_schema != 'information_schema' and table_schema != 'pg_catalog'
`;

View File

@@ -0,0 +1,11 @@
module.exports = `
select
table_name as "pureName",
table_schema as "schemaName",
view_definition as "createSql",
md5(view_definition) as "hashCode"
from
information_schema.views
where table_schema != 'information_schema' and table_schema != 'pg_catalog'
and ('views:' || table_schema || '.' || table_name) =OBJECT_ID_CONDITION
`;

View File

@@ -0,0 +1,66 @@
const { SqlDumper } = require('dbgate-tools');
class Dumper extends SqlDumper {
/** @param type {import('dbgate-types').TransformType} */
transform(type, dumpExpr) {
switch (type) {
case 'GROUP:YEAR':
case 'YEAR':
this.put('^extract(^year ^from %c)', dumpExpr);
break;
case 'MONTH':
this.put('^extract(^month ^from %c)', dumpExpr);
break;
case 'DAY':
this.put('^extract(^day ^from %c)', dumpExpr);
break;
case 'GROUP:MONTH':
this.put("^to_char(%c, '%s')", dumpExpr, 'YYYY-MM');
break;
case 'GROUP:DAY':
this.put("^to_char(%c, '%s')", dumpExpr, 'YYYY-MM-DD');
break;
default:
dumpExpr();
break;
}
}
dropRecreatedTempTable(tmptable) {
this.putCmd('^drop ^table %i ^cascade', tmptable);
}
renameTable(obj, newname) {
this.putCmd('^alter ^table %f ^rename ^to %i', obj, newname);
}
renameColumn(column, newcol) {
this.putCmd('^alter ^table %f ^rename ^column %i ^to %i', column, column.columnName, newcol);
}
dropTable(obj, options = {}) {
this.put('^drop ^table');
if (options.testIfExists) this.put(' ^if ^exists');
this.put(' %f', obj.FullName);
this.endCommand();
}
//public override void CreateIndex(IndexInfo ix)
//{
//}
enableConstraints(table, enabled) {
this.putCmd('^alter ^table %f %k ^trigger ^all', table, enabled ? 'enable' : 'disable');
}
columnDefinition(col, options) {
const { autoIncrement } = options || {};
if (col.autoIncrement) {
this.put('^serial');
return;
}
super.columnDefinition(col, options);
}
}
module.exports = Dumper;

View File

@@ -0,0 +1,27 @@
const { driverBase } = require('dbgate-tools');
const Dumper = require('./Dumper');
/** @type {import('dbgate-types').SqlDialect} */
const dialect = {
rangeSelect: true,
// stringEscapeChar: '\\',
stringEscapeChar: "'",
fallbackDataType: 'varchar',
anonymousPrimaryKey: true,
enableConstraintsPerTable: true,
quoteIdentifier(s) {
return '"' + s + '"';
},
};
/** @type {import('dbgate-types').EngineDriver} */
const driver = {
...driverBase,
dumperClass: Dumper,
dialect,
engine: 'postgres@dbgate-plugin-postgres',
title: 'Postgre SQL',
defaultPort: 5432,
};
module.exports = driver;

View File

@@ -0,0 +1,6 @@
import driver from './driver';
export default {
packageName: 'dbgate-plugin-postgres',
driver,
};