dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-23 15:06:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

engines moved to separate library

Browse Source
This commit is contained in:
Jan Prochazka
2020-02-02 22:27:49 +01:00
parent eee6c94ebc
commit 6d1f3e977c
30 changed files with 844 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
api/package.json
View File

@@ -16,7 +16,8 @@
"nedb-promises": "^4.0.1",
"pg": "^7.17.0",
"socket.io": "^2.3.0",
"uuid": "^3.4.0"
"uuid": "^3.4.0",
"@dbgate/engines": "file:../lib/engines"
},
"scripts": {
"start": "nodemon src/index.js",
@@ -26,6 +27,6 @@
"@types/lodash": "^4.14.149",
"nodemon": "^2.0.2",
"typescript": "^3.7.4",
"@types/dbgate": "file:../lib"
"@types/dbgate": "file:../types"
}
}

2
api/src/controllers/databaseConnections.js
View File

@@ -3,7 +3,7 @@ const uuidv1 = require('uuid/v1');
const connections = require('./connections');
const socket = require('../utility/socket');
const { fork } = require('child_process');
const DatabaseAnalyser = require('../engines/default/DatabaseAnalyser');
const DatabaseAnalyser = require('@dbgate/engines/default/DatabaseAnalyser');
module.exports = {
/** @type {import('dbgate').OpenedDatabaseConnection[]} */

19
api/src/engines/default/DatabaseAnalyser.js
View File

@@ -1,19 +0,0 @@
class DatabaseAnalyser {
/**
*
* @param {import('dbgate').EngineDriver} driver
*/
constructor(pool, driver) {
this.pool = pool;
this.driver = driver;
this.result = DatabaseAnalyser.createEmptyStructure();
}
async runAnalysis() {}
}
/** @returns {import('dbgate').DatabaseInfo} */
DatabaseAnalyser.createEmptyStructure = () => ({
tables: [],
});
module.exports = DatabaseAnalyser;

64
api/src/engines/default/SqlDumper.js
View File

@@ -1,64 +0,0 @@
class SqlDumper {
/** @param driver {import('dbgate').EngineDriver} */
constructor(driver) {
this.s = '';
this.driver = driver;
this.dialect = driver.dialect;
}
putRaw(text) {
this.s += text;
}
putCmd(text) {
this.putRaw(text);
this.putRaw(';\n');
}
putFormattedValue(c, value) {
switch (c) {
case 's':
if (value != null) {
this.putRaw(value.toString());
}
break;
case 'f':
{
const { schemaName, pureName } = value;
if (schemaName) {
this.putRaw(this.dialect.quoteIdentifier(schemaName));
this.putRaw('.');
}
this.putRaw(this.dialect.quoteIdentifier(pureName));
}
break;
}
}
/** @param format {string} */
put(format, ...args) {
let i = 0;
let argIndex = 0;
const length = format.length;
while (i < length) {
let c = format[i];
i++;
switch (c) {
case '^':
while (i < length && format[i].match(/[a-z]/i)) {
this.putRaw(format[i].toUpperCase());
i++;
}
break;
case '%':
c = format[i];
i++;
this.putFormattedValue(c, args[argIndex]);
argIndex++;
break;
default:
this.putRaw(c);
break;
}
}
}
}
module.exports = SqlDumper;

7
api/src/engines/index.js
View File

@@ -1,7 +0,0 @@
/** @return {import('dbgate').EngineDriver} */
function getDriver(connection) {
const { engine } = connection;
return require(`./${engine}`);
}
module.exports = getDriver;

81
api/src/engines/mssql/MsSqlAnalyser.js
View File

@@ -1,81 +0,0 @@
const fs = require('fs-extra');
const fp = require('lodash/fp');
const path = require('path');
const _ = require('lodash');
const DatabaseAnalayser = require('../default/DatabaseAnalyser');
/** @returns {Promise<string>} */
async function loadQuery(name) {
return await fs.readFile(path.join(__dirname, name), 'utf-8');
}
const byTableFilter = table => x => x.pureName == table.pureName && x.schemaName == x.schemaName;
function extractPrimaryKeys(table, pkColumns) {
const filtered = pkColumns.filter(byTableFilter(table));
if (filtered.length == 0) return undefined;
return {
..._.pick(filtered[0], ['constraintName', 'schemaName', 'pureName']),
constraintType: 'primaryKey',
columns: filtered.map(fp.pick('columnName')),
};
}
function extractForeignKeys(table, fkColumns) {
const grouped = _.groupBy(fkColumns.filter(byTableFilter(table)), 'constraintName');
return _.keys(grouped).map(constraintName => ({
constraintName,
constraintType: 'foreignKey',
..._.pick(grouped[constraintName][0], [
'constraintName',
'schemaName',
'pureName',
'refSchemaName',
'refTableName',
'updateAction',
'deleteAction',
]),
columns: grouped[constraintName].map(fp.pick(['columnName', 'refColumnName'])),
}));
}
class MsSqlAnalyser extends DatabaseAnalayser {
constructor(pool, driver) {
super(pool, driver);
}
async createQuery(
resFileName,
tables = false,
views = false,
procedures = false,
functions = false,
triggers = false
) {
let res = await loadQuery(resFileName);
res = res.replace('=[OBJECT_ID_CONDITION]', ' is not null');
return res;
}
async runAnalysis() {
const tables = await this.driver.query(this.pool, await this.createQuery('tables.sql'));
const columns = await this.driver.query(this.pool, await this.createQuery('columns.sql'));
const pkColumns = await this.driver.query(this.pool, await this.createQuery('primary_keys.sql'));
const fkColumns = await this.driver.query(this.pool, await this.createQuery('foreign_keys.sql'));
this.result.tables = tables.rows.map(table => ({
...table,
columns: columns.rows
.filter(col => col.objectId == table.objectId)
.map(({ isNullable, isIdentity, ...col }) => ({
...col,
notNull: !isNullable,
autoIncrement: !!isIdentity,
})),
primaryKey: extractPrimaryKeys(table, pkColumns.rows),
foreignKeys: extractForeignKeys(table, fkColumns.rows),
}));
}
}
module.exports = MsSqlAnalyser;

5
api/src/engines/mssql/MsSqlDumper.js
View File

@@ -1,5 +0,0 @@
const SqlDumper = require('../default/SqlDumper');
class MsSqlDumper extends SqlDumper {}
module.exports = MsSqlDumper;

13
api/src/engines/mssql/columns.sql
View File

@@ -1,13 +0,0 @@
select c.name as columnName, t.name as dataType, c.object_id as objectId, c.is_identity as isIdentity,
c.max_length as maxLength, c.precision, c.scale, c.is_nullable as isNullable,
d.definition as defaultValue, d.name as defaultConstraint,
m.definition as computedExpression, m.is_persisted as isPersisted, c.column_id as columnId,
-- TODO only if version >= 2008
c.is_sparse as isSparse
from sys.columns c
inner join sys.types t on c.system_type_id = t.system_type_id and c.user_type_id = t.user_type_id
inner join sys.objects o on c.object_id = o.object_id
left join sys.default_constraints d on c.default_object_id = d.object_id
left join sys.computed_columns m on m.object_id = c.object_id and m.column_id = c.column_id
where o.type = 'U' and o.object_id =[OBJECT_ID_CONDITION]
order by c.column_id

38
api/src/engines/mssql/foreign_keys.sql
View File

@@ -1,38 +0,0 @@
SELECT
schemaName = FK.TABLE_SCHEMA,
pureName = FK.TABLE_NAME,
columnName = CU.COLUMN_NAME,
refSchemaName = ISNULL(IXS.name, PK.TABLE_SCHEMA),
refTableName = ISNULL(IXT.name, PK.TABLE_NAME),
refColumnName = IXCC.name,
constraintName = C.CONSTRAINT_NAME,
updateAction = rc.UPDATE_RULE,
deleteAction = rc.DELETE_RULE,
objectId = o.object_id
FROM INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS C
INNER JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS FK ON C.CONSTRAINT_NAME = FK.CONSTRAINT_NAME
LEFT JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS PK ON C.UNIQUE_CONSTRAINT_NAME = PK.CONSTRAINT_NAME
LEFT JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE CU ON C.CONSTRAINT_NAME = CU.CONSTRAINT_NAME
--LEFT JOIN (
--SELECT i1.TABLE_NAME, i2.COLUMN_NAME
--FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS i1
--INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE i2 ON i1.CONSTRAINT_NAME = i2.CONSTRAINT_NAME
--WHERE i1.CONSTRAINT_TYPE = 'PRIMARY KEY'
--) PT ON PT.TABLE_NAME = PK.TABLE_NAME
INNER JOIN INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS rc ON FK.CONSTRAINT_NAME = rc.CONSTRAINT_NAME
LEFT JOIN sys.indexes IX ON IX.name = C.UNIQUE_CONSTRAINT_NAME
LEFT JOIN sys.objects IXT ON IXT.object_id = IX.object_id
LEFT JOIN sys.index_columns IXC ON IX.index_id = IXC.index_id and IX.object_id = IXC.object_id
LEFT JOIN sys.columns IXCC ON IXCC.column_id = IXC.column_id AND IXCC.object_id = IXC.object_id
LEFT JOIN sys.schemas IXS ON IXT.schema_id = IXS.schema_id
inner join sys.objects o on FK.TABLE_NAME = o.name
inner join sys.schemas s on o.schema_id = s.schema_id and FK.TABLE_SCHEMA = s.name
where o.object_id =[OBJECT_ID_CONDITION]

46
api/src/engines/mssql/index.js
View File

@@ -1,46 +0,0 @@
const _ = require('lodash');
const mssql = require('mssql');
const MsSqlAnalyser = require('./MsSqlAnalyser');
const MsSqlDumper = require('./MsSqlDumper');
/** @type {import('dbgate').SqlDialect} */
const dialect = {
limitSelect: true,
quoteIdentifier(s) {
return `[${s}]`;
},
};
/** @type {import('dbgate').EngineDriver} */
const driver = {
async connect({ server, port, user, password, database }) {
const pool = await mssql.connect({ server, port, user, password, database });
return pool;
},
async query(pool, sql) {
const resp = await pool.request().query(sql);
// console.log(Object.keys(resp.recordset));
const columns = _.sortBy(_.values(resp.recordset.columns), 'index');
return { rows: resp.recordset, columns };
},
async getVersion(pool) {
const { version } = (await this.query(pool, 'SELECT @@VERSION AS version')).rows[0];
return { version };
},
async listDatabases(pool) {
const { rows } = await this.query(pool, 'SELECT name FROM sys.databases order by name');
return rows;
},
async analyseFull(pool) {
const analyser = new MsSqlAnalyser(pool, this);
await analyser.runAnalysis();
return analyser.result;
},
// async analyseIncremental(pool) {},
createDumper() {
return new MsSqlDumper(this);
},
dialect,
};
module.exports = driver;

12
api/src/engines/mssql/primary_keys.sql
View File

@@ -1,12 +0,0 @@
select o.object_id, pureName = t.Table_Name, schemaName = t.Table_Schema, columnName = c.Column_Name, constraintName=t.constraint_name from
INFORMATION_SCHEMA.TABLE_CONSTRAINTS t,
sys.objects o,
sys.schemas s,
INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE c
where
c.Constraint_Name = t.Constraint_Name
and t.table_name = o.name
and o.schema_id = s.schema_id and t.Table_Schema = s.name
and c.Table_Name = t.Table_Name
and Constraint_Type = 'PRIMARY KEY'
and o.object_id =[OBJECT_ID_CONDITION]

6
api/src/engines/mssql/tables.sql
View File

@@ -1,6 +0,0 @@
select
o.name as pureName, s.name as schemaName, o.object_id as objectId,
o.create_date, o.modify_date
from sys.tables o
inner join sys.schemas s on o.schema_id = s.schema_id
where o.object_id =[OBJECT_ID_CONDITION]

53
api/src/engines/mysql/MySqlAnalyser.js
View File

@@ -1,53 +0,0 @@
const fs = require('fs-extra');
const fp = require('lodash/fp');
const path = require('path');
const _ = require('lodash');
const DatabaseAnalayser = require('../default/DatabaseAnalyser');
/** @returns {Promise<string>} */
async function loadQuery(name) {
return await fs.readFile(path.join(__dirname, name), 'utf-8');
}
class MySqlAnalyser extends DatabaseAnalayser {
constructor(pool, driver) {
super(pool, driver);
}
async createQuery(
resFileName,
tables = false,
views = false,
procedures = false,
functions = false,
triggers = false
) {
let res = await loadQuery(resFileName);
res = res.replace('=[OBJECT_NAME_CONDITION]', ' is not null');
res = res.replace('#DATABASE#', this.pool._database_name);
return res;
}
async runAnalysis() {
const tables = await this.driver.query(this.pool, await this.createQuery('tables.sql'));
const columns = await this.driver.query(this.pool, await this.createQuery('columns.sql'));
// const pkColumns = await this.driver.query(this.pool, await this.createQuery('primary_keys.sql'));
// const fkColumns = await this.driver.query(this.pool, await this.createQuery('foreign_keys.sql'));
this.result.tables = tables.rows.map(table => ({
...table,
columns: columns.rows
.filter(col => col.pureName == table.pureName)
.map(({ isNullable, extra, ...col }) => ({
...col,
notNull: !isNullable,
autoIncrement: extra && extra.toLowerCase().includes('auto_increment'),
})),
foreignKeys: [],
// primaryKey: extractPrimaryKeys(table, pkColumns.rows),
// foreignKeys: extractForeignKeys(table, fkColumns.rows),
}));
}
}
module.exports = MySqlAnalyser;

5
api/src/engines/mysql/MySqlDumper.js
View File

@@ -1,5 +0,0 @@
const SqlDumper = require('../default/SqlDumper');
class MySqlDumper extends SqlDumper {}
module.exports = MySqlDumper;

13
api/src/engines/mysql/columns.sql
View File

@@ -1,13 +0,0 @@
select
TABLE_NAME as pureName,
COLUMN_NAME as columnName,
IS_NULLABLE as isNullable,
DATA_TYPE as dataType,
CHARACTER_MAXIMUM_LENGTH,
NUMERIC_PRECISION,
NUMERIC_SCALE,
COLUMN_DEFAULT,
EXTRA as extra
from INFORMATION_SCHEMA.COLUMNS
where TABLE_SCHEMA = '#DATABASE#' and TABLE_NAME =[OBJECT_NAME_CONDITION]
order by ORDINAL_POSITION

48
api/src/engines/mysql/index.js
View File

@@ -1,48 +0,0 @@
const mysql = require('mysql');
const MySqlAnalyser = require('./MySqlAnalyser');
const MySqlDumper = require('./MySqlDumper');
/** @type {import('dbgate').SqlDialect} */
const dialect = {
rangeSelect: true,
quoteIdentifier(s) {
return '`' + s + '`';
},
};
/** @type {import('dbgate').EngineDriver} */
const driver = {
async connect({ server, port, user, password, database }) {
const connection = mysql.createConnection({ host: server, port, user, password, database });
connection._database_name = database;
return connection;
},
async query(connection, sql) {
return new Promise((resolve, reject) => {
connection.query(sql, function(error, results, fields) {
if (error) reject(error);
resolve({ rows: results, columns: fields });
});
});
},
async getVersion(connection) {
const { rows } = await this.query(connection, "show variables like 'version'");
const version = rows[0].Value;
return { version };
},
async analyseFull(pool) {
const analyser = new MySqlAnalyser(pool, this);
await analyser.runAnalysis();
return analyser.result;
},
async listDatabases(connection) {
const { rows } = await this.query(connection, 'show databases');
return rows.map(x => ({ name: x.Database }));
},
createDumper() {
return new MySqlDumper(this);
},
dialect,
};
module.exports = driver;

5
api/src/engines/mysql/tables.sql
View File

@@ -1,5 +0,0 @@
select
TABLE_NAME as pureName,
case when ENGINE='InnoDB' then CREATE_TIME else coalesce(UPDATE_TIME, CREATE_TIME) end as alterTime
from information_schema.tables
where TABLE_SCHEMA = '#DATABASE#' and TABLE_NAME =[OBJECT_NAME_CONDITION];

51
api/src/engines/postgres/PostgreAnalyser.js
View File

@@ -1,51 +0,0 @@
const fs = require('fs-extra');
const fp = require('lodash/fp');
const path = require('path');
const _ = require('lodash');
const DatabaseAnalayser = require('../default/DatabaseAnalyser');
/** @returns {Promise<string>} */
async function loadQuery(name) {
return await fs.readFile(path.join(__dirname, name), 'utf-8');
}
class MySqlAnalyser extends DatabaseAnalayser {
constructor(pool, driver) {
super(pool, driver);
}
async createQuery(
resFileName,
tables = false,
views = false,
procedures = false,
functions = false,
triggers = false
) {
let res = await loadQuery(resFileName);
res = res.replace('=[OBJECT_ID_CONDITION]', ' is not null');
return res;
}
async runAnalysis() {
const tables = await this.driver.query(this.pool, await this.createQuery('table_modifications.psql'));
const columns = await this.driver.query(this.pool, await this.createQuery('columns.psql'));
// const pkColumns = await this.driver.query(this.pool, await this.createQuery('primary_keys.sql'));
// const fkColumns = await this.driver.query(this.pool, await this.createQuery('foreign_keys.sql'));
this.result.tables = tables.rows.map(table => ({
...table,
columns: columns.rows
.filter(col => col.pureName == table.pureName && col.schemaName == table.schemaName)
.map(({ isNullable, ...col }) => ({
...col,
notNull: !isNullable,
})),
foreignKeys: [],
// primaryKey: extractPrimaryKeys(table, pkColumns.rows),
// foreignKeys: extractForeignKeys(table, fkColumns.rows),
}));
}
}
module.exports = MySqlAnalyser;

5
api/src/engines/postgres/PostgreDumper.js
View File

@@ -1,5 +0,0 @@
const SqlDumper = require('../default/SqlDumper');
class PostgreDumper extends SqlDumper {}
module.exports = PostgreDumper;

17
api/src/engines/postgres/columns.psql
View File

@@ -1,17 +0,0 @@
select
table_schema as "schemaName",
table_name as "pureName",
column_name as "columnName",
is_nullable as "isNullable",
data_type as dataType,
character_maximum_length,
numeric_precision,
numeric_scale,
column_default
from information_schema.columns
where
table_schema <> 'information_schema'
and table_schema <> 'pg_catalog'
and table_schema !~ '^pg_toast'
and 'table:' || table_schema || '.' || table_name =[OBJECT_ID_CONDITION]
order by ordinal_position

44
api/src/engines/postgres/index.js
View File

@@ -1,44 +0,0 @@
const { Client } = require('pg');
const PostgreAnalyser = require('./PostgreAnalyser');
const PostgreDumper = require('./PostgreDumper');
/** @type {import('dbgate').SqlDialect} */
const dialect = {
rangeSelect: true,
quoteIdentifier(s) {
return '"' + s + '"';
},
};
/** @type {import('dbgate').EngineDriver} */
const driver = {
async connect({ server, port, user, password, database }) {
const client = new Client({ host: server, port, user, password, database: database || 'postgres' });
await client.connect();
return client;
},
async query(client, sql) {
const res = await client.query(sql);
return { rows: res.rows, columns: res.fields };
},
async getVersion(client) {
const { rows } = await this.query(client, 'SELECT version()');
const { version } = rows[0];
return { version };
},
async analyseFull(pool) {
const analyser = new PostgreAnalyser(pool, this);
await analyser.runAnalysis();
return analyser.result;
},
createDumper() {
return new PostgreDumper(this);
},
async listDatabases(client) {
const { rows } = await this.query(client, 'SELECT datname AS name FROM pg_database WHERE datistemplate = false');
return rows;
},
dialect,
};
module.exports = driver;

50
api/src/engines/postgres/table_modifications.psql
View File

@@ -1,50 +0,0 @@
with pkey as
(
select cc.conrelid, format(E'create constraint %I primary key(%s);\n', cc.conname,
string_agg(a.attname, ', '
order by array_position(cc.conkey, a.attnum))) pkey
from pg_catalog.pg_constraint cc
join pg_catalog.pg_class c on c.oid = cc.conrelid
join pg_catalog.pg_attribute a on a.attrelid = cc.conrelid
and a.attnum = any(cc.conkey)
where cc.contype = 'p'
group by cc.conrelid, cc.conname
)
SELECT oid as "objectId", nspname as "schemaName", relname as "pureName",
md5('CREATE TABLE ' || nspname || '.' || relname || E'\n(\n' ||
array_to_string(
array_agg(
' ' || column_name || ' ' || type || ' '|| not_null
)
, E',\n'
) || E'\n);\n' || (select pkey from pkey where pkey.conrelid = oid)) as "hash"
from
(
SELECT
c.relname, a.attname AS column_name, c.oid,
n.nspname,
pg_catalog.format_type(a.atttypid, a.atttypmod) as type,
case
when a.attnotnull
then 'NOT NULL'
else 'NULL'
END as not_null
FROM pg_class c,
pg_namespace n,
pg_attribute a,
pg_type t
WHERE c.relkind = 'r'
AND a.attnum > 0
AND a.attrelid = c.oid
AND a.atttypid = t.oid
AND n.oid = c.relnamespace
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
ORDER BY a.attnum
) as tabledefinition
where 'table:' || nspname || '.' || relname =[OBJECT_ID_CONDITION]
group by relname, nspname, oid

5
api/src/proc/connectProcess.js
View File

@@ -1,9 +1,10 @@
const engines = require('../engines');
const engines = require('@dbgate/engines');
const driverConnect = require('../utility/driverConnect')
process.on('message', async connection => {
try {
const driver = engines(connection);
const conn = await driver.connect(connection);
const conn = await driverConnect(driver, connection);
const res = await driver.getVersion(conn);
process.send(res);
} catch (e) {

5
api/src/proc/databaseConnectionProcess.js
View File

@@ -1,5 +1,6 @@
const engines = require('../engines');
const engines = require('@dbgate/engines');
const Select = require('../dmlf/select');
const driverConnect = require('../utility/driverConnect')
let systemConnection;
let storedConnection;
@@ -15,7 +16,7 @@ async function handleConnect(connection) {
storedConnection = connection;
const driver = engines(storedConnection);
systemConnection = await driver.connect(storedConnection);
systemConnection = await driverConnect(driver, storedConnection);
handleFullRefresh();
setInterval(handleFullRefresh, 30 * 1000);
for (const [resolve, reject] of afterConnectCallbacks) {

5
api/src/proc/serverConnectionProcess.js
View File

@@ -1,4 +1,5 @@
const engines = require('../engines');
const engines = require('@dbgate/engines');
const driverConnect = require('../utility/driverConnect')
let systemConnection;
let storedConnection;
@@ -13,7 +14,7 @@ async function handleConnect(connection) {
storedConnection = connection;
const driver = engines(storedConnection);
systemConnection = await driver.connect(storedConnection);
systemConnection = await driverConnect(driver, storedConnection);
handleRefreshDatabases();
setInterval(handleRefreshDatabases, 30 * 1000);
}

14
api/src/utility/driverConnect.js Normal file
View File

@@ -0,0 +1,14 @@
const mssql = require('mssql');
const mysql = require('mysql');
const pg = require('pg');
function driverConnect(driver, connection) {
const driverModules = {
mssql,
mysql,
pg,
};
return driver.connect(driverModules, connection);
}
module.exports = driverConnect;