dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-30 22:03:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor - default dbaget engine mvoed to dbgate-tools

Browse Source
This commit is contained in:
Jan Prochazka
2020-11-23 20:49:25 +01:00
parent d3cfc44fd9
commit 88cf6d35ed
16 changed files with 115 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
packages/engines/default/DatabaseAnalyser.js
View File

@@ -1,125 +0,0 @@
const _ = require('lodash');
const fp = require('lodash/fp');
class DatabaseAnalyser {
/**
*
* @param {import('dbgate-types').EngineDriver} driver
*/
constructor(pool, driver) {
this.pool = pool;
this.driver = driver;
// this.result = DatabaseAnalyser.createEmptyStructure();
/** @type {import('dbgate-types').DatabaseInfo} */
this.structure = null;
/** import('dbgate-types').DatabaseModification[]) */
this.modifications = null;
this.singleObjectFilter = null;
}
async _runAnalysis() {
return DatabaseAnalyser.createEmptyStructure();
}
/** @returns {Promise<import('dbgate-types').DatabaseModification[]>} */
async getModifications() {
if (this.structure == null) throw new Error('DatabaseAnalyse.getModifications - structure must be filled');
return null;
}
async fullAnalysis() {
return this._runAnalysis();
}
async incrementalAnalysis(structure) {
this.structure = structure;
this.modifications = await this.getModifications();
if (this.modifications == null) {
// modifications not implemented, perform full analysis
this.structure = null;
return this._runAnalysis();
}
if (this.modifications.length == 0) return null;
console.log('DB modifications detected:', this.modifications);
return this._runAnalysis();
}
mergeAnalyseResult(newlyAnalysed, extractObjectId) {
if (this.structure == null) {
return {
...DatabaseAnalyser.createEmptyStructure(),
...newlyAnalysed,
};
}
const res = {};
for (const field of ['tables', 'views', 'functions', 'procedures', 'triggers']) {
const removedIds = this.modifications
.filter((x) => x.action == 'remove' && x.objectTypeField == field)
.map((x) => extractObjectId(x));
const newArray = newlyAnalysed[field] || [];
const addedChangedIds = newArray.map((x) => extractObjectId(x));
const removeAllIds = [...removedIds, ...addedChangedIds];
res[field] = _.sortBy(
[...this.structure[field].filter((x) => !removeAllIds.includes(extractObjectId(x))), ...newArray],
(x) => x.pureName
);
}
return res;
// const {tables,views, functions, procedures, triggers} = this.structure;
// return {
// tables:
// }
}
// findObjectById(id) {
// return this.structure.tables.find((x) => x.objectId == id);
// }
}
/** @returns {import('dbgate-types').DatabaseInfo} */
DatabaseAnalyser.createEmptyStructure = () => ({
tables: [],
views: [],
functions: [],
procedures: [],
triggers: [],
schemas: [],
});
DatabaseAnalyser.byTableFilter = (table) => (x) => x.pureName == table.pureName && x.schemaName == x.schemaName;
DatabaseAnalyser.extractPrimaryKeys = (table, pkColumns) => {
const filtered = pkColumns.filter(DatabaseAnalyser.byTableFilter(table));
if (filtered.length == 0) return undefined;
return {
..._.pick(filtered[0], ['constraintName', 'schemaName', 'pureName']),
constraintType: 'primaryKey',
columns: filtered.map(fp.pick('columnName')),
};
};
DatabaseAnalyser.extractForeignKeys = (table, fkColumns) => {
const grouped = _.groupBy(fkColumns.filter(DatabaseAnalyser.byTableFilter(table)), 'constraintName');
return _.keys(grouped).map((constraintName) => ({
constraintName,
constraintType: 'foreignKey',
..._.pick(grouped[constraintName][0], [
'constraintName',
'schemaName',
'pureName',
'refSchemaName',
'refTableName',
'updateAction',
'deleteAction',
]),
columns: grouped[constraintName].map(fp.pick(['columnName', 'refColumnName'])),
}));
};
module.exports = DatabaseAnalyser;

273
packages/engines/default/SqlDumper.js
View File

@@ -1,273 +0,0 @@
const _ = require('lodash');
const moment = require('moment');
class SqlDumper {
/** @param driver {import('dbgate-types').EngineDriver} */
constructor(driver) {
this.s = '';
this.driver = driver;
this.dialect = driver.dialect;
this.indentLevel = 0;
}
endCommand() {
this.putRaw(';\n');
}
putRaw(text) {
this.s += text;
}
escapeString(value) {
const esc = this.dialect.stringEscapeChar;
let res = '';
for (let i = 0; i < value.length; i++) {
const c = value[i];
if (c == esc || c == "'") {
res += esc;
}
res += c;
}
return res;
}
putStringValue(value) {
this.putRaw("'");
this.putRaw(this.escapeString(value));
this.putRaw("'");
}
putValue(value) {
if (value === null) this.putRaw('NULL');
if (value === true) this.putRaw('1');
if (value === false) this.putRaw('0');
else if (_.isString(value)) this.putStringValue(value);
else if (_.isNumber(value)) this.putRaw(value.toString());
else if (_.isDate(value)) this.putStringValue(moment(value).toISOString());
}
putCmd(format, ...args) {
this.put(format, ...args);
this.endCommand();
}
putFormattedValue(c, value) {
switch (c) {
case 's':
if (value != null) {
this.putRaw(value.toString());
}
break;
case 'i':
{
this.putRaw(this.dialect.quoteIdentifier(value));
}
break;
case 'k':
{
if (value) {
this.putRaw(value.toUpperCase());
}
}
break;
case 'f':
{
const { schemaName, pureName } = value;
if (schemaName) {
this.putRaw(this.dialect.quoteIdentifier(schemaName));
this.putRaw('.');
}
this.putRaw(this.dialect.quoteIdentifier(pureName));
}
break;
case 'v':
this.putValue(value);
break;
case 'c':
value(this);
break;
}
}
putFormattedList(c, collection) {
if (!collection) return;
this.putCollection(', ', collection, (item) => this.putFormattedValue(c, item));
}
/** @param format {string} */
put(format, ...args) {
let i = 0;
let argIndex = 0;
const length = format.length;
while (i < length) {
let c = format[i];
i++;
switch (c) {
case '^':
while (i < length && format[i].match(/[a-z0-9_]/i)) {
this.putRaw(format[i].toUpperCase());
i++;
}
break;
case '%':
c = format[i];
i++;
switch (c) {
case '%':
this.putRaw('%');
break;
case ',':
c = format[i];
i++;
this.putFormattedList(c, args[argIndex]);
break;
default:
this.putFormattedValue(c, args[argIndex]);
break;
}
argIndex++;
break;
case '&':
c = format[i];
i++;
switch (c) {
case '&':
this.putRaw('&');
break;
case '>':
this.indentLevel++;
break;
case '<':
this.indentLevel--;
break;
case 'n':
this.putRaw('\n');
this.putRaw(' '.repeat(2 * this.indentLevel));
break;
}
break;
default:
this.putRaw(c);
break;
}
}
}
autoIncrement() {
this.put(' ^auto_increment');
}
/**
* @param column {import('dbgate-types').ColumnInfo}
*/
columnDefinition(column, { includeDefault = true, includeNullable = true, includeCollate = true } = {}) {
if (column.computedExpression) {
this.put('^as %s', column.computedExpression);
if (column.isPersisted) this.put(' ^persisted');
return;
}
this.put('%k', column.dataType || this.dialect.fallbackDataType);
if (column.autoIncrement) {
this.autoIncrement();
}
this.putRaw(' ');
if (column.isSparse) {
this.put(' ^sparse ');
}
if (includeNullable) {
this.put(column.notNull ? '^not ^null' : '^null');
}
if (includeDefault && column.defaultValue != null) {
this.columnDefault(column);
}
}
/**
* @param column {import('dbgate-types').ColumnInfo}
*/
columnDefault(column) {
if (column.defaultConstraint != null) {
this.put(' ^constraint %i ^default %s ', column.defaultConstraint, column.defaultValue);
} else {
this.put(' ^default %s ', column.defaultValue);
}
}
/**
* @template T
* @param {string} delimiter
* @param {T[]} collection
* @param {(col: T) => void} lambda
*/
putCollection(delimiter, collection, lambda) {
if (!collection) return;
let first = true;
for (const item of collection) {
if (!first) this.put(delimiter);
first = false;
lambda(item);
}
}
/** @param table {import('dbgate-types').TableInfo} */
createTable(table) {
this.put('^create ^table %f ( &>&n', table);
this.putCollection(',&n', table.columns, (col) => {
this.put('%i ', col.columnName);
this.columnDefinition(col);
});
if (table.primaryKey) {
this.put(',&n');
if (table.primaryKey.constraintName) {
this.put('^constraint %i', table.primaryKey.constraintName);
}
this.put(
' ^primary ^key (%,i)',
table.primaryKey.columns.map((x) => x.columnName)
);
}
if (table.foreignKeys) {
table.foreignKeys.forEach((fk) => {
this.put(',&n');
this.createForeignKeyFore(fk);
});
}
// foreach (var cnt in table.Uniques)
// {
// if (!first) this.put(", &n");
// first = false;
// CreateUniqueCore(cnt);
// }
// foreach (var cnt in table.Checks)
// {
// if (!first) this.put(", &n");
// first = false;
// CreateCheckCore(cnt);
// }
this.put('&<&n)');
this.endCommand();
// foreach (var ix in table.Indexes)
// {
// CreateIndex(ix);
// }
}
/** @param fk {import('dbgate-types').ForeignKeyInfo} */
createForeignKeyFore(fk) {
if (fk.constraintName != null) this.put('^constraint %i ', fk.constraintName);
this.put(
'^foreign ^key (%,i) ^references %f (%,i)',
fk.columns.map((x) => x.columnName),
{ schemaName: fk.refSchemaName, pureName: fk.refTableName },
fk.columns.map((x) => x.refColumnName)
);
if (fk.deleteAction) this.put(' ^on ^delete %k', fk.deleteAction);
if (fk.updateAction) this.put(' ^on ^update %k', fk.updateAction);
}
/** @param type {import('dbgate-types').TransformType} */
transform(type, dumpExpr) {
dumpExpr();
}
/**
* @param table {import('dbgate-types').NamedObjectInfo}
* @param allow {boolean}
*/
allowIdentityInsert(table, allow) {}
}
module.exports = SqlDumper;

98
packages/engines/default/createBulkInsertStreamBase.js
View File

@@ -1,98 +0,0 @@
const { prepareTableForImport } = require('dbgate-tools');
const _ = require('lodash');
/**
*
* @param {import('dbgate-types').EngineDriver} driver
*/
function createBulkInsertStreamBase(driver, stream, pool, name, options) {
const fullNameQuoted = name.schemaName
? `${driver.dialect.quoteIdentifier(name.schemaName)}.${driver.dialect.quoteIdentifier(name.pureName)}`
: driver.dialect.quoteIdentifier(name.pureName);
const writable = new stream.Writable({
objectMode: true,
});
writable.buffer = [];
writable.structure = null;
writable.columnNames = null;
writable.addRow = async (row) => {
if (writable.structure) {
writable.buffer.push(row);
} else {
writable.structure = row;
await writable.checkStructure();
}
};
writable.checkStructure = async () => {
let structure = await driver.analyseSingleTable(pool, name);
// console.log('ANALYSING', name, structure);
if (structure && options.dropIfExists) {
console.log(`Dropping table ${fullNameQuoted}`);
await driver.query(pool, `DROP TABLE ${fullNameQuoted}`);
}
if (options.createIfNotExists && (!structure || options.dropIfExists)) {
console.log(`Creating table ${fullNameQuoted}`);
const dmp = driver.createDumper();
dmp.createTable(prepareTableForImport({ ...writable.structure, ...name }));
console.log(dmp.s);
await driver.query(pool, dmp.s);
structure = await driver.analyseSingleTable(pool, name);
}
if (options.truncate) {
await driver.query(pool, `TRUNCATE TABLE ${fullNameQuoted}`);
}
this.columnNames = _.intersection(
structure.columns.map((x) => x.columnName),
writable.structure.columns.map((x) => x.columnName)
);
};
writable.send = async () => {
const rows = writable.buffer;
writable.buffer = [];
const dmp = driver.createDumper();
dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`);
dmp.putCollection(',', this.columnNames, (col) => dmp.putRaw(driver.dialect.quoteIdentifier(col)));
dmp.putRaw(')\n VALUES\n');
let wasRow = false;
for (const row of rows) {
if (wasRow) dmp.putRaw(',\n');
dmp.putRaw('(');
dmp.putCollection(',', this.columnNames, (col) => dmp.putValue(row[col]));
dmp.putRaw(')');
wasRow = true;
}
dmp.putRaw(';');
// require('fs').writeFileSync('/home/jena/test.sql', dmp.s);
await driver.query(pool, dmp.s);
};
writable.sendIfFull = async () => {
if (writable.buffer.length > 100) {
await writable.send();
}
};
writable._write = async (chunk, encoding, callback) => {
await writable.addRow(chunk);
await writable.sendIfFull();
callback();
};
writable._final = async (callback) => {
await writable.send();
callback();
};
return writable;
}
module.exports = createBulkInsertStreamBase;

34
packages/engines/default/driverBase.js
View File

@@ -1,34 +0,0 @@
const createBulkInsertStreamBase = require('./createBulkInsertStreamBase');
const driverBase = {
analyserClass: null,
dumperClass: null,
async analyseFull(pool) {
const analyser = new this.analyserClass(pool, this);
return analyser.fullAnalysis();
},
async analyseSingleObject(pool, name, typeField = 'tables') {
const analyser = new this.analyserClass(pool, this);
analyser.singleObjectFilter = { ...name, typeField };
const res = await analyser.fullAnalysis();
return res.tables[0];
},
analyseSingleTable(pool, name) {
return this.analyseSingleObject(pool, name, 'tables');
},
async analyseIncremental(pool, structure) {
const analyser = new this.analyserClass(pool, this);
return analyser.incrementalAnalysis(structure);
},
createDumper() {
return new this.dumperClass(this);
},
async writeTable(pool, name, options) {
const { stream, mssql } = pool._nativeModules;
// @ts-ignore
return createBulkInsertStreamBase(this, stream, pool, name, options);
},
};
module.exports = driverBase;

2
packages/engines/mssql/MsSqlAnalyser.js
View File

@@ -2,7 +2,7 @@ const fp = require('lodash/fp');
const _ = require('lodash');
const sql = require('./sql');
const DatabaseAnalyser = require('../default/DatabaseAnalyser');
const { DatabaseAnalyser } = require('dbgate-tools');
const { filter } = require('lodash');
const { isTypeString, isTypeNumeric } = require('dbgate-tools');

2
packages/engines/mssql/MsSqlDumper.js
View File

@@ -1,4 +1,4 @@
const SqlDumper = require('../default/SqlDumper');
const { SqlDumper } = require('dbgate-tools');
class MsSqlDumper extends SqlDumper {
autoIncrement() {

2
packages/engines/mssql/createBulkInsertStream.js
View File

@@ -1,4 +1,4 @@
const createBulkInsertStreamBase = require('../default/createBulkInsertStreamBase');
const { createBulkInsertStreamBase } = require('dbgate-tools');
/**
*

2
packages/engines/mssql/index.js
View File

@@ -1,8 +1,8 @@
const _ = require('lodash');
const { driverBase } = require('dbgate-tools');
const MsSqlAnalyser = require('./MsSqlAnalyser');
const MsSqlDumper = require('./MsSqlDumper');
const createBulkInsertStream = require('./createBulkInsertStream');
const driverBase = require('../default/driverBase');
/** @type {import('dbgate-types').SqlDialect} */
const dialect = {

8
packages/engines/mysql/MySqlAnalyser.js
View File

@@ -2,7 +2,7 @@ const fp = require('lodash/fp');
const _ = require('lodash');
const sql = require('./sql');
const DatabaseAnalayser = require('../default/DatabaseAnalyser');
const { DatabaseAnalyser } = require('dbgate-tools');
const { isTypeString, isTypeNumeric } = require('dbgate-tools');
const { rangeStep } = require('lodash/fp');
@@ -35,7 +35,7 @@ function objectTypeToField(type) {
return null;
}
class MySqlAnalyser extends DatabaseAnalayser {
class MySqlAnalyser extends DatabaseAnalyser {
constructor(pool, driver) {
super(pool, driver);
}
@@ -103,8 +103,8 @@ class MySqlAnalyser extends DatabaseAnalayser {
tables: tables.rows.map((table) => ({
...table,
columns: columns.rows.filter((col) => col.pureName == table.pureName).map(getColumnInfo),
primaryKey: DatabaseAnalayser.extractPrimaryKeys(table, pkColumns.rows),
foreignKeys: DatabaseAnalayser.extractForeignKeys(table, fkColumns.rows),
primaryKey: DatabaseAnalyser.extractPrimaryKeys(table, pkColumns.rows),
foreignKeys: DatabaseAnalyser.extractForeignKeys(table, fkColumns.rows),
})),
views: views.rows.map((view) => ({
...view,

2
packages/engines/mysql/MySqlDumper.js
View File

@@ -1,4 +1,4 @@
const SqlDumper = require('../default/SqlDumper');
const { SqlDumper } = require('dbgate-tools');
class MySqlDumper extends SqlDumper {
/** @param type {import('dbgate-types').TransformType} */

8
packages/engines/mysql/index.js
View File

@@ -1,4 +1,4 @@
const driverBase = require('../default/driverBase');
const { driverBase } = require('dbgate-tools');
const MySqlAnalyser = require('./MySqlAnalyser');
const MySqlDumper = require('./MySqlDumper');
@@ -143,6 +143,12 @@ const driver = {
const { rows } = await this.query(connection, 'show databases');
return rows.map((x) => ({ name: x.Database }));
},
async writeTable(pool, name, options) {
const { stream } = pool._nativeModules;
// @ts-ignore
return createBulkInsertStreamBase(this, stream, pool, name, options);
},
// createDumper() {
// return new MySqlDumper(this);
// },

8
packages/engines/postgres/PostgreAnalyser.js
View File

@@ -2,7 +2,7 @@ const fp = require('lodash/fp');
const _ = require('lodash');
const sql = require('./sql');
const DatabaseAnalayser = require('../default/DatabaseAnalyser');
const { DatabaseAnalyser } = require('dbgate-tools');
const { isTypeString, isTypeNumeric } = require('dbgate-tools');
function normalizeTypeName(dataType) {
@@ -35,7 +35,7 @@ function getColumnInfo({
};
}
class PostgreAnalyser extends DatabaseAnalayser {
class PostgreAnalyser extends DatabaseAnalyser {
constructor(pool, driver) {
super(pool, driver);
}
@@ -84,8 +84,8 @@ class PostgreAnalyser extends DatabaseAnalayser {
columns: columns.rows
.filter((col) => col.pureName == table.pureName && col.schemaName == table.schemaName)
.map(getColumnInfo),
primaryKey: DatabaseAnalayser.extractPrimaryKeys(table, pkColumns.rows),
foreignKeys: DatabaseAnalayser.extractForeignKeys(table, fkColumns.rows),
primaryKey: DatabaseAnalyser.extractPrimaryKeys(table, pkColumns.rows),
foreignKeys: DatabaseAnalyser.extractForeignKeys(table, fkColumns.rows),
})),
views: views.rows.map((view) => ({
...view,

2
packages/engines/postgres/PostgreDumper.js
View File

@@ -1,4 +1,4 @@
const SqlDumper = require('../default/SqlDumper');
const { SqlDumper } = require('dbgate-tools');
class PostgreDumper extends SqlDumper {
/** @param type {import('dbgate-types').TransformType} */

7
packages/engines/postgres/index.js
View File

@@ -1,5 +1,5 @@
const _ = require('lodash');
const driverBase = require('../default/driverBase');
const { driverBase } = require('dbgate-tools');
const PostgreAnalyser = require('./PostgreAnalyser');
const PostgreDumper = require('./PostgreDumper');
@@ -174,6 +174,11 @@ const driver = {
// createDumper() {
// return new PostgreDumper(this);
// },
async writeTable(pool, name, options) {
const { stream } = pool._nativeModules;
// @ts-ignore
return createBulkInsertStreamBase(this, stream, pool, name, options);
},
async listDatabases(client) {
const { rows } = await this.query(client, 'SELECT datname AS name FROM pg_database WHERE datistemplate = false');
return rows;