mirror of
https://github.com/DeNNiiInc/dbgate.git
synced 2026-04-21 23:25:59 +00:00
refactor - default dbaget engine mvoed to dbgate-tools
This commit is contained in:
Jan Prochazka
117
packages/tools/src/DatabaseAnalyser.ts
Normal file
117
packages/tools/src/DatabaseAnalyser.ts
Normal file
@@ -0,0 +1,117 @@
|
||||
import { DatabaseInfo, DatabaseModification, EngineDriver } from 'dbgate-types';
|
||||
import _ from 'lodash';
|
||||
import fp from 'lodash/fp';
|
||||
|
||||
export class DatabaseAnalyser {
|
||||
structure: DatabaseInfo;
|
||||
modifications: DatabaseModification[];
|
||||
singleObjectFilter: any;
|
||||
|
||||
constructor(public pool, public driver: EngineDriver) {}
|
||||
|
||||
async _runAnalysis() {
|
||||
return DatabaseAnalyser.createEmptyStructure();
|
||||
}
|
||||
|
||||
/** @returns {Promise<import('dbgate-types').DatabaseModification[]>} */
|
||||
async getModifications() {
|
||||
if (this.structure == null) throw new Error('DatabaseAnalyse.getModifications - structure must be filled');
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async fullAnalysis() {
|
||||
return this._runAnalysis();
|
||||
}
|
||||
|
||||
async incrementalAnalysis(structure) {
|
||||
this.structure = structure;
|
||||
|
||||
this.modifications = await this.getModifications();
|
||||
if (this.modifications == null) {
|
||||
// modifications not implemented, perform full analysis
|
||||
this.structure = null;
|
||||
return this._runAnalysis();
|
||||
}
|
||||
if (this.modifications.length == 0) return null;
|
||||
console.log('DB modifications detected:', this.modifications);
|
||||
return this._runAnalysis();
|
||||
}
|
||||
|
||||
mergeAnalyseResult(newlyAnalysed, extractObjectId) {
|
||||
if (this.structure == null) {
|
||||
return {
|
||||
...DatabaseAnalyser.createEmptyStructure(),
|
||||
...newlyAnalysed,
|
||||
};
|
||||
}
|
||||
|
||||
const res = {};
|
||||
for (const field of ['tables', 'views', 'functions', 'procedures', 'triggers']) {
|
||||
const removedIds = this.modifications
|
||||
.filter((x) => x.action == 'remove' && x.objectTypeField == field)
|
||||
.map((x) => extractObjectId(x));
|
||||
const newArray = newlyAnalysed[field] || [];
|
||||
const addedChangedIds = newArray.map((x) => extractObjectId(x));
|
||||
const removeAllIds = [...removedIds, ...addedChangedIds];
|
||||
res[field] = _.sortBy(
|
||||
[...this.structure[field].filter((x) => !removeAllIds.includes(extractObjectId(x))), ...newArray],
|
||||
(x) => x.pureName
|
||||
);
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
// const {tables,views, functions, procedures, triggers} = this.structure;
|
||||
|
||||
// return {
|
||||
// tables:
|
||||
// }
|
||||
}
|
||||
|
||||
// findObjectById(id) {
|
||||
// return this.structure.tables.find((x) => x.objectId == id);
|
||||
// }
|
||||
|
||||
static createEmptyStructure(): DatabaseInfo {
|
||||
return {
|
||||
tables: [],
|
||||
views: [],
|
||||
functions: [],
|
||||
procedures: [],
|
||||
triggers: [],
|
||||
schemas: [],
|
||||
};
|
||||
}
|
||||
|
||||
static byTableFilter(table) {
|
||||
return (x) => x.pureName == table.pureName && x.schemaName == x.schemaName;
|
||||
}
|
||||
|
||||
static extractPrimaryKeys(table, pkColumns) {
|
||||
const filtered = pkColumns.filter(DatabaseAnalyser.byTableFilter(table));
|
||||
if (filtered.length == 0) return undefined;
|
||||
return {
|
||||
..._.pick(filtered[0], ['constraintName', 'schemaName', 'pureName']),
|
||||
constraintType: 'primaryKey',
|
||||
columns: filtered.map(fp.pick('columnName')),
|
||||
};
|
||||
}
|
||||
static extractForeignKeys(table, fkColumns) {
|
||||
const grouped = _.groupBy(fkColumns.filter(DatabaseAnalyser.byTableFilter(table)), 'constraintName');
|
||||
return _.keys(grouped).map((constraintName) => ({
|
||||
constraintName,
|
||||
constraintType: 'foreignKey',
|
||||
..._.pick(grouped[constraintName][0], [
|
||||
'constraintName',
|
||||
'schemaName',
|
||||
'pureName',
|
||||
'refSchemaName',
|
||||
'refTableName',
|
||||
'updateAction',
|
||||
'deleteAction',
|
||||
]),
|
||||
columns: grouped[constraintName].map(fp.pick(['columnName', 'refColumnName'])),
|
||||
}));
|
||||
}
|
||||
}
|
||||
262
packages/tools/src/SqlDumper.ts
Normal file
262
packages/tools/src/SqlDumper.ts
Normal file
@@ -0,0 +1,262 @@
|
||||
import {
|
||||
ColumnInfo,
|
||||
EngineDriver,
|
||||
ForeignKeyInfo,
|
||||
NamedObjectInfo,
|
||||
SqlDialect,
|
||||
TableInfo,
|
||||
TransformType,
|
||||
} from 'dbgate-types';
|
||||
import _ from 'lodash';
|
||||
import moment from 'moment';
|
||||
|
||||
export class SqlDumper {
|
||||
s = '';
|
||||
driver: EngineDriver;
|
||||
dialect: SqlDialect;
|
||||
indentLevel = 0;
|
||||
|
||||
constructor(driver: EngineDriver) {
|
||||
this.driver = driver;
|
||||
this.dialect = driver.dialect;
|
||||
}
|
||||
endCommand() {
|
||||
this.putRaw(';\n');
|
||||
}
|
||||
putRaw(text) {
|
||||
this.s += text;
|
||||
}
|
||||
escapeString(value) {
|
||||
const esc = this.dialect.stringEscapeChar;
|
||||
let res = '';
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
const c = value[i];
|
||||
if (c == esc || c == "'") {
|
||||
res += esc;
|
||||
}
|
||||
res += c;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
putStringValue(value) {
|
||||
this.putRaw("'");
|
||||
this.putRaw(this.escapeString(value));
|
||||
this.putRaw("'");
|
||||
}
|
||||
putValue(value) {
|
||||
if (value === null) this.putRaw('NULL');
|
||||
if (value === true) this.putRaw('1');
|
||||
if (value === false) this.putRaw('0');
|
||||
else if (_.isString(value)) this.putStringValue(value);
|
||||
else if (_.isNumber(value)) this.putRaw(value.toString());
|
||||
else if (_.isDate(value)) this.putStringValue(moment(value).toISOString());
|
||||
}
|
||||
putCmd(format, ...args) {
|
||||
this.put(format, ...args);
|
||||
this.endCommand();
|
||||
}
|
||||
putFormattedValue(c, value) {
|
||||
switch (c) {
|
||||
case 's':
|
||||
if (value != null) {
|
||||
this.putRaw(value.toString());
|
||||
}
|
||||
break;
|
||||
case 'i':
|
||||
{
|
||||
this.putRaw(this.dialect.quoteIdentifier(value));
|
||||
}
|
||||
break;
|
||||
case 'k':
|
||||
{
|
||||
if (value) {
|
||||
this.putRaw(value.toUpperCase());
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'f':
|
||||
{
|
||||
const { schemaName, pureName } = value;
|
||||
if (schemaName) {
|
||||
this.putRaw(this.dialect.quoteIdentifier(schemaName));
|
||||
this.putRaw('.');
|
||||
}
|
||||
this.putRaw(this.dialect.quoteIdentifier(pureName));
|
||||
}
|
||||
break;
|
||||
case 'v':
|
||||
this.putValue(value);
|
||||
break;
|
||||
case 'c':
|
||||
value(this);
|
||||
break;
|
||||
}
|
||||
}
|
||||
putFormattedList(c, collection) {
|
||||
if (!collection) return;
|
||||
this.putCollection(', ', collection, (item) => this.putFormattedValue(c, item));
|
||||
}
|
||||
put(format: string, ...args) {
|
||||
let i = 0;
|
||||
let argIndex = 0;
|
||||
const length = format.length;
|
||||
while (i < length) {
|
||||
let c = format[i];
|
||||
i++;
|
||||
switch (c) {
|
||||
case '^':
|
||||
while (i < length && format[i].match(/[a-z0-9_]/i)) {
|
||||
this.putRaw(format[i].toUpperCase());
|
||||
i++;
|
||||
}
|
||||
break;
|
||||
case '%':
|
||||
c = format[i];
|
||||
i++;
|
||||
switch (c) {
|
||||
case '%':
|
||||
this.putRaw('%');
|
||||
break;
|
||||
case ',':
|
||||
c = format[i];
|
||||
i++;
|
||||
this.putFormattedList(c, args[argIndex]);
|
||||
break;
|
||||
default:
|
||||
this.putFormattedValue(c, args[argIndex]);
|
||||
break;
|
||||
}
|
||||
|
||||
argIndex++;
|
||||
break;
|
||||
case '&':
|
||||
c = format[i];
|
||||
i++;
|
||||
switch (c) {
|
||||
case '&':
|
||||
this.putRaw('&');
|
||||
break;
|
||||
case '>':
|
||||
this.indentLevel++;
|
||||
break;
|
||||
case '<':
|
||||
this.indentLevel--;
|
||||
break;
|
||||
case 'n':
|
||||
this.putRaw('\n');
|
||||
this.putRaw(' '.repeat(2 * this.indentLevel));
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
this.putRaw(c);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
autoIncrement() {
|
||||
this.put(' ^auto_increment');
|
||||
}
|
||||
|
||||
columnDefinition(column: ColumnInfo, { includeDefault = true, includeNullable = true, includeCollate = true } = {}) {
|
||||
if (column.computedExpression) {
|
||||
this.put('^as %s', column.computedExpression);
|
||||
if (column.isPersisted) this.put(' ^persisted');
|
||||
return;
|
||||
}
|
||||
this.put('%k', column.dataType || this.dialect.fallbackDataType);
|
||||
if (column.autoIncrement) {
|
||||
this.autoIncrement();
|
||||
}
|
||||
|
||||
this.putRaw(' ');
|
||||
if (column.isSparse) {
|
||||
this.put(' ^sparse ');
|
||||
}
|
||||
if (includeNullable) {
|
||||
this.put(column.notNull ? '^not ^null' : '^null');
|
||||
}
|
||||
if (includeDefault && column.defaultValue != null) {
|
||||
this.columnDefault(column);
|
||||
}
|
||||
}
|
||||
|
||||
columnDefault(column: ColumnInfo) {
|
||||
if (column.defaultConstraint != null) {
|
||||
this.put(' ^constraint %i ^default %s ', column.defaultConstraint, column.defaultValue);
|
||||
} else {
|
||||
this.put(' ^default %s ', column.defaultValue);
|
||||
}
|
||||
}
|
||||
|
||||
putCollection<T>(delimiter: string, collection: T[], lambda: (col: T) => void) {
|
||||
if (!collection) return;
|
||||
let first = true;
|
||||
for (const item of collection) {
|
||||
if (!first) this.put(delimiter);
|
||||
first = false;
|
||||
lambda(item);
|
||||
}
|
||||
}
|
||||
|
||||
createTable(table: TableInfo) {
|
||||
this.put('^create ^table %f ( &>&n', table);
|
||||
this.putCollection(',&n', table.columns, (col) => {
|
||||
this.put('%i ', col.columnName);
|
||||
this.columnDefinition(col);
|
||||
});
|
||||
if (table.primaryKey) {
|
||||
this.put(',&n');
|
||||
if (table.primaryKey.constraintName) {
|
||||
this.put('^constraint %i', table.primaryKey.constraintName);
|
||||
}
|
||||
this.put(
|
||||
' ^primary ^key (%,i)',
|
||||
table.primaryKey.columns.map((x) => x.columnName)
|
||||
);
|
||||
}
|
||||
if (table.foreignKeys) {
|
||||
table.foreignKeys.forEach((fk) => {
|
||||
this.put(',&n');
|
||||
this.createForeignKeyFore(fk);
|
||||
});
|
||||
}
|
||||
// foreach (var cnt in table.Uniques)
|
||||
// {
|
||||
// if (!first) this.put(", &n");
|
||||
// first = false;
|
||||
// CreateUniqueCore(cnt);
|
||||
// }
|
||||
// foreach (var cnt in table.Checks)
|
||||
// {
|
||||
// if (!first) this.put(", &n");
|
||||
// first = false;
|
||||
// CreateCheckCore(cnt);
|
||||
// }
|
||||
this.put('&<&n)');
|
||||
this.endCommand();
|
||||
// foreach (var ix in table.Indexes)
|
||||
// {
|
||||
// CreateIndex(ix);
|
||||
// }
|
||||
}
|
||||
|
||||
createForeignKeyFore(fk: ForeignKeyInfo) {
|
||||
if (fk.constraintName != null) this.put('^constraint %i ', fk.constraintName);
|
||||
this.put(
|
||||
'^foreign ^key (%,i) ^references %f (%,i)',
|
||||
fk.columns.map((x) => x.columnName),
|
||||
{ schemaName: fk.refSchemaName, pureName: fk.refTableName },
|
||||
fk.columns.map((x) => x.refColumnName)
|
||||
);
|
||||
if (fk.deleteAction) this.put(' ^on ^delete %k', fk.deleteAction);
|
||||
if (fk.updateAction) this.put(' ^on ^update %k', fk.updateAction);
|
||||
}
|
||||
|
||||
transform(type: TransformType, dumpExpr) {
|
||||
dumpExpr();
|
||||
}
|
||||
|
||||
allowIdentityInsert(table: NamedObjectInfo, allow: boolean) {}
|
||||
}
|
||||
93
packages/tools/src/createBulkInsertStreamBase.ts
Normal file
93
packages/tools/src/createBulkInsertStreamBase.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
import { EngineDriver } from 'dbgate-types';
|
||||
import _ from 'lodash';
|
||||
import { prepareTableForImport } from './tableTransforms';
|
||||
|
||||
export function createBulkInsertStreamBase(driver, stream, pool, name, options): any {
|
||||
const fullNameQuoted = name.schemaName
|
||||
? `${driver.dialect.quoteIdentifier(name.schemaName)}.${driver.dialect.quoteIdentifier(name.pureName)}`
|
||||
: driver.dialect.quoteIdentifier(name.pureName);
|
||||
|
||||
const writable = new stream.Writable({
|
||||
objectMode: true,
|
||||
});
|
||||
|
||||
writable.buffer = [];
|
||||
writable.structure = null;
|
||||
writable.columnNames = null;
|
||||
|
||||
writable.addRow = async (row) => {
|
||||
if (writable.structure) {
|
||||
writable.buffer.push(row);
|
||||
} else {
|
||||
writable.structure = row;
|
||||
await writable.checkStructure();
|
||||
}
|
||||
};
|
||||
|
||||
writable.checkStructure = async () => {
|
||||
let structure = await driver.analyseSingleTable(pool, name);
|
||||
// console.log('ANALYSING', name, structure);
|
||||
if (structure && options.dropIfExists) {
|
||||
console.log(`Dropping table ${fullNameQuoted}`);
|
||||
await driver.query(pool, `DROP TABLE ${fullNameQuoted}`);
|
||||
}
|
||||
if (options.createIfNotExists && (!structure || options.dropIfExists)) {
|
||||
console.log(`Creating table ${fullNameQuoted}`);
|
||||
const dmp = driver.createDumper();
|
||||
dmp.createTable(prepareTableForImport({ ...writable.structure, ...name }));
|
||||
console.log(dmp.s);
|
||||
await driver.query(pool, dmp.s);
|
||||
structure = await driver.analyseSingleTable(pool, name);
|
||||
}
|
||||
if (options.truncate) {
|
||||
await driver.query(pool, `TRUNCATE TABLE ${fullNameQuoted}`);
|
||||
}
|
||||
|
||||
this.columnNames = _.intersection(
|
||||
structure.columns.map((x) => x.columnName),
|
||||
writable.structure.columns.map((x) => x.columnName)
|
||||
);
|
||||
};
|
||||
|
||||
writable.send = async () => {
|
||||
const rows = writable.buffer;
|
||||
writable.buffer = [];
|
||||
|
||||
const dmp = driver.createDumper();
|
||||
|
||||
dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`);
|
||||
dmp.putCollection(',', this.columnNames, (col) => dmp.putRaw(driver.dialect.quoteIdentifier(col)));
|
||||
dmp.putRaw(')\n VALUES\n');
|
||||
|
||||
let wasRow = false;
|
||||
for (const row of rows) {
|
||||
if (wasRow) dmp.putRaw(',\n');
|
||||
dmp.putRaw('(');
|
||||
dmp.putCollection(',', this.columnNames, (col) => dmp.putValue(row[col]));
|
||||
dmp.putRaw(')');
|
||||
wasRow = true;
|
||||
}
|
||||
dmp.putRaw(';');
|
||||
// require('fs').writeFileSync('/home/jena/test.sql', dmp.s);
|
||||
await driver.query(pool, dmp.s);
|
||||
};
|
||||
|
||||
writable.sendIfFull = async () => {
|
||||
if (writable.buffer.length > 100) {
|
||||
await writable.send();
|
||||
}
|
||||
};
|
||||
|
||||
writable._write = async (chunk, encoding, callback) => {
|
||||
await writable.addRow(chunk);
|
||||
await writable.sendIfFull();
|
||||
callback();
|
||||
};
|
||||
|
||||
writable._final = async (callback) => {
|
||||
await writable.send();
|
||||
callback();
|
||||
};
|
||||
|
||||
return writable;
|
||||
}
|
||||
27
packages/tools/src/driverBase.ts
Normal file
27
packages/tools/src/driverBase.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
import { createBulkInsertStreamBase } from './createBulkInsertStreamBase';
|
||||
|
||||
export const driverBase = {
|
||||
analyserClass: null,
|
||||
dumperClass: null,
|
||||
|
||||
async analyseFull(pool) {
|
||||
const analyser = new this.analyserClass(pool, this);
|
||||
return analyser.fullAnalysis();
|
||||
},
|
||||
async analyseSingleObject(pool, name, typeField = 'tables') {
|
||||
const analyser = new this.analyserClass(pool, this);
|
||||
analyser.singleObjectFilter = { ...name, typeField };
|
||||
const res = await analyser.fullAnalysis();
|
||||
return res.tables[0];
|
||||
},
|
||||
analyseSingleTable(pool, name) {
|
||||
return this.analyseSingleObject(pool, name, 'tables');
|
||||
},
|
||||
async analyseIncremental(pool, structure) {
|
||||
const analyser = new this.analyserClass(pool, this);
|
||||
return analyser.incrementalAnalysis(structure);
|
||||
},
|
||||
createDumper() {
|
||||
return new this.dumperClass(this);
|
||||
},
|
||||
};
|
||||
@@ -2,3 +2,7 @@ export * from './commonTypeParser';
|
||||
export * from './nameTools';
|
||||
export * from './tableTransforms';
|
||||
export * from './packageTools';
|
||||
export * from './createBulkInsertStreamBase';
|
||||
export * from './DatabaseAnalyser';
|
||||
export * from './driverBase';
|
||||
export * from './SqlDumper';
|
||||
|
||||
Reference in New Issue
Block a user