diff --git a/packages/api/package.json b/packages/api/package.json index b672f80b2..275ae8f3d 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -49,7 +49,7 @@ "uuid": "^3.4.0" }, "scripts": { - "start": "nodemon src/index.js", + "start": "node src/index.js", "start:portal": "env-cmd nodemon src/index.js", "start:covid": "env-cmd -f .covid-env nodemon src/index.js", "ts": "tsc", diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index 54cd8c943..8ddc9616e 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -1,11 +1,20 @@ import { ColumnInfo, + ConstraintInfo, EngineDriver, ForeignKeyInfo, + FunctionInfo, NamedObjectInfo, + PrimaryKeyInfo, + ProcedureInfo, SqlDialect, TableInfo, TransformType, + TriggerInfo, + ViewInfo, + IndexInfo, + UniqueInfo, + CheckInfo, } from 'dbgate-types'; import _isString from 'lodash/isString'; import _isNumber from 'lodash/isNumber'; @@ -261,4 +270,189 @@ export class SqlDumper { } allowIdentityInsert(table: NamedObjectInfo, allow: boolean) {} + enableConstraints(table: NamedObjectInfo, enabled: boolean) {} + + comment(value: string) { + if (!value) return; + for (const line of value.split('\n')) { + this.put(' -- %s', line.trimRight()); + } + } + + createView(obj: ViewInfo) { + this.putRaw(obj.createSql); + this.endCommand(); + } + dropView(obj: ViewInfo, { testIfExists = false }) { + this.putCmd('^drop ^view %f', obj); + } + alterView(obj: ViewInfo) { + this.putRaw(obj.createSql.replace(/create\s+view/i, 'ALTER VIEW')); + this.endCommand(); + } + changeViewSchema(obj: ViewInfo, newSchema: string) {} + renameView(obj: ViewInfo, newSchema: string) {} + + createProcedure(obj: ProcedureInfo) { + this.putRaw(obj.createSql); + this.endCommand(); + } + dropProcedure(obj: ProcedureInfo, { testIfExists = false }) { + this.putCmd('^drop ^procedure %f', obj); + } + alterProcedure(obj: ProcedureInfo) { + this.putRaw(obj.createSql.replace(/create\s+procedure/i, 'ALTER PROCEDURE')); + this.endCommand(); + } + changeProcedureSchema(obj: ProcedureInfo, newSchema: string) {} + renameProcedure(obj: ProcedureInfo, newSchema: string) {} + + createFunction(obj: FunctionInfo) { + this.putRaw(obj.createSql); + this.endCommand(); + } + dropFunction(obj: FunctionInfo, { testIfExists = false }) { + this.putCmd('^drop ^function %f', obj); + } + alterFunction(obj: FunctionInfo) { + this.putRaw(obj.createSql.replace(/create\s+function/i, 'ALTER FUNCTION')); + this.endCommand(); + } + changeFunctionSchema(obj: FunctionInfo, newSchema: string) {} + renameFunction(obj: FunctionInfo, newSchema: string) {} + + createTrigger(obj: TriggerInfo) { + this.putRaw(obj.createSql); + this.endCommand(); + } + dropTrigger(obj: TriggerInfo, { testIfExists = false }) { + this.putCmd('^drop ^trigger %f', obj); + } + alterTrigger(obj: TriggerInfo) { + this.putRaw(obj.createSql.replace(/create\s+trigger/i, 'ALTER TRIGGER')); + this.endCommand(); + } + changeTriggerSchema(obj: TriggerInfo, newSchema: string) {} + renameTrigger(obj: TriggerInfo, newSchema: string) {} + + dropConstraint(cnt: ConstraintInfo) { + this.putCmd('^alter ^table %f ^drop ^constraint %i', cnt, cnt.constraintName); + } + dropForeignKey(fk: ForeignKeyInfo) { + if (this.dialect.explicitDropConstraint) { + this.putCmd('^alter ^table %f ^drop ^foreign ^key %i', fk, fk.constraintName); + } else { + this.dropConstraint(fk); + } + } + createForeignKey(fk: ForeignKeyInfo) { + this.put('^alter ^table %f ^add ', fk); + this.createForeignKeyFore(fk); + this.endCommand(); + } + dropPrimaryKey(pk: PrimaryKeyInfo) { + if (this.dialect.explicitDropConstraint) { + this.putCmd('^alter ^table %f ^drop ^primary ^key', pk); + } else { + this.dropConstraint(pk); + } + } + createPrimaryKey(pk: PrimaryKeyInfo) { + this.putCmd( + '^alter ^table %f ^add ^constraint %i ^primary ^key (%,i)', + pk, + pk.constraintName, + pk.columns.map(x => x.columnName) + ); + } + + dropIndex(ix: IndexInfo) {} + createIndex(ix: IndexInfo) {} + + dropUnique(uq: UniqueInfo) { + this.dropConstraint(uq); + } + createUniqueCore(uq: UniqueInfo) { + this.put( + '^constraint %i ^unique (%,i)', + uq.constraintName, + uq.columns.map(x => x.columnName) + ); + } + + createUnique(uq: UniqueInfo) { + this.put('^alter ^table %f ^add ', uq); + this.createUniqueCore(uq); + this.endCommand(); + } + + dropCheck(ch: CheckInfo) { + this.dropConstraint(ch); + } + + createCheckCore(ch: CheckInfo) { + this.put('^constraint %i ^check (%s)', ch.constraintName, ch.definition); + } + + createCheck(ch: CheckInfo) { + this.put('^alter ^table %f ^add ', ch); + this.createCheckCore(ch); + this.endCommand(); + } + + renameConstraint(constraint: ConstraintInfo, newName: string) {} + + createColumn(table: TableInfo, column: ColumnInfo, constraints: ConstraintInfo[]) { + this.put('^alter ^table %f ^add %i ', table, column.columnName); + this.columnDefinition(column); + this.inlineConstraints(constraints); + this.endCommand(); + } + + inlineConstraints(constrains: ConstraintInfo[]) { + if (constrains == null) return; + for (const cnt of constrains) { + if (cnt.constraintType == 'primaryKey') { + if (cnt.constraintName != null && !this.dialect.anonymousPrimaryKey) { + this.put(' ^constraint %i', cnt.constraintName); + } + this.put(' ^primary ^key '); + } + } + } + + dropColumn(column: ColumnInfo) { + this.putCmd('^alter ^table %f ^drop ^column %i', column, column.columnName); + } + + renameColumn(column: ColumnInfo, newName: string) {} + + changeColumn(oldcol: ColumnInfo, newcol: ColumnInfo, constraints: ConstraintInfo[]) {} + + dropTable(obj: TableInfo, { testIfExists = false }) { + this.putCmd('^drop ^table %f', obj); + } + + changeTableSchema(obj: TableInfo, schema: string) {} + + renameTable(obj: TableInfo, newname: string) {} + + beginTransaction() { + this.putCmd('^begin ^transaction'); + } + + commitTransaction() { + this.putCmd('^commit'); + } + + alterProlog() {} + alterEpilog() {} + + selectTableIntoNewTable(sourceName: NamedObjectInfo, targetName: NamedObjectInfo) { + this.putCmd('^select * ^into %f ^from %f', targetName, sourceName); + } + + truncateTable(name: NamedObjectInfo) { + this.putCmd('^delete ^from %f', name); + } } diff --git a/packages/tools/src/SqlGenerator.ts b/packages/tools/src/SqlGenerator.ts index 3fb3f7b22..485e237e6 100644 --- a/packages/tools/src/SqlGenerator.ts +++ b/packages/tools/src/SqlGenerator.ts @@ -1,5 +1,15 @@ -import { DatabaseInfo, EngineDriver, FunctionInfo, ProcedureInfo, TableInfo, ViewInfo } from 'dbgate-types'; +import { + DatabaseInfo, + EngineDriver, + FunctionInfo, + ProcedureInfo, + TableInfo, + TriggerInfo, + ViewInfo, +} from 'dbgate-types'; +import _ from 'lodash'; import { SqlDumper } from './SqlDumper'; +import { extendDatabaseInfo } from './structureTools'; interface SqlGeneratorOptions { dropTables: boolean; @@ -14,6 +24,22 @@ interface SqlGeneratorOptions { disableConstraints: boolean; omitNulls: boolean; truncate: boolean; + + dropViews: boolean; + checkIfViewExists: boolean; + createViews: boolean; + + dropProcedures: boolean; + checkIfProcedureExists: boolean; + createProcedures: boolean; + + dropFunctions: boolean; + checkIfFunctionExists: boolean; + createFunctions: boolean; + + dropTriggers: boolean; + checkIfTriggerExists: boolean; + createTriggers: boolean; } interface SqlGeneratorObject { @@ -27,57 +53,177 @@ export class SqlGenerator { private views: ViewInfo[]; private procedures: ProcedureInfo[]; private functions: FunctionInfo[]; + private triggers: TriggerInfo[]; + public dbinfo: DatabaseInfo; + constructor( - public dbinfo: DatabaseInfo, + dbinfo: DatabaseInfo, public options: SqlGeneratorOptions, public objects: SqlGeneratorObject[], public dmp: SqlDumper, public driver: EngineDriver, public pool ) { + this.dbinfo = extendDatabaseInfo(dbinfo); this.tables = this.extract('tables'); this.views = this.extract('views'); this.procedures = this.extract('procedures'); this.functions = this.extract('functions'); + this.triggers = this.extract('triggers'); + } + + async dump() { + this.dropObjects(this.procedures, 'Procedure'); + if (this.checkDumper()) return; + this.dropObjects(this.functions, 'Function'); + if (this.checkDumper()) return; + this.dropObjects(this.views, 'View'); + if (this.checkDumper()) return; + this.dropObjects(this.triggers, 'Trigger'); + if (this.checkDumper()) return; + + this.dropTables(); + if (this.checkDumper()) return; + + this.createTables(); + if (this.checkDumper()) return; + + this.truncateTables(); + if (this.checkDumper()) return; + + await this.insertData(); + if (this.checkDumper()) return; + + this.createForeignKeys(); + if (this.checkDumper()) return; + + this.createObjects(this.procedures, 'Procedure'); + if (this.checkDumper()) return; + this.createObjects(this.functions, 'Function'); + if (this.checkDumper()) return; + this.createObjects(this.views, 'View'); + if (this.checkDumper()) return; + this.createObjects(this.triggers, 'Trigger'); + if (this.checkDumper()) return; + } + + createForeignKeys() { + const fks = []; + if (this.options.createForeignKeys) fks.push(..._.flatten(this.tables.map(x => x.foreignKeys || []))); + if (this.options.createReferences) fks.push(..._.flatten(this.tables.map(x => x.dependencies || []))); + for (const fk of _.uniqBy(fks, 'constraintName')) { + this.dmp.createForeignKey(fk); + if (this.checkDumper()) return; + } + } + + truncateTables() { + if (this.options.truncate) { + for (const table of this.tables) { + this.dmp.truncateTable(table); + if (this.checkDumper()) return; + } + } + } + + createTables() { + if (this.options.createTables) { + for (const table of this.tables) { + this.dmp.createTable({ + ...table, + foreignKeys: [], + dependencies: [], + indexes: [], + }); + if (this.checkDumper()) return; + } + } + if (this.options.createIndexes) { + for (const index of _.flatten(this.tables.map(x => x.indexes || []))) { + this.dmp.createIndex(index); + } + } + } + + async insertData() { + if (!this.options.insert) return; + + this.enableConstraints(false); + + for (const table of this.tables) { + await this.insertTableData(table); + if (this.checkDumper()) return; + } + + this.enableConstraints(true); } checkDumper() { return false; } - async dump() { - if (this.options.createTables) { - for (const table of this.tables) { - this.dmp.createTable(table); + dropObjects(list, name) { + if (this.options[`drop${name}s`]) { + for (const item of list) { + this.dmp[`drop${name}`](item, { testIfExists: this.options[`checkIf${name}Exists`] }); if (this.checkDumper()) return; } } - if (this.options.insert) { - for (const table of this.tables) { - await this.insertTableData(table); + } + + createObjects(list, name) { + if (this.options[`create${name}s`]) { + for (const item of list) { + this.dmp[`create${name}`](item); if (this.checkDumper()) return; } } } + dropTables() { + if (this.options.dropReferences) { + for (const fk of _.flatten(this.tables.map(x => x.dependencies || []))) { + this.dmp.dropForeignKey(fk); + } + } + + if (this.options.dropTables) { + for (const table of this.tables) { + this.dmp.dropTable(table, { testIfExists: this.options.checkIfTableExists }); + } + } + } + async insertTableData(table: TableInfo) { - const dmp = this.driver.createDumper(); - dmp.put('^select * ^from %f', table); - const readable = await this.driver.readQuery(this.pool, dmp.s, table); + const dmpLocal = this.driver.createDumper(); + dmpLocal.put('^select * ^from %f', table); + + const autoinc = table.columns.find(x => x.autoIncrement); + if (autoinc && !this.options.skipAutoincrementColumn) { + this.dmp.allowIdentityInsert(table, true); + } + + const readable = await this.driver.readQuery(this.pool, dmpLocal.s, table); await this.processReadable(table, readable); + + if (autoinc && !this.options.skipAutoincrementColumn) { + this.dmp.allowIdentityInsert(table, false); + } } processReadable(table: TableInfo, readable) { - const columnNames = table.columns.map(x => x.columnName); + const columnsFiltered = this.options.skipAutoincrementColumn + ? table.columns.filter(x => !x.autoIncrement) + : table.columns; + const columnNames = columnsFiltered.map(x => x.columnName); return new Promise(resolve => { readable.on('data', chunk => { - // const chunk = readable.read(); - // if (!chunk) return; + const columnNamesCopy = this.options.omitNulls ? columnNames.filter(col => chunk[col] != null) : columnNames; this.dmp.put( '^insert ^into %f (%,i) ^values (%,v);&n', table, - columnNames, - columnNames.map(col => chunk[col]) + columnNamesCopy, + columnNamesCopy.map(col => chunk[col]) ); }); readable.on('end', () => { @@ -93,4 +239,16 @@ export class SqlGenerator { ) ); } + + enableConstraints(enabled) { + if (this.options.disableConstraints) { + if (this.driver.dialect.enableConstraintsPerTable) { + for (const table of this.tables) { + this.dmp.enableConstraints(table, enabled); + } + } else { + this.dmp.enableConstraints(null, enabled); + } + } + } } diff --git a/packages/tools/src/index.ts b/packages/tools/src/index.ts index a5204138f..956184ec4 100644 --- a/packages/tools/src/index.ts +++ b/packages/tools/src/index.ts @@ -9,3 +9,4 @@ export * from './SqlDumper'; export * from './testPermission'; export * from './splitPostgresQuery'; export * from './SqlGenerator'; +export * from './structureTools'; diff --git a/packages/tools/src/structureTools.ts b/packages/tools/src/structureTools.ts new file mode 100644 index 000000000..5c1111d92 --- /dev/null +++ b/packages/tools/src/structureTools.ts @@ -0,0 +1,31 @@ +import { DatabaseInfo } from 'dbgate-types'; +import _ from 'lodash'; + +export function addTableDependencies(db: DatabaseInfo): DatabaseInfo { + const allForeignKeys = _.flatten(db.tables.map(x => x.foreignKeys)); + return { + ...db, + tables: db.tables.map(table => ({ + ...table, + dependencies: allForeignKeys.filter(x => x.refSchemaName == table.schemaName && x.refTableName == table.pureName), + })), + }; +} + +function fillTableExtendedInfo(db: DatabaseInfo) { + return { + ...db, + tables: db.tables.map(table => ({ + ...table, + columns: (table.columns || []).map(column => ({ + pureName: table.pureName, + schemaName: table.schemaName, + ...column, + })), + })), + }; +} + +export function extendDatabaseInfo(db: DatabaseInfo): DatabaseInfo { + return fillTableExtendedInfo(addTableDependencies(db)); +} diff --git a/packages/types/dbinfo.d.ts b/packages/types/dbinfo.d.ts index cbf43fba2..7c9c74cf2 100644 --- a/packages/types/dbinfo.d.ts +++ b/packages/types/dbinfo.d.ts @@ -10,7 +10,7 @@ export interface ColumnReference { export interface ConstraintInfo extends NamedObjectInfo { constraintName: string; - constraintType: string; + constraintType: 'primaryKey' | 'foreignKey' | 'index' | 'check' | 'unique'; } export interface ColumnsConstraintInfo extends ConstraintInfo { @@ -26,7 +26,18 @@ export interface ForeignKeyInfo extends ColumnsConstraintInfo { deleteAction: string; } -export interface ColumnInfo { +export interface IndexInfo extends ColumnsConstraintInfo { + isUnique: boolean; + indexType: 'normal' | 'clustered' | 'xml' | 'spatial' | 'fulltext'; +} + +export interface UniqueInfo extends ColumnsConstraintInfo {} + +export interface CheckInfo extends ConstraintInfo { + definition: string; +} + +export interface ColumnInfo extends NamedObjectInfo { columnName: string; notNull: boolean; autoIncrement: boolean; @@ -58,6 +69,8 @@ export interface TableInfo extends DatabaseObjectInfo { primaryKey?: PrimaryKeyInfo; foreignKeys: ForeignKeyInfo[]; dependencies?: ForeignKeyInfo[]; + indexes?: IndexInfo[]; + checks?: CheckInfo[]; } export interface ViewInfo extends SqlObjectInfo { diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index 4cb44fed4..00e3387c4 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -5,4 +5,7 @@ export interface SqlDialect { offsetFetchRangeSyntax?: boolean; quoteIdentifier(s: string): string; fallbackDataType?: string; + explicitDropConstraint?: boolean; + anonymousPrimaryKey?: boolean; + enableConstraintsPerTable?: boolean; } diff --git a/packages/web/src/modals/SqlGeneratorModal.svelte b/packages/web/src/modals/SqlGeneratorModal.svelte index 691dbb91a..1a7940306 100644 --- a/packages/web/src/modals/SqlGeneratorModal.svelte +++ b/packages/web/src/modals/SqlGeneratorModal.svelte @@ -3,6 +3,7 @@ import { writable } from 'svelte/store'; import AppObjectList from '../appobj/AppObjectList.svelte'; import * as databaseObjectAppObject from '../appobj/DatabaseObjectAppObject.svelte'; + import uuidv1 from 'uuid/v1'; import HorizontalSplitter from '../elements/HorizontalSplitter.svelte'; @@ -18,6 +19,7 @@ import FontIcon from '../icons/FontIcon.svelte'; import SqlEditor from '../query/SqlEditor.svelte'; import axiosInstance from '../utility/axiosInstance'; + import createRef from '../utility/createRef'; import { useDatabaseInfo } from '../utility/metadataLoaders'; import WidgetColumnBar from '../widgets/WidgetColumnBar.svelte'; import WidgetColumnBarItem from '../widgets/WidgetColumnBarItem.svelte'; @@ -35,7 +37,11 @@ let sqlPreview = ''; const checkedObjectsStore = writable([]); - const valuesStore = writable({}); + const valuesStore = writable({ + checkIfTableExists: true, + disableConstraints: true, + }); + const loadRef = createRef(null); $: console.log('checkedObjectsStore', $checkedObjectsStore); @@ -53,8 +59,19 @@ ); async function generatePreview(options, objects) { + const loadid = uuidv1(); + loadRef.set(loadid); busy = true; - const response = await axiosInstance.post('database-connections/sql-preview', { conid, database, objects, options }); + const response = await axiosInstance.post('database-connections/sql-preview', { + conid, + database, + objects, + options, + }); + if (loadRef.get() != loadid) { + // newer load exists + return; + } if (_.isString(response.data)) { sqlPreview = response.data; } @@ -96,8 +113,8 @@ {#if values.dropTables} -
- +
+
{/if} @@ -109,9 +126,9 @@ {#if values.insert} -
+
- +
{/if} diff --git a/packages/web/src/utility/metadataLoaders.ts b/packages/web/src/utility/metadataLoaders.ts index 7e05e0a52..e7e1056af 100644 --- a/packages/web/src/utility/metadataLoaders.ts +++ b/packages/web/src/utility/metadataLoaders.ts @@ -7,23 +7,13 @@ import socket from './socket'; import getAsArray from './getAsArray'; import { DatabaseInfo } from 'dbgate-types'; import { derived } from 'svelte/store'; +import { extendDatabaseInfo } from 'dbgate-tools'; const databaseInfoLoader = ({ conid, database }) => ({ url: 'database-connections/structure', params: { conid, database }, reloadTrigger: `database-structure-changed-${conid}-${database}`, - transform: (db: DatabaseInfo) => { - const allForeignKeys = _.flatten(db.tables.map(x => x.foreignKeys)); - return { - ...db, - tables: db.tables.map(table => ({ - ...table, - dependencies: allForeignKeys.filter( - x => x.refSchemaName == table.schemaName && x.refTableName == table.pureName - ), - })), - }; - }, + transform: extendDatabaseInfo, }); // const tableInfoLoader = ({ conid, database, schemaName, pureName }) => ({