diff --git a/integration-tests/__tests__/deploy-database.spec.js b/integration-tests/__tests__/deploy-database.spec.js index 56edc855c..c2e58031c 100644 --- a/integration-tests/__tests__/deploy-database.spec.js +++ b/integration-tests/__tests__/deploy-database.spec.js @@ -612,4 +612,65 @@ describe('Deploy database', () => { ); }) ); + + test.each(engines.map(engine => [engine.label, engine]))( + 'Script drived deploy - basic predeploy - %s', + testWrapper(async (conn, driver, engine) => { + await testDatabaseDeploy(engine, conn, driver, [ + [ + { + name: '1.predeploy.sql', + text: 'create table t1 (id int primary key); insert into t1 (id) values (1);', + }, + ], + ]); + + const res1 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1'); + expect(res1.rows[0].cnt == 1).toBeTruthy(); + + const res2 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM dbgate_deploy_journal'); + expect(res2.rows[0].cnt == 1).toBeTruthy(); + }) + ); + + test.each(engines.map(engine => [engine.label, engine]))( + 'Script drived deploy - install+uninstall - %s', + testWrapper(async (conn, driver, engine) => { + await testDatabaseDeploy(engine, conn, driver, [ + [ + { + name: 't1.install.sql', + text: 'create table t1 (id int primary key); insert into t1 (id) values (1)', + }, + { + name: 't2.once.sql', + text: 'create table t2 (id int primary key); insert into t2 (id) values (1)', + }, + ], + [ + { + name: 't1.uninstall.sql', + text: 'drop table t1', + }, + { + name: 't1.install.sql', + text: 'create table t1 (id int primary key, val int); insert into t1 (id, val) values (1, 11)', + }, + { + name: 't2.once.sql', + text: 'insert into t2 (id) values (2)', + }, + ], + ]); + + const res1 = await driver.query(conn, 'SELECT val from t1 where id = 1'); + expect(res1.rows[0].val == 11).toBeTruthy(); + + const res2 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t2'); + expect(res2.rows[0].cnt == 1).toBeTruthy(); + + const res3 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM dbgate_deploy_journal'); + expect(res3.rows[0].cnt == 3).toBeTruthy(); + }) + ); }); diff --git a/packages/api/src/shell/deployDb.js b/packages/api/src/shell/deployDb.js index 3141bc077..e90279686 100644 --- a/packages/api/src/shell/deployDb.js +++ b/packages/api/src/shell/deployDb.js @@ -1,5 +1,9 @@ const generateDeploySql = require('./generateDeploySql'); const executeQuery = require('./executeQuery'); +const { ScriptDrivedDeployer } = require('dbgate-datalib'); +const connectUtility = require('../utility/connectUtility'); +const requireEngineDriver = require('../utility/requireEngineDriver'); +const loadModelFolder = require('../utility/loadModelFolder'); async function deployDb({ connection, @@ -13,9 +17,15 @@ async function deployDb({ ignoreNameRegex = '', targetSchema = null, }) { + const dbhan = systemConnection || (await connectUtility(driver, connection, 'read')); + if (!driver) driver = requireEngineDriver(connection); + + const scriptDeployer = new ScriptDrivedDeployer(dbhan, driver, loadedDbModel ?? (await loadModelFolder(modelFolder))); + await scriptDeployer.runPre(); + const { sql } = await generateDeploySql({ connection, - systemConnection, + systemConnection: dbhan, driver, analysedStructure, modelFolder, @@ -26,7 +36,9 @@ async function deployDb({ targetSchema, }); // console.log('RUNNING DEPLOY SCRIPT:', sql); - await executeQuery({ connection, systemConnection, driver, sql, logScriptItems: true }); + await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true }); + + await scriptDeployer.runPost(); } module.exports = deployDb; diff --git a/packages/api/src/shell/generateDeploySql.js b/packages/api/src/shell/generateDeploySql.js index 0d0901f03..8bc60487b 100644 --- a/packages/api/src/shell/generateDeploySql.js +++ b/packages/api/src/shell/generateDeploySql.js @@ -9,6 +9,7 @@ const { skipNamesInStructureByRegex, replaceSchemaInStructure, filterStructureBySchema, + skipDbGateInternalObjects, } = require('dbgate-tools'); const importDbModel = require('../utility/importDbModel'); const requireEngineDriver = require('../utility/requireEngineDriver'); @@ -38,6 +39,7 @@ async function generateDeploySql({ if (ignoreNameRegex) { analysedStructure = skipNamesInStructureByRegex(analysedStructure, new RegExp(ignoreNameRegex, 'i')); } + analysedStructure = skipDbGateInternalObjects(analysedStructure); let deployedModelSource = loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index 54b9da1aa..dc0503382 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -35,6 +35,7 @@ const sqlTextReplacementTransform = require('./sqlTextReplacementTransform'); const autoIndexForeignKeysTransform = require('./autoIndexForeignKeysTransform'); const generateDeploySql = require('./generateDeploySql'); const dropAllDbObjects = require('./dropAllDbObjects'); +const scriptDrivedDeploy = require('./scriptDrivedDeploy'); const dbgateApi = { queryReader, @@ -73,6 +74,7 @@ const dbgateApi = { autoIndexForeignKeysTransform, generateDeploySql, dropAllDbObjects, + scriptDrivedDeploy, }; requirePlugin.initializeDbgateApi(dbgateApi); diff --git a/packages/api/src/utility/importDbModel.js b/packages/api/src/utility/importDbModel.js index 655fe6ac1..4c078adc9 100644 --- a/packages/api/src/utility/importDbModel.js +++ b/packages/api/src/utility/importDbModel.js @@ -1,28 +1,8 @@ -const fs = require('fs-extra'); -const path = require('path'); -const yaml = require('js-yaml'); const { databaseInfoFromYamlModel, DatabaseAnalyser } = require('dbgate-tools'); -const { startsWith } = require('lodash'); -const { archivedir, resolveArchiveFolder } = require('./directories'); -const loadFilesRecursive = require('./loadFilesRecursive'); +const loadModelFolder = require('./loadModelFolder'); async function importDbModel(inputDir) { - const files = []; - - const dir = inputDir.startsWith('archive:') ? resolveArchiveFolder(inputDir.substring('archive:'.length)) : inputDir; - - for (const name of await loadFilesRecursive(dir)) { - if (name.endsWith('.table.yaml') || name.endsWith('.sql')) { - const text = await fs.readFile(path.join(dir, name), { encoding: 'utf-8' }); - - files.push({ - name: path.parse(name).base, - text, - json: name.endsWith('.yaml') ? yaml.load(text) : null, - }); - } - } - + const files = await loadModelFolder(inputDir); return databaseInfoFromYamlModel(files); } diff --git a/packages/api/src/utility/loadModelFolder.js b/packages/api/src/utility/loadModelFolder.js new file mode 100644 index 000000000..8478e712d --- /dev/null +++ b/packages/api/src/utility/loadModelFolder.js @@ -0,0 +1,27 @@ +const fs = require('fs-extra'); +const path = require('path'); +const yaml = require('js-yaml'); +const { resolveArchiveFolder } = require('./directories'); +const loadFilesRecursive = require('./loadFilesRecursive'); + +async function loadModelFolder(inputDir) { + const files = []; + + const dir = inputDir.startsWith('archive:') ? resolveArchiveFolder(inputDir.substring('archive:'.length)) : inputDir; + + for (const name of await loadFilesRecursive(dir)) { + if (name.endsWith('.table.yaml') || name.endsWith('.sql')) { + const text = await fs.readFile(path.join(dir, name), { encoding: 'utf-8' }); + + files.push({ + name: path.parse(name).base, + text, + json: name.endsWith('.yaml') ? yaml.load(text) : null, + }); + } + } + + return files; +} + +module.exports = loadModelFolder; diff --git a/packages/datalib/src/ScriptDrivedDeployer.ts b/packages/datalib/src/ScriptDrivedDeployer.ts new file mode 100644 index 000000000..4377097fc --- /dev/null +++ b/packages/datalib/src/ScriptDrivedDeployer.ts @@ -0,0 +1,174 @@ +import { DatabaseModelFile, extractErrorLogData, getLogger, runCommandOnDriver, runQueryOnDriver } from 'dbgate-tools'; +import { EngineDriver } from 'dbgate-types'; +import crypto from 'crypto'; +import _sortBy from 'lodash/sortBy'; + +const logger = getLogger('ScriptDrivedDeployer'); + +interface DeployScriptJournalItem { + id: number; + name: string; + category: string; + first_run_date: string; + last_run_date: string; + script_hash: string; +} + +export class ScriptDrivedDeployer { + predeploy: DatabaseModelFile[] = []; + uninstall: DatabaseModelFile[] = []; + install: DatabaseModelFile[] = []; + once: DatabaseModelFile[] = []; + postdeploy: DatabaseModelFile[] = []; + isEmpty = false; + + journalItems: DeployScriptJournalItem[] = []; + + constructor(public dbhan: any, public driver: EngineDriver, public files: DatabaseModelFile[]) { + this.predeploy = files.filter(x => x.name.endsWith('.predeploy.sql')); + this.uninstall = files.filter(x => x.name.endsWith('.uninstall.sql')); + this.install = files.filter(x => x.name.endsWith('.install.sql')); + this.once = files.filter(x => x.name.endsWith('.once.sql')); + this.postdeploy = files.filter(x => x.name.endsWith('.postdeploy.sql')); + this.isEmpty = + this.predeploy.length === 0 && + this.uninstall.length === 0 && + this.install.length === 0 && + this.once.length === 0 && + this.postdeploy.length === 0; + } + + async loadJournalItems() { + try { + const { rows } = await runQueryOnDriver(this.dbhan, this.driver, dmp => + dmp.put('select * from ~dbgate_deploy_journal') + ); + this.journalItems = rows; + logger.debug(`Loaded ${rows.length} items from DbGate deploy journal`); + } catch (err) { + logger.error( + extractErrorLogData(err), + 'Error loading DbGate deploy journal, createing table dbgate_deploy_journal' + ); + const dmp = this.driver.createDumper(); + dmp.createTable({ + pureName: 'dbgate_deploy_journal', + columns: [ + { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true, pureName: 'dbgate_deploy_journal' }, + { columnName: 'name', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' }, + { columnName: 'category', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' }, + { columnName: 'first_run_date', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' }, + { columnName: 'last_run_date', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' }, + { columnName: 'script_hash', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' }, + ], + foreignKeys: [], + primaryKey: { + columns: [{ columnName: 'id' }], + constraintType: 'primaryKey', + pureName: 'dbgate_deploy_journal', + }, + }); + await this.driver.query(this.dbhan, dmp.s, { discardResult: true }); + } + } + + async runPre() { + // don't create journal table if no scripts are present + if (this.isEmpty) return; + await this.loadJournalItems(); + await this.runFiles(this.predeploy, 'predeploy'); + } + + async runPost() { + await this.runFiles(this.install, 'install'); + await this.runFiles(this.once, 'once'); + await this.runFiles(this.postdeploy, 'postdeploy'); + } + + async run() { + await this.runPre(); + await this.runPost(); + } + + async runFiles(files: DatabaseModelFile[], category: string) { + for (const file of _sortBy(files, x => x.name)) { + await this.runFile(file, category); + } + } + + async saveToJournal(file: DatabaseModelFile, category: string, hash: string) { + const existing = this.journalItems.find(x => x.name == file.name); + if (existing) { + await runCommandOnDriver(this.dbhan, this.driver, dmp => { + dmp.put( + 'update ~dbgate_deploy_journal set ~last_run_date = %v, ~script_hash = %v where ~id = %v', + new Date().toISOString(), + hash, + existing.id + ); + }); + } else { + await runCommandOnDriver(this.dbhan, this.driver, dmp => { + dmp.put( + 'insert into ~dbgate_deploy_journal (~name, ~category, ~first_run_date, ~last_run_date, ~script_hash) values (%v, %v, %v, %v, %v)', + file.name, + category, + new Date().toISOString(), + new Date().toISOString(), + hash + ); + }); + } + } + + async runFileCore(file: DatabaseModelFile, category: string, hash: string) { + if (this.driver.supportsTransactions) { + runCommandOnDriver(this.dbhan, this.driver, dmp => dmp.beginTransaction()); + } + + logger.debug(`Running ${category} script ${file.name}`); + try { + await this.driver.script(this.dbhan, file.text); + await this.saveToJournal(file, category, hash); + } catch (err) { + logger.error(extractErrorLogData(err), `Error running ${category} script ${file.name}`); + if (this.driver.supportsTransactions) { + runCommandOnDriver(this.dbhan, this.driver, dmp => dmp.rollbackTransaction()); + return; + } + } + + if (this.driver.supportsTransactions) { + runCommandOnDriver(this.dbhan, this.driver, dmp => dmp.commitTransaction()); + } + } + + async runFile(file: DatabaseModelFile, category: string) { + const hash = crypto.createHash('md5').update(file.text.trim()).digest('hex'); + const journalItem = this.journalItems.find(x => x.name == file.name); + const isEqual = journalItem && journalItem.script_hash == hash; + + switch (category) { + case 'predeploy': + case 'postdeploy': + await this.runFileCore(file, category, hash); + break; + case 'once': + if (journalItem) return; + await this.runFileCore(file, category, hash); + break; + case 'install': + if (isEqual) return; + const uninstallFile = this.uninstall.find(x => x.name == file.name.replace('.install.sql', '.uninstall.sql')); + if (uninstallFile) { + await this.runFileCore( + uninstallFile, + 'uninstall', + crypto.createHash('md5').update(uninstallFile.text.trim()).digest('hex') + ); + } + await this.runFileCore(file, category, hash); + break; + } + } +} diff --git a/packages/datalib/src/index.ts b/packages/datalib/src/index.ts index 85f4e40f8..ab407eeed 100644 --- a/packages/datalib/src/index.ts +++ b/packages/datalib/src/index.ts @@ -22,3 +22,4 @@ export * from './DataDuplicator'; export * from './FreeTableGridDisplay'; export * from './FreeTableModel'; export * from './CustomGridDisplay'; +export * from './ScriptDrivedDeployer'; diff --git a/packages/tools/src/structureTools.ts b/packages/tools/src/structureTools.ts index ba3c7d0f0..b04f423da 100644 --- a/packages/tools/src/structureTools.ts +++ b/packages/tools/src/structureTools.ts @@ -286,3 +286,10 @@ export function removePreloadedRowsFromStructure(db: DatabaseInfo): DatabaseInfo })), }; } + +export function skipDbGateInternalObjects(db: DatabaseInfo) { + return { + ...db, + tables: (db.tables || []).filter(tbl => tbl.pureName != 'dbgate_deploy_journal'), + }; +}