script base deployer

integration-tests/__tests__/deploy-database.spec.js

@@ -612,4 +612,65 @@ describe('Deploy database', () => {
       );
     })
   );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script drived deploy - basic predeploy - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(engine, conn, driver, [
+        [
+          {
+            name: '1.predeploy.sql',
+            text: 'create table t1 (id int primary key); insert into t1 (id) values (1);',
+          },
+        ],
+      ]);
+
+      const res1 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
+      expect(res1.rows[0].cnt == 1).toBeTruthy();
+
+      const res2 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM dbgate_deploy_journal');
+      expect(res2.rows[0].cnt == 1).toBeTruthy();
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script drived deploy - install+uninstall - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(engine, conn, driver, [
+        [
+          {
+            name: 't1.install.sql',
+            text: 'create table t1 (id int primary key); insert into t1 (id) values (1)',
+          },
+          {
+            name: 't2.once.sql',
+            text: 'create table t2 (id int primary key); insert into t2 (id) values (1)',
+          },
+        ],
+        [
+          {
+            name: 't1.uninstall.sql',
+            text: 'drop table t1',
+          },
+          {
+            name: 't1.install.sql',
+            text: 'create table t1 (id int primary key, val int); insert into t1 (id, val) values (1, 11)',
+          },
+          {
+            name: 't2.once.sql',
+            text: 'insert into t2 (id) values (2)',
+          },
+        ],
+      ]);
+
+      const res1 = await driver.query(conn, 'SELECT val from t1 where id = 1');
+      expect(res1.rows[0].val == 11).toBeTruthy();
+
+      const res2 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t2');
+      expect(res2.rows[0].cnt == 1).toBeTruthy();
+
+      const res3 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM dbgate_deploy_journal');
+      expect(res3.rows[0].cnt == 3).toBeTruthy();
+    })
+  );
 });

packages/api/src/shell/deployDb.js

@@ -1,5 +1,9 @@
 const generateDeploySql = require('./generateDeploySql');
 const executeQuery = require('./executeQuery');
+const { ScriptDrivedDeployer } = require('dbgate-datalib');
+const connectUtility = require('../utility/connectUtility');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const loadModelFolder = require('../utility/loadModelFolder');
 
 async function deployDb({
   connection,
@@ -13,9 +17,15 @@ async function deployDb({
   ignoreNameRegex = '',
   targetSchema = null,
 }) {
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
+  if (!driver) driver = requireEngineDriver(connection);
+
+  const scriptDeployer = new ScriptDrivedDeployer(dbhan, driver, loadedDbModel ?? (await loadModelFolder(modelFolder)));
+  await scriptDeployer.runPre();
+
   const { sql } = await generateDeploySql({
     connection,
-    systemConnection,
+    systemConnection: dbhan,
     driver,
     analysedStructure,
     modelFolder,
@@ -26,7 +36,9 @@ async function deployDb({
     targetSchema,
   });
   // console.log('RUNNING DEPLOY SCRIPT:', sql);
-  await executeQuery({ connection, systemConnection, driver, sql, logScriptItems: true });
+  await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });
+
+  await scriptDeployer.runPost();
 }
 
 module.exports = deployDb;

packages/api/src/shell/generateDeploySql.js

@@ -9,6 +9,7 @@ const {
   skipNamesInStructureByRegex,
   replaceSchemaInStructure,
   filterStructureBySchema,
+  skipDbGateInternalObjects,
 } = require('dbgate-tools');
 const importDbModel = require('../utility/importDbModel');
 const requireEngineDriver = require('../utility/requireEngineDriver');
@@ -38,6 +39,7 @@ async function generateDeploySql({
     if (ignoreNameRegex) {
       analysedStructure = skipNamesInStructureByRegex(analysedStructure, new RegExp(ignoreNameRegex, 'i'));
     }
+    analysedStructure = skipDbGateInternalObjects(analysedStructure);
 
     let deployedModelSource = loadedDbModel
       ? databaseInfoFromYamlModel(loadedDbModel)

packages/api/src/shell/index.js

@@ -35,6 +35,7 @@ const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
 const autoIndexForeignKeysTransform = require('./autoIndexForeignKeysTransform');
 const generateDeploySql = require('./generateDeploySql');
 const dropAllDbObjects = require('./dropAllDbObjects');
+const scriptDrivedDeploy = require('./scriptDrivedDeploy');
 
 const dbgateApi = {
   queryReader,
@@ -73,6 +74,7 @@ const dbgateApi = {
   autoIndexForeignKeysTransform,
   generateDeploySql,
   dropAllDbObjects,
+  scriptDrivedDeploy,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

packages/api/src/utility/importDbModel.js

@@ -1,28 +1,8 @@
-const fs = require('fs-extra');
-const path = require('path');
-const yaml = require('js-yaml');
 const { databaseInfoFromYamlModel, DatabaseAnalyser } = require('dbgate-tools');
-const { startsWith } = require('lodash');
-const { archivedir, resolveArchiveFolder } = require('./directories');
-const loadFilesRecursive = require('./loadFilesRecursive');
+const loadModelFolder = require('./loadModelFolder');
 
 async function importDbModel(inputDir) {
-  const files = [];
-
-  const dir = inputDir.startsWith('archive:') ? resolveArchiveFolder(inputDir.substring('archive:'.length)) : inputDir;
-
-  for (const name of await loadFilesRecursive(dir)) {
-    if (name.endsWith('.table.yaml') || name.endsWith('.sql')) {
-      const text = await fs.readFile(path.join(dir, name), { encoding: 'utf-8' });
-
-      files.push({
-        name: path.parse(name).base,
-        text,
-        json: name.endsWith('.yaml') ? yaml.load(text) : null,
-      });
-    }
-  }
-
+  const files = await loadModelFolder(inputDir);
   return databaseInfoFromYamlModel(files);
 }
 

packages/api/src/utility/loadModelFolder.js

@@ -0,0 +1,27 @@
+const fs = require('fs-extra');
+const path = require('path');
+const yaml = require('js-yaml');
+const { resolveArchiveFolder } = require('./directories');
+const loadFilesRecursive = require('./loadFilesRecursive');
+
+async function loadModelFolder(inputDir) {
+  const files = [];
+
+  const dir = inputDir.startsWith('archive:') ? resolveArchiveFolder(inputDir.substring('archive:'.length)) : inputDir;
+
+  for (const name of await loadFilesRecursive(dir)) {
+    if (name.endsWith('.table.yaml') || name.endsWith('.sql')) {
+      const text = await fs.readFile(path.join(dir, name), { encoding: 'utf-8' });
+
+      files.push({
+        name: path.parse(name).base,
+        text,
+        json: name.endsWith('.yaml') ? yaml.load(text) : null,
+      });
+    }
+  }
+
+  return files;
+}
+
+module.exports = loadModelFolder;

packages/datalib/src/ScriptDrivedDeployer.ts

@@ -0,0 +1,174 @@
+import { DatabaseModelFile, extractErrorLogData, getLogger, runCommandOnDriver, runQueryOnDriver } from 'dbgate-tools';
+import { EngineDriver } from 'dbgate-types';
+import crypto from 'crypto';
+import _sortBy from 'lodash/sortBy';
+
+const logger = getLogger('ScriptDrivedDeployer');
+
+interface DeployScriptJournalItem {
+  id: number;
+  name: string;
+  category: string;
+  first_run_date: string;
+  last_run_date: string;
+  script_hash: string;
+}
+
+export class ScriptDrivedDeployer {
+  predeploy: DatabaseModelFile[] = [];
+  uninstall: DatabaseModelFile[] = [];
+  install: DatabaseModelFile[] = [];
+  once: DatabaseModelFile[] = [];
+  postdeploy: DatabaseModelFile[] = [];
+  isEmpty = false;
+
+  journalItems: DeployScriptJournalItem[] = [];
+
+  constructor(public dbhan: any, public driver: EngineDriver, public files: DatabaseModelFile[]) {
+    this.predeploy = files.filter(x => x.name.endsWith('.predeploy.sql'));
+    this.uninstall = files.filter(x => x.name.endsWith('.uninstall.sql'));
+    this.install = files.filter(x => x.name.endsWith('.install.sql'));
+    this.once = files.filter(x => x.name.endsWith('.once.sql'));
+    this.postdeploy = files.filter(x => x.name.endsWith('.postdeploy.sql'));
+    this.isEmpty =
+      this.predeploy.length === 0 &&
+      this.uninstall.length === 0 &&
+      this.install.length === 0 &&
+      this.once.length === 0 &&
+      this.postdeploy.length === 0;
+  }
+
+  async loadJournalItems() {
+    try {
+      const { rows } = await runQueryOnDriver(this.dbhan, this.driver, dmp =>
+        dmp.put('select * from ~dbgate_deploy_journal')
+      );
+      this.journalItems = rows;
+      logger.debug(`Loaded ${rows.length} items from DbGate deploy journal`);
+    } catch (err) {
+      logger.error(
+        extractErrorLogData(err),
+        'Error loading DbGate deploy journal, createing table dbgate_deploy_journal'
+      );
+      const dmp = this.driver.createDumper();
+      dmp.createTable({
+        pureName: 'dbgate_deploy_journal',
+        columns: [
+          { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true, pureName: 'dbgate_deploy_journal' },
+          { columnName: 'name', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+          { columnName: 'category', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+          { columnName: 'first_run_date', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+          { columnName: 'last_run_date', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+          { columnName: 'script_hash', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+        ],
+        foreignKeys: [],
+        primaryKey: {
+          columns: [{ columnName: 'id' }],
+          constraintType: 'primaryKey',
+          pureName: 'dbgate_deploy_journal',
+        },
+      });
+      await this.driver.query(this.dbhan, dmp.s, { discardResult: true });
+    }
+  }
+
+  async runPre() {
+    // don't create journal table if no scripts are present
+    if (this.isEmpty) return;
+    await this.loadJournalItems();
+    await this.runFiles(this.predeploy, 'predeploy');
+  }
+
+  async runPost() {
+    await this.runFiles(this.install, 'install');
+    await this.runFiles(this.once, 'once');
+    await this.runFiles(this.postdeploy, 'postdeploy');
+  }
+
+  async run() {
+    await this.runPre();
+    await this.runPost();
+  }
+
+  async runFiles(files: DatabaseModelFile[], category: string) {
+    for (const file of _sortBy(files, x => x.name)) {
+      await this.runFile(file, category);
+    }
+  }
+
+  async saveToJournal(file: DatabaseModelFile, category: string, hash: string) {
+    const existing = this.journalItems.find(x => x.name == file.name);
+    if (existing) {
+      await runCommandOnDriver(this.dbhan, this.driver, dmp => {
+        dmp.put(
+          'update ~dbgate_deploy_journal set ~last_run_date = %v, ~script_hash = %v where ~id = %v',
+          new Date().toISOString(),
+          hash,
+          existing.id
+        );
+      });
+    } else {
+      await runCommandOnDriver(this.dbhan, this.driver, dmp => {
+        dmp.put(
+          'insert into ~dbgate_deploy_journal (~name, ~category, ~first_run_date, ~last_run_date, ~script_hash) values (%v, %v, %v, %v, %v)',
+          file.name,
+          category,
+          new Date().toISOString(),
+          new Date().toISOString(),
+          hash
+        );
+      });
+    }
+  }
+
+  async runFileCore(file: DatabaseModelFile, category: string, hash: string) {
+    if (this.driver.supportsTransactions) {
+      runCommandOnDriver(this.dbhan, this.driver, dmp => dmp.beginTransaction());
+    }
+
+    logger.debug(`Running ${category} script ${file.name}`);
+    try {
+      await this.driver.script(this.dbhan, file.text);
+      await this.saveToJournal(file, category, hash);
+    } catch (err) {
+      logger.error(extractErrorLogData(err), `Error running ${category} script ${file.name}`);
+      if (this.driver.supportsTransactions) {
+        runCommandOnDriver(this.dbhan, this.driver, dmp => dmp.rollbackTransaction());
+        return;
+      }
+    }
+
+    if (this.driver.supportsTransactions) {
+      runCommandOnDriver(this.dbhan, this.driver, dmp => dmp.commitTransaction());
+    }
+  }
+
+  async runFile(file: DatabaseModelFile, category: string) {
+    const hash = crypto.createHash('md5').update(file.text.trim()).digest('hex');
+    const journalItem = this.journalItems.find(x => x.name == file.name);
+    const isEqual = journalItem && journalItem.script_hash == hash;
+
+    switch (category) {
+      case 'predeploy':
+      case 'postdeploy':
+        await this.runFileCore(file, category, hash);
+        break;
+      case 'once':
+        if (journalItem) return;
+        await this.runFileCore(file, category, hash);
+        break;
+      case 'install':
+        if (isEqual) return;
+        const uninstallFile = this.uninstall.find(x => x.name == file.name.replace('.install.sql', '.uninstall.sql'));
+        if (uninstallFile) {
+          await this.runFileCore(
+            uninstallFile,
+            'uninstall',
+            crypto.createHash('md5').update(uninstallFile.text.trim()).digest('hex')
+          );
+        }
+        await this.runFileCore(file, category, hash);
+        break;
+    }
+  }
+}

packages/datalib/src/index.ts

@@ -22,3 +22,4 @@ export * from './DataDuplicator';
 export * from './FreeTableGridDisplay';
 export * from './FreeTableModel';
 export * from './CustomGridDisplay';
+export * from './ScriptDrivedDeployer';

packages/tools/src/structureTools.ts

@@ -286,3 +286,10 @@ export function removePreloadedRowsFromStructure(db: DatabaseInfo): DatabaseInfo
     })),
   };
 }
+
+export function skipDbGateInternalObjects(db: DatabaseInfo) {
+  return {
+    ...db,
+    tables: (db.tables || []).filter(tbl => tbl.pureName != 'dbgate_deploy_journal'),
+  };
+}