dbmodel tool initial import

packages/api/src/shell/index.js

@@ -23,6 +23,7 @@ const deployDb = require('./deployDb');
 const initializeApiEnvironment = require('./initializeApiEnvironment');
 const dumpDatabase = require('./dumpDatabase');
 const importDatabase = require('./importDatabase');
+const loadDatabase = require('./loadDatabase');
 
 const dbgateApi = {
   queryReader,
@@ -49,6 +50,7 @@ const dbgateApi = {
   initializeApiEnvironment,
   dumpDatabase,
   importDatabase,
+  loadDatabase,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

packages/api/src/shell/loadDatabase.js

@@ -0,0 +1,21 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+const exportDbModel = require('../utility/exportDbModel');
+
+const logger = getLogger('analyseDb');
+
+async function loadDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, outputDir }) {
+  logger.info(`Analysing database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
+  logger.info(`Connected.`);
+
+  const dbInfo = await driver.analyseFull(pool);
+  logger.info(`Analyse finished`);
+
+  await exportDbModel(dbInfo, outputDir);
+}
+
+module.exports = loadDatabase;

packages/dbmodel/.gitignore

@@ -0,0 +1 @@
+testdata/db

packages/dbmodel/README.md

@@ -0,0 +1,114 @@
+# dbmodel
+Deploy, load or build script from model of SQL database. Can be used as command-line tool or as javascript functions
+
+Model is stored as a collection of files:
+* tables - stored as YAML files
+  * columns
+  * indexes
+  * primary keys
+  * foreign keys
+* views - stored as SQL file with extension **.view.sql**
+* stored procedures - stored as SQL file with extension **.proc.sql**
+* functions - stored as SQL file with extension **.func.sql**
+
+## Installation - as global tool
+
+    npm install --global dbmodel
+
+## Installation - as regular package
+
+    npm install --save dbmodel
+
+## Available commands
+* **load** - loads structure of database, saves it to local directory (called *project*). Also can download data of enlisted tables (use --load-data-condition options)
+* **deploy** - deploys structure from local directory (*project*) to database. *Deploy does not perform any actions leading to data loss, these changes must be made manually.*
+  * creates not existing tables
+  * creates not existing columns of existing tables
+  * checks column NULL/NOT NULL flag, alters colums
+  * checks tables, which are in database, but not in project, list of these tables are reported
+  * checks columns, which are in database, but not in project, list of these columns are reported
+  * checks indexes and its definitions, indexes are created or recreated, if neccessary (*but not deleted*)
+  * checks and creates foreign keys
+  * checks, creates new or changes existing views, stored procedures and functions
+  * updates and creates static data (included in table yaml files)
+* **build** - builds script from project folder. This operation is complete offline, no database connection is needed. Built script makes subset of deploy command. It can be executed on empty database, but also it can convert existing database to current structure (but only using operations below).
+  * creates not existing tables
+  * creates not existing columns of existing tables
+  * creates not existing indexes (checked only by name)
+  * creates not existing foreign keys
+  * creates new or changes existing views, stored procedures and functions
+  * updates and creates static data (included in table yaml files)
+
+## Command line interface
+
+```sh
+# load from existing database
+dbmodel load -s localhost -u USERNAME -p PASSWORD -d DATABASE -c mssql OUTPUT_FOLDER
+
+# deploy project to database
+dbmodel deploy -s localhost -u USERNAME -p PASSWORD -d DATABASE -c mssql PROJECT_FOLDER
+
+# build SQL script from project
+dbmodel build -c mssql PROJECT_FOLDER OUTPUT_FILE.sql
+```
+
+## JavaScript interface
+
+```javascript
+const dbmodel = require('dbmodel');
+
+await dbmodel.deploy({
+  client: 'mssql',
+  connection: {
+    server: '127.0.0.1',
+    user: process.env.DB_USER,
+    password: process.env.DB_PASSWORD,
+    database: 'Chinook_Model',
+  },
+  hooks: [dbmodel.hooks.autoIndexForeignKeys], // this hook adds indexes to all foreign keys
+  projectDir: 'model',
+})
+```
+
+list of dbmodel exported functions:
+* build - builds SQL script
+* deploy - deploys model to database 
+* dump - dumps loaded model into directory
+* load - loads model from database
+* read - reads model from directory
+* connect - creates database connection defined in options
+
+## Table yaml file documentation
+
+```yaml
+name: Album # table name
+columns:
+  - name: AlbumId # column name
+    type: int # data type. is used directly in target SQL engine 
+    autoIncrement: true # column is autoincrement
+    notNull: true # column is not nullable (default: is nullable)
+  - name: Title
+    type: nvarchar
+    length: 160 # maximum character length
+    notNull: true
+  - name: ArtistId
+    type: int
+    references: Artist # name of table. Is used for creating foreign key
+  - name: isDeleted
+    type: bit
+    notNull: true
+    default: 0 # default value
+primaryKey:
+  - AlbumId # list of primary key column names
+indexes:
+  - name: UQ_AlbumTitleArtistId # index name
+    unique: true # whether index is unique. default=false
+    columns: # list of index columns
+      - Title
+      - ArtistId
+    filter: isDeleted=0 # if defined, filtered index (with WHERE condition) is created
+    continueOnError: true # if true and there was error in creating this index, continue (suitable for lately added unique indexes)
+data: # static data (only for list tables)
+  - AlbumId: -1 # values for all columns, which should be filled
+    Title: Predefined static album
+```

packages/dbmodel/bin/dbmodel.js

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+const path = require('path');
+require('dotenv').config();
+
+global.API_PACKAGE = path.dirname(path.dirname(require.resolve('dbgate-api')));
+global.PLUGINS_DIR = process.env.DEVMODE
+  ? path.join(path.dirname(path.dirname(global.API_PACKAGE)), 'plugins')
+  : path.dirname(global.API_PACKAGE);
+global.IS_NPM_DIST = true;
+
+const program = require('commander');
+const dbmodel = require('../lib');
+const dbgateApi = require('dbgate-api');
+
+program
+  .option('-s, --server <server>', 'server host')
+  .option('-u, --user <user>', 'user name')
+  .option('-p, --password <password>', 'password')
+  .option('-d, --database <database>', 'database name')
+  .option('--auto-index-foreign-keys', 'automatically adds indexes to all foreign keys')
+  .option(
+    '--load-data-condition <condition>',
+    'regex, which table data will be loaded and stored in model (in load command)'
+  )
+  .requiredOption('-e, --engine <engine>', 'engine name, eg. mysql@dbgate-plugin-mysql');
+
+program
+  .command('deploy <projectDir>')
+  .description('Deploys model to database')
+  .action(projectDir => {
+    const { engine, server, user, password, database } = program;
+    const hooks = [];
+    if (program.autoIndexForeignKeys) hooks.push(dbmodel.hooks.autoIndexForeignKeys);
+    dbmodel.runAndExit(
+      dbmodel.deploy({
+        connection: {
+          engine,
+          server,
+          user,
+          password,
+          database,
+        },
+        hooks,
+        projectDir,
+      })
+    );
+  });
+
+program
+  .command('load <outputDir>')
+  .description('Loads model from database')
+  .action(outputDir => {
+    const { engine, server, user, password, database } = program.opts();
+    // const loadDataCondition = program.loadDataCondition
+    //   ? table => table.name.match(new RegExp(program.loadDataCondition, 'i'))
+    //   : null;
+    // const hooks = [];
+
+    dbmodel.runAndExit(
+      dbgateApi.loadDatabase({
+        connection: {
+          engine,
+          server,
+          user,
+          password,
+          database,
+        },
+        outputDir,
+      })
+    );
+  });
+
+program
+  .command('build <projectDir> <outputFile>')
+  .description('Builds single SQL script from project')
+  .action((projectDir, outputFile) => {
+    const { client } = program;
+    const hooks = [];
+    dbmodel.runAndExit(
+      dbmodel.build({
+        client,
+        hooks,
+        projectDir,
+        outputFile,
+      })
+    );
+  });
+
+program.parse(process.argv);

packages/dbmodel/lib/index.js

@@ -0,0 +1,20 @@
+const load = require('./load');
+// const connect = require('./connect');
+// const dump = require('./dump');
+// const deploy = require('./deploy');
+// const build = require('./build');
+// const efmodel = require('./efmodel');
+
+// const hooks = require('./hooks');
+const runAndExit = require('./runAndExit');
+
+module.exports = {
+  load,
+  // connect,
+  // dump,
+  // deploy,
+  // hooks,
+  // build,
+  // efmodel,
+  runAndExit,
+};

packages/dbmodel/lib/load.js

@@ -0,0 +1,23 @@
+const _ = require('lodash');
+const dbgateApi = require('dbgate-api');
+
+async function load({ connection, outputDir }) {
+  await dbgateApi.loadDatabase(connection, outputDir);
+
+  // await connect(options);
+  // const { client } = options;
+  // const loadFunc = require(`./clients/${client}/load`);
+  // options.databaseStructure = await loadFunc(options);
+  // if (options.loadDataCondition) {
+  //   const { tables } = options.databaseStructure;
+  //   for (const tableName of _.keys(tables)) {
+  //     const table = tables[tableName];
+  //     if (!options.loadDataCondition(table)) continue;
+  //     const data = await query(options, `SELECT * FROM [${tableName}]`);
+  //     table.data = data;
+  //   }
+  // }
+  // return options;
+}
+
+module.exports = load;

packages/dbmodel/lib/runAndExit.js

@@ -0,0 +1,16 @@
+const { createLogger } = require('pinomin');
+
+const logger = createLogger('runAndExit');
+
+async function runAndExit(promise) {
+  try {
+    await promise;
+    logger.info('Success');
+    process.exit();
+  } catch (err) {
+    logger.error({ err }, 'Processing failed');
+    process.exit(1);
+  }
+}
+
+module.exports = runAndExit;

packages/dbmodel/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "dbmodel",
+  "version": "5.0.0-alpha.1",
+  "homepage": "https://dbgate.org/",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate.git"
+  },
+  "description": "Deploy, load or build script from model of SQL database",
+  "author": "Jan Prochazka",
+  "license": "MIT",
+  "bin": {
+    "dbgate-serve": "./bin/dbmodel.js"
+  },
+  "keywords": [
+    "sql",
+    "dbgate",
+    "web"
+  ],
+  "scripts": {
+    "dbmodel": "node ./bin/dbmodel.js",
+    "dbmodel:load": "cross-env DEVMODE=1 node ./bin/dbmodel.js load testdata/db -e postgres@dbgate-plugin-postgres -s localhost -u postgres -p Pwd2020Db -d zradlo"
+  },
+  "dependencies": {
+    "commander": "^10.0.0",
+    "dbgate-api": "^5.0.0-alpha.1",
+    "dbgate-plugin-csv": "^5.0.0-alpha.1",
+    "dbgate-plugin-excel": "^5.0.0-alpha.1",
+    "dbgate-plugin-mongo": "^5.0.0-alpha.1",
+    "dbgate-plugin-mssql": "^5.0.0-alpha.1",
+    "dbgate-plugin-mysql": "^5.0.0-alpha.1",
+    "dbgate-plugin-postgres": "^5.0.0-alpha.1",
+    "dbgate-plugin-xml": "^5.0.0-alpha.1",
+    "dbgate-web": "^5.0.0-alpha.1",
+    "dotenv": "^16.0.0",
+    "pinomin": "^1.0.1"
+  }
+}

yarn.lock

@@ -2991,6 +2991,11 @@ combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6:
   dependencies:
     delayed-stream "~1.0.0"
 
+commander@^10.0.0:
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.0.tgz#71797971162cd3cf65f0b9d24eb28f8d303acdf1"
+  integrity sha512-zS5PnTI22FIRM6ylNW8G4Ap0IEOyk62fhLSD0+uHRT9McRCLGpkVNvao4bjimpK/GShynyQkFFxHhwMcETmduA==
+
 commander@^2.20.0:
   version "2.20.3"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"