added plugins

plugins/dbgate-plugin-mysql/src/backend/Analyser.js

@@ -0,0 +1,222 @@
+const fp = require('lodash/fp');
+const _ = require('lodash');
+const sql = require('./sql');
+
+const { DatabaseAnalyser } = require('dbgate-tools');
+const { isTypeString, isTypeNumeric } = require('dbgate-tools');
+const { rangeStep } = require('lodash/fp');
+
+function getColumnInfo({
+  isNullable,
+  extra,
+  columnName,
+  dataType,
+  charMaxLength,
+  numericPrecision,
+  numericScale,
+  defaultValue,
+}) {
+  let fullDataType = dataType;
+  if (charMaxLength && isTypeString(dataType)) fullDataType = `${dataType}(${charMaxLength})`;
+  if (numericPrecision && numericScale && isTypeNumeric(dataType))
+    fullDataType = `${dataType}(${numericPrecision},${numericScale})`;
+  return {
+    notNull: !isNullable || isNullable == 'NO' || isNullable == 'no',
+    autoIncrement: extra && extra.toLowerCase().includes('auto_increment'),
+    columnName,
+    dataType: fullDataType,
+    defaultValue,
+  };
+}
+
+function objectTypeToField(type) {
+  if (type == 'VIEW') return 'views';
+  if (type == 'BASE TABLE') return 'tables';
+  return null;
+}
+
+class Analyser extends DatabaseAnalyser {
+  constructor(pool, driver) {
+    super(pool, driver);
+  }
+
+  createQuery(resFileName, typeFields) {
+    let res = sql[resFileName];
+    if (this.singleObjectFilter) {
+      const { typeField, pureName } = this.singleObjectFilter;
+      if (!typeFields || !typeFields.includes(typeField)) return null;
+      res = res.replace('=[OBJECT_NAME_CONDITION]', ` = '${pureName}'`).replace('#DATABASE#', this.pool._database_name);
+      return res;
+    }
+    if (!this.modifications || !typeFields || this.modifications.length == 0) {
+      res = res.replace('=[OBJECT_NAME_CONDITION]', ' is not null');
+    } else {
+      const filterNames = this.modifications
+        .filter(x => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change'))
+        .map(x => x.newName && x.newName.pureName)
+        .filter(Boolean);
+      if (filterNames.length == 0) {
+        res = res.replace('=[OBJECT_NAME_CONDITION]', ' IS NULL');
+      } else {
+        res = res.replace('=[OBJECT_NAME_CONDITION]', ` in (${filterNames.map(x => `'${x}'`).join(',')})`);
+      }
+    }
+    res = res.replace('#DATABASE#', this.pool._database_name);
+    return res;
+  }
+
+  getRequestedViewNames(allViewNames) {
+    if (this.singleObjectFilter) {
+      const { typeField, pureName } = this.singleObjectFilter;
+      if (typeField == 'views') return [pureName];
+    }
+    if (this.modifications) {
+      return this.modifications.filter(x => x.objectTypeField == 'views').map(x => x.newName.pureName);
+    }
+    return allViewNames;
+  }
+
+  async getViewTexts(allViewNames) {
+    const res = {};
+    for (const viewName of this.getRequestedViewNames(allViewNames)) {
+      const resp = await this.driver.query(this.pool, `SHOW CREATE VIEW \`${viewName}\``);
+      res[viewName] = resp.rows[0]['Create View'];
+    }
+    return res;
+  }
+
+  async _runAnalysis() {
+    const tables = await this.driver.query(this.pool, this.createQuery('tables', ['tables']));
+    const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables', 'views']));
+    const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables']));
+    const fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables']));
+    const views = await this.driver.query(this.pool, this.createQuery('views', ['views']));
+    const programmables = await this.driver.query(
+      this.pool,
+      this.createQuery('programmables', ['procedures', 'functions'])
+    );
+
+    const viewTexts = await this.getViewTexts(views.rows.map(x => x.pureName));
+
+    return this.mergeAnalyseResult({
+      tables: tables.rows.map(table => ({
+        ...table,
+        objectId: table.pureName,
+        columns: columns.rows.filter(col => col.pureName == table.pureName).map(getColumnInfo),
+        primaryKey: DatabaseAnalyser.extractPrimaryKeys(table, pkColumns.rows),
+        foreignKeys: DatabaseAnalyser.extractForeignKeys(table, fkColumns.rows),
+      })),
+      views: views.rows.map(view => ({
+        ...view,
+        objectId: view.pureName,
+        columns: columns.rows.filter(col => col.pureName == view.pureName).map(getColumnInfo),
+        createSql: viewTexts[view.pureName],
+        requiresFormat: true,
+      })),
+      procedures: programmables.rows
+        .filter(x => x.objectType == 'PROCEDURE')
+        .map(fp.omit(['objectType']))
+        .map(x => ({ ...x, objectId: x.pureName })),
+      functions: programmables.rows
+        .filter(x => x.objectType == 'FUNCTION')
+        .map(fp.omit(['objectType']))
+        .map(x => ({ ...x, objectId: x.pureName })),
+    });
+  }
+
+  getDeletedObjectsForField(nameArray, objectTypeField) {
+    return this.structure[objectTypeField]
+      .filter(x => !nameArray.includes(x.pureName))
+      .map(x => ({
+        oldName: _.pick(x, ['pureName']),
+        action: 'remove',
+        objectTypeField,
+        objectId: x.pureName,
+      }));
+  }
+
+  getDeletedObjects(nameArray) {
+    return [
+      ...this.getDeletedObjectsForField(nameArray, 'tables'),
+      ...this.getDeletedObjectsForField(nameArray, 'views'),
+      ...this.getDeletedObjectsForField(nameArray, 'procedures'),
+      ...this.getDeletedObjectsForField(nameArray, 'functions'),
+      ...this.getDeletedObjectsForField(nameArray, 'triggers'),
+    ];
+  }
+
+  async getModifications() {
+    const tableModificationsQueryData = await this.driver.query(this.pool, this.createQuery('tableModifications'));
+    const procedureModificationsQueryData = await this.driver.query(
+      this.pool,
+      this.createQuery('procedureModifications')
+    );
+    const functionModificationsQueryData = await this.driver.query(
+      this.pool,
+      this.createQuery('functionModifications')
+    );
+
+    const allModifications = _.compact([
+      ...tableModificationsQueryData.rows.map(x => {
+        if (x.objectType == 'BASE TABLE') return { ...x, objectTypeField: 'tables' };
+        if (x.objectType == 'VIEW') return { ...x, objectTypeField: 'views' };
+        return null;
+      }),
+      ...procedureModificationsQueryData.rows.map(x => ({
+        objectTypeField: 'procedures',
+        modifyDate: x.Modified,
+        pureName: x.Name,
+      })),
+      ...functionModificationsQueryData.rows.map(x => ({
+        objectTypeField: 'functions',
+        modifyDate: x.Modified,
+        pureName: x.Name,
+      })),
+    ]);
+
+    // console.log('allModifications', allModifications);
+    // console.log(
+    //   'DATES',
+    //   this.structure.procedures.map((x) => x.modifyDate)
+    // );
+    // console.log('MOD - SRC', modifications);
+    // console.log(
+    //   'MODs',
+    //   this.structure.tables.map((x) => x.modifyDate)
+    // );
+    const modifications = allModifications.map(x => {
+      const { objectType, modifyDate, pureName } = x;
+      const field = objectTypeToField(objectType);
+
+      if (!field || !this.structure[field]) return null;
+      // @ts-ignore
+      const obj = this.structure[field].find(x => x.pureName == pureName);
+
+      // object not modified
+      if (obj && Math.abs(new Date(modifyDate).getTime() - new Date(obj.modifyDate).getTime()) < 1000) return null;
+
+      // console.log('MODIFICATION OF ', field, pureName, modifyDate, obj.modifyDate);
+
+      /** @type {import('dbgate-types').DatabaseModification} */
+      const action = obj
+        ? {
+            newName: { pureName },
+            oldName: _.pick(obj, ['pureName']),
+            action: 'change',
+            objectTypeField: field,
+            objectId: pureName,
+          }
+        : {
+            newName: { pureName },
+            action: 'add',
+            objectTypeField: field,
+            objectId: pureName,
+          };
+      return action;
+    });
+
+    return [..._.compact(modifications), ...this.getDeletedObjects([...allModifications.map(x => x.pureName)])];
+  }
+}
+
+module.exports = Analyser;

plugins/dbgate-plugin-mysql/src/backend/driver.js

@@ -0,0 +1,186 @@
+const _ = require('lodash');
+const stream = require('stream');
+const driverBase = require('../frontend/driver');
+const Analyser = require('./Analyser');
+const mysql2 = require('mysql2');
+const { createBulkInsertStreamBase, makeUniqueColumnNames } = require('dbgate-tools');
+const mysqlSplitter = require('@verycrazydog/mysql-parser');
+
+function extractColumns(fields) {
+  if (fields) {
+    const res = fields.map(col => ({
+      columnName: col.name,
+    }));
+    makeUniqueColumnNames(res);
+    return res;
+  }
+  return null;
+}
+
+function zipDataRow(rowArray, columns) {
+  return _.zipObject(
+    columns.map(x => x.columnName),
+    rowArray
+  );
+}
+
+async function runQueryItem(connection, sql) {
+  return new Promise((resolve, reject) => {
+    connection.query(sql, function (error, results, fields) {
+      if (error) reject(error);
+      const columns = extractColumns(fields);
+      resolve({ rows: results && columns && results.map && results.map(row => zipDataRow(row, columns)), columns });
+    });
+  });
+}
+
+async function runStreamItem(connection, sql, options) {
+  return new Promise((resolve, reject) => {
+    const query = connection.query(sql);
+    let columns = [];
+
+    // const handleInfo = (info) => {
+    //   const { message, lineNumber, procName } = info;
+    //   options.info({
+    //     message,
+    //     line: lineNumber,
+    //     procedure: procName,
+    //     time: new Date(),
+    //     severity: 'info',
+    //   });
+    // };
+
+    const handleEnd = () => {
+      resolve();
+    };
+
+    const handleRow = row => {
+      if (row && row.constructor && row.constructor.name == 'OkPacket') {
+        options.info({
+          message: `${row.affectedRows} rows affected`,
+          time: new Date(),
+          severity: 'info',
+        });
+      } else {
+        if (columns) {
+          options.row(zipDataRow(row, columns));
+        }
+      }
+    };
+
+    const handleFields = fields => {
+      columns = extractColumns(fields);
+      if (columns) options.recordset(columns);
+    };
+
+    const handleError = error => {
+      console.log('ERROR', error);
+      const { message, lineNumber, procName } = error;
+      options.info({
+        message,
+        line: lineNumber,
+        procedure: procName,
+        time: new Date(),
+        severity: 'error',
+      });
+    };
+
+    query.on('error', handleError).on('fields', handleFields).on('result', handleRow).on('end', handleEnd);
+  });
+}
+
+/** @type {import('dbgate-types').EngineDriver} */
+const driver = {
+  ...driverBase,
+  analyserClass: Analyser,
+
+  async connect({ server, port, user, password, database, ssl }) {
+    const connection = mysql2.createConnection({
+      host: server,
+      port,
+      user,
+      password,
+      database,
+      ssl,
+      rowsAsArray: true,
+      supportBigNumbers: true,
+      bigNumberStrings: true,
+      // TODO: test following options
+      // multipleStatements: true,
+      // dateStrings: true,
+    });
+    connection._database_name = database;
+    return connection;
+  },
+  async query(connection, sql) {
+    if (sql == null) {
+      return {
+        rows: [],
+        columns: [],
+      };
+    }
+
+    const sqlSplitted = mysqlSplitter.split(sql);
+    let res = {
+      rows: [],
+      columns: [],
+    };
+    for (const sqlItem of sqlSplitted) {
+      const resultItem = await runQueryItem(connection, sqlItem);
+      if (resultItem.rows && resultItem.columns && resultItem.columns.length > 0) {
+        res = resultItem;
+      }
+    }
+    return res;
+  },
+  async stream(connection, sql, options) {
+    const sqlSplitted = mysqlSplitter.split(sql);
+
+    for (const sqlItem of sqlSplitted) {
+      await runStreamItem(connection, sqlItem, options);
+    }
+
+    options.done();
+  },
+  async readQuery(connection, sql, structure) {
+    const query = connection.query(sql);
+
+    const pass = new stream.PassThrough({
+      objectMode: true,
+      highWaterMark: 100,
+    });
+
+    let columns = [];
+    query
+      .on('error', err => {
+        console.error(err);
+        pass.end();
+      })
+      .on('fields', fields => {
+        columns = extractColumns(fields);
+        pass.write({
+          __isStreamHeader: true,
+          ...(structure || { columns }),
+        });
+      })
+      .on('result', row => pass.write(zipDataRow(row, columns)))
+      .on('end', () => pass.end());
+
+    return pass;
+  },
+  async getVersion(connection) {
+    const { rows } = await this.query(connection, "show variables like 'version'");
+    const version = rows[0].Value;
+    return { version };
+  },
+  async listDatabases(connection) {
+    const { rows } = await this.query(connection, 'show databases');
+    return rows.map(x => ({ name: x.Database }));
+  },
+  async writeTable(pool, name, options) {
+    // @ts-ignore
+    return createBulkInsertStreamBase(this, stream, pool, name, options);
+  },
+};
+
+module.exports = driver;

plugins/dbgate-plugin-mysql/src/backend/index.js

@@ -0,0 +1,6 @@
+const driver = require('./driver');
+
+module.exports = {
+  packageName: 'dbgate-plugin-mysql',
+  driver,
+};

plugins/dbgate-plugin-mysql/src/backend/sql/columns.js

@@ -0,0 +1,15 @@
+module.exports = `
+select 
+	TABLE_NAME as pureName, 
+	COLUMN_NAME as columnName,
+	IS_NULLABLE as isNullable,
+	DATA_TYPE as dataType,
+	CHARACTER_MAXIMUM_LENGTH as charMaxLength,
+	NUMERIC_PRECISION as numericPrecision,
+	NUMERIC_SCALE as numericScale,
+	COLUMN_DEFAULT as defaultValue,
+	EXTRA as extra
+from INFORMATION_SCHEMA.COLUMNS
+where TABLE_SCHEMA = '#DATABASE#' and TABLE_NAME =[OBJECT_NAME_CONDITION]
+order by ORDINAL_POSITION
+`;

plugins/dbgate-plugin-mysql/src/backend/sql/foreignKeys.js

@@ -0,0 +1,17 @@
+module.exports = `
+select 
+	REFERENTIAL_CONSTRAINTS.CONSTRAINT_NAME as constraintName,
+	REFERENTIAL_CONSTRAINTS.TABLE_NAME as pureName,
+	REFERENTIAL_CONSTRAINTS.UPDATE_RULE as updateAction,
+	REFERENTIAL_CONSTRAINTS.DELETE_RULE as deleteAction,
+	REFERENTIAL_CONSTRAINTS.REFERENCED_TABLE_NAME as refTableName,
+	KEY_COLUMN_USAGE.COLUMN_NAME as columnName,
+	KEY_COLUMN_USAGE.REFERENCED_COLUMN_NAME as refColumnName
+from INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS
+inner join INFORMATION_SCHEMA.KEY_COLUMN_USAGE 
+	on REFERENTIAL_CONSTRAINTS.TABLE_NAME = KEY_COLUMN_USAGE.TABLE_NAME 
+	and REFERENTIAL_CONSTRAINTS.CONSTRAINT_NAME = KEY_COLUMN_USAGE.CONSTRAINT_NAME
+	and REFERENTIAL_CONSTRAINTS.CONSTRAINT_SCHEMA = KEY_COLUMN_USAGE.CONSTRAINT_SCHEMA
+where REFERENTIAL_CONSTRAINTS.CONSTRAINT_SCHEMA = '#DATABASE#' and REFERENTIAL_CONSTRAINTS.TABLE_NAME =[OBJECT_NAME_CONDITION]
+order by KEY_COLUMN_USAGE.ORDINAL_POSITION
+`;

plugins/dbgate-plugin-mysql/src/backend/sql/functionModifications.js

@@ -0,0 +1,3 @@
+module.exports = `
+SHOW FUNCTION STATUS WHERE Db = '#DATABASE#'
+`;

plugins/dbgate-plugin-mysql/src/backend/sql/index.js

@@ -0,0 +1,21 @@
+const columns = require('./columns');
+const tables = require('./tables');
+const primaryKeys = require('./primaryKeys');
+const foreignKeys = require('./foreignKeys');
+const tableModifications = require('./tableModifications');
+const views = require('./views');
+const programmables = require('./programmables');
+const procedureModifications = require('./procedureModifications');
+const functionModifications = require('./functionModifications');
+
+module.exports = {
+  columns,
+  tables,
+  primaryKeys,
+  foreignKeys,
+  tableModifications,
+  views,
+  programmables,
+  procedureModifications,
+  functionModifications,
+};

plugins/dbgate-plugin-mysql/src/backend/sql/primaryKeys.js

@@ -0,0 +1,12 @@
+module.exports = `select 
+	TABLE_CONSTRAINTS.CONSTRAINT_NAME as constraintName,
+	TABLE_CONSTRAINTS.TABLE_NAME as pureName,
+	KEY_COLUMN_USAGE.COLUMN_NAME as columnName
+from INFORMATION_SCHEMA.TABLE_CONSTRAINTS
+inner join INFORMATION_SCHEMA.KEY_COLUMN_USAGE 
+	on TABLE_CONSTRAINTS.TABLE_NAME = KEY_COLUMN_USAGE.TABLE_NAME 
+		and TABLE_CONSTRAINTS.CONSTRAINT_NAME = KEY_COLUMN_USAGE.CONSTRAINT_NAME
+		and TABLE_CONSTRAINTS.CONSTRAINT_SCHEMA = KEY_COLUMN_USAGE.CONSTRAINT_SCHEMA
+where TABLE_CONSTRAINTS.CONSTRAINT_SCHEMA = '#DATABASE#' and TABLE_CONSTRAINTS.TABLE_NAME =[OBJECT_NAME_CONDITION] AND TABLE_CONSTRAINTS.CONSTRAINT_TYPE = 'PRIMARY KEY'
+order by KEY_COLUMN_USAGE.ORDINAL_POSITION
+`;

plugins/dbgate-plugin-mysql/src/backend/sql/procedureModifications.js

@@ -0,0 +1,3 @@
+module.exports = `
+SHOW PROCEDURE STATUS WHERE Db = '#DATABASE#'
+`;

plugins/dbgate-plugin-mysql/src/backend/sql/programmables.js

@@ -0,0 +1,9 @@
+module.exports = `
+select 
+    ROUTINE_NAME as pureName,
+    ROUTINE_TYPE as objectType,
+    COALESCE(LAST_ALTERED, CREATED) as modifyDate,
+    ROUTINE_DEFINITION as createSql
+from information_schema.routines
+where ROUTINE_SCHEMA = '#DATABASE#' and ROUTINE_NAME =[OBJECT_NAME_CONDITION]
+`;

plugins/dbgate-plugin-mysql/src/backend/sql/tableModifications.js

@@ -0,0 +1,8 @@
+module.exports = `
+select 
+	TABLE_NAME as pureName, 
+	TABLE_TYPE as objectType,
+	case when ENGINE='InnoDB' then CREATE_TIME else coalesce(UPDATE_TIME, CREATE_TIME) end as modifyDate 
+from information_schema.tables 
+where TABLE_SCHEMA = '#DATABASE#'
+`;

plugins/dbgate-plugin-mysql/src/backend/sql/tables.js

@@ -0,0 +1,7 @@
+module.exports = `
+select 
+	TABLE_NAME as pureName, 
+	case when ENGINE='InnoDB' then CREATE_TIME else coalesce(UPDATE_TIME, CREATE_TIME) end as modifyDate 
+from information_schema.tables 
+where TABLE_SCHEMA = '#DATABASE#' and TABLE_TYPE='BASE TABLE' and TABLE_NAME =[OBJECT_NAME_CONDITION];
+`;

plugins/dbgate-plugin-mysql/src/backend/sql/views.js

@@ -0,0 +1,7 @@
+module.exports = `
+select 
+	TABLE_NAME as pureName, 
+    coalesce(UPDATE_TIME, CREATE_TIME) as modifyDate
+from information_schema.tables 
+where TABLE_SCHEMA = '#DATABASE#' and TABLE_NAME =[OBJECT_NAME_CONDITION] and TABLE_TYPE = 'VIEW';
+`;

plugins/dbgate-plugin-mysql/src/frontend/Dumper.js

@@ -0,0 +1,68 @@
+const { SqlDumper } = require('dbgate-tools');
+
+class Dumper extends SqlDumper {
+  /** @param type {import('dbgate-types').TransformType} */
+  transform(type, dumpExpr) {
+    switch (type) {
+      case 'GROUP:YEAR':
+      case 'YEAR':
+        this.put('^year(%c)', dumpExpr);
+        break;
+      case 'MONTH':
+        this.put('^month(%c)', dumpExpr);
+        break;
+      case 'DAY':
+        this.put('^day(%c)', dumpExpr);
+        break;
+      case 'GROUP:MONTH':
+        this.put("^date_format(%c, '%s')", dumpExpr, '%Y-%m');
+        break;
+      case 'GROUP:DAY':
+        this.put("^date_format(%c, '%s')", dumpExpr, '%Y-%m-%d');
+        break;
+      default:
+        dumpExpr();
+        break;
+    }
+  }
+
+  renameTable(obj, newName) {
+    this.putCmd('^rename ^table %f ^to %i', obj, newName);
+  }
+
+  changeColumn(oldcol, newcol, constraints) {
+    this.put('^alter ^table %f ^change ^column %i %i ', oldcol, oldcol.columnName, newcol.columnName);
+    this.columnDefinition(newcol, true, true, true);
+    this.inlineConstraints(constraints);
+    this.endCommand();
+  }
+
+  renameColumn(column, newcol) {
+    this.changeColumn(
+      column,
+      {
+        ...column,
+        columnName: newcol,
+      },
+      []
+    );
+  }
+
+  enableConstraints(table, enabled) {
+    this.putCmd('^set FOREIGN_KEY_CHECKS = %s', enabled ? '1' : '0');
+  }
+
+  comment(value) {
+    this.put('/* %s */', value);
+  }
+
+  beginTransaction() {
+    this.putCmd('^start ^transaction');
+  }
+
+  selectTableIntoNewTable(sourceName, targetName) {
+    this.putCmd('^create ^table %f (^select * ^from %f)', targetName, sourceName);
+  }
+}
+
+module.exports = Dumper;

plugins/dbgate-plugin-mysql/src/frontend/driver.js

@@ -0,0 +1,27 @@
+const { driverBase } = require('dbgate-tools');
+const Dumper = require('./Dumper');
+
+/** @type {import('dbgate-types').SqlDialect} */
+const dialect = {
+  rangeSelect: true,
+  stringEscapeChar: '\\',
+  fallbackDataType: 'longtext',
+  enableConstraintsPerTable: false,
+  anonymousPrimaryKey: true,
+  explicitDropConstraint: true,
+  quoteIdentifier(s) {
+    return '`' + s + '`';
+  },
+};
+
+/** @type {import('dbgate-types').EngineDriver} */
+const driver = {
+  ...driverBase,
+  dumperClass: Dumper,
+  dialect,
+  engine: 'mysql@dbgate-plugin-mysql',
+  title: 'MySQL / MariaDB',
+  defaultPort: 3306,
+};
+
+module.exports = driver;

plugins/dbgate-plugin-mysql/src/frontend/index.js

@@ -0,0 +1,6 @@
+import driver from './driver';
+
+export default {
+  packageName: 'dbgate-plugin-mysql',
+  driver,
+};