introduced yarn workspace

packages/api/src/controllers/connections.js

@@ -0,0 +1,59 @@
+
+const path = require('path');
+const { fork } = require('child_process');
+const _ = require('lodash');
+const nedb = require('nedb-promises');
+
+const datadir = require('../utility/datadir');
+const socket = require('../utility/socket');
+
+module.exports = {
+  datastore: null,
+  opened: [],
+
+  async _init() {
+    const dir = await datadir();
+    // @ts-ignore
+    this.datastore = nedb.create(path.join(dir, 'connections.jsonl'));
+  },
+
+  list_meta: 'get',
+  async list() {
+    return this.datastore.find();
+  },
+
+  test_meta: {
+    method: 'post',
+    raw: true,
+  },
+  test(req, res) {
+    const subprocess = fork(`${__dirname}/../proc/connectProcess.js`);
+    subprocess.on('message', resp => res.json(resp));
+    subprocess.send(req.body);
+  },
+
+  save_meta: 'post',
+  async save(connection) {
+    let res;
+    if (connection._id) {
+      res = await this.datastore.update(_.pick(connection, '_id'), connection);
+    } else {
+      res = await this.datastore.insert(connection);
+    }
+    socket.emit('connection-list-changed');
+    return res;
+  },
+
+  delete_meta: 'post',
+  async delete(connection) {
+    const res = await this.datastore.remove(_.pick(connection, '_id'));
+    socket.emit('connection-list-changed');
+    return res;
+  },
+
+  get_meta: 'get',
+  async get({ conid }) {
+    const res = await this.datastore.find({ _id: conid });
+    return res[0];
+  },
+};

packages/api/src/controllers/databaseConnections.js

@@ -0,0 +1,69 @@
+const _ = require('lodash');
+const uuidv1 = require('uuid/v1');
+const connections = require('./connections');
+const socket = require('../utility/socket');
+const { fork } = require('child_process');
+const DatabaseAnalyser = require('@dbgate/engines/default/DatabaseAnalyser');
+
+module.exports = {
+  /** @type {import('dbgate').OpenedDatabaseConnection[]} */
+  opened: [],
+  requests: {},
+
+  handle_structure(conid, database, { structure }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.structure = structure;
+    conid;
+    socket.emit(`database-structure-changed-${conid}-${database}`);
+  },
+  handle_error(conid, database, props) {
+    const { error } = props;
+    console.log(`Error in database connection ${conid}, database ${database}: ${error}`);
+  },
+  handle_response(conid, database, { msgid, ...response }) {
+    const [resolve, reject] = this.requests[msgid];
+    resolve(response);
+    delete this.requests[msgid];
+  },
+
+  async ensureOpened(conid, database) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (existing) return existing;
+    const connection = await connections.get({ conid });
+    const subprocess = fork(`${__dirname}/../proc/databaseConnectionProcess.js`);
+    const newOpened = {
+      conid,
+      database,
+      subprocess,
+      structure: DatabaseAnalyser.createEmptyStructure(),
+      connection,
+    };
+    this.opened.push(newOpened);
+    // @ts-ignore
+    subprocess.on('message', ({ msgtype, ...message }) => {
+      this[`handle_${msgtype}`](conid, database, message);
+    });
+    subprocess.send({ msgtype: 'connect', ...connection, database });
+    return newOpened;
+  },
+
+  /** @param {import('dbgate').OpenedDatabaseConnection} conn */
+  async sendRequest(conn, message) {
+    const msgid = uuidv1();
+    const promise = new Promise((resolve, reject) => {
+      this.requests[msgid] = [resolve, reject];
+      conn.subprocess.send({ msgid, ...message });
+    });
+    return promise;
+  },
+
+  listObjects_meta: 'get',
+  async listObjects({ conid, database }) {
+    const opened = await this.ensureOpened(conid, database);
+    const { tables } = opened.structure;
+    return {
+      tables: _.sortBy(tables, x => `${x.schemaName}.${x.pureName}`),
+    }; // .map(fp.pick(['tableName', 'schemaName']));
+  },
+};

packages/api/src/controllers/serverConnections.js

@@ -0,0 +1,44 @@
+
+const connections = require('./connections');
+const socket = require('../utility/socket');
+const { fork } = require('child_process');
+
+module.exports = {
+  opened: [],
+
+  handle_databases(conid, { databases }) {
+    const existing = this.opened.find(x => x.conid == conid);
+    if (!existing) return;
+    existing.databases = databases;
+    socket.emit(`database-list-changed-${conid}`);
+  },
+  handle_error(conid, { error }) {
+    console.log(`Error in server connection ${conid}: ${error}`);
+  },
+
+  async ensureOpened(conid) {
+    const existing = this.opened.find(x => x.conid == conid);
+    if (existing) return existing;
+    const connection = await connections.get({ conid });
+    const subprocess = fork(`${__dirname}/../proc/serverConnectionProcess.js`);
+    const newOpened = {
+      conid,
+      subprocess,
+      databases: [],
+      connection,
+    };
+    this.opened.push(newOpened);
+    // @ts-ignore
+    subprocess.on('message', ({ msgtype, ...message }) => {
+      this[`handle_${msgtype}`](conid, message);
+    });
+    subprocess.send({ msgtype: 'connect', ...connection });
+    return newOpened;
+  },
+
+  listDatabases_meta: 'get',
+  async listDatabases({ conid }) {
+    const opened = await this.ensureOpened(conid);
+    return opened.databases;
+  },
+};

packages/api/src/controllers/tables.js

@@ -0,0 +1,22 @@
+const _ = require('lodash');
+const databaseConnections = require('./databaseConnections');
+
+module.exports = {
+  tableData_meta: 'get',
+  async tableData({ conid, database, schemaName, pureName }) {
+    const opened = await databaseConnections.ensureOpened(conid, database);
+    const res = await databaseConnections.sendRequest(opened, { msgtype: 'tableData', schemaName, pureName });
+    return res;
+  },
+
+  tableInfo_meta: 'get',
+  async tableInfo({ conid, database, schemaName, pureName }) {
+    const opened = await databaseConnections.ensureOpened(conid, database);
+    const table = opened.structure.tables.find(x => x.pureName == pureName && x.schemaName == schemaName);
+    const allForeignKeys = _.flatten(opened.structure.tables.map(x => x.foreignKeys));
+    return {
+      ...table,
+      dependencies: allForeignKeys.filter(x => x.refSchemaName == schemaName && x.refTableName == pureName),
+    };
+  },
+};

packages/api/src/dmlf/command.js

@@ -0,0 +1,13 @@
+class Command {
+  /** @param driver {import('dbgate').EngineDriver}  */
+  toSql(driver) {
+    const dumper = driver.createDumper();
+    this.dumpSql(dumper);
+    return dumper.s;
+  }
+
+  /** @param dumper {import('dbgate').SqlDumper}  */
+  dumpSql(dumper) {}
+}
+
+module.exports = Command;

packages/api/src/dmlf/select.js

@@ -0,0 +1,37 @@
+const Command = require('./command');
+
+class Select extends Command {
+  constructor() {
+    super();
+    /** @type {number} */
+    this.topRecords = undefined;
+    /** @type {import('dbgate').NamedObjectInfo} */
+    this.from = undefined;
+    /** @type {import('dbgate').RangeDefinition} */
+    this.range = undefined;
+    this.distinct = false;
+    this.selectAll = false;
+  }
+
+  /** @param dumper {import('dbgate').SqlDumper}  */
+  dumpSql(dumper) {
+    dumper.put('^select ');
+    if (this.topRecords) {
+      dumper.put('^top %s ', this.topRecords);
+    }
+    if (this.distinct) {
+      dumper.put('^distinct ');
+    }
+    if (this.selectAll) {
+      dumper.put('* ');
+    } else {
+      // TODO
+    }
+    dumper.put('^from %f ', this.from);
+    if (this.range) {
+      dumper.put('^limit %s ^offset %s ', this.range.limit, this.range.offset);
+    }
+  }
+}
+
+module.exports = Select;

packages/api/src/index.js

@@ -0,0 +1,31 @@
+const express = require('express');
+const bodyParser = require('body-parser');
+const http = require('http');
+const cors = require('cors');
+const io = require('socket.io');
+
+const useController = require('./utility/useController');
+const connections = require('./controllers/connections');
+const serverConnections = require('./controllers/serverConnections');
+const databaseConnections = require('./controllers/databaseConnections');
+const tables = require('./controllers/tables');
+const socket = require('./utility/socket');
+
+const app = express();
+
+const server = http.createServer(app);
+socket.set(io(server));
+
+app.use(cors());
+app.use(bodyParser.json());
+
+app.get('/', (req, res) => {
+  res.send('DbGate API');
+});
+
+useController(app, '/connections', connections);
+useController(app, '/server-connections', serverConnections);
+useController(app, '/database-connections', databaseConnections);
+useController(app, '/tables', tables);
+
+server.listen(3000);

packages/api/src/proc/connectProcess.js

@@ -0,0 +1,14 @@
+const engines = require('@dbgate/engines');
+const driverConnect = require('../utility/driverConnect')
+
+process.on('message', async connection => {
+  try {
+    const driver = engines(connection);
+    const conn = await driverConnect(driver, connection);
+    const res = await driver.getVersion(conn);
+    process.send(res);
+  } catch (e) {
+    console.log(e);
+    process.send({ msgtype: 'error', error: e.message });
+  }
+});

packages/api/src/proc/databaseConnectionProcess.js

@@ -0,0 +1,66 @@
+const engines = require('@dbgate/engines');
+const Select = require('../dmlf/select');
+const driverConnect = require('../utility/driverConnect')
+
+let systemConnection;
+let storedConnection;
+let afterConnectCallbacks = [];
+
+async function handleFullRefresh() {
+  const driver = engines(storedConnection);
+  const structure = await driver.analyseFull(systemConnection);
+  process.send({ msgtype: 'structure', structure });
+}
+
+async function handleConnect(connection) {
+  storedConnection = connection;
+
+  const driver = engines(storedConnection);
+  systemConnection = await driverConnect(driver, storedConnection);
+  handleFullRefresh();
+  setInterval(handleFullRefresh, 30 * 1000);
+  for (const [resolve, reject] of afterConnectCallbacks) {
+    resolve();
+  }
+  afterConnectCallbacks = [];
+}
+
+function waitConnected() {
+  if (systemConnection) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    afterConnectCallbacks.push([resolve, reject]);
+  });
+}
+
+async function handleTableData({ msgid, schemaName, pureName }) {
+  await waitConnected();
+  const driver = engines(storedConnection);
+
+  const select = new Select();
+  if (driver.dialect.limitSelect) select.topRecords = 100;
+  if (driver.dialect.rangeSelect) select.range = { offset: 0, limit: 100 };
+  select.from = { schemaName, pureName };
+  select.selectAll = true;
+  const sql = select.toSql(driver);
+  const res = await driver.query(systemConnection, sql);
+
+  process.send({ msgtype: 'response', msgid, ...res });
+}
+
+const messageHandlers = {
+  connect: handleConnect,
+  tableData: handleTableData,
+};
+
+async function handleMessage({ msgtype, ...other }) {
+  const handler = messageHandlers[msgtype];
+  await handler(other);
+}
+
+process.on('message', async message => {
+  try {
+    await handleMessage(message);
+  } catch (e) {
+    process.send({ msgtype: 'error', error: e.message });
+  }
+});

packages/api/src/proc/serverConnectionProcess.js

@@ -0,0 +1,37 @@
+const engines = require('@dbgate/engines');
+const driverConnect = require('../utility/driverConnect')
+
+let systemConnection;
+let storedConnection;
+
+async function handleRefreshDatabases() {
+  const driver = engines(storedConnection);
+  const databases = await driver.listDatabases(systemConnection);
+  process.send({ msgtype: 'databases', databases });
+}
+
+async function handleConnect(connection) {
+  storedConnection = connection;
+
+  const driver = engines(storedConnection);
+  systemConnection = await driverConnect(driver, storedConnection);
+  handleRefreshDatabases();
+  setInterval(handleRefreshDatabases, 30 * 1000);
+}
+
+const messageHandlers = {
+  connect: handleConnect,
+};
+
+async function handleMessage({ msgtype, ...other }) {
+  const handler = messageHandlers[msgtype];
+  await handler(other);
+}
+
+process.on('message', async message => {
+  try {
+    await handleMessage(message);
+  } catch (e) {
+    process.send({ msgtype: 'error', error: e.message });
+  }
+});

packages/api/src/utility/datadir.js

@@ -0,0 +1,18 @@
+const os = require('os');
+const path = require('path');
+const fs = require('fs-extra');
+
+let created = false;
+
+module.exports = async function datadir() {
+  const dir = path.join(os.homedir(), 'dbgate-data');
+  if (!created) {
+    const stat = await fs.stat(dir);
+    if (!stat.isDirectory) {
+      await fs.mkdir(dir);
+    }
+    created = true;
+  }
+
+  return dir;
+};

packages/api/src/utility/driverConnect.js

@@ -0,0 +1,14 @@
+const mssql = require('mssql');
+const mysql = require('mysql');
+const pg = require('pg');
+
+function driverConnect(driver, connection) {
+  const driverModules = {
+    mssql,
+    mysql,
+    pg,
+  };
+  return driver.connect(driverModules, connection);
+}
+
+module.exports = driverConnect;

packages/api/src/utility/socket.js

@@ -0,0 +1,13 @@
+let socket = null;
+
+module.exports = {
+  set(value) {
+    socket = value;
+  },
+  get() {
+    return socket;
+  },
+  emit(message, data) {
+    socket.emit(message, data);
+  },
+};

packages/api/src/utility/useController.js

@@ -0,0 +1,52 @@
+const _ = require('lodash');
+const express = require('express');
+
+/**
+ * @param {string} route
+ */
+module.exports = function useController(app, route, controller) {
+  const router = express.Router();
+
+  for (const key of _.keys(controller)) {
+    const obj = controller[key];
+    if (!_.isFunction(obj)) continue;
+    const meta = controller[`${key}_meta`];
+    if (!meta) continue;
+
+    let method = 'get';
+    let raw = false;
+    let rawParams = false;
+
+    if (_.isString(meta)) {
+      method = meta;
+    }
+    if (_.isPlainObject(meta)) {
+      method = meta.method;
+      raw = meta.raw;
+      rawParams = meta.rawParams;
+    }
+
+    const route = `/${_.kebabCase(key)}`;
+    if (raw) {
+      router[method](route, controller[key]);
+    } else {
+      router[method](route, async (req, res) => {
+        if (controller._init && !controller._init_called) {
+          await controller._init();
+          controller._init_called = true;
+        }
+        try {
+          let params = [{ ...req.body, ...req.query }];
+          if (rawParams) params = [req, res];
+          const data = await controller[key](...params);
+          res.json(data);
+        } catch (e) {
+          console.log(e);
+          res.status(500).json({ error: e.message });
+        }
+      });
+    }
+  }
+
+  app.use(route, router);
+};