clickhouse plugin - initial import

plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js

@@ -0,0 +1,37 @@
+const { DatabaseAnalyser } = require('dbgate-tools');
+const sql = require('./sql');
+
+class Analyser extends DatabaseAnalyser {
+  constructor(connection, driver) {
+    super(connection, driver);
+  }
+
+  createQuery(resFileName, typeFields, replacements = {}) {
+    let res = sql[resFileName];
+    res = res.replace('#DATABASE#', this.pool._database_name);
+    return super.createQuery(res, typeFields, replacements);
+  }
+
+  async _runAnalysis() {
+    this.feedback({ analysingMessage: 'Loading tables' });
+    const tables = await this.analyserQuery('tables', ['tables']);
+    this.feedback({ analysingMessage: 'Loading columns' });
+    const columns = await this.analyserQuery('columns', ['tables', 'views']);
+
+    const res = {
+      tables: tables.rows.map((table) => ({
+        ...table,
+        primaryKeyColumns: undefined,
+        sortingKeyColumns: undefined,
+        columns: columns.rows.filter((col) => col.pureName == table.pureName),
+        primaryKey: (table.primaryKeyColumns || '').split(',').map((columnName) => ({ columnName })),
+        sortingKey: (table.sortingKeyColumns || '').split(',').map((columnName) => ({ columnName })),
+        foreignKeys: [],
+      })),
+    };
+    this.feedback({ analysingMessage: null });
+    return res;
+  }
+}
+
+module.exports = Analyser;

plugins/dbgate-plugin-clickhouse/src/backend/driver.js

@@ -0,0 +1,146 @@
+const _ = require('lodash');
+const stream = require('stream');
+const driverBase = require('../frontend/driver');
+const Analyser = require('./Analyser');
+const { createClient } = require('@clickhouse/client');
+
+/** @type {import('dbgate-types').EngineDriver} */
+const driver = {
+  ...driverBase,
+  analyserClass: Analyser,
+  // creating connection
+  async connect({ server, port, user, password, database, useDatabaseUrl, databaseUrl }) {
+    const client = createClient({
+      url: databaseUrl,
+      username: user,
+      password: password,
+      database: database,
+    });
+
+    client._database_name = database;
+    return client;
+  },
+  // called for retrieve data (eg. browse in data grid) and for update database
+  async query(client, query) {
+    const resultSet = await client.query({
+      query,
+      format: 'JSONCompactEachRowWithNamesAndTypes',
+    });
+
+    const dataSet = await resultSet.json();
+
+    const columns = dataSet[0].map((columnName, i) => ({
+      columnName,
+      dataType: dataSet[1][i],
+    }));
+
+    return {
+      rows: dataSet.slice(2).map((row) => _.zipObject(dataSet[0], row)),
+      columns,
+    };
+  },
+  // called in query console
+  async stream(client, query, options) {
+    try {
+      const resultSet = await client.query({
+        query,
+        format: 'JSONCompactEachRowWithNamesAndTypes',
+      });
+
+      let columnNames = null;
+      let dataTypes = null;
+
+      const stream = resultSet.stream();
+
+      stream.on('data', (rows) => {
+        rows.forEach((row) => {
+          const json = row.json();
+          if (!columnNames) {
+            columnNames = json;
+            return;
+          }
+          if (!dataTypes) {
+            dataTypes = json;
+
+            const columns = columnNames.map((columnName, i) => ({
+              columnName,
+              dataType: dataTypes[i],
+            }));
+
+            options.recordset(columns);
+            return;
+          }
+          const data = _.zipObject(columnNames, json);
+          options.row(data);
+        });
+      });
+
+      stream.on('end', () => {
+        options.done();
+      });
+
+      stream.on('error', (err) => {
+        options.info({
+          message: err.toString(),
+          time: new Date(),
+          severity: 'error',
+        });
+        options.done();
+      });
+    } catch (err) {
+      const mLine = err.message.match(/\(line (\d+)\,/);
+      let line = undefined;
+      if (mLine) {
+        line = parseInt(mLine[1]) - 1;
+      }
+
+      options.info({
+        message: err.message,
+        time: new Date(),
+        severity: 'error',
+        line,
+      });
+      options.done();
+    }
+  },
+  // called when exporting table or view
+  async readQuery(connection, sql, structure) {
+    const pass = new stream.PassThrough({
+      objectMode: true,
+      highWaterMark: 100,
+    });
+
+    // pass.write(structure)
+    // pass.write(row1)
+    // pass.write(row2)
+    // pass.end()
+
+    return pass;
+  },
+  // called when importing into table or view
+  async writeTable(connection, name, options) {
+    return createBulkInsertStreamBase(this, stream, pool, name, options);
+  },
+  // detect server version
+  async getVersion(client) {
+    const resultSet = await client.query({
+      query: 'SELECT version() as version',
+      format: 'JSONEachRow',
+    });
+    const dataset = await resultSet.json();
+    return { version: dataset[0].version };
+  },
+  // list databases on server
+  async listDatabases(client) {
+    const resultSet = await client.query({
+      query: `SELECT name
+              FROM system.databases
+              WHERE name NOT IN ('system', 'information_schema', 'information_schema_ro', 'INFORMATION_SCHEMA')`,
+      format: 'JSONEachRow',
+    });
+    const dataset = await resultSet.json();
+    return dataset;
+  },
+};
+
+module.exports = driver;

plugins/dbgate-plugin-clickhouse/src/backend/index.js

@@ -0,0 +1,6 @@
+const driver = require('./driver');
+
+module.exports = {
+  packageName: 'dbgate-plugin-clickhouse',
+  drivers: [driver],
+};

plugins/dbgate-plugin-clickhouse/src/backend/sql/columns.js

@@ -0,0 +1,12 @@
+module.exports = `
+select 
+    columns.table as "pureName",
+    tables.uuid as "objectId",
+    columns.name as "columnName",
+    columns.type as "dataType",
+    columns.comment as "columnComment"
+from system.columns 
+inner join system.tables on columns.table = tables.name and columns.database = tables.database
+where columns.database='#DATABASE#' and tables.uuid =OBJECT_ID_CONDITION
+order by toInt32(columns.position)
+`;

plugins/dbgate-plugin-clickhouse/src/backend/sql/index.js

@@ -0,0 +1,7 @@
+const columns = require('./columns');
+const tables = require('./tables');
+
+module.exports = {
+  columns,
+  tables,
+};

plugins/dbgate-plugin-clickhouse/src/backend/sql/tables.js

@@ -0,0 +1,6 @@
+module.exports = `
+select name as "pureName", metadata_modification_time as "contentHash", total_rows as "tableRowCount", uuid as "objectId", comment as "objectComment",
+engine as "tableEngine", primary_key as "primaryKeyColumns", sorting_key as "sortingKeyColumns" 
+from system.tables 
+where database='#DATABASE#' and uuid =OBJECT_ID_CONDITION;
+`;

plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js

@@ -0,0 +1,6 @@
+const { SqlDumper } = require('dbgate-tools');
+
+class Dumper extends SqlDumper {
+}
+
+module.exports = Dumper;

plugins/dbgate-plugin-clickhouse/src/frontend/driver.js

@@ -0,0 +1,33 @@
+const { driverBase } = require('dbgate-tools');
+const Dumper = require('./Dumper');
+const { mysqlSplitterOptions } = require('dbgate-query-splitter/lib/options');
+
+/** @type {import('dbgate-types').SqlDialect} */
+const dialect = {
+  limitSelect: true,
+  rangeSelect: true,
+  offsetFetchRangeSyntax: true,
+  stringEscapeChar: "'",
+  fallbackDataType: 'nvarchar(max)',
+  quoteIdentifier(s) {
+    return `[${s}]`;
+  },
+};
+
+/** @type {import('dbgate-types').EngineDriver} */
+const driver = {
+  ...driverBase,
+  dumperClass: Dumper,
+  dialect,
+  engine: 'clickhouse@dbgate-plugin-clickhouse',
+  title: 'ClickHouse',
+  showConnectionField: (field, values) => {
+    return ['databaseUrl', 'defaultDatabase', 'singleDatabase', 'isReadOnly', 'user', 'password'].includes(field);
+  },
+  getQuerySplitterOptions: (usage) =>
+    usage == 'editor'
+      ? { ...mysqlSplitterOptions, ignoreComments: true, preventSingleLineSplit: true }
+      : mysqlSplitterOptions,
+};
+
+module.exports = driver;

plugins/dbgate-plugin-clickhouse/src/frontend/index.js

@@ -0,0 +1,6 @@
+import driver from './driver';
+
+export default {
+  packageName: 'dbgate-plugin-clickhouse',
+  drivers: [driver],
+};