clickhouse plugin - initial import

2026-04-18 00:56:02 +00:00 · 2024-09-10 14:29:51 +02:00
parent d75f533b76
commit f2d29f97dc
17 changed files with 407 additions and 2 deletions
--- a/plugins/dbgate-plugin-clickhouse/README.md
+++ b/plugins/dbgate-plugin-clickhouse/README.md
@@ -0,0 +1,6 @@
+[![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
+[![NPM version](https://img.shields.io/npm/v/dbgate-plugin-clickhouse.svg)](https://www.npmjs.com/package/dbgate-plugin-clickhouse)
+
+# dbgate-plugin-clickhouse
+
+Use DbGate for install of this plugin
--- a/plugins/dbgate-plugin-clickhouse/icon.svg
+++ b/plugins/dbgate-plugin-clickhouse/icon.svg
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 384 384" style="enable-background:new 0 0 384 384;" xml:space="preserve">
+<polygon style="fill:#EFEEEE;" points="64,0 64,384 288,384 384,288 384,0 "/>
+<polygon style="fill:#ABABAB;" points="288,288 288,384 384,288 "/>
+<polygon style="fill:#DEDEDD;" points="192,384 288,384 288,288 "/>
+<path style="fill:#448E47;" d="M0,96v112h256V96L0,96L0,96z"/>
+<g>
+	<path style="fill:#FFFFFF;" d="M64.32,130.112c-1.184-2.288-3.344-3.424-6.48-3.424c-1.728,0-3.152,0.464-4.272,1.408
+		c-1.12,0.928-2,2.416-2.64,4.496s-1.088,4.8-1.344,8.176c-0.272,3.36-0.384,7.472-0.384,12.336c0,5.184,0.176,9.376,0.528,12.576
+		c0.336,3.2,0.896,5.664,1.632,7.44s1.664,2.96,2.784,3.552c1.12,0.608,2.416,0.928,3.888,0.928c1.216,0,2.352-0.208,3.408-0.624
+		s1.968-1.248,2.736-2.496c0.784-1.248,1.392-3.008,1.824-5.28c0.448-2.272,0.672-5.264,0.672-8.976H80.48
+		c0,3.696-0.288,7.232-0.864,10.56s-1.664,6.24-3.216,8.736c-1.584,2.48-3.776,4.432-6.624,5.84
+		c-2.848,1.408-6.544,2.128-11.088,2.128c-5.168,0-9.312-0.848-12.368-2.496c-3.072-1.664-5.424-4.064-7.056-7.2
+		s-2.688-6.88-3.168-11.232c-0.464-4.336-0.72-9.152-0.72-14.384c0-5.184,0.256-9.968,0.72-14.352
+		c0.48-4.368,1.552-8.144,3.168-11.28c1.648-3.12,3.984-5.584,7.056-7.344c3.056-1.744,7.2-2.64,12.368-2.64
+		c4.944,0,8.816,0.8,11.664,2.4c2.848,1.6,4.976,3.632,6.368,6.096s2.304,5.12,2.64,7.968c0.352,2.848,0.528,5.52,0.528,8.016H66.08
+		C66.08,136,65.488,132.368,64.32,130.112z"/>
+	<path style="fill:#FFFFFF;" d="M109.072,167.008c0,1.6,0.144,3.056,0.384,4.352c0.272,1.312,0.736,2.416,1.44,3.312
+		c0.704,0.912,1.664,1.616,2.848,2.128c1.168,0.496,2.672,0.768,4.448,0.768c2.128,0,4.016-0.688,5.712-2.064
+		c1.68-1.376,2.544-3.52,2.544-6.384c0-1.536-0.224-2.864-0.624-3.984c-0.416-1.12-1.104-2.128-2.064-3.008
+		c-0.976-0.912-2.24-1.712-3.792-2.448s-3.504-1.488-5.808-2.256c-3.056-1.024-5.712-2.16-7.968-3.376
+		c-2.24-1.2-4.112-2.624-5.616-4.272c-1.504-1.632-2.608-3.52-3.312-5.664c-0.704-2.16-1.056-4.624-1.056-7.456
+		c0-6.784,1.888-11.824,5.664-15.152c3.76-3.328,8.96-4.992,15.552-4.992c3.072,0,5.904,0.336,8.496,1.008s4.832,1.744,6.72,3.264
+		c1.888,1.504,3.36,3.424,4.416,5.744c1.04,2.336,1.584,5.136,1.584,8.4v1.92h-13.232c0-3.264-0.576-5.776-1.712-7.552
+		c-1.152-1.744-3.072-2.64-5.76-2.64c-1.536,0-2.816,0.24-3.84,0.672c-1.008,0.448-1.84,1.04-2.448,1.776s-1.04,1.616-1.264,2.576
+		c-0.24,0.96-0.336,1.952-0.336,2.976c0,2.128,0.448,3.888,1.344,5.328c0.896,1.456,2.816,2.784,5.76,3.984l10.656,4.608
+		c2.624,1.152,4.768,2.352,6.416,3.616c1.664,1.248,3.008,2.592,3.984,4.032c0.992,1.44,1.68,3.008,2.064,4.752
+		c0.384,1.712,0.576,3.648,0.576,5.744c0,7.232-2.096,12.496-6.288,15.792c-4.192,3.296-10.032,4.96-17.52,4.96
+		c-7.808,0-13.392-1.696-16.768-5.088c-3.36-3.392-5.024-8.256-5.024-14.592v-2.784h13.824L109.072,167.008L109.072,167.008z"/>
+	<path style="fill:#FFFFFF;" d="M177.344,168.544h0.304l10.176-50.688h14.32L186.4,186.4h-17.76l-15.728-68.544h14.784
+		L177.344,168.544z"/>
+</g>
+</svg>
--- a/plugins/dbgate-plugin-clickhouse/package.json
+++ b/plugins/dbgate-plugin-clickhouse/package.json
@@ -0,0 +1,38 @@
+{
+  "name": "dbgate-plugin-clickhouse",
+  "main": "dist/backend.js",
+  "version": "5.0.0-alpha.1",
+  "license": "GPL-3.0",
+  "author": "Jan Prochazka",
+  "description": "Clickhouse connector for DbGate",
+  "keywords": [
+    "dbgate",
+    "dbgateplugin",
+    "clickhouse"
+  ],
+  "files": [
+    "dist",
+    "icon.svg"
+  ],
+  "scripts": {
+    "build:frontend": "webpack --config webpack-frontend.config",
+    "build:frontend:watch": "webpack  --watch --config webpack-frontend.config",
+    "build:backend": "webpack --config webpack-backend.config.js",
+    "build": "yarn build:frontend && yarn build:backend",
+    "plugin": "yarn build && yarn pack && dbgate-plugin dbgate-plugin-clickhouse",
+    "plugout": "dbgate-plugout dbgate-plugin-clickhouse",
+    "copydist": "yarn build && yarn pack && dbgate-copydist ../dist/dbgate-plugin-clickhouse",
+    "prepublishOnly": "yarn build"
+  },
+  "devDependencies": {
+    "byline": "^5.0.0",
+    "dbgate-plugin-tools": "^1.0.8",
+    "dbgate-tools": "^5.0.0-alpha.1",
+    "json-stable-stringify": "^1.0.1",
+    "webpack": "^5.91.0",
+    "webpack-cli": "^5.1.4"
+  },
+  "dependencies": {
+    "@clickhouse/client": "^1.5.0"
+  }
+}
--- a/plugins/dbgate-plugin-clickhouse/prettier.config.js
+++ b/plugins/dbgate-plugin-clickhouse/prettier.config.js
@@ -0,0 +1,8 @@
+module.exports = {
+  trailingComma: 'es5',
+  tabWidth: 2,
+  semi: true,
+  singleQuote: true,
+  arrowParen: 'avoid',
+  printWidth: 120,
+};
--- a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js
+++ b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js
@@ -0,0 +1,37 @@
+const { DatabaseAnalyser } = require('dbgate-tools');
+const sql = require('./sql');
+
+class Analyser extends DatabaseAnalyser {
+  constructor(connection, driver) {
+    super(connection, driver);
+  }
+
+  createQuery(resFileName, typeFields, replacements = {}) {
+    let res = sql[resFileName];
+    res = res.replace('#DATABASE#', this.pool._database_name);
+    return super.createQuery(res, typeFields, replacements);
+  }
+
+  async _runAnalysis() {
+    this.feedback({ analysingMessage: 'Loading tables' });
+    const tables = await this.analyserQuery('tables', ['tables']);
+    this.feedback({ analysingMessage: 'Loading columns' });
+    const columns = await this.analyserQuery('columns', ['tables', 'views']);
+
+    const res = {
+      tables: tables.rows.map((table) => ({
+        ...table,
+        primaryKeyColumns: undefined,
+        sortingKeyColumns: undefined,
+        columns: columns.rows.filter((col) => col.pureName == table.pureName),
+        primaryKey: (table.primaryKeyColumns || '').split(',').map((columnName) => ({ columnName })),
+        sortingKey: (table.sortingKeyColumns || '').split(',').map((columnName) => ({ columnName })),
+        foreignKeys: [],
+      })),
+    };
+    this.feedback({ analysingMessage: null });
+    return res;
+  }
+}
+
+module.exports = Analyser;
--- a/plugins/dbgate-plugin-clickhouse/src/backend/driver.js
+++ b/plugins/dbgate-plugin-clickhouse/src/backend/driver.js
@@ -0,0 +1,146 @@
+const _ = require('lodash');
+const stream = require('stream');
+const driverBase = require('../frontend/driver');
+const Analyser = require('./Analyser');
+const { createClient } = require('@clickhouse/client');
+
+/** @type {import('dbgate-types').EngineDriver} */
+const driver = {
+  ...driverBase,
+  analyserClass: Analyser,
+  // creating connection
+  async connect({ server, port, user, password, database, useDatabaseUrl, databaseUrl }) {
+    const client = createClient({
+      url: databaseUrl,
+      username: user,
+      password: password,
+      database: database,
+    });
+
+    client._database_name = database;
+    return client;
+  },
+  // called for retrieve data (eg. browse in data grid) and for update database
+  async query(client, query) {
+    const resultSet = await client.query({
+      query,
+      format: 'JSONCompactEachRowWithNamesAndTypes',
+    });
+
+    const dataSet = await resultSet.json();
+
+    const columns = dataSet[0].map((columnName, i) => ({
+      columnName,
+      dataType: dataSet[1][i],
+    }));
+
+    return {
+      rows: dataSet.slice(2).map((row) => _.zipObject(dataSet[0], row)),
+      columns,
+    };
+  },
+  // called in query console
+  async stream(client, query, options) {
+    try {
+      const resultSet = await client.query({
+        query,
+        format: 'JSONCompactEachRowWithNamesAndTypes',
+      });
+
+      let columnNames = null;
+      let dataTypes = null;
+
+      const stream = resultSet.stream();
+
+      stream.on('data', (rows) => {
+        rows.forEach((row) => {
+          const json = row.json();
+          if (!columnNames) {
+            columnNames = json;
+            return;
+          }
+          if (!dataTypes) {
+            dataTypes = json;
+
+            const columns = columnNames.map((columnName, i) => ({
+              columnName,
+              dataType: dataTypes[i],
+            }));
+
+            options.recordset(columns);
+            return;
+          }
+          const data = _.zipObject(columnNames, json);
+          options.row(data);
+        });
+      });
+
+      stream.on('end', () => {
+        options.done();
+      });
+
+      stream.on('error', (err) => {
+        options.info({
+          message: err.toString(),
+          time: new Date(),
+          severity: 'error',
+        });
+        options.done();
+      });
+    } catch (err) {
+      const mLine = err.message.match(/\(line (\d+)\,/);
+      let line = undefined;
+      if (mLine) {
+        line = parseInt(mLine[1]) - 1;
+      }
+
+      options.info({
+        message: err.message,
+        time: new Date(),
+        severity: 'error',
+        line,
+      });
+      options.done();
+    }
+  },
+  // called when exporting table or view
+  async readQuery(connection, sql, structure) {
+    const pass = new stream.PassThrough({
+      objectMode: true,
+      highWaterMark: 100,
+    });
+
+    // pass.write(structure)
+    // pass.write(row1)
+    // pass.write(row2)
+    // pass.end()
+
+    return pass;
+  },
+  // called when importing into table or view
+  async writeTable(connection, name, options) {
+    return createBulkInsertStreamBase(this, stream, pool, name, options);
+  },
+  // detect server version
+  async getVersion(client) {
+    const resultSet = await client.query({
+      query: 'SELECT version() as version',
+      format: 'JSONEachRow',
+    });
+    const dataset = await resultSet.json();
+    return { version: dataset[0].version };
+  },
+  // list databases on server
+  async listDatabases(client) {
+    const resultSet = await client.query({
+      query: `SELECT name
+              FROM system.databases
+              WHERE name NOT IN ('system', 'information_schema', 'information_schema_ro', 'INFORMATION_SCHEMA')`,
+      format: 'JSONEachRow',
+    });
+    const dataset = await resultSet.json();
+    return dataset;
+  },
+};
+
+module.exports = driver;
--- a/plugins/dbgate-plugin-clickhouse/src/backend/index.js
+++ b/plugins/dbgate-plugin-clickhouse/src/backend/index.js
@@ -0,0 +1,6 @@
+const driver = require('./driver');
+
+module.exports = {
+  packageName: 'dbgate-plugin-clickhouse',
+  drivers: [driver],
+};
--- a/plugins/dbgate-plugin-clickhouse/src/backend/sql/columns.js
+++ b/plugins/dbgate-plugin-clickhouse/src/backend/sql/columns.js
@@ -0,0 +1,12 @@
+module.exports = `
+select 
+    columns.table as "pureName",
+    tables.uuid as "objectId",
+    columns.name as "columnName",
+    columns.type as "dataType",
+    columns.comment as "columnComment"
+from system.columns 
+inner join system.tables on columns.table = tables.name and columns.database = tables.database
+where columns.database='#DATABASE#' and tables.uuid =OBJECT_ID_CONDITION
+order by toInt32(columns.position)
+`;
--- a/plugins/dbgate-plugin-clickhouse/src/backend/sql/index.js
+++ b/plugins/dbgate-plugin-clickhouse/src/backend/sql/index.js
@@ -0,0 +1,7 @@
+const columns = require('./columns');
+const tables = require('./tables');
+
+module.exports = {
+  columns,
+  tables,
+};
--- a/plugins/dbgate-plugin-clickhouse/src/backend/sql/tables.js
+++ b/plugins/dbgate-plugin-clickhouse/src/backend/sql/tables.js
@@ -0,0 +1,6 @@
+module.exports = `
+select name as "pureName", metadata_modification_time as "contentHash", total_rows as "tableRowCount", uuid as "objectId", comment as "objectComment",
+engine as "tableEngine", primary_key as "primaryKeyColumns", sorting_key as "sortingKeyColumns" 
+from system.tables 
+where database='#DATABASE#' and uuid =OBJECT_ID_CONDITION;
+`;
--- a/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js
+++ b/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js
@@ -0,0 +1,6 @@
+const { SqlDumper } = require('dbgate-tools');
+
+class Dumper extends SqlDumper {
+}
+
+module.exports = Dumper;
--- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js
+++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js
@@ -0,0 +1,33 @@
+const { driverBase } = require('dbgate-tools');
+const Dumper = require('./Dumper');
+const { mysqlSplitterOptions } = require('dbgate-query-splitter/lib/options');
+
+/** @type {import('dbgate-types').SqlDialect} */
+const dialect = {
+  limitSelect: true,
+  rangeSelect: true,
+  offsetFetchRangeSyntax: true,
+  stringEscapeChar: "'",
+  fallbackDataType: 'nvarchar(max)',
+  quoteIdentifier(s) {
+    return `[${s}]`;
+  },
+};
+
+/** @type {import('dbgate-types').EngineDriver} */
+const driver = {
+  ...driverBase,
+  dumperClass: Dumper,
+  dialect,
+  engine: 'clickhouse@dbgate-plugin-clickhouse',
+  title: 'ClickHouse',
+  showConnectionField: (field, values) => {
+    return ['databaseUrl', 'defaultDatabase', 'singleDatabase', 'isReadOnly', 'user', 'password'].includes(field);
+  },
+  getQuerySplitterOptions: (usage) =>
+    usage == 'editor'
+      ? { ...mysqlSplitterOptions, ignoreComments: true, preventSingleLineSplit: true }
+      : mysqlSplitterOptions,
+};
+
+module.exports = driver;
--- a/plugins/dbgate-plugin-clickhouse/src/frontend/index.js
+++ b/plugins/dbgate-plugin-clickhouse/src/frontend/index.js
@@ -0,0 +1,6 @@
+import driver from './driver';
+
+export default {
+  packageName: 'dbgate-plugin-clickhouse',
+  drivers: [driver],
+};
--- a/plugins/dbgate-plugin-clickhouse/webpack-backend.config.js
+++ b/plugins/dbgate-plugin-clickhouse/webpack-backend.config.js
@@ -0,0 +1,23 @@
+var webpack = require('webpack');
+var path = require('path');
+
+var config = {
+  context: __dirname + '/src/backend',
+
+  entry: {
+    app: './index.js',
+  },
+  target: 'node',
+  output: {
+    path: path.resolve(__dirname, 'dist'),
+    filename: 'backend.js',
+    libraryTarget: 'commonjs2',
+  },
+
+  // uncomment for disable minimalization
+  //   optimization: {
+  //     minimize: false,
+  //   },
+};
+
+module.exports = config;
--- a/plugins/dbgate-plugin-clickhouse/webpack-frontend.config.js
+++ b/plugins/dbgate-plugin-clickhouse/webpack-frontend.config.js
@@ -0,0 +1,24 @@
+var webpack = require("webpack");
+var path = require("path");
+
+var config = {
+  context: __dirname + "/src/frontend",
+
+  entry: {
+    app: "./index.js",
+  },
+  target: "web",
+  output: {
+    path: path.resolve(__dirname, "dist"),
+    filename: "frontend.js",
+    libraryTarget: "var",
+    library: 'plugin',
+  },
+
+  // uncomment for disable minimalization
+  // optimization: {
+  //   minimize: false,
+  // },
+};
+
+module.exports = config;
--- a/plugins/dbgate-plugin-mysql/src/backend/Analyser.js
+++ b/plugins/dbgate-plugin-mysql/src/backend/Analyser.js
@@ -66,10 +66,10 @@ class Analyser extends DatabaseAnalyser {
    super(pool, driver, version);
  }

-  createQuery(resFileName, typeFields) {
+  createQuery(resFileName, typeFields, replacements = {}) {
    let res = sql[resFileName];
    res = res.replace('#DATABASE#', this.pool._database_name);
-    return super.createQuery(res, typeFields);
+    return super.createQuery(res, typeFields, replacements);
  }

  getRequestedViewNames(allViewNames) {
--- a/yarn.lock
+++ b/yarn.lock
@@ -466,6 +466,18 @@
  resolved "https://registry.yarnpkg.com/@changesets/types/-/types-0.4.0.tgz#3413badb2c3904357a36268cb9f8c7e0afc3a804"
  integrity sha512-TclHHKDVYQ8rJGZgVeWiF7c91yWzTTWdPagltgutelGu/Psup5PQlUq6svx7S8suj+jXcaE34yEEsfIvzXXB2Q==

+"@clickhouse/client-common@1.5.0":
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/@clickhouse/client-common/-/client-common-1.5.0.tgz#fa621ee4fbdf8f4b44e5548fd5d9fe1e44b07e88"
+  integrity sha512-U3vDp+PDnNVEv6kia+Mq5ygnlMZzsYU+3TX+0da3XvL926jzYLMBlIvFUxe2+/5k47ySvnINRC/2QxVK7PC2/A==
+
+"@clickhouse/client@^1.5.0":
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/@clickhouse/client/-/client-1.5.0.tgz#ce6110a710396544a2435fe9ed8f61f20bd178b8"
+  integrity sha512-Udwyoec+AHHS1TiLxDiWiJWcm2BvhZEqGjmUnvzL54NyT8D8eh2mxn5RR/W5ie64JDnsKLeZFlPYKRRhZMhkxA==
+  dependencies:
+    "@clickhouse/client-common" "1.5.0"
+
 "@cnakazawa/watch@^1.0.3":
  version "1.0.4"
  resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a"