diff --git a/plugins/dbgate-plugin-clickhouse/README.md b/plugins/dbgate-plugin-clickhouse/README.md new file mode 100644 index 000000000..bc14e2ae9 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/README.md @@ -0,0 +1,6 @@ +[![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier) +[![NPM version](https://img.shields.io/npm/v/dbgate-plugin-clickhouse.svg)](https://www.npmjs.com/package/dbgate-plugin-clickhouse) + +# dbgate-plugin-clickhouse + +Use DbGate for install of this plugin diff --git a/plugins/dbgate-plugin-clickhouse/icon.svg b/plugins/dbgate-plugin-clickhouse/icon.svg new file mode 100644 index 000000000..cfe0335e5 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/icon.svg @@ -0,0 +1,35 @@ + + + + + + + + + + + + + diff --git a/plugins/dbgate-plugin-clickhouse/package.json b/plugins/dbgate-plugin-clickhouse/package.json new file mode 100644 index 000000000..8d7c5e477 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/package.json @@ -0,0 +1,38 @@ +{ + "name": "dbgate-plugin-clickhouse", + "main": "dist/backend.js", + "version": "5.0.0-alpha.1", + "license": "GPL-3.0", + "author": "Jan Prochazka", + "description": "Clickhouse connector for DbGate", + "keywords": [ + "dbgate", + "dbgateplugin", + "clickhouse" + ], + "files": [ + "dist", + "icon.svg" + ], + "scripts": { + "build:frontend": "webpack --config webpack-frontend.config", + "build:frontend:watch": "webpack --watch --config webpack-frontend.config", + "build:backend": "webpack --config webpack-backend.config.js", + "build": "yarn build:frontend && yarn build:backend", + "plugin": "yarn build && yarn pack && dbgate-plugin dbgate-plugin-clickhouse", + "plugout": "dbgate-plugout dbgate-plugin-clickhouse", + "copydist": "yarn build && yarn pack && dbgate-copydist ../dist/dbgate-plugin-clickhouse", + "prepublishOnly": "yarn build" + }, + "devDependencies": { + "byline": "^5.0.0", + "dbgate-plugin-tools": "^1.0.8", + "dbgate-tools": "^5.0.0-alpha.1", + "json-stable-stringify": "^1.0.1", + "webpack": "^5.91.0", + "webpack-cli": "^5.1.4" + }, + "dependencies": { + "@clickhouse/client": "^1.5.0" + } +} diff --git a/plugins/dbgate-plugin-clickhouse/prettier.config.js b/plugins/dbgate-plugin-clickhouse/prettier.config.js new file mode 100644 index 000000000..406484074 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/prettier.config.js @@ -0,0 +1,8 @@ +module.exports = { + trailingComma: 'es5', + tabWidth: 2, + semi: true, + singleQuote: true, + arrowParen: 'avoid', + printWidth: 120, +}; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js new file mode 100644 index 000000000..1cdf75a0a --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js @@ -0,0 +1,37 @@ +const { DatabaseAnalyser } = require('dbgate-tools'); +const sql = require('./sql'); + +class Analyser extends DatabaseAnalyser { + constructor(connection, driver) { + super(connection, driver); + } + + createQuery(resFileName, typeFields, replacements = {}) { + let res = sql[resFileName]; + res = res.replace('#DATABASE#', this.pool._database_name); + return super.createQuery(res, typeFields, replacements); + } + + async _runAnalysis() { + this.feedback({ analysingMessage: 'Loading tables' }); + const tables = await this.analyserQuery('tables', ['tables']); + this.feedback({ analysingMessage: 'Loading columns' }); + const columns = await this.analyserQuery('columns', ['tables', 'views']); + + const res = { + tables: tables.rows.map((table) => ({ + ...table, + primaryKeyColumns: undefined, + sortingKeyColumns: undefined, + columns: columns.rows.filter((col) => col.pureName == table.pureName), + primaryKey: (table.primaryKeyColumns || '').split(',').map((columnName) => ({ columnName })), + sortingKey: (table.sortingKeyColumns || '').split(',').map((columnName) => ({ columnName })), + foreignKeys: [], + })), + }; + this.feedback({ analysingMessage: null }); + return res; + } +} + +module.exports = Analyser; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/driver.js b/plugins/dbgate-plugin-clickhouse/src/backend/driver.js new file mode 100644 index 000000000..7cdd20100 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/driver.js @@ -0,0 +1,146 @@ +const _ = require('lodash'); +const stream = require('stream'); +const driverBase = require('../frontend/driver'); +const Analyser = require('./Analyser'); +const { createClient } = require('@clickhouse/client'); + +/** @type {import('dbgate-types').EngineDriver} */ +const driver = { + ...driverBase, + analyserClass: Analyser, + // creating connection + async connect({ server, port, user, password, database, useDatabaseUrl, databaseUrl }) { + const client = createClient({ + url: databaseUrl, + username: user, + password: password, + database: database, + }); + + client._database_name = database; + return client; + }, + // called for retrieve data (eg. browse in data grid) and for update database + async query(client, query) { + const resultSet = await client.query({ + query, + format: 'JSONCompactEachRowWithNamesAndTypes', + }); + + const dataSet = await resultSet.json(); + + const columns = dataSet[0].map((columnName, i) => ({ + columnName, + dataType: dataSet[1][i], + })); + + return { + rows: dataSet.slice(2).map((row) => _.zipObject(dataSet[0], row)), + columns, + }; + }, + // called in query console + async stream(client, query, options) { + try { + const resultSet = await client.query({ + query, + format: 'JSONCompactEachRowWithNamesAndTypes', + }); + + let columnNames = null; + let dataTypes = null; + + const stream = resultSet.stream(); + + stream.on('data', (rows) => { + rows.forEach((row) => { + const json = row.json(); + if (!columnNames) { + columnNames = json; + return; + } + if (!dataTypes) { + dataTypes = json; + + const columns = columnNames.map((columnName, i) => ({ + columnName, + dataType: dataTypes[i], + })); + + options.recordset(columns); + return; + } + const data = _.zipObject(columnNames, json); + options.row(data); + }); + }); + + stream.on('end', () => { + options.done(); + }); + + stream.on('error', (err) => { + options.info({ + message: err.toString(), + time: new Date(), + severity: 'error', + }); + options.done(); + }); + } catch (err) { + const mLine = err.message.match(/\(line (\d+)\,/); + let line = undefined; + if (mLine) { + line = parseInt(mLine[1]) - 1; + } + + options.info({ + message: err.message, + time: new Date(), + severity: 'error', + line, + }); + options.done(); + } + }, + // called when exporting table or view + async readQuery(connection, sql, structure) { + const pass = new stream.PassThrough({ + objectMode: true, + highWaterMark: 100, + }); + + // pass.write(structure) + // pass.write(row1) + // pass.write(row2) + // pass.end() + + return pass; + }, + // called when importing into table or view + async writeTable(connection, name, options) { + return createBulkInsertStreamBase(this, stream, pool, name, options); + }, + // detect server version + async getVersion(client) { + const resultSet = await client.query({ + query: 'SELECT version() as version', + format: 'JSONEachRow', + }); + const dataset = await resultSet.json(); + return { version: dataset[0].version }; + }, + // list databases on server + async listDatabases(client) { + const resultSet = await client.query({ + query: `SELECT name + FROM system.databases + WHERE name NOT IN ('system', 'information_schema', 'information_schema_ro', 'INFORMATION_SCHEMA')`, + format: 'JSONEachRow', + }); + const dataset = await resultSet.json(); + return dataset; + }, +}; + +module.exports = driver; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/index.js b/plugins/dbgate-plugin-clickhouse/src/backend/index.js new file mode 100644 index 000000000..b930194a5 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/index.js @@ -0,0 +1,6 @@ +const driver = require('./driver'); + +module.exports = { + packageName: 'dbgate-plugin-clickhouse', + drivers: [driver], +}; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/sql/columns.js b/plugins/dbgate-plugin-clickhouse/src/backend/sql/columns.js new file mode 100644 index 000000000..7d9f3770a --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/sql/columns.js @@ -0,0 +1,12 @@ +module.exports = ` +select + columns.table as "pureName", + tables.uuid as "objectId", + columns.name as "columnName", + columns.type as "dataType", + columns.comment as "columnComment" +from system.columns +inner join system.tables on columns.table = tables.name and columns.database = tables.database +where columns.database='#DATABASE#' and tables.uuid =OBJECT_ID_CONDITION +order by toInt32(columns.position) +`; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/sql/index.js b/plugins/dbgate-plugin-clickhouse/src/backend/sql/index.js new file mode 100644 index 000000000..1d0b3f63e --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/sql/index.js @@ -0,0 +1,7 @@ +const columns = require('./columns'); +const tables = require('./tables'); + +module.exports = { + columns, + tables, +}; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/sql/tables.js b/plugins/dbgate-plugin-clickhouse/src/backend/sql/tables.js new file mode 100644 index 000000000..be79c3a54 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/sql/tables.js @@ -0,0 +1,6 @@ +module.exports = ` +select name as "pureName", metadata_modification_time as "contentHash", total_rows as "tableRowCount", uuid as "objectId", comment as "objectComment", +engine as "tableEngine", primary_key as "primaryKeyColumns", sorting_key as "sortingKeyColumns" +from system.tables +where database='#DATABASE#' and uuid =OBJECT_ID_CONDITION; +`; diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js b/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js new file mode 100644 index 000000000..afcc64731 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js @@ -0,0 +1,6 @@ +const { SqlDumper } = require('dbgate-tools'); + +class Dumper extends SqlDumper { +} + +module.exports = Dumper; diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js new file mode 100644 index 000000000..6c0ef39ea --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -0,0 +1,33 @@ +const { driverBase } = require('dbgate-tools'); +const Dumper = require('./Dumper'); +const { mysqlSplitterOptions } = require('dbgate-query-splitter/lib/options'); + +/** @type {import('dbgate-types').SqlDialect} */ +const dialect = { + limitSelect: true, + rangeSelect: true, + offsetFetchRangeSyntax: true, + stringEscapeChar: "'", + fallbackDataType: 'nvarchar(max)', + quoteIdentifier(s) { + return `[${s}]`; + }, +}; + +/** @type {import('dbgate-types').EngineDriver} */ +const driver = { + ...driverBase, + dumperClass: Dumper, + dialect, + engine: 'clickhouse@dbgate-plugin-clickhouse', + title: 'ClickHouse', + showConnectionField: (field, values) => { + return ['databaseUrl', 'defaultDatabase', 'singleDatabase', 'isReadOnly', 'user', 'password'].includes(field); + }, + getQuerySplitterOptions: (usage) => + usage == 'editor' + ? { ...mysqlSplitterOptions, ignoreComments: true, preventSingleLineSplit: true } + : mysqlSplitterOptions, +}; + +module.exports = driver; diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/index.js b/plugins/dbgate-plugin-clickhouse/src/frontend/index.js new file mode 100644 index 000000000..a880d9660 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/index.js @@ -0,0 +1,6 @@ +import driver from './driver'; + +export default { + packageName: 'dbgate-plugin-clickhouse', + drivers: [driver], +}; diff --git a/plugins/dbgate-plugin-clickhouse/webpack-backend.config.js b/plugins/dbgate-plugin-clickhouse/webpack-backend.config.js new file mode 100644 index 000000000..e75357dff --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/webpack-backend.config.js @@ -0,0 +1,23 @@ +var webpack = require('webpack'); +var path = require('path'); + +var config = { + context: __dirname + '/src/backend', + + entry: { + app: './index.js', + }, + target: 'node', + output: { + path: path.resolve(__dirname, 'dist'), + filename: 'backend.js', + libraryTarget: 'commonjs2', + }, + + // uncomment for disable minimalization + // optimization: { + // minimize: false, + // }, +}; + +module.exports = config; diff --git a/plugins/dbgate-plugin-clickhouse/webpack-frontend.config.js b/plugins/dbgate-plugin-clickhouse/webpack-frontend.config.js new file mode 100644 index 000000000..db07de291 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/webpack-frontend.config.js @@ -0,0 +1,24 @@ +var webpack = require("webpack"); +var path = require("path"); + +var config = { + context: __dirname + "/src/frontend", + + entry: { + app: "./index.js", + }, + target: "web", + output: { + path: path.resolve(__dirname, "dist"), + filename: "frontend.js", + libraryTarget: "var", + library: 'plugin', + }, + + // uncomment for disable minimalization + // optimization: { + // minimize: false, + // }, +}; + +module.exports = config; diff --git a/plugins/dbgate-plugin-mysql/src/backend/Analyser.js b/plugins/dbgate-plugin-mysql/src/backend/Analyser.js index 076f20ebe..118b55efc 100644 --- a/plugins/dbgate-plugin-mysql/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-mysql/src/backend/Analyser.js @@ -66,10 +66,10 @@ class Analyser extends DatabaseAnalyser { super(pool, driver, version); } - createQuery(resFileName, typeFields) { + createQuery(resFileName, typeFields, replacements = {}) { let res = sql[resFileName]; res = res.replace('#DATABASE#', this.pool._database_name); - return super.createQuery(res, typeFields); + return super.createQuery(res, typeFields, replacements); } getRequestedViewNames(allViewNames) { diff --git a/yarn.lock b/yarn.lock index 9d6bebfd2..cdd73c4fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -466,6 +466,18 @@ resolved "https://registry.yarnpkg.com/@changesets/types/-/types-0.4.0.tgz#3413badb2c3904357a36268cb9f8c7e0afc3a804" integrity sha512-TclHHKDVYQ8rJGZgVeWiF7c91yWzTTWdPagltgutelGu/Psup5PQlUq6svx7S8suj+jXcaE34yEEsfIvzXXB2Q== +"@clickhouse/client-common@1.5.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@clickhouse/client-common/-/client-common-1.5.0.tgz#fa621ee4fbdf8f4b44e5548fd5d9fe1e44b07e88" + integrity sha512-U3vDp+PDnNVEv6kia+Mq5ygnlMZzsYU+3TX+0da3XvL926jzYLMBlIvFUxe2+/5k47ySvnINRC/2QxVK7PC2/A== + +"@clickhouse/client@^1.5.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@clickhouse/client/-/client-1.5.0.tgz#ce6110a710396544a2435fe9ed8f61f20bd178b8" + integrity sha512-Udwyoec+AHHS1TiLxDiWiJWcm2BvhZEqGjmUnvzL54NyT8D8eh2mxn5RR/W5ie64JDnsKLeZFlPYKRRhZMhkxA== + dependencies: + "@clickhouse/client-common" "1.5.0" + "@cnakazawa/watch@^1.0.3": version "1.0.4" resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a"