From f2d29f97dcc1bde71ef5e1be24a441446b21d3f2 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 10 Sep 2024 14:29:51 +0200 Subject: [PATCH 01/57] clickhouse plugin - initial import --- plugins/dbgate-plugin-clickhouse/README.md | 6 + plugins/dbgate-plugin-clickhouse/icon.svg | 35 +++++ plugins/dbgate-plugin-clickhouse/package.json | 38 +++++ .../prettier.config.js | 8 + .../src/backend/Analyser.js | 37 +++++ .../src/backend/driver.js | 146 ++++++++++++++++++ .../src/backend/index.js | 6 + .../src/backend/sql/columns.js | 12 ++ .../src/backend/sql/index.js | 7 + .../src/backend/sql/tables.js | 6 + .../src/frontend/Dumper.js | 6 + .../src/frontend/driver.js | 33 ++++ .../src/frontend/index.js | 6 + .../webpack-backend.config.js | 23 +++ .../webpack-frontend.config.js | 24 +++ .../src/backend/Analyser.js | 4 +- yarn.lock | 12 ++ 17 files changed, 407 insertions(+), 2 deletions(-) create mode 100644 plugins/dbgate-plugin-clickhouse/README.md create mode 100644 plugins/dbgate-plugin-clickhouse/icon.svg create mode 100644 plugins/dbgate-plugin-clickhouse/package.json create mode 100644 plugins/dbgate-plugin-clickhouse/prettier.config.js create mode 100644 plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js create mode 100644 plugins/dbgate-plugin-clickhouse/src/backend/driver.js create mode 100644 plugins/dbgate-plugin-clickhouse/src/backend/index.js create mode 100644 plugins/dbgate-plugin-clickhouse/src/backend/sql/columns.js create mode 100644 plugins/dbgate-plugin-clickhouse/src/backend/sql/index.js create mode 100644 plugins/dbgate-plugin-clickhouse/src/backend/sql/tables.js create mode 100644 plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js create mode 100644 plugins/dbgate-plugin-clickhouse/src/frontend/driver.js create mode 100644 plugins/dbgate-plugin-clickhouse/src/frontend/index.js create mode 100644 plugins/dbgate-plugin-clickhouse/webpack-backend.config.js create mode 100644 plugins/dbgate-plugin-clickhouse/webpack-frontend.config.js diff --git a/plugins/dbgate-plugin-clickhouse/README.md b/plugins/dbgate-plugin-clickhouse/README.md new file mode 100644 index 000000000..bc14e2ae9 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/README.md @@ -0,0 +1,6 @@ +[![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier) +[![NPM version](https://img.shields.io/npm/v/dbgate-plugin-clickhouse.svg)](https://www.npmjs.com/package/dbgate-plugin-clickhouse) + +# dbgate-plugin-clickhouse + +Use DbGate for install of this plugin diff --git a/plugins/dbgate-plugin-clickhouse/icon.svg b/plugins/dbgate-plugin-clickhouse/icon.svg new file mode 100644 index 000000000..cfe0335e5 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/icon.svg @@ -0,0 +1,35 @@ + + + + + + + + + + + + + diff --git a/plugins/dbgate-plugin-clickhouse/package.json b/plugins/dbgate-plugin-clickhouse/package.json new file mode 100644 index 000000000..8d7c5e477 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/package.json @@ -0,0 +1,38 @@ +{ + "name": "dbgate-plugin-clickhouse", + "main": "dist/backend.js", + "version": "5.0.0-alpha.1", + "license": "GPL-3.0", + "author": "Jan Prochazka", + "description": "Clickhouse connector for DbGate", + "keywords": [ + "dbgate", + "dbgateplugin", + "clickhouse" + ], + "files": [ + "dist", + "icon.svg" + ], + "scripts": { + "build:frontend": "webpack --config webpack-frontend.config", + "build:frontend:watch": "webpack --watch --config webpack-frontend.config", + "build:backend": "webpack --config webpack-backend.config.js", + "build": "yarn build:frontend && yarn build:backend", + "plugin": "yarn build && yarn pack && dbgate-plugin dbgate-plugin-clickhouse", + "plugout": "dbgate-plugout dbgate-plugin-clickhouse", + "copydist": "yarn build && yarn pack && dbgate-copydist ../dist/dbgate-plugin-clickhouse", + "prepublishOnly": "yarn build" + }, + "devDependencies": { + "byline": "^5.0.0", + "dbgate-plugin-tools": "^1.0.8", + "dbgate-tools": "^5.0.0-alpha.1", + "json-stable-stringify": "^1.0.1", + "webpack": "^5.91.0", + "webpack-cli": "^5.1.4" + }, + "dependencies": { + "@clickhouse/client": "^1.5.0" + } +} diff --git a/plugins/dbgate-plugin-clickhouse/prettier.config.js b/plugins/dbgate-plugin-clickhouse/prettier.config.js new file mode 100644 index 000000000..406484074 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/prettier.config.js @@ -0,0 +1,8 @@ +module.exports = { + trailingComma: 'es5', + tabWidth: 2, + semi: true, + singleQuote: true, + arrowParen: 'avoid', + printWidth: 120, +}; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js new file mode 100644 index 000000000..1cdf75a0a --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js @@ -0,0 +1,37 @@ +const { DatabaseAnalyser } = require('dbgate-tools'); +const sql = require('./sql'); + +class Analyser extends DatabaseAnalyser { + constructor(connection, driver) { + super(connection, driver); + } + + createQuery(resFileName, typeFields, replacements = {}) { + let res = sql[resFileName]; + res = res.replace('#DATABASE#', this.pool._database_name); + return super.createQuery(res, typeFields, replacements); + } + + async _runAnalysis() { + this.feedback({ analysingMessage: 'Loading tables' }); + const tables = await this.analyserQuery('tables', ['tables']); + this.feedback({ analysingMessage: 'Loading columns' }); + const columns = await this.analyserQuery('columns', ['tables', 'views']); + + const res = { + tables: tables.rows.map((table) => ({ + ...table, + primaryKeyColumns: undefined, + sortingKeyColumns: undefined, + columns: columns.rows.filter((col) => col.pureName == table.pureName), + primaryKey: (table.primaryKeyColumns || '').split(',').map((columnName) => ({ columnName })), + sortingKey: (table.sortingKeyColumns || '').split(',').map((columnName) => ({ columnName })), + foreignKeys: [], + })), + }; + this.feedback({ analysingMessage: null }); + return res; + } +} + +module.exports = Analyser; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/driver.js b/plugins/dbgate-plugin-clickhouse/src/backend/driver.js new file mode 100644 index 000000000..7cdd20100 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/driver.js @@ -0,0 +1,146 @@ +const _ = require('lodash'); +const stream = require('stream'); +const driverBase = require('../frontend/driver'); +const Analyser = require('./Analyser'); +const { createClient } = require('@clickhouse/client'); + +/** @type {import('dbgate-types').EngineDriver} */ +const driver = { + ...driverBase, + analyserClass: Analyser, + // creating connection + async connect({ server, port, user, password, database, useDatabaseUrl, databaseUrl }) { + const client = createClient({ + url: databaseUrl, + username: user, + password: password, + database: database, + }); + + client._database_name = database; + return client; + }, + // called for retrieve data (eg. browse in data grid) and for update database + async query(client, query) { + const resultSet = await client.query({ + query, + format: 'JSONCompactEachRowWithNamesAndTypes', + }); + + const dataSet = await resultSet.json(); + + const columns = dataSet[0].map((columnName, i) => ({ + columnName, + dataType: dataSet[1][i], + })); + + return { + rows: dataSet.slice(2).map((row) => _.zipObject(dataSet[0], row)), + columns, + }; + }, + // called in query console + async stream(client, query, options) { + try { + const resultSet = await client.query({ + query, + format: 'JSONCompactEachRowWithNamesAndTypes', + }); + + let columnNames = null; + let dataTypes = null; + + const stream = resultSet.stream(); + + stream.on('data', (rows) => { + rows.forEach((row) => { + const json = row.json(); + if (!columnNames) { + columnNames = json; + return; + } + if (!dataTypes) { + dataTypes = json; + + const columns = columnNames.map((columnName, i) => ({ + columnName, + dataType: dataTypes[i], + })); + + options.recordset(columns); + return; + } + const data = _.zipObject(columnNames, json); + options.row(data); + }); + }); + + stream.on('end', () => { + options.done(); + }); + + stream.on('error', (err) => { + options.info({ + message: err.toString(), + time: new Date(), + severity: 'error', + }); + options.done(); + }); + } catch (err) { + const mLine = err.message.match(/\(line (\d+)\,/); + let line = undefined; + if (mLine) { + line = parseInt(mLine[1]) - 1; + } + + options.info({ + message: err.message, + time: new Date(), + severity: 'error', + line, + }); + options.done(); + } + }, + // called when exporting table or view + async readQuery(connection, sql, structure) { + const pass = new stream.PassThrough({ + objectMode: true, + highWaterMark: 100, + }); + + // pass.write(structure) + // pass.write(row1) + // pass.write(row2) + // pass.end() + + return pass; + }, + // called when importing into table or view + async writeTable(connection, name, options) { + return createBulkInsertStreamBase(this, stream, pool, name, options); + }, + // detect server version + async getVersion(client) { + const resultSet = await client.query({ + query: 'SELECT version() as version', + format: 'JSONEachRow', + }); + const dataset = await resultSet.json(); + return { version: dataset[0].version }; + }, + // list databases on server + async listDatabases(client) { + const resultSet = await client.query({ + query: `SELECT name + FROM system.databases + WHERE name NOT IN ('system', 'information_schema', 'information_schema_ro', 'INFORMATION_SCHEMA')`, + format: 'JSONEachRow', + }); + const dataset = await resultSet.json(); + return dataset; + }, +}; + +module.exports = driver; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/index.js b/plugins/dbgate-plugin-clickhouse/src/backend/index.js new file mode 100644 index 000000000..b930194a5 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/index.js @@ -0,0 +1,6 @@ +const driver = require('./driver'); + +module.exports = { + packageName: 'dbgate-plugin-clickhouse', + drivers: [driver], +}; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/sql/columns.js b/plugins/dbgate-plugin-clickhouse/src/backend/sql/columns.js new file mode 100644 index 000000000..7d9f3770a --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/sql/columns.js @@ -0,0 +1,12 @@ +module.exports = ` +select + columns.table as "pureName", + tables.uuid as "objectId", + columns.name as "columnName", + columns.type as "dataType", + columns.comment as "columnComment" +from system.columns +inner join system.tables on columns.table = tables.name and columns.database = tables.database +where columns.database='#DATABASE#' and tables.uuid =OBJECT_ID_CONDITION +order by toInt32(columns.position) +`; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/sql/index.js b/plugins/dbgate-plugin-clickhouse/src/backend/sql/index.js new file mode 100644 index 000000000..1d0b3f63e --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/sql/index.js @@ -0,0 +1,7 @@ +const columns = require('./columns'); +const tables = require('./tables'); + +module.exports = { + columns, + tables, +}; diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/sql/tables.js b/plugins/dbgate-plugin-clickhouse/src/backend/sql/tables.js new file mode 100644 index 000000000..be79c3a54 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/backend/sql/tables.js @@ -0,0 +1,6 @@ +module.exports = ` +select name as "pureName", metadata_modification_time as "contentHash", total_rows as "tableRowCount", uuid as "objectId", comment as "objectComment", +engine as "tableEngine", primary_key as "primaryKeyColumns", sorting_key as "sortingKeyColumns" +from system.tables +where database='#DATABASE#' and uuid =OBJECT_ID_CONDITION; +`; diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js b/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js new file mode 100644 index 000000000..afcc64731 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js @@ -0,0 +1,6 @@ +const { SqlDumper } = require('dbgate-tools'); + +class Dumper extends SqlDumper { +} + +module.exports = Dumper; diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js new file mode 100644 index 000000000..6c0ef39ea --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -0,0 +1,33 @@ +const { driverBase } = require('dbgate-tools'); +const Dumper = require('./Dumper'); +const { mysqlSplitterOptions } = require('dbgate-query-splitter/lib/options'); + +/** @type {import('dbgate-types').SqlDialect} */ +const dialect = { + limitSelect: true, + rangeSelect: true, + offsetFetchRangeSyntax: true, + stringEscapeChar: "'", + fallbackDataType: 'nvarchar(max)', + quoteIdentifier(s) { + return `[${s}]`; + }, +}; + +/** @type {import('dbgate-types').EngineDriver} */ +const driver = { + ...driverBase, + dumperClass: Dumper, + dialect, + engine: 'clickhouse@dbgate-plugin-clickhouse', + title: 'ClickHouse', + showConnectionField: (field, values) => { + return ['databaseUrl', 'defaultDatabase', 'singleDatabase', 'isReadOnly', 'user', 'password'].includes(field); + }, + getQuerySplitterOptions: (usage) => + usage == 'editor' + ? { ...mysqlSplitterOptions, ignoreComments: true, preventSingleLineSplit: true } + : mysqlSplitterOptions, +}; + +module.exports = driver; diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/index.js b/plugins/dbgate-plugin-clickhouse/src/frontend/index.js new file mode 100644 index 000000000..a880d9660 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/index.js @@ -0,0 +1,6 @@ +import driver from './driver'; + +export default { + packageName: 'dbgate-plugin-clickhouse', + drivers: [driver], +}; diff --git a/plugins/dbgate-plugin-clickhouse/webpack-backend.config.js b/plugins/dbgate-plugin-clickhouse/webpack-backend.config.js new file mode 100644 index 000000000..e75357dff --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/webpack-backend.config.js @@ -0,0 +1,23 @@ +var webpack = require('webpack'); +var path = require('path'); + +var config = { + context: __dirname + '/src/backend', + + entry: { + app: './index.js', + }, + target: 'node', + output: { + path: path.resolve(__dirname, 'dist'), + filename: 'backend.js', + libraryTarget: 'commonjs2', + }, + + // uncomment for disable minimalization + // optimization: { + // minimize: false, + // }, +}; + +module.exports = config; diff --git a/plugins/dbgate-plugin-clickhouse/webpack-frontend.config.js b/plugins/dbgate-plugin-clickhouse/webpack-frontend.config.js new file mode 100644 index 000000000..db07de291 --- /dev/null +++ b/plugins/dbgate-plugin-clickhouse/webpack-frontend.config.js @@ -0,0 +1,24 @@ +var webpack = require("webpack"); +var path = require("path"); + +var config = { + context: __dirname + "/src/frontend", + + entry: { + app: "./index.js", + }, + target: "web", + output: { + path: path.resolve(__dirname, "dist"), + filename: "frontend.js", + libraryTarget: "var", + library: 'plugin', + }, + + // uncomment for disable minimalization + // optimization: { + // minimize: false, + // }, +}; + +module.exports = config; diff --git a/plugins/dbgate-plugin-mysql/src/backend/Analyser.js b/plugins/dbgate-plugin-mysql/src/backend/Analyser.js index 076f20ebe..118b55efc 100644 --- a/plugins/dbgate-plugin-mysql/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-mysql/src/backend/Analyser.js @@ -66,10 +66,10 @@ class Analyser extends DatabaseAnalyser { super(pool, driver, version); } - createQuery(resFileName, typeFields) { + createQuery(resFileName, typeFields, replacements = {}) { let res = sql[resFileName]; res = res.replace('#DATABASE#', this.pool._database_name); - return super.createQuery(res, typeFields); + return super.createQuery(res, typeFields, replacements); } getRequestedViewNames(allViewNames) { diff --git a/yarn.lock b/yarn.lock index 9d6bebfd2..cdd73c4fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -466,6 +466,18 @@ resolved "https://registry.yarnpkg.com/@changesets/types/-/types-0.4.0.tgz#3413badb2c3904357a36268cb9f8c7e0afc3a804" integrity sha512-TclHHKDVYQ8rJGZgVeWiF7c91yWzTTWdPagltgutelGu/Psup5PQlUq6svx7S8suj+jXcaE34yEEsfIvzXXB2Q== +"@clickhouse/client-common@1.5.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@clickhouse/client-common/-/client-common-1.5.0.tgz#fa621ee4fbdf8f4b44e5548fd5d9fe1e44b07e88" + integrity sha512-U3vDp+PDnNVEv6kia+Mq5ygnlMZzsYU+3TX+0da3XvL926jzYLMBlIvFUxe2+/5k47ySvnINRC/2QxVK7PC2/A== + +"@clickhouse/client@^1.5.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@clickhouse/client/-/client-1.5.0.tgz#ce6110a710396544a2435fe9ed8f61f20bd178b8" + integrity sha512-Udwyoec+AHHS1TiLxDiWiJWcm2BvhZEqGjmUnvzL54NyT8D8eh2mxn5RR/W5ie64JDnsKLeZFlPYKRRhZMhkxA== + dependencies: + "@clickhouse/client-common" "1.5.0" + "@cnakazawa/watch@^1.0.3": version "1.0.4" resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a" From ceb51a259733be99aded2cb727aa86dae08808aa Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 10 Sep 2024 14:38:33 +0200 Subject: [PATCH 02/57] basic driver works --- plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js | 4 ++-- plugins/dbgate-plugin-clickhouse/src/frontend/driver.js | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js index 1cdf75a0a..2807ff9a1 100644 --- a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js @@ -24,8 +24,8 @@ class Analyser extends DatabaseAnalyser { primaryKeyColumns: undefined, sortingKeyColumns: undefined, columns: columns.rows.filter((col) => col.pureName == table.pureName), - primaryKey: (table.primaryKeyColumns || '').split(',').map((columnName) => ({ columnName })), - sortingKey: (table.sortingKeyColumns || '').split(',').map((columnName) => ({ columnName })), + primaryKey: { columns: (table.primaryKeyColumns || '').split(',').map((columnName) => ({ columnName })) }, + sortingKey: { columns: (table.sortingKeyColumns || '').split(',').map((columnName) => ({ columnName })) }, foreignKeys: [], })), }; diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js index 6c0ef39ea..36ed6a5eb 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -6,11 +6,10 @@ const { mysqlSplitterOptions } = require('dbgate-query-splitter/lib/options'); const dialect = { limitSelect: true, rangeSelect: true, - offsetFetchRangeSyntax: true, stringEscapeChar: "'", - fallbackDataType: 'nvarchar(max)', + fallbackDataType: 'String', quoteIdentifier(s) { - return `[${s}]`; + return `"${s}"`; }, }; From 8d865ab3b3c34fa46b990905ef244e24cbd3307e Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 10 Sep 2024 15:18:57 +0200 Subject: [PATCH 03/57] clickhouse: nullable types --- .../src/backend/Analyser.js | 22 ++++++++++++++++++- .../src/frontend/driver.js | 11 ++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js index 2807ff9a1..d2c5c8c07 100644 --- a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js @@ -1,6 +1,21 @@ const { DatabaseAnalyser } = require('dbgate-tools'); const sql = require('./sql'); +function extractDataType(dataType) { + if (!dataType) return {}; + if (dataType.startsWith('Nullable(')) { + dataType = dataType.substring('Nullable('.length, dataType.length - 1); + return { + dataType, + notNull: false, + }; + } + return { + dataType, + notNull: true, + }; +} + class Analyser extends DatabaseAnalyser { constructor(connection, driver) { super(connection, driver); @@ -23,7 +38,12 @@ class Analyser extends DatabaseAnalyser { ...table, primaryKeyColumns: undefined, sortingKeyColumns: undefined, - columns: columns.rows.filter((col) => col.pureName == table.pureName), + columns: columns.rows + .filter((col) => col.pureName == table.pureName) + .map((col) => ({ + ...col, + ...extractDataType(col.dataType), + })), primaryKey: { columns: (table.primaryKeyColumns || '').split(',').map((columnName) => ({ columnName })) }, sortingKey: { columns: (table.sortingKeyColumns || '').split(',').map((columnName) => ({ columnName })) }, foreignKeys: [], diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js index 36ed6a5eb..d542c04d3 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -8,6 +8,17 @@ const dialect = { rangeSelect: true, stringEscapeChar: "'", fallbackDataType: 'String', + + createColumn: true, + dropColumn: true, + changeColumn: true, + createIndex: true, + dropIndex: true, + + columnProperties: { + columnComment: true, + }, + quoteIdentifier(s) { return `"${s}"`; }, From 18e6200c3ba2e7107aac97ce07c6ec8dce6b3898 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 10 Sep 2024 15:42:22 +0200 Subject: [PATCH 04/57] wip --- .../src/backend/driver.js | 55 ++++++++++++++----- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/driver.js b/plugins/dbgate-plugin-clickhouse/src/backend/driver.js index 7cdd20100..911bc3b61 100644 --- a/plugins/dbgate-plugin-clickhouse/src/backend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/backend/driver.js @@ -21,27 +21,54 @@ const driver = { return client; }, // called for retrieve data (eg. browse in data grid) and for update database - async query(client, query) { - const resultSet = await client.query({ - query, - format: 'JSONCompactEachRowWithNamesAndTypes', - }); + async query(client, query, options) { + if (options?.discardResult) { + await client.command({ + query, + }); + return { + rows: [], + columns: [], + }; + } else { + const resultSet = await client.query({ + query, + format: 'JSONCompactEachRowWithNamesAndTypes', + }); - const dataSet = await resultSet.json(); + const dataSet = await resultSet.json(); - const columns = dataSet[0].map((columnName, i) => ({ - columnName, - dataType: dataSet[1][i], - })); + const columns = dataSet[0].map((columnName, i) => ({ + columnName, + dataType: dataSet[1][i], + })); - return { - rows: dataSet.slice(2).map((row) => _.zipObject(dataSet[0], row)), - columns, - }; + return { + rows: dataSet.slice(2).map((row) => _.zipObject(dataSet[0], row)), + columns, + }; + } }, // called in query console async stream(client, query, options) { try { + if (!query.match(/^\s*SELECT/i)) { + const resp = await client.command({ + query, + }); + // console.log('RESP', resp); + // const { rowsAffected } = resp || {}; + // if (rowsAffected) { + // options.info({ + // message: `${rowsAffected} rows affected`, + // time: new Date(), + // severity: 'info', + // }); + // } + options.done(); + return; + } + const resultSet = await client.query({ query, format: 'JSONCompactEachRowWithNamesAndTypes', From 5c50faa0a2320076e604458280c5d31ff115b10b Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 10 Sep 2024 16:14:25 +0200 Subject: [PATCH 05/57] clickhouse: show sorting key --- packages/types/dbinfo.d.ts | 8 +++++--- .../web/src/elements/ObjectListControl.svelte | 4 ++-- .../web/src/tableeditor/TableEditor.svelte | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/packages/types/dbinfo.d.ts b/packages/types/dbinfo.d.ts index 73b4951f6..4ee4ac41d 100644 --- a/packages/types/dbinfo.d.ts +++ b/packages/types/dbinfo.d.ts @@ -61,7 +61,7 @@ export interface ColumnInfo extends NamedObjectInfo { isUnsigned?: boolean; isZerofill?: boolean; options?: []; - canSelectMultipleOptions?: boolean, + canSelectMultipleOptions?: boolean; } export interface DatabaseObjectInfo extends NamedObjectInfo { @@ -82,6 +82,7 @@ export interface SqlObjectInfo extends DatabaseObjectInfo { export interface TableInfo extends DatabaseObjectInfo { columns: ColumnInfo[]; primaryKey?: PrimaryKeyInfo; + sortingKey?: ColumnsConstraintInfo; foreignKeys: ForeignKeyInfo[]; dependencies?: ForeignKeyInfo[]; indexes?: IndexInfo[]; @@ -91,6 +92,7 @@ export interface TableInfo extends DatabaseObjectInfo { preloadedRowsKey?: string[]; preloadedRowsInsertOnly?: string[]; tableRowCount?: number | string; + tableEngine?: string; __isDynamicStructure?: boolean; } @@ -102,10 +104,10 @@ export interface CollectionInfo extends DatabaseObjectInfo { uniqueKey?: ColumnReference[]; // partition key columns - partitionKey?: ColumnReference[] + partitionKey?: ColumnReference[]; // unique key inside partition - clusterKey?: ColumnReference[]; + clusterKey?: ColumnReference[]; } export interface ViewInfo extends SqlObjectInfo { diff --git a/packages/web/src/elements/ObjectListControl.svelte b/packages/web/src/elements/ObjectListControl.svelte index fe8022dce..06a438403 100644 --- a/packages/web/src/elements/ObjectListControl.svelte +++ b/packages/web/src/elements/ObjectListControl.svelte @@ -10,8 +10,8 @@ export let showIfEmpty = false; export let emptyMessage = null; export let hideDisplayName = false; - export let clickable; - export let onAddNew; + export let clickable = false; + export let onAddNew = null; {#if collection?.length > 0 || showIfEmpty || emptyMessage} diff --git a/packages/web/src/tableeditor/TableEditor.svelte b/packages/web/src/tableeditor/TableEditor.svelte index bb469e6db..7863e0709 100644 --- a/packages/web/src/tableeditor/TableEditor.svelte +++ b/packages/web/src/tableeditor/TableEditor.svelte @@ -144,6 +144,7 @@ $: columns = tableInfo?.columns; $: primaryKey = tableInfo?.primaryKey; + $: sortingKey = tableInfo?.sortingKey; $: foreignKeys = tableInfo?.foreignKeys; $: dependencies = tableInfo?.dependencies; $: indexes = tableInfo?.indexes; @@ -273,6 +274,23 @@ > + {#if sortingKey} + + + {row?.columns.map(x => x.columnName).join(', ')} + + {/if} + 0 ? addIndex : null} From 293ef047d0713c58582da996e2498fd226760722 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 10 Sep 2024 16:32:02 +0200 Subject: [PATCH 06/57] add column WIP --- packages/tools/src/SqlDumper.ts | 4 +++- packages/types/dialect.d.ts | 3 +++ plugins/dbgate-plugin-clickhouse/src/frontend/driver.js | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index e0a72d0c7..7f2b29cc3 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -531,7 +531,9 @@ export class SqlDumper implements AlterProcessor { renameConstraint(constraint: ConstraintInfo, newName: string) {} createColumn(column: ColumnInfo, constraints: ConstraintInfo[]) { - this.put('^alter ^table %f ^add %i ', column, column.columnName); + this.put('^alter ^table %f ^add ', column); + if (this.dialect.createColumnWithColumnKeyword) this.put('^column '); + this.put(' %i ', column.columnName); this.columnDefinition(column); this.inlineConstraints(constraints); this.endCommand(); diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index ffff7f0c2..6a28f7366 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -34,6 +34,9 @@ export interface SqlDialect { createCheck?: boolean; dropCheck?: boolean; + // syntax for create column: ALTER TABLE table ADD COLUMN column + createColumnWithColumnKeyword?: boolean; + dropReferencesWhenDropTable?: boolean; requireFromDual?: boolean; diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js index d542c04d3..819490ae1 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -14,6 +14,8 @@ const dialect = { changeColumn: true, createIndex: true, dropIndex: true, + anonymousPrimaryKey: true, + createColumnWithColumnKeyword: true, columnProperties: { columnComment: true, From 448c15c3088e4c1acc052639c5144d403be601a5 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 08:16:54 +0200 Subject: [PATCH 07/57] supportsTransactions driver parameter --- packages/tools/src/driverBase.ts | 6 +++--- packages/types/engines.d.ts | 1 + plugins/dbgate-plugin-mssql/src/frontend/driver.js | 1 + plugins/dbgate-plugin-mysql/src/frontend/drivers.js | 1 + plugins/dbgate-plugin-oracle/src/frontend/driver.js | 1 + plugins/dbgate-plugin-sqlite/src/frontend/driver.js | 1 + 6 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/tools/src/driverBase.ts b/packages/tools/src/driverBase.ts index 307146c44..a16206688 100644 --- a/packages/tools/src/driverBase.ts +++ b/packages/tools/src/driverBase.ts @@ -66,20 +66,20 @@ export const driverBase = { return new this.dumperClass(this, options); }, async script(pool, sql, options: RunScriptOptions) { - if (options?.useTransaction) { + if (options?.useTransaction && this.supportsTransactions) { runCommandOnDriver(pool, this, dmp => dmp.beginTransaction()); } for (const sqlItem of splitQuery(sql, this.getQuerySplitterOptions('script'))) { try { await this.query(pool, sqlItem, { discardResult: true }); } catch (err) { - if (options?.useTransaction) { + if (options?.useTransaction && this.supportsTransactions) { runCommandOnDriver(pool, this, dmp => dmp.rollbackTransaction()); } throw err; } } - if (options?.useTransaction) { + if (options?.useTransaction && this.supportsTransactions) { runCommandOnDriver(pool, this, dmp => dmp.commitTransaction()); } }, diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 51cfac77d..d054e6e1f 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -148,6 +148,7 @@ export interface EngineDriver extends FilterBehaviourProvider { profilerChartAggregateFunction?: string; profilerChartMeasures?: { label: string; field: string }[]; isElectronOnly?: boolean; + supportsTransactions?: boolean; collectionSingularLabel?: string; collectionPluralLabel?: string; diff --git a/plugins/dbgate-plugin-mssql/src/frontend/driver.js b/plugins/dbgate-plugin-mssql/src/frontend/driver.js index 748ad1dfd..9d58bb041 100644 --- a/plugins/dbgate-plugin-mssql/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mssql/src/frontend/driver.js @@ -142,6 +142,7 @@ const driver = { title: 'Microsoft SQL Server', defaultPort: 1433, defaultAuthTypeName: 'tedious', + supportsTransactions: true, // databaseUrlPlaceholder: 'e.g. server=localhost&authentication.type=default&authentication.type.user=myuser&authentication.type.password=pwd&options.database=mydb', getNewObjectTemplates() { diff --git a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js index 570e065c1..80e7c6869 100644 --- a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js @@ -120,6 +120,7 @@ const mysqlDriverBase = { authTypeLabel: 'Connection mode', defaultAuthTypeName: 'hostPort', defaultSocketPath: '/var/run/mysqld/mysqld.sock', + supportsTransactions: true, getNewObjectTemplates() { return [ diff --git a/plugins/dbgate-plugin-oracle/src/frontend/driver.js b/plugins/dbgate-plugin-oracle/src/frontend/driver.js index 4150c3102..b21d1a8c4 100644 --- a/plugins/dbgate-plugin-oracle/src/frontend/driver.js +++ b/plugins/dbgate-plugin-oracle/src/frontend/driver.js @@ -97,6 +97,7 @@ const oracleDriver = { // ['server', 'port', 'user', 'password', 'defaultDatabase', 'singleDatabase'].includes(field), getQuerySplitterOptions: () => oracleSplitterOptions, readOnlySessions: true, + supportsTransactions: true, databaseUrlPlaceholder: 'e.g. localhost:1521/orcl', diff --git a/plugins/dbgate-plugin-sqlite/src/frontend/driver.js b/plugins/dbgate-plugin-sqlite/src/frontend/driver.js index b804c6644..42f91fa2b 100644 --- a/plugins/dbgate-plugin-sqlite/src/frontend/driver.js +++ b/plugins/dbgate-plugin-sqlite/src/frontend/driver.js @@ -44,6 +44,7 @@ const driver = { engine: 'sqlite@dbgate-plugin-sqlite', title: 'SQLite', readOnlySessions: true, + supportsTransactions: true, showConnectionField: (field, values) => field == 'databaseFile' || field == 'isReadOnly', showConnectionTab: (field) => false, beforeConnectionSave: (connection) => ({ From 15c400747e75def1680e5bac7afb23a70ce48f94 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 08:48:15 +0200 Subject: [PATCH 08/57] table engine shown in object tree --- .../web/src/appobj/DatabaseObjectAppObject.svelte | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/web/src/appobj/DatabaseObjectAppObject.svelte b/packages/web/src/appobj/DatabaseObjectAppObject.svelte index e12c54069..4dbd0d081 100644 --- a/packages/web/src/appobj/DatabaseObjectAppObject.svelte +++ b/packages/web/src/appobj/DatabaseObjectAppObject.svelte @@ -860,6 +860,18 @@ return createDatabaseObjectMenu(data, passProps?.connection); } + function getExtInfo(data) { + const res = []; + if (data.tableRowCount != null) { + res.push(`${formatRowCount(data.tableRowCount)} rows`); + } + if (data.tableEngine) { + res.push(data.tableEngine); + } + if (res.length > 0) return res.join(', '); + return null; + } + $: isPinned = !!$pinnedTables.find(x => testEqual(data, x)); @@ -873,7 +885,7 @@ showPinnedInsteadOfUnpin={passProps?.showPinnedInsteadOfUnpin} onPin={isPinned ? null : () => pinnedTables.update(list => [...list, data])} onUnpin={isPinned ? () => pinnedTables.update(list => list.filter(x => !testEqual(x, data))) : null} - extInfo={data.tableRowCount != null ? `${formatRowCount(data.tableRowCount)} rows` : null} + extInfo={getExtInfo(data)} on:click={() => handleClick()} on:middleclick={() => handleClick(true)} on:expand From ae9ffe1aef8516810767fe0a76d5350e21a2f869 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 09:16:08 +0200 Subject: [PATCH 09/57] editing table works --- packages/sqltree/src/dumpSqlCommand.ts | 21 +++++++++---- packages/sqltree/src/types.ts | 5 ++++ packages/tools/src/driverBase.ts | 4 +++ packages/types/engines.d.ts | 5 ++++ packages/web/src/tabs/TableDataTab.svelte | 9 +++--- .../src/frontend/driver.js | 30 +++++++++++++++++++ 6 files changed, 64 insertions(+), 10 deletions(-) diff --git a/packages/sqltree/src/dumpSqlCommand.ts b/packages/sqltree/src/dumpSqlCommand.ts index b16f3c34d..0c3f2c667 100644 --- a/packages/sqltree/src/dumpSqlCommand.ts +++ b/packages/sqltree/src/dumpSqlCommand.ts @@ -62,10 +62,13 @@ export function dumpSqlSelect(dmp: SqlDumper, cmd: Select) { } export function dumpSqlUpdate(dmp: SqlDumper, cmd: Update) { - dmp.put('^update '); - dumpSqlSourceRef(dmp, cmd.from); - - dmp.put('&n^set '); + if (cmd.alterTableUpdateSyntax) { + dmp.put('^alter ^table %f &n^update ', cmd.from?.name); + } else { + dmp.put('^update '); + dumpSqlSourceRef(dmp, cmd.from); + dmp.put('&n^set '); + } dmp.put('&>'); dmp.putCollection(', ', cmd.fields, col => { dmp.put('%i=', col.targetColumn); @@ -81,8 +84,14 @@ export function dumpSqlUpdate(dmp: SqlDumper, cmd: Update) { } export function dumpSqlDelete(dmp: SqlDumper, cmd: Delete) { - dmp.put('^delete ^from '); - dumpSqlSourceRef(dmp, cmd.from); + if (cmd.alterTableDeleteSyntax) { + dmp.put('^alter ^table '); + dumpSqlSourceRef(dmp, cmd.from); + dmp.put('^delete '); + } else { + dmp.put('^delete ^from '); + dumpSqlSourceRef(dmp, cmd.from); + } if (cmd.where) { dmp.put('&n^where '); diff --git a/packages/sqltree/src/types.ts b/packages/sqltree/src/types.ts index 44a1e2ea7..c2fe24674 100644 --- a/packages/sqltree/src/types.ts +++ b/packages/sqltree/src/types.ts @@ -26,12 +26,17 @@ export interface Update { fields: UpdateField[]; from: FromDefinition; where?: Condition; + // ALTER TABLE xxx UPDATE col1=val1 - syntax for ClickHouse + alterTableUpdateSyntax?: boolean; } export interface Delete { commandType: 'delete'; from: FromDefinition; where?: Condition; + + // ALTER TABLE xxx DELETE - syntax for ClickHouse + alterTableDeleteSyntax?: boolean; } export interface Insert { diff --git a/packages/tools/src/driverBase.ts b/packages/tools/src/driverBase.ts index a16206688..e67433fc3 100644 --- a/packages/tools/src/driverBase.ts +++ b/packages/tools/src/driverBase.ts @@ -173,4 +173,8 @@ export const driverBase = { parseSqlNull: true, parseHexAsBuffer: true, }, + + createSaveChangeSetScript(changeSet, dbinfo, defaultCreator) { + return defaultCreator(changeSet, dbinfo); + }, }; diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index d054e6e1f..477bb784e 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -223,6 +223,11 @@ export interface EngineDriver extends FilterBehaviourProvider { getCollectionExportQueryJson(collection: string, condition: any, sort?: CollectionSortDefinition): {}; getScriptTemplates(objectTypeField: keyof DatabaseInfo): { label: string; scriptTemplate: string }[]; getScriptTemplateContent(scriptTemplate: string, props: any): Promise; + createSaveChangeSetScript( + changeSet: any, + dbinfo: DatabaseInfo, + defaultCreator: (changeSet: any, dbinfo: DatabaseInfo) => any + ): any[]; analyserClass?: any; dumperClass?: any; diff --git a/packages/web/src/tabs/TableDataTab.svelte b/packages/web/src/tabs/TableDataTab.svelte index d2094f96d..83a6f13df 100644 --- a/packages/web/src/tabs/TableDataTab.svelte +++ b/packages/web/src/tabs/TableDataTab.svelte @@ -71,10 +71,7 @@ changeSetToSql, createChangeSet, createGridCache, - createGridConfig, getDeleteCascades, - TableFormViewDisplay, - TableGridDisplay, } from 'dbgate-datalib'; import { findEngineDriver } from 'dbgate-tools'; import { reloadDataCacheFunc } from 'dbgate-datalib'; @@ -160,7 +157,11 @@ export function save() { const driver = findEngineDriver($connection, $extensions); - const script = changeSetToSql($changeSetStore?.value, $dbinfo); + + const script = driver.createSaveChangeSetScript($changeSetStore?.value, $dbinfo, () => + changeSetToSql($changeSetStore?.value, $dbinfo) + ); + const deleteCascades = getDeleteCascades($changeSetStore?.value, $dbinfo); const sql = scriptToSql(driver, script); const deleteCascadesScripts = _.map(deleteCascades, ({ title, commands }) => ({ diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js index 819490ae1..242c858df 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -1,6 +1,7 @@ const { driverBase } = require('dbgate-tools'); const Dumper = require('./Dumper'); const { mysqlSplitterOptions } = require('dbgate-query-splitter/lib/options'); +const _cloneDeepWith = require('lodash/cloneDeepWith'); /** @type {import('dbgate-types').SqlDialect} */ const dialect = { @@ -40,6 +41,35 @@ const driver = { usage == 'editor' ? { ...mysqlSplitterOptions, ignoreComments: true, preventSingleLineSplit: true } : mysqlSplitterOptions, + + createSaveChangeSetScript(changeSet, dbinfo, defaultCreator) { + function removeConditionSource(cmd) { + cmd.where = _cloneDeepWith(cmd.where, (expr) => { + if (expr.exprType == 'column') { + return { + ...expr, + source: undefined, + }; + } + }); + } + + const res = defaultCreator(changeSet, dbinfo); + for (const cmd of res) { + if (cmd.commandType == 'update') { + cmd.alterTableUpdateSyntax = true; + removeConditionSource(cmd); + } + if (cmd.commandType == 'delete') { + const table = dbinfo?.tables?.find((x) => x.pureName == cmd?.from?.name?.pureName); + if (table?.tableEngine != 'MergeTree') { + cmd.alterTableDeleteSyntax = true; + } + removeConditionSource(cmd); + } + } + return res; + }, }; module.exports = driver; From 36a65ea13a54aa506f323f7375c4547e56668600 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 09:34:26 +0200 Subject: [PATCH 10/57] mysql - engine --- plugins/dbgate-plugin-mysql/src/backend/sql/tables.js | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/dbgate-plugin-mysql/src/backend/sql/tables.js b/plugins/dbgate-plugin-mysql/src/backend/sql/tables.js index de865c61a..993f9dbbc 100644 --- a/plugins/dbgate-plugin-mysql/src/backend/sql/tables.js +++ b/plugins/dbgate-plugin-mysql/src/backend/sql/tables.js @@ -2,6 +2,7 @@ module.exports = ` select TABLE_NAME as pureName, TABLE_ROWS as tableRowCount, + ENGINE as tableEngine, case when ENGINE='InnoDB' then CREATE_TIME else coalesce(UPDATE_TIME, CREATE_TIME) end as modifyDate from information_schema.tables where TABLE_SCHEMA = '#DATABASE#' and (TABLE_TYPE='BASE TABLE' or TABLE_TYPE='SYSTEM VERSIONED') and TABLE_NAME =OBJECT_ID_CONDITION; From f6e0b634f00017771b71b79eceac7712b2c7ca68 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 09:53:16 +0200 Subject: [PATCH 11/57] collapsible table editor parts --- .../web/src/elements/ObjectListControl.svelte | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/web/src/elements/ObjectListControl.svelte b/packages/web/src/elements/ObjectListControl.svelte index 06a438403..15ddaaf2a 100644 --- a/packages/web/src/elements/ObjectListControl.svelte +++ b/packages/web/src/elements/ObjectListControl.svelte @@ -12,11 +12,21 @@ export let hideDisplayName = false; export let clickable = false; export let onAddNew = null; + + let collapsed = false; {#if collection?.length > 0 || showIfEmpty || emptyMessage}
+ { + collapsed = !collapsed; + }} + > + + {title} {#if onAddNew} Add new @@ -27,7 +37,7 @@ {emptyMessage}
{/if} - {#if collection?.length > 0 || showIfEmpty} + {#if !collapsed && (collection?.length > 0 || showIfEmpty)}
.wrapper { margin-bottom: 20px; + user-select: none; } .header { @@ -93,4 +104,13 @@ .body { margin: 20px; } + + .collapse { + cursor: pointer; + } + + .collapse:hover { + color: var(--theme-font-hover); + background: var(--theme-bg-3); + } From ff33ec668b963ccc67678e3833b998c11cd01822 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 12:51:09 +0200 Subject: [PATCH 12/57] clickhouse: edit table options --- packages/tools/src/SqlDumper.ts | 13 ++++ packages/types/engines.d.ts | 5 ++ .../src/elements/ObjectFieldsEditor.svelte | 67 +++++++++++++++++++ packages/web/src/forms/FormArgument.svelte | 24 +++++-- .../web/src/tableeditor/TableEditor.svelte | 16 +++++ .../src/frontend/driver.js | 67 +++++++++++++++++++ 6 files changed, 186 insertions(+), 6 deletions(-) create mode 100644 packages/web/src/elements/ObjectFieldsEditor.svelte diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index 7f2b29cc3..9c1a07325 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -294,12 +294,25 @@ export class SqlDumper implements AlterProcessor { }); this.put('&<&n)'); + + this.tableOptions(table); + this.endCommand(); (table.indexes || []).forEach(ix => { this.createIndex(ix); }); } + tableOptions(table: TableInfo) { + const options = this.driver.getTableFormOptions('sqlCreateTable'); + for (const option of options) { + if (table[option.name]) { + this.put('&n'); + this.put(option.sqlFormatString, table[option.name]); + } + } + } + createTablePrimaryKeyCore(table: TableInfo) { if (table.primaryKey) { this.put(',&n'); diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 477bb784e..e3166bd0d 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -155,6 +155,11 @@ export interface EngineDriver extends FilterBehaviourProvider { collectionNameLabel?: string; newCollectionFormParams?: any[]; + getTableFormOptions(intent: 'newTableForm' | 'editTableForm' | 'sqlCreateTable' | 'sqlAlterTable'): { + name: string; + sqlFormatString: string; + }[]; + supportedCreateDatabase?: boolean; showConnectionField?: ( field: string, diff --git a/packages/web/src/elements/ObjectFieldsEditor.svelte b/packages/web/src/elements/ObjectFieldsEditor.svelte new file mode 100644 index 000000000..cc5dcf8ab --- /dev/null +++ b/packages/web/src/elements/ObjectFieldsEditor.svelte @@ -0,0 +1,67 @@ + + +
+
+ { + collapsed = !collapsed; + }} + > + + + {title} +
+ {#if !collapsed} + + + + {/if} +
+ + diff --git a/packages/web/src/forms/FormArgument.svelte b/packages/web/src/forms/FormArgument.svelte index ea1bd923d..a40fcfced 100644 --- a/packages/web/src/forms/FormArgument.svelte +++ b/packages/web/src/forms/FormArgument.svelte @@ -5,11 +5,15 @@ import FormSelectField from './FormSelectField.svelte'; import FormTextField from './FormTextField.svelte'; import FormStringList from './FormStringList.svelte'; + import FormDropDownTextField from './FormDropDownTextField.svelte'; + import { getFormContext } from './FormProviderCore.svelte'; export let arg; export let namePrefix; $: name = `${namePrefix}${arg.name}`; + + const { setFieldValue } = getFormContext(); {#if arg.type == 'text'} @@ -19,14 +23,10 @@ defaultValue={arg.default} focused={arg.focused} placeholder={arg.placeholder} + disabled={arg.disabled} /> {:else if arg.type == 'stringlist'} - + {:else if arg.type == 'number'} +{:else if arg.type == 'dropdowntext'} + { + return arg.options.map(opt => ({ + text: _.isString(opt) ? opt : opt.name, + onClick: () => setFieldValue(name, _.isString(opt) ? opt : opt.value), + })); + }} + /> {/if} diff --git a/packages/web/src/tableeditor/TableEditor.svelte b/packages/web/src/tableeditor/TableEditor.svelte index 7863e0709..27d24d5a2 100644 --- a/packages/web/src/tableeditor/TableEditor.svelte +++ b/packages/web/src/tableeditor/TableEditor.svelte @@ -81,6 +81,7 @@ import IndexEditorModal from './IndexEditorModal.svelte'; import PrimaryKeyEditorModal from './PrimaryKeyEditorModal.svelte'; import UniqueEditorModal from './UniqueEditorModal.svelte'; + import ObjectFieldsEditor from '../elements/ObjectFieldsEditor.svelte'; export const activator = createActivator('TableEditor', true); @@ -154,9 +155,24 @@ tableInfo; invalidateCommands(); } + + $: tableFormOptions = driver?.getTableFormOptions(tableInfo?.objectId ? 'editTableForm' : 'newTableForm');
+ {#if tableFormOptions} + { + if (!_.isEmpty(vals)) { + setTableInfo(tbl => ({ ...tbl, ...vals })); + } + }} + /> + {/if} + ({ ...x, ordinal: index + 1 }))} title={`Columns (${columns?.length || 0})`} diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js index 242c858df..2d24f47aa 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -3,6 +3,53 @@ const Dumper = require('./Dumper'); const { mysqlSplitterOptions } = require('dbgate-query-splitter/lib/options'); const _cloneDeepWith = require('lodash/cloneDeepWith'); +const clickhouseEngines = [ + 'MergeTree', + 'ReplacingMergeTree', + 'SummingMergeTree', + 'AggregatingMergeTree', + 'CollapsingMergeTree', + 'VersionedCollapsingMergeTree', + 'GraphiteMergeTree', + 'Distributed', + 'Log', + 'TinyLog', + 'StripeLog', + 'Memory', + 'File', + 'URL', + 'JDBC', + 'ODBC', + 'Buffer', + 'Null', + 'Kafka', + 'HDFS', + 'S3', + 'Merge', + 'Join', + 'MaterializedView', + 'Dictionary', + 'MySQL', + 'PostgreSQL', + 'MongoDB', + 'EmbeddedRocksDB', + 'View', + 'MaterializeMySQL', + 'MaterializePostgreSQL', + 'ReplicatedMergeTree', + 'ReplicatedReplacingMergeTree', + 'ReplicatedSummingMergeTree', + 'ReplicatedAggregatingMergeTree', + 'ReplicatedCollapsingMergeTree', + 'ReplicatedVersionedCollapsingMergeTree', + 'ReplicatedGraphiteMergeTree', + 'ExternalDistributed', + 'Iceberg', + 'Parquet', + 'ORC', + 'DeltaLake', +]; + /** @type {import('dbgate-types').SqlDialect} */ const dialect = { limitSelect: true, @@ -70,6 +117,26 @@ const driver = { } return res; }, + + getTableFormOptions: (intent) => { + const isNewTable = intent == 'newTableForm'; + return [ + { + type: isNewTable ? 'dropdowntext' : text, + options: clickhouseEngines, + label: 'Engine', + name: 'tableEngine', + sqlFormatString: '^engine = %s', + disabled: !isNewTable, + }, + { + type: 'text', + label: 'Comment', + name: 'objectComment', + sqlFormatString: '^comment %v', + }, + ]; + }, }; module.exports = driver; From 4f429c27c0977b270bc2f24c0fa806989b86d515 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 13:05:42 +0200 Subject: [PATCH 13/57] fix --- packages/sqltree/src/dumpSqlCommand.ts | 2 +- plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/sqltree/src/dumpSqlCommand.ts b/packages/sqltree/src/dumpSqlCommand.ts index 0c3f2c667..d73d7d690 100644 --- a/packages/sqltree/src/dumpSqlCommand.ts +++ b/packages/sqltree/src/dumpSqlCommand.ts @@ -87,7 +87,7 @@ export function dumpSqlDelete(dmp: SqlDumper, cmd: Delete) { if (cmd.alterTableDeleteSyntax) { dmp.put('^alter ^table '); dumpSqlSourceRef(dmp, cmd.from); - dmp.put('^delete '); + dmp.put(' ^delete '); } else { dmp.put('^delete ^from '); dumpSqlSourceRef(dmp, cmd.from); diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js index d2c5c8c07..91118d28e 100644 --- a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js @@ -44,8 +44,12 @@ class Analyser extends DatabaseAnalyser { ...col, ...extractDataType(col.dataType), })), - primaryKey: { columns: (table.primaryKeyColumns || '').split(',').map((columnName) => ({ columnName })) }, - sortingKey: { columns: (table.sortingKeyColumns || '').split(',').map((columnName) => ({ columnName })) }, + primaryKey: table.primaryKeyColumns + ? { columns: (table.primaryKeyColumns || '').split(',').map((columnName) => ({ columnName })) } + : null, + sortingKey: table.sortingKeyColumns + ? { columns: (table.sortingKeyColumns || '').split(',').map((columnName) => ({ columnName })) } + : null, foreignKeys: [], })), }; From b0165c14e9d272bb9ad3843cccf61a1b1be445bf Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 13:41:04 +0200 Subject: [PATCH 14/57] tabl eoptions for mysql - comment, engine --- packages/tools/src/driverBase.ts | 4 ++ .../web/src/tableeditor/TableEditor.svelte | 5 +- .../src/frontend/driver.js | 4 +- .../src/backend/sql/tables.js | 1 + .../src/frontend/drivers.js | 70 +++++++++++++++++++ 5 files changed, 81 insertions(+), 3 deletions(-) diff --git a/packages/tools/src/driverBase.ts b/packages/tools/src/driverBase.ts index e67433fc3..3c9c8d8f1 100644 --- a/packages/tools/src/driverBase.ts +++ b/packages/tools/src/driverBase.ts @@ -177,4 +177,8 @@ export const driverBase = { createSaveChangeSetScript(changeSet, dbinfo, defaultCreator) { return defaultCreator(changeSet, dbinfo); }, + + getTableFormOptions() { + return null; + }, }; diff --git a/packages/web/src/tableeditor/TableEditor.svelte b/packages/web/src/tableeditor/TableEditor.svelte index 27d24d5a2..aa8e6cb19 100644 --- a/packages/web/src/tableeditor/TableEditor.svelte +++ b/packages/web/src/tableeditor/TableEditor.svelte @@ -164,7 +164,10 @@ x.name) + )} onChangeValues={vals => { if (!_.isEmpty(vals)) { setTableInfo(tbl => ({ ...tbl, ...vals })); diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js index 2d24f47aa..baff6fa23 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -118,11 +118,11 @@ const driver = { return res; }, - getTableFormOptions: (intent) => { + getTableFormOptions(intent) { const isNewTable = intent == 'newTableForm'; return [ { - type: isNewTable ? 'dropdowntext' : text, + type: isNewTable ? 'dropdowntext' : 'text', options: clickhouseEngines, label: 'Engine', name: 'tableEngine', diff --git a/plugins/dbgate-plugin-mysql/src/backend/sql/tables.js b/plugins/dbgate-plugin-mysql/src/backend/sql/tables.js index 993f9dbbc..216e8c43b 100644 --- a/plugins/dbgate-plugin-mysql/src/backend/sql/tables.js +++ b/plugins/dbgate-plugin-mysql/src/backend/sql/tables.js @@ -3,6 +3,7 @@ select TABLE_NAME as pureName, TABLE_ROWS as tableRowCount, ENGINE as tableEngine, + TABLE_COMMENT as objectComment, case when ENGINE='InnoDB' then CREATE_TIME else coalesce(UPDATE_TIME, CREATE_TIME) end as modifyDate from information_schema.tables where TABLE_SCHEMA = '#DATABASE#' and (TABLE_TYPE='BASE TABLE' or TABLE_TYPE='SYSTEM VERSIONED') and TABLE_NAME =OBJECT_ID_CONDITION; diff --git a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js index 80e7c6869..8b49c9451 100644 --- a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js @@ -132,6 +132,30 @@ const mysqlDriverBase = { { label: 'New function', sql: 'CREATE FUNCTION myfunc (arg1 INT)\nRETURNS INT DETERMINISTIC\nRETURN 1' }, ]; }, + + getSupportedEngines() { + return []; + }, + + getTableFormOptions(intent) { + const isNewTable = intent == 'newTableForm'; + return [ + { + type: isNewTable ? 'dropdowntext' : 'text', + options: this.getSupportedEngines(), + label: 'Engine', + name: 'tableEngine', + sqlFormatString: '^engine = %s', + disabled: !isNewTable, + }, + { + type: 'text', + label: 'Comment', + name: 'objectComment', + sqlFormatString: '^comment %v', + }, + ]; + }, }; /** @type {import('dbgate-types').EngineDriver} */ @@ -142,6 +166,27 @@ const mysqlDriver = { __analyserInternals: { quoteDefaultValues: true, }, + + getSupportedEngines() { + const mysqlEngines = [ + 'InnoDB', // Default and most commonly used engine with ACID transaction support and referential integrity. + 'MyISAM', // Older engine without transaction or referential integrity support. + 'MEMORY', // Tables stored in memory, very fast but volatile, used for temporary data. + 'CSV', // Tables stored in CSV format, useful for import/export of data. + 'ARCHIVE', // Engine for storing large amounts of historical data with compression. + 'BLACKHOLE', // Engine that discards data, useful for replication. + 'FEDERATED', // Access tables on remote MySQL servers. + 'MRG_MYISAM', // Merges multiple MyISAM tables into one. + 'NDB', // Cluster storage engine for MySQL Cluster. + 'EXAMPLE', // Example engine for developers, has no real functionality. + 'PERFORMANCE_SCHEMA', // Engine used for performance monitoring in MySQL. + 'SEQUENCE', // Special engine for sequences, used in MariaDB. + 'SPIDER', // Engine for horizontal partitioning, often used in MariaDB. + 'ROCKSDB', // Engine optimized for read-heavy workloads, commonly used in Facebook MySQL. + 'TokuDB', // Engine with high data compression and SSD optimization. + ]; + return mysqlEngines; + }, }; /** @type {import('dbgate-types').EngineDriver} */ @@ -152,6 +197,31 @@ const mariaDriver = { __analyserInternals: { quoteDefaultValues: false, }, + getSupportedEngines() { + const mariaDBEngines = [ + 'InnoDB', // Main transactional engine, similar to MySQL, supports ACID transactions and referential integrity. + 'Aria', // Replacement for MyISAM, supports crash recovery and optimized for high speed. + 'MyISAM', // Older engine without transaction support, still supported for compatibility. + 'MEMORY', // Tables stored in memory, suitable for temporary data. + 'CSV', // Stores data in CSV format, easy for export/import. + 'ARCHIVE', // Stores compressed data, suitable for historical records. + 'BLACKHOLE', // Engine that does not store data, often used for replication. + 'FEDERATED', // Allows access to tables on remote MariaDB/MySQL servers. + 'MRG_MyISAM', // Allows merging multiple MyISAM tables into one. + 'SEQUENCE', // Special engine for generating sequences. + 'SphinxSE', // Engine for full-text search using Sphinx. + 'SPIDER', // Engine for sharding, supports horizontal partitioning. + 'TokuDB', // High-compression engine optimized for large data sets and SSDs. + 'RocksDB', // Read-optimized engine focused on performance with large data. + 'CONNECT', // Engine for accessing external data sources (e.g., files, web services). + 'OQGRAPH', // Graph engine, suitable for hierarchical and graph structures. + 'ColumnStore', // Analytical engine for columnar data storage, suitable for Big Data. + 'Mroonga', // Engine supporting full-text search in Japanese and other languages. + 'S3', // Allows storing data in Amazon S3-compatible storage. + 'XtraDB', // Enhanced InnoDB engine with optimizations from Percona (commonly used in older MariaDB versions). + ]; + return mariaDBEngines; + }, }; module.exports = [mysqlDriver, mariaDriver]; From 7ad195077794f121191ed733296a48ff572fa7c2 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 14:01:11 +0200 Subject: [PATCH 15/57] getTableFormOptions moved to dialect --- packages/tools/src/driverBase.ts | 4 - packages/types/dialect.d.ts | 5 + packages/types/engines.d.ts | 5 - .../web/src/tableeditor/TableEditor.svelte | 2 +- .../src/frontend/driver.js | 40 +++---- .../src/frontend/drivers.js | 106 ++++++++++-------- 6 files changed, 83 insertions(+), 79 deletions(-) diff --git a/packages/tools/src/driverBase.ts b/packages/tools/src/driverBase.ts index 3c9c8d8f1..e67433fc3 100644 --- a/packages/tools/src/driverBase.ts +++ b/packages/tools/src/driverBase.ts @@ -177,8 +177,4 @@ export const driverBase = { createSaveChangeSetScript(changeSet, dbinfo, defaultCreator) { return defaultCreator(changeSet, dbinfo); }, - - getTableFormOptions() { - return null; - }, }; diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index 6a28f7366..9a6460d30 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -44,4 +44,9 @@ export interface SqlDialect { // create sql-tree expression createColumnViewExpression(columnName: string, dataType: string, source: { alias: string }, alias?: string): any; + + getTableFormOptions(intent: 'newTableForm' | 'editTableForm' | 'sqlCreateTable' | 'sqlAlterTable'): { + name: string; + sqlFormatString: string; + }[]; } diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index e3166bd0d..477bb784e 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -155,11 +155,6 @@ export interface EngineDriver extends FilterBehaviourProvider { collectionNameLabel?: string; newCollectionFormParams?: any[]; - getTableFormOptions(intent: 'newTableForm' | 'editTableForm' | 'sqlCreateTable' | 'sqlAlterTable'): { - name: string; - sqlFormatString: string; - }[]; - supportedCreateDatabase?: boolean; showConnectionField?: ( field: string, diff --git a/packages/web/src/tableeditor/TableEditor.svelte b/packages/web/src/tableeditor/TableEditor.svelte index aa8e6cb19..63b9d32f2 100644 --- a/packages/web/src/tableeditor/TableEditor.svelte +++ b/packages/web/src/tableeditor/TableEditor.svelte @@ -156,7 +156,7 @@ invalidateCommands(); } - $: tableFormOptions = driver?.getTableFormOptions(tableInfo?.objectId ? 'editTableForm' : 'newTableForm'); + $: tableFormOptions = driver?.dialect?.getTableFormOptions?.(tableInfo?.objectId ? 'editTableForm' : 'newTableForm');
diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js index baff6fa23..f7607b3b3 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -72,6 +72,26 @@ const dialect = { quoteIdentifier(s) { return `"${s}"`; }, + + getTableFormOptions(intent) { + const isNewTable = intent == 'newTableForm'; + return [ + { + type: isNewTable ? 'dropdowntext' : 'text', + options: clickhouseEngines, + label: 'Engine', + name: 'tableEngine', + sqlFormatString: '^engine = %s', + disabled: !isNewTable, + }, + { + type: 'text', + label: 'Comment', + name: 'objectComment', + sqlFormatString: '^comment %v', + }, + ]; + }, }; /** @type {import('dbgate-types').EngineDriver} */ @@ -117,26 +137,6 @@ const driver = { } return res; }, - - getTableFormOptions(intent) { - const isNewTable = intent == 'newTableForm'; - return [ - { - type: isNewTable ? 'dropdowntext' : 'text', - options: clickhouseEngines, - label: 'Engine', - name: 'tableEngine', - sqlFormatString: '^engine = %s', - disabled: !isNewTable, - }, - { - type: 'text', - label: 'Comment', - name: 'objectComment', - sqlFormatString: '^comment %v', - }, - ]; - }, }; module.exports = driver; diff --git a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js index 8b49c9451..1131c2cc7 100644 --- a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js @@ -99,39 +99,6 @@ const dialect = { }; } }, -}; - -const mysqlDriverBase = { - ...driverBase, - showConnectionField: (field, values) => - ['authType', 'user', 'password', 'defaultDatabase', 'singleDatabase', 'isReadOnly'].includes(field) || - (values.authType == 'socket' && ['socketPath'].includes(field)) || - (values.authType != 'socket' && ['server', 'port'].includes(field)), - dumperClass: Dumper, - dialect, - defaultPort: 3306, - getQuerySplitterOptions: usage => - usage == 'editor' - ? { ...mysqlSplitterOptions, ignoreComments: true, preventSingleLineSplit: true } - : mysqlSplitterOptions, - - readOnlySessions: true, - supportsDatabaseDump: true, - authTypeLabel: 'Connection mode', - defaultAuthTypeName: 'hostPort', - defaultSocketPath: '/var/run/mysqld/mysqld.sock', - supportsTransactions: true, - - getNewObjectTemplates() { - return [ - { label: 'New view', sql: 'CREATE VIEW myview\nAS\nSELECT * FROM table1' }, - { - label: 'New procedure', - sql: 'DELIMITER //\n\nCREATE PROCEDURE myproc (IN arg1 INT)\nBEGIN\n SELECT * FROM table1;\nEND\n\nDELIMITER ;', - }, - { label: 'New function', sql: 'CREATE FUNCTION myfunc (arg1 INT)\nRETURNS INT DETERMINISTIC\nRETURN 1' }, - ]; - }, getSupportedEngines() { return []; @@ -158,15 +125,8 @@ const mysqlDriverBase = { }, }; -/** @type {import('dbgate-types').EngineDriver} */ -const mysqlDriver = { - ...mysqlDriverBase, - engine: 'mysql@dbgate-plugin-mysql', - title: 'MySQL', - __analyserInternals: { - quoteDefaultValues: true, - }, - +const mysqlDialect = { + ...dialect, getSupportedEngines() { const mysqlEngines = [ 'InnoDB', // Default and most commonly used engine with ACID transaction support and referential integrity. @@ -189,14 +149,51 @@ const mysqlDriver = { }, }; -/** @type {import('dbgate-types').EngineDriver} */ -const mariaDriver = { - ...mysqlDriverBase, - engine: 'mariadb@dbgate-plugin-mysql', - title: 'MariaDB', - __analyserInternals: { - quoteDefaultValues: false, +const mysqlDriverBase = { + ...driverBase, + showConnectionField: (field, values) => + ['authType', 'user', 'password', 'defaultDatabase', 'singleDatabase', 'isReadOnly'].includes(field) || + (values.authType == 'socket' && ['socketPath'].includes(field)) || + (values.authType != 'socket' && ['server', 'port'].includes(field)), + dumperClass: Dumper, + defaultPort: 3306, + getQuerySplitterOptions: usage => + usage == 'editor' + ? { ...mysqlSplitterOptions, ignoreComments: true, preventSingleLineSplit: true } + : mysqlSplitterOptions, + + readOnlySessions: true, + supportsDatabaseDump: true, + authTypeLabel: 'Connection mode', + defaultAuthTypeName: 'hostPort', + defaultSocketPath: '/var/run/mysqld/mysqld.sock', + supportsTransactions: true, + + getNewObjectTemplates() { + return [ + { label: 'New view', sql: 'CREATE VIEW myview\nAS\nSELECT * FROM table1' }, + { + label: 'New procedure', + sql: 'DELIMITER //\n\nCREATE PROCEDURE myproc (IN arg1 INT)\nBEGIN\n SELECT * FROM table1;\nEND\n\nDELIMITER ;', + }, + { label: 'New function', sql: 'CREATE FUNCTION myfunc (arg1 INT)\nRETURNS INT DETERMINISTIC\nRETURN 1' }, + ]; }, +}; + +/** @type {import('dbgate-types').EngineDriver} */ +const mysqlDriver = { + ...mysqlDriverBase, + dialect: mysqlDialect, + engine: 'mysql@dbgate-plugin-mysql', + title: 'MySQL', + __analyserInternals: { + quoteDefaultValues: true, + }, +}; + +const mariaDbDialect = { + ...dialect, getSupportedEngines() { const mariaDBEngines = [ 'InnoDB', // Main transactional engine, similar to MySQL, supports ACID transactions and referential integrity. @@ -224,4 +221,15 @@ const mariaDriver = { }, }; +/** @type {import('dbgate-types').EngineDriver} */ +const mariaDriver = { + ...mysqlDriverBase, + dialect: mariaDbDialect, + engine: 'mariadb@dbgate-plugin-mysql', + title: 'MariaDB', + __analyserInternals: { + quoteDefaultValues: false, + }, +}; + module.exports = [mysqlDriver, mariaDriver]; From fb39cd13023660f27f45ee80f645623f98a17660 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 15:09:16 +0200 Subject: [PATCH 16/57] clickhouse + mysql: modify table option --- packages/tools/src/SqlDumper.ts | 19 +++++++++++- packages/tools/src/alterPlan.ts | 22 +++++++++++++- .../src/database-info-alter-processor.ts | 5 ++++ packages/tools/src/diffTools.ts | 14 +++++++++ packages/types/alter-processor.d.ts | 1 + packages/types/dialect.d.ts | 1 + .../src/elements/ObjectFieldsEditor.svelte | 1 + .../web/src/tableeditor/TableEditor.svelte | 29 ++++++++++--------- .../web/src/tabs/TableStructureTab.svelte | 5 +++- .../src/frontend/Dumper.js | 4 +++ .../src/frontend/driver.js | 2 +- .../src/frontend/drivers.js | 6 ++-- 12 files changed, 88 insertions(+), 21 deletions(-) diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index 9c1a07325..ec45907a8 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -304,7 +304,7 @@ export class SqlDumper implements AlterProcessor { } tableOptions(table: TableInfo) { - const options = this.driver.getTableFormOptions('sqlCreateTable'); + const options = this.driver?.dialect?.getTableFormOptions?.('sqlCreateTable') || []; for (const option of options) { if (table[option.name]) { this.put('&n'); @@ -686,6 +686,23 @@ export class SqlDumper implements AlterProcessor { this.putCmd('^drop %s %f', this.getSqlObjectSqlName(obj.objectTypeField), obj); } + setTableOption(table: TableInfo, optionName: string, optionValue: string) { + const options = this?.dialect?.getTableFormOptions?.('sqlAlterTable'); + const option = options?.find(x => x.name == optionName && !x.disabled); + if (!option) { + return; + } + + this.setTableOptionCore(table, optionName, optionValue, option.sqlFormatString); + + this.endCommand(); + } + + setTableOptionCore(table: TableInfo, optionName: string, optionValue: string, formatString: string) { + this.put('^alter ^table %f ', table); + this.put(formatString, optionValue); + } + fillPreloadedRows( table: NamedObjectInfo, oldRows: any[], diff --git a/packages/tools/src/alterPlan.ts b/packages/tools/src/alterPlan.ts index c47ecdc95..b87209780 100644 --- a/packages/tools/src/alterPlan.ts +++ b/packages/tools/src/alterPlan.ts @@ -97,6 +97,13 @@ interface AlterOperation_FillPreloadedRows { autoIncrementColumn: string; } +interface AlterOperation_SetTableOption { + operationType: 'setTableOption'; + table: TableInfo; + optionName: string; + optionValue: string; +} + type AlterOperation = | AlterOperation_CreateColumn | AlterOperation_ChangeColumn @@ -112,7 +119,8 @@ type AlterOperation = | AlterOperation_CreateSqlObject | AlterOperation_DropSqlObject | AlterOperation_RecreateTable - | AlterOperation_FillPreloadedRows; + | AlterOperation_FillPreloadedRows + | AlterOperation_SetTableOption; export class AlterPlan { recreates = { @@ -253,6 +261,15 @@ export class AlterPlan { }); } + setTableOption(table: TableInfo, optionName: string, optionValue: string) { + this.operations.push({ + operationType: 'setTableOption', + table, + optionName, + optionValue, + }); + } + run(processor: AlterProcessor) { for (const op of this.operations) { runAlterOperation(op, processor); @@ -575,6 +592,9 @@ export function runAlterOperation(op: AlterOperation, processor: AlterProcessor) case 'dropSqlObject': processor.dropSqlObject(op.oldObject); break; + case 'setTableOption': + processor.setTableOption(op.table, op.optionName, op.optionValue); + break; case 'fillPreloadedRows': processor.fillPreloadedRows(op.table, op.oldRows, op.newRows, op.key, op.insertOnly, op.autoIncrementColumn); break; diff --git a/packages/tools/src/database-info-alter-processor.ts b/packages/tools/src/database-info-alter-processor.ts index 2b8c2b23f..533252435 100644 --- a/packages/tools/src/database-info-alter-processor.ts +++ b/packages/tools/src/database-info-alter-processor.ts @@ -129,4 +129,9 @@ export class DatabaseInfoAlterProcessor { tableInfo.preloadedRowsKey = key; tableInfo.preloadedRowsInsertOnly = insertOnly; } + + setTableOption(table: TableInfo, optionName: string, optionValue: string) { + const tableInfo = this.db.tables.find(x => x.pureName == table.pureName && x.schemaName == table.schemaName); + tableInfo[optionName] = optionValue; + } } diff --git a/packages/tools/src/diffTools.ts b/packages/tools/src/diffTools.ts index 6f939cb89..a92506eed 100644 --- a/packages/tools/src/diffTools.ts +++ b/packages/tools/src/diffTools.ts @@ -425,6 +425,20 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf constraintPairs.filter(x => x[0] == null).forEach(x => plan.createConstraint(x[1])); planTablePreload(plan, oldTable, newTable); + + planChangeTableOptions(plan, oldTable, newTable, opts); +} + +function planChangeTableOptions(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo, opts: DbDiffOptions) { + for(const option of plan.dialect?.getTableFormOptions?.('sqlAlterTable') || []) { + if (option.disabled) { + continue; + } + const name = option.name; + if (oldTable[name] != newTable[name] && (oldTable[name]||newTable[name])) { + plan.setTableOption(newTable, name, newTable[name]); + } + } } export function testEqualTables( diff --git a/packages/types/alter-processor.d.ts b/packages/types/alter-processor.d.ts index e2d1d6d71..fae6d4fe9 100644 --- a/packages/types/alter-processor.d.ts +++ b/packages/types/alter-processor.d.ts @@ -15,6 +15,7 @@ export interface AlterProcessor { recreateTable(oldTable: TableInfo, newTable: TableInfo); createSqlObject(obj: SqlObjectInfo); dropSqlObject(obj: SqlObjectInfo); + setTableOption(table: TableInfo, optionName: string, optionValue: string); fillPreloadedRows( table: NamedObjectInfo, oldRows: any[], diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index 9a6460d30..2173a9cbe 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -48,5 +48,6 @@ export interface SqlDialect { getTableFormOptions(intent: 'newTableForm' | 'editTableForm' | 'sqlCreateTable' | 'sqlAlterTable'): { name: string; sqlFormatString: string; + disabled?: boolean; }[]; } diff --git a/packages/web/src/elements/ObjectFieldsEditor.svelte b/packages/web/src/elements/ObjectFieldsEditor.svelte index cc5dcf8ab..caa2fbf63 100644 --- a/packages/web/src/elements/ObjectFieldsEditor.svelte +++ b/packages/web/src/elements/ObjectFieldsEditor.svelte @@ -4,6 +4,7 @@ import FormArgumentList from '../forms/FormArgumentList.svelte'; import { writable } from 'svelte/store'; import FormProviderCore from '../forms/FormProviderCore.svelte'; + import createRef from '../utility/createRef'; export let title; export let fieldDefinitions; diff --git a/packages/web/src/tableeditor/TableEditor.svelte b/packages/web/src/tableeditor/TableEditor.svelte index 63b9d32f2..953ec9cf9 100644 --- a/packages/web/src/tableeditor/TableEditor.svelte +++ b/packages/web/src/tableeditor/TableEditor.svelte @@ -89,6 +89,7 @@ export let setTableInfo; export let dbInfo; export let driver; + export let resetCounter; $: isWritable = !!setTableInfo; @@ -161,19 +162,21 @@
{#if tableFormOptions} - x.name) - )} - onChangeValues={vals => { - if (!_.isEmpty(vals)) { - setTableInfo(tbl => ({ ...tbl, ...vals })); - } - }} - /> + {#key resetCounter} + x.name) + )} + onChangeValues={vals => { + if (!_.isEmpty(vals)) { + setTableInfo(tbl => ({ ...tbl, ...vals })); + } + }} + /> + {/key} {/if} setEditorData(tbl => diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js b/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js index afcc64731..f7842c044 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js @@ -1,6 +1,10 @@ const { SqlDumper } = require('dbgate-tools'); class Dumper extends SqlDumper { + setTableOptionCore(table, optionName, optionValue, formatString) { + this.put('^alter ^table %f ^modify ', table); + this.put(formatString, optionValue); + } } module.exports = Dumper; diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js index f7607b3b3..d0364ce43 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -74,7 +74,7 @@ const dialect = { }, getTableFormOptions(intent) { - const isNewTable = intent == 'newTableForm'; + const isNewTable = intent == 'newTableForm' || intent == 'sqlCreateTable'; return [ { type: isNewTable ? 'dropdowntext' : 'text', diff --git a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js index 1131c2cc7..7be9aaf52 100644 --- a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js @@ -105,21 +105,19 @@ const dialect = { }, getTableFormOptions(intent) { - const isNewTable = intent == 'newTableForm'; return [ { - type: isNewTable ? 'dropdowntext' : 'text', + type: 'dropdowntext', options: this.getSupportedEngines(), label: 'Engine', name: 'tableEngine', sqlFormatString: '^engine = %s', - disabled: !isNewTable, }, { type: 'text', label: 'Comment', name: 'objectComment', - sqlFormatString: '^comment %v', + sqlFormatString: '^comment = %v', }, ]; }, From f74533b42f969cea0e2fda7877baab74754b5a52 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 15:41:26 +0200 Subject: [PATCH 17/57] clickhouse: added specifcNotNull dialect option --- packages/tools/src/SqlDumper.ts | 2 +- packages/types/dbinfo.d.ts | 1 + packages/types/dialect.d.ts | 2 ++ packages/web/src/appobj/ColumnAppObject.svelte | 2 +- packages/web/src/datagrid/ColumnHeaderControl.svelte | 4 ++-- packages/web/src/designer/ColumnLine.svelte | 2 +- packages/web/src/elements/ColumnLabel.svelte | 3 ++- packages/web/src/tableeditor/ColumnEditorModal.svelte | 4 +++- packages/web/src/tableeditor/TableEditor.svelte | 2 +- plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js | 3 ++- plugins/dbgate-plugin-clickhouse/src/frontend/driver.js | 1 + 11 files changed, 17 insertions(+), 9 deletions(-) diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index ec45907a8..82e0eac49 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -246,7 +246,7 @@ export class SqlDumper implements AlterProcessor { this.putRaw(' '); this.specialColumnOptions(column); - if (includeNullable) { + if (includeNullable && !this.dialect?.specificNotNull) { this.put(column.notNull ? '^not ^null' : '^null'); } if (includeDefault && column.defaultValue?.trim()) { diff --git a/packages/types/dbinfo.d.ts b/packages/types/dbinfo.d.ts index 4ee4ac41d..ffa25dd6f 100644 --- a/packages/types/dbinfo.d.ts +++ b/packages/types/dbinfo.d.ts @@ -49,6 +49,7 @@ export interface ColumnInfo extends NamedObjectInfo { notNull?: boolean; autoIncrement?: boolean; dataType: string; + displayedDataType?: string; precision?: number; scale?: number; length?: number; diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index 2173a9cbe..654bf9539 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -34,6 +34,8 @@ export interface SqlDialect { createCheck?: boolean; dropCheck?: boolean; + specificNotNull?: boolean; + // syntax for create column: ALTER TABLE table ADD COLUMN column createColumnWithColumnKeyword?: boolean; diff --git a/packages/web/src/appobj/ColumnAppObject.svelte b/packages/web/src/appobj/ColumnAppObject.svelte index 8ee9f79db..37aa9b6f9 100644 --- a/packages/web/src/appobj/ColumnAppObject.svelte +++ b/packages/web/src/appobj/ColumnAppObject.svelte @@ -31,7 +31,7 @@ return [ { text: 'Rename column', onClick: handleRenameColumn }, { text: 'Drop column', onClick: handleDropColumn }, - { text: 'Copy name', onClick: () => navigator.clipboard.writeText(data.columnName)}, + { text: 'Copy name', onClick: () => navigator.clipboard.writeText(data.columnName) }, ]; } diff --git a/packages/web/src/datagrid/ColumnHeaderControl.svelte b/packages/web/src/datagrid/ColumnHeaderControl.svelte index 9e367a0fa..6083c680d 100644 --- a/packages/web/src/datagrid/ColumnHeaderControl.svelte +++ b/packages/web/src/datagrid/ColumnHeaderControl.svelte @@ -88,9 +88,9 @@ {/if} - {#if _.isString(column.dataType) && !order} + {#if _.isString(column.displayedDataType || column.dataType) && !order} - {column.dataType.toLowerCase()} + {(column.displayedDataType || column.dataType).toLowerCase()} {/if}
diff --git a/packages/web/src/designer/ColumnLine.svelte b/packages/web/src/designer/ColumnLine.svelte index 7128ff89c..26065169d 100644 --- a/packages/web/src/designer/ColumnLine.svelte +++ b/packages/web/src/designer/ColumnLine.svelte @@ -196,7 +196,7 @@
{#if designer?.style?.showDataType && column?.dataType}
- {column?.dataType.toLowerCase()} + {(column?.displayedDataType || column?.dataType).toLowerCase()}
{/if} {#if designer?.style?.showNullability} diff --git a/packages/web/src/elements/ColumnLabel.svelte b/packages/web/src/elements/ColumnLabel.svelte index ffb77ba86..14d74c972 100644 --- a/packages/web/src/elements/ColumnLabel.svelte +++ b/packages/web/src/elements/ColumnLabel.svelte @@ -19,6 +19,7 @@ export let columnName = ''; export let extInfo = null; export let dataType = null; + export let displayedDataType = null; export let showDataType = false; export let foreignKey; export let conid = undefined; @@ -59,7 +60,7 @@ {/if} {:else if dataType} - {dataType.toLowerCase()} + {(displayedDataType || dataType).toLowerCase()} {/if} {/if} diff --git a/packages/web/src/tableeditor/ColumnEditorModal.svelte b/packages/web/src/tableeditor/ColumnEditorModal.svelte index 3411f40ca..5d7a4ffc6 100644 --- a/packages/web/src/tableeditor/ColumnEditorModal.svelte +++ b/packages/web/src/tableeditor/ColumnEditorModal.svelte @@ -32,7 +32,9 @@ - + {#if !driver?.dialect?.specificNotNull} + + {/if} showModal(ColumnEditorModal, { columnInfo: e.detail, tableInfo, setTableInfo, driver })} onAddNew={isWritable ? addColumn : null} columns={[ - { + !driver?.dialect?.specificNotNull && { fieldName: 'notNull', header: 'Nullability', sortable: true, diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js index 91118d28e..662c09dcd 100644 --- a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js @@ -4,9 +4,10 @@ const sql = require('./sql'); function extractDataType(dataType) { if (!dataType) return {}; if (dataType.startsWith('Nullable(')) { - dataType = dataType.substring('Nullable('.length, dataType.length - 1); + const displayedDataType = dataType.substring('Nullable('.length, dataType.length - 1); return { dataType, + displayedDataType, notNull: false, }; } diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js index d0364ce43..3fa0f61ee 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -64,6 +64,7 @@ const dialect = { dropIndex: true, anonymousPrimaryKey: true, createColumnWithColumnKeyword: true, + specificNotNull: true, columnProperties: { columnComment: true, From 08fce96691c9ac1a8eb87a33a5033ef3021e1ffb Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 15:43:14 +0200 Subject: [PATCH 18/57] specificNullabilityImplementation --- packages/tools/src/SqlDumper.ts | 2 +- packages/types/dialect.d.ts | 2 +- packages/web/src/tableeditor/ColumnEditorModal.svelte | 2 +- packages/web/src/tableeditor/TableEditor.svelte | 2 +- plugins/dbgate-plugin-clickhouse/src/frontend/driver.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index 82e0eac49..25ea9c279 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -246,7 +246,7 @@ export class SqlDumper implements AlterProcessor { this.putRaw(' '); this.specialColumnOptions(column); - if (includeNullable && !this.dialect?.specificNotNull) { + if (includeNullable && !this.dialect?.specificNullabilityImplementation) { this.put(column.notNull ? '^not ^null' : '^null'); } if (includeDefault && column.defaultValue?.trim()) { diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index 654bf9539..eff2efe4e 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -34,7 +34,7 @@ export interface SqlDialect { createCheck?: boolean; dropCheck?: boolean; - specificNotNull?: boolean; + specificNullabilityImplementation?: boolean; // syntax for create column: ALTER TABLE table ADD COLUMN column createColumnWithColumnKeyword?: boolean; diff --git a/packages/web/src/tableeditor/ColumnEditorModal.svelte b/packages/web/src/tableeditor/ColumnEditorModal.svelte index 5d7a4ffc6..00060325b 100644 --- a/packages/web/src/tableeditor/ColumnEditorModal.svelte +++ b/packages/web/src/tableeditor/ColumnEditorModal.svelte @@ -32,7 +32,7 @@ - {#if !driver?.dialect?.specificNotNull} + {#if !driver?.dialect?.specificNullabilityImplementation} {/if} diff --git a/packages/web/src/tableeditor/TableEditor.svelte b/packages/web/src/tableeditor/TableEditor.svelte index e330e79df..902963982 100644 --- a/packages/web/src/tableeditor/TableEditor.svelte +++ b/packages/web/src/tableeditor/TableEditor.svelte @@ -187,7 +187,7 @@ on:clickrow={e => showModal(ColumnEditorModal, { columnInfo: e.detail, tableInfo, setTableInfo, driver })} onAddNew={isWritable ? addColumn : null} columns={[ - !driver?.dialect?.specificNotNull && { + !driver?.dialect?.specificNullabilityImplementation && { fieldName: 'notNull', header: 'Nullability', sortable: true, diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js index 3fa0f61ee..68958acc4 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -64,7 +64,7 @@ const dialect = { dropIndex: true, anonymousPrimaryKey: true, createColumnWithColumnKeyword: true, - specificNotNull: true, + specificNullabilityImplementation: true, columnProperties: { columnComment: true, From 33eed816aac4a659efe61cd655e71d885773a03a Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 16:28:35 +0200 Subject: [PATCH 19/57] clickhouse: rename & change column --- packages/tools/src/diffTools.ts | 6 +-- .../src/frontend/Dumper.js | 18 +++++++++ .../src/frontend/driver.js | 37 +++++++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/packages/tools/src/diffTools.ts b/packages/tools/src/diffTools.ts index a92506eed..75101f7af 100644 --- a/packages/tools/src/diffTools.ts +++ b/packages/tools/src/diffTools.ts @@ -407,7 +407,7 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf // console.log('PLAN RENAME COLUMN') plan.renameColumn(x[0], x[1].columnName); } else { - // console.log('PLAN CHANGE COLUMN') + // console.log('PLAN CHANGE COLUMN', x[0], x[1]); plan.changeColumn(x[0], x[1]); } } @@ -430,12 +430,12 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf } function planChangeTableOptions(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo, opts: DbDiffOptions) { - for(const option of plan.dialect?.getTableFormOptions?.('sqlAlterTable') || []) { + for (const option of plan.dialect?.getTableFormOptions?.('sqlAlterTable') || []) { if (option.disabled) { continue; } const name = option.name; - if (oldTable[name] != newTable[name] && (oldTable[name]||newTable[name])) { + if (oldTable[name] != newTable[name] && (oldTable[name] || newTable[name])) { plan.setTableOption(newTable, name, newTable[name]); } } diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js b/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js index f7842c044..e74514cd8 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js @@ -5,6 +5,24 @@ class Dumper extends SqlDumper { this.put('^alter ^table %f ^modify ', table); this.put(formatString, optionValue); } + + changeColumn(oldcol, newcol, constraints) { + if (oldcol.columnName != newcol.columnName) { + this.putCmd('^alter ^table %f ^rename ^column %i ^to %i', oldcol, oldcol.columnName, newcol.columnName); + } + + this.put('^alter ^table %f ^modify ^column %i ', newcol, newcol.columnName); + this.columnDefinition(newcol); + this.endCommand(); + } + + columnType(dataType) { + this.putRaw(dataType); + } + + renameColumn(column, newcol) { + this.putCmd('^alter ^table %f ^rename ^column %i ^to %i', column, column.columnName, newcol); + } } module.exports = Dumper; diff --git a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js index 68958acc4..fc721eae1 100644 --- a/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/frontend/driver.js @@ -50,6 +50,41 @@ const clickhouseEngines = [ 'DeltaLake', ]; +const clickhouseDataTypes = [ + 'Int8', + 'Int16', + 'Int32', + 'Int64', + 'UInt8', + 'UInt16', + 'UInt32', + 'UInt64', + 'Float32', + 'Float64', + 'Decimal', + 'String', + 'FixedString', + 'UUID', + 'Date', + 'DateTime', + 'DateTime64', + "DateTime('UTC')", + 'Date32', + 'Enum8', + 'Enum16', + 'Array', + 'Tuple', + 'Nullable', + 'LowCardinality', + 'Map', + 'JSON', + 'IPv4', + 'IPv6', + 'Nested', + 'AggregateFunction', + 'SimpleAggregateFunction', +]; + /** @type {import('dbgate-types').SqlDialect} */ const dialect = { limitSelect: true, @@ -93,6 +128,8 @@ const dialect = { }, ]; }, + + predefinedDataTypes: clickhouseDataTypes, }; /** @type {import('dbgate-types').EngineDriver} */ From 575f8f23a744b2c46290a910a3ab973b66323f78 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Wed, 11 Sep 2024 16:53:11 +0200 Subject: [PATCH 20/57] clickhouse: sorting key editor support --- packages/tools/src/schemaEditorTools.ts | 4 + packages/tools/src/structureTools.ts | 10 +- packages/types/dbinfo.d.ts | 2 +- packages/types/dialect.d.ts | 3 + .../ForeignKeyObjectListControl.svelte | 2 +- .../tableeditor/PrimaryKeyEditorModal.svelte | 6 +- .../PrimaryKeyLikeListControl.svelte | 72 +++++++++ .../web/src/tableeditor/TableEditor.svelte | 146 +++++++----------- .../src/frontend/driver.js | 3 + 9 files changed, 153 insertions(+), 95 deletions(-) create mode 100644 packages/web/src/tableeditor/PrimaryKeyLikeListControl.svelte diff --git a/packages/tools/src/schemaEditorTools.ts b/packages/tools/src/schemaEditorTools.ts index 18e095198..89e143b91 100644 --- a/packages/tools/src/schemaEditorTools.ts +++ b/packages/tools/src/schemaEditorTools.ts @@ -266,6 +266,10 @@ export function editorDeleteConstraint(table: TableInfo, constraint: ConstraintI res.primaryKey = null; } + if (constraint.constraintType == 'sortingKey') { + res.sortingKey = null; + } + if (constraint.constraintType == 'foreignKey') { res.foreignKeys = table.foreignKeys.filter(x => x.pairingId != constraint.pairingId); } diff --git a/packages/tools/src/structureTools.ts b/packages/tools/src/structureTools.ts index 6583d0cdd..0ef7b9aca 100644 --- a/packages/tools/src/structureTools.ts +++ b/packages/tools/src/structureTools.ts @@ -5,7 +5,7 @@ export function addTableDependencies(db: DatabaseInfo): DatabaseInfo { if (!db.tables) { return db; } - + const allForeignKeys = _flatten(db.tables.map(x => x.foreignKeys || [])); return { ...db, @@ -33,6 +33,14 @@ export function extendTableInfo(table: TableInfo): TableInfo { constraintType: 'primaryKey', } : undefined, + sortingKey: table.sortingKey + ? { + ...table.sortingKey, + pureName: table.pureName, + schemaName: table.schemaName, + constraintType: 'sortingKey', + } + : undefined, foreignKeys: (table.foreignKeys || []).map(cnt => ({ ...cnt, pureName: table.pureName, diff --git a/packages/types/dbinfo.d.ts b/packages/types/dbinfo.d.ts index ffa25dd6f..038ba345e 100644 --- a/packages/types/dbinfo.d.ts +++ b/packages/types/dbinfo.d.ts @@ -15,7 +15,7 @@ export interface ColumnReference { export interface ConstraintInfo extends NamedObjectInfo { pairingId?: string; constraintName?: string; - constraintType: 'primaryKey' | 'foreignKey' | 'index' | 'check' | 'unique'; + constraintType: 'primaryKey' | 'foreignKey' | 'sortingKey' | 'index' | 'check' | 'unique'; } export interface ColumnsConstraintInfo extends ConstraintInfo { diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index eff2efe4e..fdb43eadf 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -35,6 +35,9 @@ export interface SqlDialect { dropCheck?: boolean; specificNullabilityImplementation?: boolean; + omitForeignKeys?: boolean; + omitUniqueConstraints?: boolean; + sortingKeys?: boolean; // syntax for create column: ALTER TABLE table ADD COLUMN column createColumnWithColumnKeyword?: boolean; diff --git a/packages/web/src/elements/ForeignKeyObjectListControl.svelte b/packages/web/src/elements/ForeignKeyObjectListControl.svelte index b56fb3eea..da4417c82 100644 --- a/packages/web/src/elements/ForeignKeyObjectListControl.svelte +++ b/packages/web/src/elements/ForeignKeyObjectListControl.svelte @@ -8,7 +8,7 @@ export let collection; export let title; - export let clickable; + export let clickable = false; export let onRemove = null; export let onAddNew = null; export let emptyMessage = null; diff --git a/packages/web/src/tableeditor/PrimaryKeyEditorModal.svelte b/packages/web/src/tableeditor/PrimaryKeyEditorModal.svelte index fc86db830..05e46c8bb 100644 --- a/packages/web/src/tableeditor/PrimaryKeyEditorModal.svelte +++ b/packages/web/src/tableeditor/PrimaryKeyEditorModal.svelte @@ -5,12 +5,14 @@ export let setTableInfo; export let tableInfo; + export let constraintLabel = 'primary key'; + export let constraintType = 'primaryKey'; + import { editorDeleteConstraint } from 'dbgate-tools'; + + import _ from 'lodash'; + import ConstraintLabel from '../elements/ConstraintLabel.svelte'; + import Link from '../elements/Link.svelte'; + + import ObjectListControl from '../elements/ObjectListControl.svelte'; + import { showModal } from '../modals/modalTools'; + + import PrimaryKeyEditorModal from './PrimaryKeyEditorModal.svelte'; + + export let tableInfo; + export let setTableInfo; + export let isWritable; + + export let constraintLabel = 'primary key'; + export let constraintType = 'primaryKey'; + + $: columns = tableInfo?.columns; + $: keyConstraint = tableInfo?.[constraintType]; + + function addKeyConstraint() { + showModal(PrimaryKeyEditorModal, { + setTableInfo, + tableInfo, + constraintLabel, + constraintType, + }); + } + + + 0 ? addKeyConstraint : null} + clickable + on:clickrow={e => + showModal(PrimaryKeyEditorModal, { + constraintInfo: e.detail, + tableInfo, + setTableInfo, + constraintLabel, + constraintType, + })} + columns={[ + { + fieldName: 'columns', + header: 'Columns', + slot: 0, + }, + isWritable + ? { + fieldName: 'actions', + sortable: true, + slot: 1, + } + : null, + ]} +> + + {row?.columns.map(x => x.columnName).join(', ')} + { + e.stopPropagation(); + setTableInfo(tbl => editorDeleteConstraint(tbl, row)); + }}>Remove + diff --git a/packages/web/src/tableeditor/TableEditor.svelte b/packages/web/src/tableeditor/TableEditor.svelte index 902963982..1ff8dc05c 100644 --- a/packages/web/src/tableeditor/TableEditor.svelte +++ b/packages/web/src/tableeditor/TableEditor.svelte @@ -82,6 +82,7 @@ import PrimaryKeyEditorModal from './PrimaryKeyEditorModal.svelte'; import UniqueEditorModal from './UniqueEditorModal.svelte'; import ObjectFieldsEditor from '../elements/ObjectFieldsEditor.svelte'; + import PrimaryKeyLikeListControl from './PrimaryKeyLikeListControl.svelte'; export const activator = createActivator('TableEditor', true); @@ -262,55 +263,16 @@ - 0 ? addPrimaryKey : null} - clickable - on:clickrow={e => showModal(PrimaryKeyEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo })} - columns={[ - { - fieldName: 'columns', - header: 'Columns', - slot: 0, - }, - isWritable - ? { - fieldName: 'actions', - sortable: true, - slot: 1, - } - : null, - ]} - > - - {row?.columns.map(x => x.columnName).join(', ')} - { - e.stopPropagation(); - setTableInfo(tbl => editorDeleteConstraint(tbl, row)); - }}>Remove - + - {#if sortingKey} - - - {row?.columns.map(x => x.columnName).join(', ')} - + {#if driver?.dialect?.sortingKeys} + {/if} - 0 ? addUnique : null} - title={`Unique constraints (${uniques?.length || 0})`} - emptyMessage={isWritable ? 'No unique defined' : null} - clickable - on:clickrow={e => showModal(UniqueEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo })} - columns={[ - { - fieldName: 'columns', - header: 'Columns', - slot: 0, - }, - isWritable - ? { - fieldName: 'actions', - sortable: true, - slot: 1, - } - : null, - ]} - > - - {row?.columns.map(x => x.columnName).join(', ')} - { - e.stopPropagation(); - setTableInfo(tbl => editorDeleteConstraint(tbl, row)); - }}>Remove 0 ? addUnique : null} + title={`Unique constraints (${uniques?.length || 0})`} + emptyMessage={isWritable ? 'No unique defined' : null} + clickable + on:clickrow={e => showModal(UniqueEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo })} + columns={[ + { + fieldName: 'columns', + header: 'Columns', + slot: 0, + }, + isWritable + ? { + fieldName: 'actions', + sortable: true, + slot: 1, + } + : null, + ]} > - + + {row?.columns.map(x => x.columnName).join(', ')} + { + e.stopPropagation(); + setTableInfo(tbl => editorDeleteConstraint(tbl, row)); + }}>Remove + + {/if} - 0 ? addForeignKey : null} - title={`Foreign keys (${foreignKeys?.length || 0})`} - emptyMessage={isWritable ? 'No foreign key defined' : null} - clickable - onRemove={row => setTableInfo(tbl => editorDeleteConstraint(tbl, row))} - on:clickrow={e => showModal(ForeignKeyEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo, dbInfo })} - /> - + {#if !driver?.dialect?.omitForeignKeys} + 0 ? addForeignKey : null} + title={`Foreign keys (${foreignKeys?.length || 0})`} + emptyMessage={isWritable ? 'No foreign key defined' : null} + clickable + onRemove={row => setTableInfo(tbl => editorDeleteConstraint(tbl, row))} + on:clickrow={e => showModal(ForeignKeyEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo, dbInfo })} + /> + + {/if}