Merge branch 'develop'

This commit is contained in:
SPRINX0\prochazka
2024-09-16 10:10:54 +02:00
80 changed files with 1767 additions and 339 deletions

View File

@@ -108,7 +108,7 @@ jobs:
rm artifacts/builder-debug.yml
- name: Upload artifacts
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.os }}
path: artifacts

View File

@@ -138,7 +138,7 @@ jobs:
rm artifacts/builder-debug.yml
- name: Upload artifacts
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.os }}
path: artifacts

View File

@@ -141,7 +141,7 @@ jobs:
rm artifacts/builder-debug.yml
- name: Upload artifacts
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.os }}
path: artifacts

View File

@@ -144,7 +144,7 @@ jobs:
mv app/dist/dbgate-pad.xml artifacts/ || true
- name: Upload artifacts
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.os }}
path: artifacts

View File

@@ -154,3 +154,8 @@ jobs:
working-directory: plugins/dbgate-plugin-oracle
run: |
npm publish
- name: Publish dbgate-plugin-clickhouse
working-directory: plugins/dbgate-plugin-clickhouse
run: |
npm publish

View File

@@ -77,6 +77,11 @@ jobs:
ACCEPT_EULA: Y
SA_PASSWORD: Pwd2020Db
MSSQL_PID: Express
clickhouse:
image: bitnami/clickhouse:24.8.4
env:
CLICKHOUSE_ADMIN_PASSWORD: Pwd2020Db
# cockroachdb:
# image: cockroachdb/cockroach

View File

@@ -7,7 +7,9 @@ const { getAlterDatabaseScript, extendDatabaseInfo, generateDbPairingId } = requ
function flatSource() {
return _.flatten(
engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
engines
.filter(x => !x.skipReferences)
.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
);
}
@@ -41,7 +43,7 @@ async function testDatabaseDiff(conn, driver, mangle, createObject = null) {
}
describe('Alter database', () => {
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
'Drop referenced table - %s',
testWrapper(async (conn, driver, engine) => {
await testDatabaseDiff(conn, driver, db => {

View File

@@ -6,39 +6,44 @@ const engines = require('../engines');
const crypto = require('crypto');
const { getAlterTableScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
function pickImportantTableInfo(table) {
function pickImportantTableInfo(engine, table) {
const props = ['columnName'];
if (!engine.skipNullability) props.push('notNull');
if (!engine.skipAutoIncrement) props.push('autoIncrement');
return {
pureName: table.pureName,
columns: table.columns
.filter(x => x.columnName != 'rowid')
.map(fp.pick(['columnName', 'notNull', 'autoIncrement'])),
columns: table.columns.filter(x => x.columnName != 'rowid').map(fp.pick(props)),
};
}
function checkTableStructure(t1, t2) {
function checkTableStructure(engine, t1, t2) {
// expect(t1.pureName).toEqual(t2.pureName)
expect(pickImportantTableInfo(t1)).toEqual(pickImportantTableInfo(t2));
expect(pickImportantTableInfo(engine, t1)).toEqual(pickImportantTableInfo(engine, t2));
}
async function testTableDiff(conn, driver, mangle) {
async function testTableDiff(engine, conn, driver, mangle) {
await driver.query(conn, `create table t0 (id int not null primary key)`);
await driver.query(
conn,
`create table t1 (
col_pk int not null primary key,
col_std int null,
col_def int null default 12,
col_fk int null references t0(id),
col_idx int null,
col_uq int null unique,
col_ref int null unique
col_std int,
col_def int default 12,
${engine.skipReferences ? '' : 'col_fk int references t0(id),'}
col_idx int,
col_uq int ${engine.skipUnique ? '' : 'unique'} ,
col_ref int ${engine.skipUnique ? '' : 'unique'}
)`
);
await driver.query(conn, `create index idx1 on t1(col_idx)`);
if (!engine.skipIndexes) {
await driver.query(conn, `create index idx1 on t1(col_idx)`);
}
await driver.query(conn, `create table t2 (id int not null primary key, fkval int null references t1(col_ref))`);
if (!engine.skipReferences) {
await driver.query(conn, `create table t2 (id int not null primary key, fkval int null references t1(col_ref))`);
}
const tget = x => x.tables.find(y => y.pureName == 't1');
const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
@@ -53,7 +58,7 @@ async function testTableDiff(conn, driver, mangle) {
const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));
checkTableStructure(tget(structure2Real), tget(structure2));
checkTableStructure(engine, tget(structure2Real), tget(structure2));
// expect(stableStringify(structure2)).toEqual(stableStringify(structure2Real));
}
@@ -65,14 +70,22 @@ const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'co
// const TESTED_COLUMNS = ['col_ref'];
function engines_columns_source() {
return _.flatten(engines.map(engine => TESTED_COLUMNS.map(column => [engine.label, column, engine])));
return _.flatten(
engines.map(engine =>
TESTED_COLUMNS.filter(col => !col.endsWith('_pk') || !engine.skipPkColumnTesting).map(column => [
engine.label,
column,
engine,
])
)
);
}
describe('Alter table', () => {
test.each(engines.map(engine => [engine.label, engine]))(
'Add column - %s',
testWrapper(async (conn, driver, engine) => {
await testTableDiff(conn, driver, tbl => {
await testTableDiff(engine, conn, driver, tbl => {
tbl.columns.push({
columnName: 'added',
dataType: 'int',
@@ -87,7 +100,7 @@ describe('Alter table', () => {
test.each(engines_columns_source())(
'Drop column - %s - %s',
testWrapper(async (conn, driver, column, engine) => {
await testTableDiff(conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)));
await testTableDiff(engine, conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)));
})
);
@@ -95,6 +108,7 @@ describe('Alter table', () => {
'Change nullability - %s - %s',
testWrapper(async (conn, driver, column, engine) => {
await testTableDiff(
engine,
conn,
driver,
tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, notNull: true } : x)))
@@ -106,6 +120,7 @@ describe('Alter table', () => {
'Rename column - %s - %s',
testWrapper(async (conn, driver, column, engine) => {
await testTableDiff(
engine,
conn,
driver,
tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x)))
@@ -116,7 +131,7 @@ describe('Alter table', () => {
test.each(engines.map(engine => [engine.label, engine]))(
'Drop index - %s',
testWrapper(async (conn, driver, engine) => {
await testTableDiff(conn, driver, tbl => {
await testTableDiff(engine, conn, driver, tbl => {
tbl.indexes = [];
});
})

View File

@@ -5,7 +5,7 @@ const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator');
const { runCommandOnDriver } = require('dbgate-tools');
describe('Data duplicator', () => {
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipDataDuplicator).map(engine => [engine.label, engine]))(
'Insert simple data - %s',
testWrapper(async (conn, driver, engine) => {
runCommandOnDriver(conn, driver, dmp =>

View File

@@ -167,7 +167,7 @@ describe('Deploy database', () => {
})
);
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
'Foreign keys - %s',
testWrapper(async (conn, driver, engine) => {
await testDatabaseDeploy(
@@ -222,7 +222,7 @@ describe('Deploy database', () => {
})
);
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
'Deploy preloaded data - %s',
testWrapper(async (conn, driver, engine) => {
await testDatabaseDeploy(conn, driver, [
@@ -251,7 +251,7 @@ describe('Deploy database', () => {
})
);
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
'Deploy preloaded data - update - %s',
testWrapper(async (conn, driver, engine) => {
await testDatabaseDeploy(conn, driver, [

View File

@@ -2,7 +2,7 @@ const { testWrapper } = require('../tools');
const engines = require('../engines');
const _ = require('lodash');
const initSql = ['CREATE TABLE t1 (id int)', 'CREATE TABLE t2 (id int)'];
const initSql = ['CREATE TABLE t1 (id int primary key)', 'CREATE TABLE t2 (id int primary key)'];
function flatSource() {
return _.flatten(
@@ -26,9 +26,9 @@ describe('Object analyse', () => {
test.each(flatSource())(
'Full analysis - %s - %s',
testWrapper(async (conn, driver, type, object, engine) => {
for (const sql of initSql) await driver.query(conn, sql);
for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
await driver.query(conn, object.create1);
await driver.query(conn, object.create1, { discardResult: true });
const structure = await driver.analyseFull(conn);
expect(structure[type].length).toEqual(1);
@@ -39,11 +39,11 @@ describe('Object analyse', () => {
test.each(flatSource())(
'Incremental analysis - add - %s - %s',
testWrapper(async (conn, driver, type, object, engine) => {
for (const sql of initSql) await driver.query(conn, sql);
for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
await driver.query(conn, object.create2);
await driver.query(conn, object.create2, { discardResult: true });
const structure1 = await driver.analyseFull(conn);
await driver.query(conn, object.create1);
await driver.query(conn, object.create1, { discardResult: true });
const structure2 = await driver.analyseIncremental(conn, structure1);
expect(structure2[type].length).toEqual(2);
@@ -54,12 +54,12 @@ describe('Object analyse', () => {
test.each(flatSource())(
'Incremental analysis - drop - %s - %s',
testWrapper(async (conn, driver, type, object, engine) => {
for (const sql of initSql) await driver.query(conn, sql);
for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
await driver.query(conn, object.create1);
await driver.query(conn, object.create2);
await driver.query(conn, object.create1, { discardResult: true });
await driver.query(conn, object.create2, { discardResult: true });
const structure1 = await driver.analyseFull(conn);
await driver.query(conn, object.drop2);
await driver.query(conn, object.drop2, { discardResult: true });
const structure2 = await driver.analyseIncremental(conn, structure1);
expect(structure2[type].length).toEqual(1);
@@ -70,15 +70,15 @@ describe('Object analyse', () => {
test.each(flatSource())(
'Create SQL - add - %s - %s',
testWrapper(async (conn, driver, type, object, engine) => {
for (const sql of initSql) await driver.query(conn, sql);
for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
await driver.query(conn, object.create1);
await driver.query(conn, object.create1, { discardResult: true });
const structure1 = await driver.analyseFull(conn);
await driver.query(conn, object.drop1);
await driver.query(conn, object.drop1, { discardResult: true });
const structure2 = await driver.analyseIncremental(conn, structure1);
expect(structure2[type].length).toEqual(0);
await driver.query(conn, structure1[type][0].createSql);
await driver.query(conn, structure1[type][0].createSql, { discardResult: true });
const structure3 = await driver.analyseIncremental(conn, structure2);

View File

@@ -2,7 +2,11 @@ const engines = require('../engines');
const { splitQuery } = require('dbgate-query-splitter');
const { testWrapper } = require('../tools');
const initSql = ['CREATE TABLE t1 (id int)', 'INSERT INTO t1 (id) VALUES (1)', 'INSERT INTO t1 (id) VALUES (2)'];
const initSql = [
'CREATE TABLE t1 (id int primary key)',
'INSERT INTO t1 (id) VALUES (1)',
'INSERT INTO t1 (id) VALUES (2)',
];
expect.extend({
dataRow(row, expected) {
@@ -64,7 +68,7 @@ describe('Query', () => {
test.each(engines.map(engine => [engine.label, engine]))(
'Simple query - %s',
testWrapper(async (conn, driver, engine) => {
for (const sql of initSql) await driver.query(conn, sql);
for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
const res = await driver.query(conn, 'SELECT id FROM t1 ORDER BY id');
expect(res.columns).toEqual([
@@ -87,7 +91,7 @@ describe('Query', () => {
test.each(engines.map(engine => [engine.label, engine]))(
'Simple stream query - %s',
testWrapper(async (conn, driver, engine) => {
for (const sql of initSql) await driver.query(conn, sql);
for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
const results = await executeStream(driver, conn, 'SELECT id FROM t1 ORDER BY id');
expect(results.length).toEqual(1);
const res = results[0];
@@ -100,7 +104,7 @@ describe('Query', () => {
test.each(engines.map(engine => [engine.label, engine]))(
'More queries - %s',
testWrapper(async (conn, driver, engine) => {
for (const sql of initSql) await driver.query(conn, sql);
for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
const results = await executeStream(
driver,
conn,
@@ -124,7 +128,7 @@ describe('Query', () => {
const results = await executeStream(
driver,
conn,
'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
'CREATE TABLE t1 (id int primary key); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
);
expect(results.length).toEqual(1);
@@ -146,14 +150,15 @@ describe('Query', () => {
})
);
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
'Save data query - %s',
testWrapper(async (conn, driver, engine) => {
for (const sql of initSql) await driver.query(conn, sql);
for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
await driver.script(
conn,
'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;'
'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;',
{ discardResult: true }
);
const res = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
// console.log(res);

View File

@@ -1,32 +1,37 @@
const engines = require('../engines');
const { testWrapper } = require('../tools');
const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50) null)';
const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50))';
const ix1Sql = 'CREATE index ix1 ON t1(val1, id)';
const t2Sql = 'CREATE TABLE t2 (id int not null primary key, val2 varchar(50) null unique)';
const t2Sql = engine =>
`CREATE TABLE t2 (id int not null primary key, val2 varchar(50) ${engine.skipUnique ? '' : 'unique'})`;
const t3Sql = 'CREATE TABLE t3 (id int not null primary key, valfk int, foreign key (valfk) references t2(id))';
// const fkSql = 'ALTER TABLE t3 ADD FOREIGN KEY (valfk) REFERENCES t2(id)'
const txMatch = (tname, vcolname, nextcol) =>
const txMatch = (engine, tname, vcolname, nextcol) =>
expect.objectContaining({
pureName: tname,
columns: [
expect.objectContaining({
columnName: 'id',
notNull: true,
dataType: expect.stringMatching(/int/i),
dataType: expect.stringMatching(/int.*/i),
...(engine.skipNullability ? {} : { notNull: true }),
}),
expect.objectContaining({
columnName: vcolname,
notNull: false,
dataType: expect.stringMatching(/.*char.*\(50\)/),
...(engine.skipNullability ? {} : { notNull: false }),
dataType: engine.skipStringLength
? expect.stringMatching(/.*string|char.*/i)
: expect.stringMatching(/.*char.*\(50\)/i),
}),
...(nextcol
? [
expect.objectContaining({
columnName: 'nextcol',
notNull: false,
dataType: expect.stringMatching(/.*char.*\(50\)/),
...(engine.skipNullability ? {} : { notNull: false }),
dataType: engine.skipStringLength
? expect.stringMatching(/.*string.*|char.*/i)
: expect.stringMatching(/.*char.*\(50\).*/i),
}),
]
: []),
@@ -40,9 +45,9 @@ const txMatch = (tname, vcolname, nextcol) =>
}),
});
const t1Match = txMatch('t1', 'val1');
const t2Match = txMatch('t2', 'val2');
const t2NextColMatch = txMatch('t2', 'val2', true);
const t1Match = engine => txMatch(engine, 't1', 'val1');
const t2Match = engine => txMatch(engine, 't2', 'val2');
const t2NextColMatch = engine => txMatch(engine, 't2', 'val2', true);
describe('Table analyse', () => {
test.each(engines.map(engine => [engine.label, engine]))(
@@ -53,25 +58,25 @@ describe('Table analyse', () => {
const structure = await driver.analyseFull(conn);
expect(structure.tables.length).toEqual(1);
expect(structure.tables[0]).toEqual(t1Match);
expect(structure.tables[0]).toEqual(t1Match(engine));
})
);
test.each(engines.map(engine => [engine.label, engine]))(
'Table add - incremental analysis - %s',
testWrapper(async (conn, driver, engine) => {
await driver.query(conn, t2Sql);
await driver.query(conn, t2Sql(engine));
const structure1 = await driver.analyseFull(conn);
expect(structure1.tables.length).toEqual(1);
expect(structure1.tables[0]).toEqual(t2Match);
expect(structure1.tables[0]).toEqual(t2Match(engine));
await driver.query(conn, t1Sql);
const structure2 = await driver.analyseIncremental(conn, structure1);
expect(structure2.tables.length).toEqual(2);
expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match(engine));
expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match(engine));
})
);
@@ -79,17 +84,17 @@ describe('Table analyse', () => {
'Table remove - incremental analysis - %s',
testWrapper(async (conn, driver, engine) => {
await driver.query(conn, t1Sql);
await driver.query(conn, t2Sql);
await driver.query(conn, t2Sql(engine));
const structure1 = await driver.analyseFull(conn);
expect(structure1.tables.length).toEqual(2);
expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match(engine));
expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match(engine));
await driver.query(conn, 'DROP TABLE t2');
const structure2 = await driver.analyseIncremental(conn, structure1);
expect(structure2.tables.length).toEqual(1);
expect(structure2.tables[0]).toEqual(t1Match);
expect(structure2.tables[0]).toEqual(t1Match(engine));
})
);
@@ -97,23 +102,26 @@ describe('Table analyse', () => {
'Table change - incremental analysis - %s',
testWrapper(async (conn, driver, engine) => {
await driver.query(conn, t1Sql);
await driver.query(conn, t2Sql);
await driver.query(conn, t2Sql(engine));
const structure1 = await driver.analyseFull(conn);
if (engine.dbSnapshotBySeconds) await new Promise(resolve => setTimeout(resolve, 1100));
await driver.query(conn, 'ALTER TABLE t2 ADD nextcol varchar(50)');
await driver.query(
conn,
`ALTER TABLE t2 ADD ${engine.alterTableAddColumnSyntax ? 'COLUMN' : ''} nextcol varchar(50)`
);
const structure2 = await driver.analyseIncremental(conn, structure1);
expect(structure2).toBeTruthy(); // if falsy, no modification is detected
expect(structure2.tables.length).toEqual(2);
expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch);
expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match(engine));
expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch(engine));
})
);
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipIndexes).map(engine => [engine.label, engine]))(
'Index - full analysis - %s',
testWrapper(async (conn, driver, engine) => {
await driver.query(conn, t1Sql);
@@ -128,10 +136,10 @@ describe('Table analyse', () => {
})
);
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipUnique).map(engine => [engine.label, engine]))(
'Unique - full analysis - %s',
testWrapper(async (conn, driver, engine) => {
await driver.query(conn, t2Sql);
await driver.query(conn, t2Sql(engine));
const structure = await driver.analyseFull(conn);
const t2 = structure.tables.find(x => x.pureName == 't2');
@@ -142,10 +150,10 @@ describe('Table analyse', () => {
})
);
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
'Foreign key - full analysis - %s',
testWrapper(async (conn, driver, engine) => {
await driver.query(conn, t2Sql);
await driver.query(conn, t2Sql(engine));
await driver.query(conn, t3Sql);
// await driver.query(conn, fkSql);

View File

@@ -62,7 +62,7 @@ describe('Table create', () => {
})
);
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipIndexes).map(engine => [engine.label, engine]))(
'Table with index - %s',
testWrapper(async (conn, driver, engine) => {
await testTableCreate(conn, driver, {
@@ -92,7 +92,7 @@ describe('Table create', () => {
})
);
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
'Table with foreign key - %s',
testWrapper(async (conn, driver, engine) => {
await testTableCreate(conn, driver, {
@@ -122,7 +122,7 @@ describe('Table create', () => {
})
);
test.each(engines.map(engine => [engine.label, engine]))(
test.each(engines.filter(x => !x.skipUnique).map(engine => [engine.label, engine]))(
'Table with unique - %s',
testWrapper(async (conn, driver, engine) => {
await testTableCreate(conn, driver, {

View File

@@ -26,15 +26,23 @@ services:
# environment:
# - MYSQL_ROOT_PASSWORD=Pwd2020Db
mssql:
image: mcr.microsoft.com/mssql/server
clickhouse:
image: bitnami/clickhouse:24.8.4
restart: always
ports:
- 15002:1433
- 15005:8123
environment:
- ACCEPT_EULA=Y
- SA_PASSWORD=Pwd2020Db
- MSSQL_PID=Express
- CLICKHOUSE_ADMIN_PASSWORD=Pwd2020Db
# mssql:
# image: mcr.microsoft.com/mssql/server
# restart: always
# ports:
# - 15002:1433
# environment:
# - ACCEPT_EULA=Y
# - SA_PASSWORD=Pwd2020Db
# - MSSQL_PID=Express
# cockroachdb:
# image: cockroachdb/cockroach

View File

@@ -129,6 +129,30 @@ const engines = [
skipOnCI: true,
objects: [views, matviews],
},
{
label: 'ClickHouse',
connection: {
engine: 'clickhouse@dbgate-plugin-clickhouse',
databaseUrl: 'http://clickhouse:8123',
password: 'Pwd2020Db',
},
local: {
databaseUrl: 'http://localhost:15005',
},
skipOnCI: false,
objects: [views],
skipDataModifications: true,
skipReferences: true,
skipIndexes: true,
skipNullability: true,
skipUnique: true,
skipAutoIncrement: true,
skipPkColumnTesting: true,
skipDataDuplicator: true,
skipStringLength: true,
alterTableAddColumnSyntax: true,
dbSnapshotBySeconds: true,
},
];
const filterLocal = [
@@ -137,8 +161,9 @@ const filterLocal = [
'-MariaDB',
'-PostgreSQL',
'-SQL Server',
'SQLite',
'-SQLite',
'-CockroachDB',
'ClickHouse',
];
const enginesPostgre = engines.filter(x => x.label == 'PostgreSQL');

View File

@@ -0,0 +1,3 @@
module.exports = {
setupFilesAfterEnv: ['<rootDir>/setupTests.js'],
};

View File

@@ -11,12 +11,9 @@
"scripts": {
"wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js",
"wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
"test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
"test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-duplicator.spec.js",
"test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
"run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
},
"jest": {
@@ -24,7 +21,7 @@
},
"devDependencies": {
"cross-env": "^7.0.3",
"jest": "^27.0.1"
},
"dependencies": {}
"jest": "^27.0.1",
"pino-pretty": "^11.2.2"
}
}

View File

@@ -0,0 +1,22 @@
const { prettyFactory } = require('pino-pretty');
const pretty = prettyFactory({
colorize: true,
translateTime: 'SYS:standard',
ignore: 'pid,hostname',
});
global.console = {
...console,
log: (...messages) => {
try {
const parsedMessage = JSON.parse(messages[0]);
process.stdout.write(pretty(parsedMessage));
} catch (error) {
process.stdout.write(messages.join(' ') + '\n');
}
},
debug: (...messages) => {
process.stdout.write(messages.join(' ') + '\n');
},
};

View File

@@ -1,6 +1,6 @@
{
"private": true,
"version": "5.4.4",
"version": "5.4.5-alpha.5",
"name": "dbgate-all",
"workspaces": [
"packages/*",

View File

@@ -115,7 +115,7 @@ async function handleDatabaseOp(op, { msgid, name }) {
const dmp = driver.createDumper();
dmp[op](name);
logger.info({ sql: dmp.s }, 'Running script');
await driver.query(systemConnection, dmp.s);
await driver.query(systemConnection, dmp.s, { discardResult: true });
}
await handleRefresh();

View File

@@ -62,10 +62,13 @@ export function dumpSqlSelect(dmp: SqlDumper, cmd: Select) {
}
export function dumpSqlUpdate(dmp: SqlDumper, cmd: Update) {
dmp.put('^update ');
dumpSqlSourceRef(dmp, cmd.from);
dmp.put('&n^set ');
if (cmd.alterTableUpdateSyntax) {
dmp.put('^alter ^table %f &n^update ', cmd.from?.name);
} else {
dmp.put('^update ');
dumpSqlSourceRef(dmp, cmd.from);
dmp.put('&n^set ');
}
dmp.put('&>');
dmp.putCollection(', ', cmd.fields, col => {
dmp.put('%i=', col.targetColumn);
@@ -81,8 +84,14 @@ export function dumpSqlUpdate(dmp: SqlDumper, cmd: Update) {
}
export function dumpSqlDelete(dmp: SqlDumper, cmd: Delete) {
dmp.put('^delete ^from ');
dumpSqlSourceRef(dmp, cmd.from);
if (cmd.alterTableDeleteSyntax) {
dmp.put('^alter ^table ');
dumpSqlSourceRef(dmp, cmd.from);
dmp.put(' ^delete ');
} else {
dmp.put('^delete ^from ');
dumpSqlSourceRef(dmp, cmd.from);
}
if (cmd.where) {
dmp.put('&n^where ');

View File

@@ -26,12 +26,17 @@ export interface Update {
fields: UpdateField[];
from: FromDefinition;
where?: Condition;
// ALTER TABLE xxx UPDATE col1=val1 - syntax for ClickHouse
alterTableUpdateSyntax?: boolean;
}
export interface Delete {
commandType: 'delete';
from: FromDefinition;
where?: Condition;
// ALTER TABLE xxx DELETE - syntax for ClickHouse
alterTableDeleteSyntax?: boolean;
}
export interface Insert {

View File

@@ -246,7 +246,7 @@ export class SqlDumper implements AlterProcessor {
this.putRaw(' ');
this.specialColumnOptions(column);
if (includeNullable) {
if (includeNullable && !this.dialect?.specificNullabilityImplementation) {
this.put(column.notNull ? '^not ^null' : '^null');
}
if (includeDefault && column.defaultValue?.trim()) {
@@ -294,12 +294,25 @@ export class SqlDumper implements AlterProcessor {
});
this.put('&<&n)');
this.tableOptions(table);
this.endCommand();
(table.indexes || []).forEach(ix => {
this.createIndex(ix);
});
}
tableOptions(table: TableInfo) {
const options = this.driver?.dialect?.getTableFormOptions?.('sqlCreateTable') || [];
for (const option of options) {
if (table[option.name]) {
this.put('&n');
this.put(option.sqlFormatString, table[option.name]);
}
}
}
createTablePrimaryKeyCore(table: TableInfo) {
if (table.primaryKey) {
this.put(',&n');
@@ -531,7 +544,9 @@ export class SqlDumper implements AlterProcessor {
renameConstraint(constraint: ConstraintInfo, newName: string) {}
createColumn(column: ColumnInfo, constraints: ConstraintInfo[]) {
this.put('^alter ^table %f ^add %i ', column, column.columnName);
this.put('^alter ^table %f ^add ', column);
if (this.dialect.createColumnWithColumnKeyword) this.put('^column ');
this.put(' %i ', column.columnName);
this.columnDefinition(column);
this.inlineConstraints(constraints);
this.endCommand();
@@ -607,10 +622,8 @@ export class SqlDumper implements AlterProcessor {
if (!oldTable.pairingId || !newTable.pairingId || oldTable.pairingId != newTable.pairingId) {
throw new Error('Recreate is not possible: oldTable.paringId != newTable.paringId');
}
const tmpTable = `temp_${uuidv1()}`;
// console.log('oldTable', oldTable);
// console.log('newTable', newTable);
const tmpTable = `temp_${uuidv1()}`;
const columnPairs = oldTable.columns
.map(oldcol => ({
@@ -619,33 +632,49 @@ export class SqlDumper implements AlterProcessor {
}))
.filter(x => x.newcol);
this.dropConstraints(oldTable, true);
this.renameTable(oldTable, tmpTable);
if (this.driver.supportsTransactions) {
this.dropConstraints(oldTable, true);
this.renameTable(oldTable, tmpTable);
this.createTable(newTable);
this.createTable(newTable);
const autoinc = newTable.columns.find(x => x.autoIncrement);
if (autoinc) {
this.allowIdentityInsert(newTable, true);
const autoinc = newTable.columns.find(x => x.autoIncrement);
if (autoinc) {
this.allowIdentityInsert(newTable, true);
}
this.putCmd(
'^insert ^into %f (%,i) select %,s ^from %f',
newTable,
columnPairs.map(x => x.newcol.columnName),
columnPairs.map(x => x.oldcol.columnName),
{ ...oldTable, pureName: tmpTable }
);
if (autoinc) {
this.allowIdentityInsert(newTable, false);
}
if (this.dialect.dropForeignKey) {
newTable.dependencies.forEach(cnt => this.createConstraint(cnt));
}
this.dropTable({ ...oldTable, pureName: tmpTable });
} else {
// we have to preserve old table as long as possible
this.createTable({ ...newTable, pureName: tmpTable });
this.putCmd(
'^insert ^into %f (%,i) select %,s ^from %f',
{ ...newTable, pureName: tmpTable },
columnPairs.map(x => x.newcol.columnName),
columnPairs.map(x => x.oldcol.columnName),
oldTable
);
this.dropTable(oldTable);
this.renameTable({ ...newTable, pureName: tmpTable }, newTable.pureName);
}
this.putCmd(
'^insert ^into %f (%,i) select %,s ^from %f',
newTable,
columnPairs.map(x => x.newcol.columnName),
columnPairs.map(x => x.oldcol.columnName),
{ ...oldTable, pureName: tmpTable }
);
if (autoinc) {
this.allowIdentityInsert(newTable, false);
}
if (this.dialect.dropForeignKey) {
newTable.dependencies.forEach(cnt => this.createConstraint(cnt));
}
this.dropTable({ ...oldTable, pureName: tmpTable });
}
createSqlObject(obj: SqlObjectInfo) {
@@ -671,6 +700,23 @@ export class SqlDumper implements AlterProcessor {
this.putCmd('^drop %s %f', this.getSqlObjectSqlName(obj.objectTypeField), obj);
}
setTableOption(table: TableInfo, optionName: string, optionValue: string) {
const options = this?.dialect?.getTableFormOptions?.('sqlAlterTable');
const option = options?.find(x => x.name == optionName && !x.disabled);
if (!option) {
return;
}
this.setTableOptionCore(table, optionName, optionValue, option.sqlFormatString);
this.endCommand();
}
setTableOptionCore(table: TableInfo, optionName: string, optionValue: string, formatString: string) {
this.put('^alter ^table %f ', table);
this.put(formatString, optionValue);
}
fillPreloadedRows(
table: NamedObjectInfo,
oldRows: any[],

View File

@@ -97,6 +97,13 @@ interface AlterOperation_FillPreloadedRows {
autoIncrementColumn: string;
}
interface AlterOperation_SetTableOption {
operationType: 'setTableOption';
table: TableInfo;
optionName: string;
optionValue: string;
}
type AlterOperation =
| AlterOperation_CreateColumn
| AlterOperation_ChangeColumn
@@ -112,7 +119,8 @@ type AlterOperation =
| AlterOperation_CreateSqlObject
| AlterOperation_DropSqlObject
| AlterOperation_RecreateTable
| AlterOperation_FillPreloadedRows;
| AlterOperation_FillPreloadedRows
| AlterOperation_SetTableOption;
export class AlterPlan {
recreates = {
@@ -253,6 +261,15 @@ export class AlterPlan {
});
}
setTableOption(table: TableInfo, optionName: string, optionValue: string) {
this.operations.push({
operationType: 'setTableOption',
table,
optionName,
optionValue,
});
}
run(processor: AlterProcessor) {
for (const op of this.operations) {
runAlterOperation(op, processor);
@@ -267,6 +284,7 @@ export class AlterPlan {
: [];
const constraints = _.compact([
dependencyDefinition?.includes('primaryKey') ? table.primaryKey : null,
dependencyDefinition?.includes('sortingKey') ? table.sortingKey : null,
...(dependencyDefinition?.includes('foreignKeys') ? table.foreignKeys : []),
...(dependencyDefinition?.includes('indexes') ? table.indexes : []),
...(dependencyDefinition?.includes('uniques') ? table.uniques : []),
@@ -297,35 +315,40 @@ export class AlterPlan {
return res;
}
if (op.operationType == 'changeColumn') {
const constraints = this._getDependendColumnConstraints(op.oldObject, this.dialect.changeColumnDependencies);
for (const [testedOperationType, testedDependencies, testedObject] of [
['changeColumn', this.dialect.changeColumnDependencies, (op as AlterOperation_ChangeColumn).oldObject],
['renameColumn', this.dialect.renameColumnDependencies, (op as AlterOperation_RenameColumn).object],
]) {
if (op.operationType == testedOperationType) {
const constraints = this._getDependendColumnConstraints(testedObject as ColumnInfo, testedDependencies);
if (constraints.length > 0 && this.opts.noDropConstraint) {
return [];
if (constraints.length > 0 && this.opts.noDropConstraint) {
return [];
}
const res: AlterOperation[] = [
...constraints.map(oldObject => {
const opRes: AlterOperation = {
operationType: 'dropConstraint',
oldObject,
};
return opRes;
}),
op,
..._.reverse([...constraints]).map(newObject => {
const opRes: AlterOperation = {
operationType: 'createConstraint',
newObject,
};
return opRes;
}),
];
if (constraints.length > 0) {
this.recreates.constraints += 1;
}
return res;
}
const res: AlterOperation[] = [
...constraints.map(oldObject => {
const opRes: AlterOperation = {
operationType: 'dropConstraint',
oldObject,
};
return opRes;
}),
op,
..._.reverse([...constraints]).map(newObject => {
const opRes: AlterOperation = {
operationType: 'createConstraint',
newObject,
};
return opRes;
}),
];
if (constraints.length > 0) {
this.recreates.constraints += 1;
}
return res;
}
if (op.operationType == 'dropTable') {
@@ -374,7 +397,8 @@ export class AlterPlan {
this._testTableRecreate(op, 'dropColumn', this.dialect.dropColumn, 'oldObject') ||
this._testTableRecreate(op, 'createConstraint', obj => this._canCreateConstraint(obj), 'newObject') ||
this._testTableRecreate(op, 'dropConstraint', obj => this._canDropConstraint(obj), 'oldObject') ||
this._testTableRecreate(op, 'changeColumn', this.dialect.changeColumn, 'newObject') || [op]
this._testTableRecreate(op, 'changeColumn', this.dialect.changeColumn, 'newObject') ||
this._testTableRecreate(op, 'renameColumn', true, 'object') || [op]
);
});
@@ -383,6 +407,7 @@ export class AlterPlan {
_canCreateConstraint(cnt: ConstraintInfo) {
if (cnt.constraintType == 'primaryKey') return this.dialect.createPrimaryKey;
if (cnt.constraintType == 'sortingKey') return this.dialect.createPrimaryKey;
if (cnt.constraintType == 'foreignKey') return this.dialect.createForeignKey;
if (cnt.constraintType == 'index') return this.dialect.createIndex;
if (cnt.constraintType == 'unique') return this.dialect.createUnique;
@@ -392,6 +417,7 @@ export class AlterPlan {
_canDropConstraint(cnt: ConstraintInfo) {
if (cnt.constraintType == 'primaryKey') return this.dialect.dropPrimaryKey;
if (cnt.constraintType == 'sortingKey') return this.dialect.dropPrimaryKey;
if (cnt.constraintType == 'foreignKey') return this.dialect.dropForeignKey;
if (cnt.constraintType == 'index') return this.dialect.dropIndex;
if (cnt.constraintType == 'unique') return this.dialect.dropUnique;
@@ -453,7 +479,7 @@ export class AlterPlan {
}
} else {
// @ts-ignore
const oldObject: TableInfo = op.oldObject;
const oldObject: TableInfo = op.oldObject || op.object;
if (oldObject) {
const recreated = recreates[`${oldObject.schemaName}||${oldObject.pureName}`];
if (recreated) {
@@ -575,6 +601,9 @@ export function runAlterOperation(op: AlterOperation, processor: AlterProcessor)
case 'dropSqlObject':
processor.dropSqlObject(op.oldObject);
break;
case 'setTableOption':
processor.setTableOption(op.table, op.optionName, op.optionValue);
break;
case 'fillPreloadedRows':
processor.fillPreloadedRows(op.table, op.oldRows, op.newRows, op.key, op.insertOnly, op.autoIncrementColumn);
break;

View File

@@ -37,7 +37,8 @@ export function createBulkInsertStreamBase(driver: EngineDriver, stream, pool, n
}
if (options.createIfNotExists && (!structure || options.dropIfExists)) {
const dmp = driver.createDumper();
dmp.createTable(prepareTableForImport({ ...writable.structure, ...name }));
const createdTableInfo = driver.adaptTableInfo(prepareTableForImport({ ...writable.structure, ...name }));
dmp.createTable(createdTableInfo);
logger.info({ sql: dmp.s }, `Creating table ${fullNameQuoted}`);
await driver.script(pool, dmp.s);
structure = await driver.analyseSingleTable(pool, name);

View File

@@ -11,6 +11,7 @@ import {
UniqueInfo,
SqlObjectInfo,
NamedObjectInfo,
ColumnsConstraintInfo,
} from '../../types';
export class DatabaseInfoAlterProcessor {
@@ -59,6 +60,9 @@ export class DatabaseInfoAlterProcessor {
case 'primaryKey':
table.primaryKey = constraint as PrimaryKeyInfo;
break;
case 'sortingKey':
table.sortingKey = constraint as ColumnsConstraintInfo;
break;
case 'foreignKey':
table.foreignKeys.push(constraint as ForeignKeyInfo);
break;
@@ -86,6 +90,9 @@ export class DatabaseInfoAlterProcessor {
case 'primaryKey':
table.primaryKey = null;
break;
case 'sortingKey':
table.sortingKey = null;
break;
case 'foreignKey':
table.foreignKeys = table.foreignKeys.filter(x => x.constraintName != constraint.constraintName);
break;
@@ -129,4 +136,9 @@ export class DatabaseInfoAlterProcessor {
tableInfo.preloadedRowsKey = key;
tableInfo.preloadedRowsInsertOnly = insertOnly;
}
setTableOption(table: TableInfo, optionName: string, optionValue: string) {
const tableInfo = this.db.tables.find(x => x.pureName == table.pureName && x.schemaName == table.schemaName);
tableInfo[optionName] = optionValue;
}
}

View File

@@ -46,6 +46,14 @@ export function generateTablePairingId(table: TableInfo): TableInfo {
if (!table.pairingId) {
return {
...table,
primaryKey: table.primaryKey && {
...table.primaryKey,
pairingId: table.primaryKey.pairingId || uuidv1(),
},
sortingKey: table.sortingKey && {
...table.sortingKey,
pairingId: table.sortingKey.pairingId || uuidv1(),
},
columns: table.columns?.map(col => ({
...col,
pairingId: col.pairingId || uuidv1(),
@@ -335,6 +343,7 @@ export function testEqualTypes(a: ColumnInfo, b: ColumnInfo, opts: DbDiffOptions
function getTableConstraints(table: TableInfo) {
const res = [];
if (table.primaryKey) res.push(table.primaryKey);
if (table.sortingKey) res.push(table.sortingKey);
if (table.foreignKeys) res.push(...table.foreignKeys);
if (table.indexes) res.push(...table.indexes);
if (table.uniques) res.push(...table.uniques);
@@ -345,7 +354,9 @@ function getTableConstraints(table: TableInfo) {
function createPairs(oldList, newList, additionalCondition = null) {
const res = [];
for (const a of oldList) {
const b = newList.find(x => x.pairingId == a.pairingId || (additionalCondition && additionalCondition(a, x)));
const b = newList.find(
x => (a.pairingId && x.pairingId == a.pairingId) || (additionalCondition && additionalCondition(a, x))
);
if (b) {
res.push([a, b]);
} else {
@@ -381,9 +392,14 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
const constraintPairs = createPairs(
getTableConstraints(oldTable),
getTableConstraints(newTable),
(a, b) => a.constraintType == 'primaryKey' && b.constraintType == 'primaryKey'
(a, b) =>
(a.constraintType == 'primaryKey' && b.constraintType == 'primaryKey') ||
(a.constraintType == 'sortingKey' && b.constraintType == 'sortingKey')
);
// console.log('constraintPairs SOURCE', getTableConstraints(oldTable), getTableConstraints(newTable));
// console.log('constraintPairs OLD TABLE', oldTable);
// console.log('constraintPairs NEW TABLE', newTable);
// console.log('constraintPairs SOURCE OLD', getTableConstraints(oldTable));
// console.log('constraintPairs SOURCE NEW', getTableConstraints(newTable));
// console.log('constraintPairs', constraintPairs);
if (!opts.noDropConstraint) {
@@ -407,7 +423,7 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
// console.log('PLAN RENAME COLUMN')
plan.renameColumn(x[0], x[1].columnName);
} else {
// console.log('PLAN CHANGE COLUMN')
// console.log('PLAN CHANGE COLUMN', x[0], x[1]);
plan.changeColumn(x[0], x[1]);
}
}
@@ -425,6 +441,28 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
constraintPairs.filter(x => x[0] == null).forEach(x => plan.createConstraint(x[1]));
planTablePreload(plan, oldTable, newTable);
planChangeTableOptions(plan, oldTable, newTable, opts);
// console.log('oldTable', oldTable);
// console.log('newTable', newTable);
// console.log('plan.operations', plan.operations);
}
function planChangeTableOptions(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo, opts: DbDiffOptions) {
for (const option of plan.dialect?.getTableFormOptions?.('sqlAlterTable') || []) {
if (option.disabled) {
continue;
}
const name = option.name;
if (
oldTable[name] != newTable[name] &&
(oldTable[name] || newTable[name]) &&
(newTable[name] || option.allowEmptyValue)
) {
plan.setTableOption(newTable, name, newTable[name]);
}
}
}
export function testEqualTables(

View File

@@ -66,20 +66,20 @@ export const driverBase = {
return new this.dumperClass(this, options);
},
async script(pool, sql, options: RunScriptOptions) {
if (options?.useTransaction) {
if (options?.useTransaction && this.supportsTransactions) {
runCommandOnDriver(pool, this, dmp => dmp.beginTransaction());
}
for (const sqlItem of splitQuery(sql, this.getQuerySplitterOptions('script'))) {
try {
await this.query(pool, sqlItem, { discardResult: true });
} catch (err) {
if (options?.useTransaction) {
if (options?.useTransaction && this.supportsTransactions) {
runCommandOnDriver(pool, this, dmp => dmp.rollbackTransaction());
}
throw err;
}
}
if (options?.useTransaction) {
if (options?.useTransaction && this.supportsTransactions) {
runCommandOnDriver(pool, this, dmp => dmp.commitTransaction());
}
},
@@ -173,4 +173,12 @@ export const driverBase = {
parseSqlNull: true,
parseHexAsBuffer: true,
},
createSaveChangeSetScript(changeSet, dbinfo, defaultCreator) {
return defaultCreator(changeSet, dbinfo);
},
adaptTableInfo(table) {
return table;
}
};

View File

@@ -2,6 +2,7 @@ import uuidv1 from 'uuid/v1';
import _omit from 'lodash/omit';
import type {
ColumnInfo,
ColumnsConstraintInfo,
ConstraintInfo,
ForeignKeyInfo,
IndexInfo,
@@ -195,6 +196,13 @@ export function editorAddConstraint(table: TableInfo, constraint: ConstraintInfo
} as PrimaryKeyInfo;
}
if (constraint.constraintType == 'sortingKey') {
res.sortingKey = {
pairingId: uuidv1(),
...constraint,
} as ColumnsConstraintInfo;
}
if (constraint.constraintType == 'foreignKey') {
res.foreignKeys = [
...(res.foreignKeys || []),
@@ -240,6 +248,13 @@ export function editorModifyConstraint(table: TableInfo, constraint: ConstraintI
};
}
if (constraint.constraintType == 'sortingKey') {
res.sortingKey = {
...res.sortingKey,
...constraint,
};
}
if (constraint.constraintType == 'foreignKey') {
res.foreignKeys = table.foreignKeys.map(fk =>
fk.pairingId == constraint.pairingId ? { ...fk, ...constraint } : fk
@@ -266,6 +281,10 @@ export function editorDeleteConstraint(table: TableInfo, constraint: ConstraintI
res.primaryKey = null;
}
if (constraint.constraintType == 'sortingKey') {
res.sortingKey = null;
}
if (constraint.constraintType == 'foreignKey') {
res.foreignKeys = table.foreignKeys.filter(x => x.pairingId != constraint.pairingId);
}

View File

@@ -5,7 +5,7 @@ export function addTableDependencies(db: DatabaseInfo): DatabaseInfo {
if (!db.tables) {
return db;
}
const allForeignKeys = _flatten(db.tables.map(x => x.foreignKeys || []));
return {
...db,
@@ -33,6 +33,14 @@ export function extendTableInfo(table: TableInfo): TableInfo {
constraintType: 'primaryKey',
}
: undefined,
sortingKey: table.sortingKey
? {
...table.sortingKey,
pureName: table.pureName,
schemaName: table.schemaName,
constraintType: 'sortingKey',
}
: undefined,
foreignKeys: (table.foreignKeys || []).map(cnt => ({
...cnt,
pureName: table.pureName,

View File

@@ -10,6 +10,7 @@ export function prepareTableForImport(table: TableInfo): TableInfo {
res.uniques = [];
res.checks = [];
if (res.primaryKey) res.primaryKey.constraintName = null;
res.tableEngine = null;
return res;
}

View File

@@ -27,6 +27,7 @@ export interface TableInfoYaml {
// schema?: string;
columns: ColumnInfoYaml[];
primaryKey?: string[];
sortingKey?: string[];
insertKey?: string[];
insertOnly?: string[];
@@ -91,6 +92,9 @@ export function tableInfoToYaml(table: TableInfo): TableInfoYaml {
if (tableCopy.primaryKey && !tableCopy.primaryKey['_dumped']) {
res.primaryKey = tableCopy.primaryKey.columns.map(x => x.columnName);
}
if (tableCopy.sortingKey && !tableCopy.sortingKey['_dumped']) {
res.sortingKey = tableCopy.sortingKey.columns.map(x => x.columnName);
}
// const foreignKeys = (tableCopy.foreignKeys || []).filter(x => !x['_dumped']).map(foreignKeyInfoToYaml);
return res;
}
@@ -132,6 +136,13 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
columns: table.primaryKey.map(columnName => ({ columnName })),
};
}
if (table.sortingKey) {
res.sortingKey = {
pureName: table.name,
constraintType: 'sortingKey',
columns: table.sortingKey.map(columnName => ({ columnName })),
};
}
res.preloadedRows = table.data;
res.preloadedRowsKey = table.insertKey;
res.preloadedRowsInsertOnly = table.insertOnly;

View File

@@ -15,6 +15,7 @@ export interface AlterProcessor {
recreateTable(oldTable: TableInfo, newTable: TableInfo);
createSqlObject(obj: SqlObjectInfo);
dropSqlObject(obj: SqlObjectInfo);
setTableOption(table: TableInfo, optionName: string, optionValue: string);
fillPreloadedRows(
table: NamedObjectInfo,
oldRows: any[],

View File

@@ -15,7 +15,7 @@ export interface ColumnReference {
export interface ConstraintInfo extends NamedObjectInfo {
pairingId?: string;
constraintName?: string;
constraintType: 'primaryKey' | 'foreignKey' | 'index' | 'check' | 'unique';
constraintType: 'primaryKey' | 'foreignKey' | 'sortingKey' | 'index' | 'check' | 'unique';
}
export interface ColumnsConstraintInfo extends ConstraintInfo {
@@ -49,6 +49,7 @@ export interface ColumnInfo extends NamedObjectInfo {
notNull?: boolean;
autoIncrement?: boolean;
dataType: string;
displayedDataType?: string;
precision?: number;
scale?: number;
length?: number;
@@ -61,7 +62,7 @@ export interface ColumnInfo extends NamedObjectInfo {
isUnsigned?: boolean;
isZerofill?: boolean;
options?: [];
canSelectMultipleOptions?: boolean,
canSelectMultipleOptions?: boolean;
}
export interface DatabaseObjectInfo extends NamedObjectInfo {
@@ -82,6 +83,7 @@ export interface SqlObjectInfo extends DatabaseObjectInfo {
export interface TableInfo extends DatabaseObjectInfo {
columns: ColumnInfo[];
primaryKey?: PrimaryKeyInfo;
sortingKey?: ColumnsConstraintInfo;
foreignKeys: ForeignKeyInfo[];
dependencies?: ForeignKeyInfo[];
indexes?: IndexInfo[];
@@ -91,6 +93,7 @@ export interface TableInfo extends DatabaseObjectInfo {
preloadedRowsKey?: string[];
preloadedRowsInsertOnly?: string[];
tableRowCount?: number | string;
tableEngine?: string;
__isDynamicStructure?: boolean;
}
@@ -102,10 +105,10 @@ export interface CollectionInfo extends DatabaseObjectInfo {
uniqueKey?: ColumnReference[];
// partition key columns
partitionKey?: ColumnReference[]
partitionKey?: ColumnReference[];
// unique key inside partition
clusterKey?: ColumnReference[];
clusterKey?: ColumnReference[];
}
export interface ViewInfo extends SqlObjectInfo {

View File

@@ -17,6 +17,7 @@ export interface SqlDialect {
dropColumnDependencies?: string[];
changeColumnDependencies?: string[];
renameColumnDependencies?: string[];
dropIndexContainsTableSpec?: boolean;
@@ -34,6 +35,15 @@ export interface SqlDialect {
createCheck?: boolean;
dropCheck?: boolean;
specificNullabilityImplementation?: boolean;
omitForeignKeys?: boolean;
omitUniqueConstraints?: boolean;
omitIndexes?: boolean;
sortingKeys?: boolean;
// syntax for create column: ALTER TABLE table ADD COLUMN column
createColumnWithColumnKeyword?: boolean;
dropReferencesWhenDropTable?: boolean;
requireFromDual?: boolean;
@@ -41,4 +51,11 @@ export interface SqlDialect {
// create sql-tree expression
createColumnViewExpression(columnName: string, dataType: string, source: { alias: string }, alias?: string): any;
getTableFormOptions(intent: 'newTableForm' | 'editTableForm' | 'sqlCreateTable' | 'sqlAlterTable'): {
name: string;
sqlFormatString: string;
disabled?: boolean;
allowEmptyValue?: boolean;
}[];
}

View File

@@ -148,6 +148,7 @@ export interface EngineDriver extends FilterBehaviourProvider {
profilerChartAggregateFunction?: string;
profilerChartMeasures?: { label: string; field: string }[];
isElectronOnly?: boolean;
supportsTransactions?: boolean;
collectionSingularLabel?: string;
collectionPluralLabel?: string;
@@ -222,6 +223,13 @@ export interface EngineDriver extends FilterBehaviourProvider {
getCollectionExportQueryJson(collection: string, condition: any, sort?: CollectionSortDefinition): {};
getScriptTemplates(objectTypeField: keyof DatabaseInfo): { label: string; scriptTemplate: string }[];
getScriptTemplateContent(scriptTemplate: string, props: any): Promise<string>;
createSaveChangeSetScript(
changeSet: any,
dbinfo: DatabaseInfo,
defaultCreator: (changeSet: any, dbinfo: DatabaseInfo) => any
): any[];
// adapts table info from different source (import, other database) to be suitable for this database
adaptTableInfo(table: TableInfo): TableInfo;
analyserClass?: any;
dumperClass?: any;

View File

@@ -31,7 +31,7 @@
return [
{ text: 'Rename column', onClick: handleRenameColumn },
{ text: 'Drop column', onClick: handleDropColumn },
{ text: 'Copy name', onClick: () => navigator.clipboard.writeText(data.columnName)},
{ text: 'Copy name', onClick: () => navigator.clipboard.writeText(data.columnName) },
];
}

View File

@@ -860,6 +860,18 @@
return createDatabaseObjectMenu(data, passProps?.connection);
}
function getExtInfo(data) {
const res = [];
if (data.tableRowCount != null) {
res.push(`${formatRowCount(data.tableRowCount)} rows`);
}
if (data.tableEngine) {
res.push(data.tableEngine);
}
if (res.length > 0) return res.join(', ');
return null;
}
$: isPinned = !!$pinnedTables.find(x => testEqual(data, x));
</script>
@@ -873,7 +885,7 @@
showPinnedInsteadOfUnpin={passProps?.showPinnedInsteadOfUnpin}
onPin={isPinned ? null : () => pinnedTables.update(list => [...list, data])}
onUnpin={isPinned ? () => pinnedTables.update(list => list.filter(x => !testEqual(x, data))) : null}
extInfo={data.tableRowCount != null ? `${formatRowCount(data.tableRowCount)} rows` : null}
extInfo={getExtInfo(data)}
on:click={() => handleClick()}
on:middleclick={() => handleClick(true)}
on:expand

View File

@@ -88,9 +88,9 @@
{/if}
<ColumnLabel {...column} />
{#if _.isString(column.dataType) && !order}
{#if _.isString(column.displayedDataType || column.dataType) && !order}
<span class="data-type" title={column.dataType}>
{column.dataType.toLowerCase()}
{(column.displayedDataType || column.dataType).toLowerCase()}
</span>
{/if}
</div>

View File

@@ -196,7 +196,7 @@
<div class="space" />
{#if designer?.style?.showDataType && column?.dataType}
<div class="ml-2">
{column?.dataType.toLowerCase()}
{(column?.displayedDataType || column?.dataType).toLowerCase()}
</div>
{/if}
{#if designer?.style?.showNullability}

View File

@@ -19,6 +19,7 @@
export let columnName = '';
export let extInfo = null;
export let dataType = null;
export let displayedDataType = null;
export let showDataType = false;
export let foreignKey;
export let conid = undefined;
@@ -59,7 +60,7 @@
{/if}
</span>
{:else if dataType}
<span class="extinfo">{dataType.toLowerCase()}</span>
<span class="extinfo">{(displayedDataType || dataType).toLowerCase()}</span>
{/if}
{/if}
</span>

View File

@@ -8,7 +8,7 @@
export let collection;
export let title;
export let clickable;
export let clickable = false;
export let onRemove = null;
export let onAddNew = null;
export let emptyMessage = null;

View File

@@ -0,0 +1,68 @@
<script lang="ts">
import _ from 'lodash';
import FontIcon from '../icons/FontIcon.svelte';
import FormArgumentList from '../forms/FormArgumentList.svelte';
import { writable } from 'svelte/store';
import FormProviderCore from '../forms/FormProviderCore.svelte';
import createRef from '../utility/createRef';
export let title;
export let fieldDefinitions;
export let values;
export let onChangeValues;
let collapsed = false;
const valuesStore = writable(values || {});
$: onChangeValues($valuesStore);
</script>
<div class="wrapper">
<div class="header">
<span
class="collapse"
on:click={() => {
collapsed = !collapsed;
}}
>
<FontIcon icon={collapsed ? 'icon chevron-down' : 'icon chevron-up'} />
</span>
<span class="title mr-1">{title}</span>
</div>
{#if !collapsed}
<FormProviderCore values={valuesStore}>
<FormArgumentList args={fieldDefinitions} />
</FormProviderCore>
{/if}
</div>
<style>
.wrapper {
margin-bottom: 20px;
user-select: none;
}
.header {
background-color: var(--theme-bg-1);
padding: 5px;
}
.title {
font-weight: bold;
margin-left: 5px;
}
.body {
margin: 20px;
}
.collapse {
cursor: pointer;
}
.collapse:hover {
color: var(--theme-font-hover);
background: var(--theme-bg-3);
}
</style>

View File

@@ -10,13 +10,23 @@
export let showIfEmpty = false;
export let emptyMessage = null;
export let hideDisplayName = false;
export let clickable;
export let onAddNew;
export let clickable = false;
export let onAddNew = null;
let collapsed = false;
</script>
{#if collection?.length > 0 || showIfEmpty || emptyMessage}
<div class="wrapper">
<div class="header">
<span
class="collapse"
on:click={() => {
collapsed = !collapsed;
}}
>
<FontIcon icon={collapsed ? 'icon chevron-down' : 'icon chevron-up'} />
</span>
<span class="title mr-1">{title}</span>
{#if onAddNew}
<Link onClick={onAddNew}><FontIcon icon="icon add" /> Add new</Link>
@@ -27,7 +37,7 @@
{emptyMessage}
</div>
{/if}
{#if collection?.length > 0 || showIfEmpty}
{#if !collapsed && (collection?.length > 0 || showIfEmpty)}
<div class="body">
<TableControl
rows={collection || []}
@@ -78,6 +88,7 @@
<style>
.wrapper {
margin-bottom: 20px;
user-select: none;
}
.header {
@@ -93,4 +104,13 @@
.body {
margin: 20px;
}
.collapse {
cursor: pointer;
}
.collapse:hover {
color: var(--theme-font-hover);
background: var(--theme-bg-3);
}
</style>

View File

@@ -5,11 +5,15 @@
import FormSelectField from './FormSelectField.svelte';
import FormTextField from './FormTextField.svelte';
import FormStringList from './FormStringList.svelte';
import FormDropDownTextField from './FormDropDownTextField.svelte';
import { getFormContext } from './FormProviderCore.svelte';
export let arg;
export let namePrefix;
$: name = `${namePrefix}${arg.name}`;
const { setFieldValue } = getFormContext();
</script>
{#if arg.type == 'text'}
@@ -19,14 +23,10 @@
defaultValue={arg.default}
focused={arg.focused}
placeholder={arg.placeholder}
disabled={arg.disabled}
/>
{:else if arg.type == 'stringlist'}
<FormStringList
label={arg.label}
addButtonLabel={arg.addButtonLabel}
{name}
placeholder={arg.placeholder}
/>
<FormStringList label={arg.label} addButtonLabel={arg.addButtonLabel} {name} placeholder={arg.placeholder} />
{:else if arg.type == 'number'}
<FormTextField
label={arg.label}
@@ -48,4 +48,16 @@
_.isString(opt) ? { label: opt, value: opt } : { label: opt.name, value: opt.value }
)}
/>
{:else if arg.type == 'dropdowntext'}
<FormDropDownTextField
label={arg.label}
{name}
defaultValue={arg.default}
menu={() => {
return arg.options.map(opt => ({
text: _.isString(opt) ? opt : opt.name,
onClick: () => setFieldValue(name, _.isString(opt) ? opt : opt.value),
}));
}}
/>
{/if}

View File

@@ -32,7 +32,9 @@
<FormTextField name="columnName" label="Column name" focused disabled={isReadOnly} />
<DataTypeEditor dialect={driver?.dialect} disabled={isReadOnly} />
<FormCheckboxField name="notNull" label="NOT NULL" disabled={isReadOnly} />
{#if !driver?.dialect?.specificNullabilityImplementation}
<FormCheckboxField name="notNull" label="NOT NULL" disabled={isReadOnly} />
{/if}
<FormCheckboxField name="isPrimaryKey" label="Is Primary Key" disabled={isReadOnly} />
<FormCheckboxField name="autoIncrement" label="Is Autoincrement" disabled={isReadOnly} />
<FormTextField

View File

@@ -17,6 +17,7 @@
export let constraintType;
export let constraintNameLabel = 'Constraint name';
export let getExtractConstraintProps;
export let hideConstraintName = false;
let constraintName = constraintInfo?.constraintName;
let columns = constraintInfo?.columns || [];
@@ -44,17 +45,19 @@
>
<div class="largeFormMarker">
<div class="row">
<div class="label col-3">{constraintNameLabel}</div>
<div class="col-9">
<TextField
value={constraintName}
on:input={e => (constraintName = e.target['value'])}
focused
disabled={isReadOnly}
/>
{#if !hideConstraintName}
<div class="row">
<div class="label col-3">{constraintNameLabel}</div>
<div class="col-9">
<TextField
value={constraintName}
on:input={e => (constraintName = e.target['value'])}
focused
disabled={isReadOnly}
/>
</div>
</div>
</div>
{/if}
{#if $$slots.constraintProps}
<slot name="constraintProps" />

View File

@@ -4,14 +4,18 @@
export let constraintInfo;
export let setTableInfo;
export let tableInfo;
export let driver;
export let constraintLabel = 'primary key';
export let constraintType = 'primaryKey';
</script>
<ColumnsConstraintEditorModal
{...$$restProps}
constraintLabel="primary key"
constraintType="primaryKey"
{constraintLabel}
{constraintType}
{constraintInfo}
{setTableInfo}
{tableInfo}
hideConstraintName={driver?.dialect?.anonymousPrimaryKey}
/>

View File

@@ -0,0 +1,76 @@
<script lang="ts">
import { editorDeleteConstraint } from 'dbgate-tools';
import _ from 'lodash';
import ConstraintLabel from '../elements/ConstraintLabel.svelte';
import Link from '../elements/Link.svelte';
import ObjectListControl from '../elements/ObjectListControl.svelte';
import { showModal } from '../modals/modalTools';
import PrimaryKeyEditorModal from './PrimaryKeyEditorModal.svelte';
export let tableInfo;
export let setTableInfo;
export let isWritable;
export let driver;
export let constraintLabel = 'primary key';
export let constraintType = 'primaryKey';
$: columns = tableInfo?.columns;
$: keyConstraint = tableInfo?.[constraintType];
function addKeyConstraint() {
showModal(PrimaryKeyEditorModal, {
setTableInfo,
tableInfo,
constraintLabel,
constraintType,
driver,
});
}
</script>
<ObjectListControl
collection={_.compact([keyConstraint])}
title={_.startCase(constraintLabel)}
emptyMessage={isWritable ? `No ${constraintLabel} defined` : null}
onAddNew={isWritable && !keyConstraint && columns?.length > 0 ? addKeyConstraint : null}
hideDisplayName={driver?.dialect?.anonymousPrimaryKey}
clickable
on:clickrow={e =>
showModal(PrimaryKeyEditorModal, {
constraintInfo: e.detail,
tableInfo,
setTableInfo,
constraintLabel,
constraintType,
driver,
})}
columns={[
{
fieldName: 'columns',
header: 'Columns',
slot: 0,
},
isWritable
? {
fieldName: 'actions',
sortable: true,
slot: 1,
}
: null,
]}
>
<svelte:fragment slot="name" let:row><ConstraintLabel {...row} /></svelte:fragment>
<svelte:fragment slot="0" let:row>{row?.columns.map(x => x.columnName).join(', ')}</svelte:fragment>
<svelte:fragment slot="1" let:row
><Link
onClick={e => {
e.stopPropagation();
setTableInfo(tbl => editorDeleteConstraint(tbl, row));
}}>Remove</Link
></svelte:fragment
>
</ObjectListControl>

View File

@@ -30,7 +30,7 @@
icon: 'icon add-key',
toolbar: true,
isRelatedToTab: true,
testEnabled: () => getCurrentEditor()?.getIsWritable(),
testEnabled: () => getCurrentEditor()?.getIsWritable() && !getCurrentEditor()?.getDialect()?.omitForeignKeys,
onClick: () => getCurrentEditor().addForeignKey(),
});
@@ -41,7 +41,7 @@
icon: 'icon add-key',
toolbar: true,
isRelatedToTab: true,
testEnabled: () => getCurrentEditor()?.getIsWritable(),
testEnabled: () => getCurrentEditor()?.getIsWritable() && !getCurrentEditor()?.getDialect()?.omitIndexes,
onClick: () => getCurrentEditor().addIndex(),
});
@@ -52,7 +52,7 @@
icon: 'icon add-key',
toolbar: true,
isRelatedToTab: true,
testEnabled: () => getCurrentEditor()?.getIsWritable(),
testEnabled: () => getCurrentEditor()?.getIsWritable() && !getCurrentEditor()?.getDialect()?.omitUniqueConstraints,
onClick: () => getCurrentEditor().addUnique(),
});
</script>
@@ -81,6 +81,8 @@
import IndexEditorModal from './IndexEditorModal.svelte';
import PrimaryKeyEditorModal from './PrimaryKeyEditorModal.svelte';
import UniqueEditorModal from './UniqueEditorModal.svelte';
import ObjectFieldsEditor from '../elements/ObjectFieldsEditor.svelte';
import PrimaryKeyLikeListControl from './PrimaryKeyLikeListControl.svelte';
export const activator = createActivator('TableEditor', true);
@@ -88,6 +90,7 @@
export let setTableInfo;
export let dbInfo;
export let driver;
export let resetCounter;
$: isWritable = !!setTableInfo;
@@ -95,6 +98,10 @@
return isWritable;
}
export function getDialect() {
return driver?.dialect;
}
export function addColumn() {
showModal(ColumnEditorModal, {
setTableInfo,
@@ -115,6 +122,7 @@
showModal(PrimaryKeyEditorModal, {
setTableInfo,
tableInfo,
driver,
});
}
@@ -143,7 +151,6 @@
}
$: columns = tableInfo?.columns;
$: primaryKey = tableInfo?.primaryKey;
$: foreignKeys = tableInfo?.foreignKeys;
$: dependencies = tableInfo?.dependencies;
$: indexes = tableInfo?.indexes;
@@ -153,9 +160,29 @@
tableInfo;
invalidateCommands();
}
$: tableFormOptions = driver?.dialect?.getTableFormOptions?.(tableInfo?.objectId ? 'editTableForm' : 'newTableForm');
</script>
<div class="wrapper">
{#if tableFormOptions}
{#key resetCounter}
<ObjectFieldsEditor
title="Table properties"
fieldDefinitions={tableFormOptions}
values={_.pick(
tableInfo,
tableFormOptions.map(x => x.name)
)}
onChangeValues={vals => {
if (!_.isEmpty(vals)) {
setTableInfo(tbl => ({ ...tbl, ...vals }));
}
}}
/>
{/key}
{/if}
<ObjectListControl
collection={columns?.map((x, index) => ({ ...x, ordinal: index + 1 }))}
title={`Columns (${columns?.length || 0})`}
@@ -164,7 +191,7 @@
on:clickrow={e => showModal(ColumnEditorModal, { columnInfo: e.detail, tableInfo, setTableInfo, driver })}
onAddNew={isWritable ? addColumn : null}
columns={[
{
!driver?.dialect?.specificNullabilityImplementation && {
fieldName: 'notNull',
header: 'Nullability',
sortable: true,
@@ -239,124 +266,109 @@
<svelte:fragment slot="name" let:row><ColumnLabel {...row} forceIcon /></svelte:fragment>
</ObjectListControl>
<ObjectListControl
collection={_.compact([primaryKey])}
title="Primary key"
emptyMessage={isWritable ? 'No primary key defined' : null}
onAddNew={isWritable && !primaryKey && columns?.length > 0 ? addPrimaryKey : null}
clickable
on:clickrow={e => showModal(PrimaryKeyEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo })}
columns={[
{
fieldName: 'columns',
header: 'Columns',
slot: 0,
},
isWritable
? {
fieldName: 'actions',
sortable: true,
slot: 1,
}
: null,
]}
>
<svelte:fragment slot="name" let:row><ConstraintLabel {...row} /></svelte:fragment>
<svelte:fragment slot="0" let:row>{row?.columns.map(x => x.columnName).join(', ')}</svelte:fragment>
<svelte:fragment slot="1" let:row
><Link
onClick={e => {
e.stopPropagation();
setTableInfo(tbl => editorDeleteConstraint(tbl, row));
}}>Remove</Link
></svelte:fragment
>
</ObjectListControl>
<PrimaryKeyLikeListControl {tableInfo} {setTableInfo} {isWritable} {driver} />
<ObjectListControl
collection={indexes}
onAddNew={isWritable && columns?.length > 0 ? addIndex : null}
title={`Indexes (${indexes?.length || 0})`}
emptyMessage={isWritable ? 'No index defined' : null}
clickable
on:clickrow={e => showModal(IndexEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo })}
columns={[
{
fieldName: 'columns',
header: 'Columns',
slot: 0,
},
{
fieldName: 'unique',
header: 'Unique',
slot: 1,
},
isWritable
? {
fieldName: 'actions',
sortable: true,
slot: 2,
}
: null,
]}
>
<svelte:fragment slot="name" let:row><ConstraintLabel {...row} /></svelte:fragment>
<svelte:fragment slot="0" let:row>{row?.columns.map(x => x.columnName).join(', ')}</svelte:fragment>
<svelte:fragment slot="1" let:row>{row?.isUnique ? 'YES' : 'NO'}</svelte:fragment>
<svelte:fragment slot="2" let:row
><Link
onClick={e => {
e.stopPropagation();
setTableInfo(tbl => editorDeleteConstraint(tbl, row));
}}>Remove</Link
></svelte:fragment
>
</ObjectListControl>
{#if driver?.dialect?.sortingKeys}
<PrimaryKeyLikeListControl
{tableInfo}
{setTableInfo}
{isWritable}
{driver}
constraintLabel="sorting key"
constraintType="sortingKey"
/>
{/if}
<ObjectListControl
collection={uniques}
onAddNew={isWritable && columns?.length > 0 ? addUnique : null}
title={`Unique constraints (${uniques?.length || 0})`}
emptyMessage={isWritable ? 'No unique defined' : null}
clickable
on:clickrow={e => showModal(UniqueEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo })}
columns={[
{
fieldName: 'columns',
header: 'Columns',
slot: 0,
},
isWritable
? {
fieldName: 'actions',
sortable: true,
slot: 1,
}
: null,
]}
>
<svelte:fragment slot="name" let:row><ConstraintLabel {...row} /></svelte:fragment>
<svelte:fragment slot="0" let:row>{row?.columns.map(x => x.columnName).join(', ')}</svelte:fragment>
<svelte:fragment slot="1" let:row
><Link
onClick={e => {
e.stopPropagation();
setTableInfo(tbl => editorDeleteConstraint(tbl, row));
}}>Remove</Link
></svelte:fragment
{#if !driver?.dialect?.omitIndexes}
<ObjectListControl
collection={indexes}
onAddNew={isWritable && columns?.length > 0 ? addIndex : null}
title={`Indexes (${indexes?.length || 0})`}
emptyMessage={isWritable ? 'No index defined' : null}
clickable
on:clickrow={e => showModal(IndexEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo })}
columns={[
{
fieldName: 'columns',
header: 'Columns',
slot: 0,
},
{
fieldName: 'unique',
header: 'Unique',
slot: 1,
},
isWritable
? {
fieldName: 'actions',
sortable: true,
slot: 2,
}
: null,
]}
>
</ObjectListControl>
<svelte:fragment slot="name" let:row><ConstraintLabel {...row} /></svelte:fragment>
<svelte:fragment slot="0" let:row>{row?.columns.map(x => x.columnName).join(', ')}</svelte:fragment>
<svelte:fragment slot="1" let:row>{row?.isUnique ? 'YES' : 'NO'}</svelte:fragment>
<svelte:fragment slot="2" let:row
><Link
onClick={e => {
e.stopPropagation();
setTableInfo(tbl => editorDeleteConstraint(tbl, row));
}}>Remove</Link
></svelte:fragment
>
</ObjectListControl>
{/if}
<ForeignKeyObjectListControl
collection={foreignKeys}
onAddNew={isWritable && columns?.length > 0 ? addForeignKey : null}
title={`Foreign keys (${foreignKeys?.length || 0})`}
emptyMessage={isWritable ? 'No foreign key defined' : null}
clickable
onRemove={row => setTableInfo(tbl => editorDeleteConstraint(tbl, row))}
on:clickrow={e => showModal(ForeignKeyEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo, dbInfo })}
/>
<ForeignKeyObjectListControl collection={dependencies} title="Dependencies" />
{#if !driver?.dialect?.omitUniqueConstraints}
<ObjectListControl
collection={uniques}
onAddNew={isWritable && columns?.length > 0 ? addUnique : null}
title={`Unique constraints (${uniques?.length || 0})`}
emptyMessage={isWritable ? 'No unique defined' : null}
clickable
on:clickrow={e => showModal(UniqueEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo })}
columns={[
{
fieldName: 'columns',
header: 'Columns',
slot: 0,
},
isWritable
? {
fieldName: 'actions',
sortable: true,
slot: 1,
}
: null,
]}
>
<svelte:fragment slot="name" let:row><ConstraintLabel {...row} /></svelte:fragment>
<svelte:fragment slot="0" let:row>{row?.columns.map(x => x.columnName).join(', ')}</svelte:fragment>
<svelte:fragment slot="1" let:row
><Link
onClick={e => {
e.stopPropagation();
setTableInfo(tbl => editorDeleteConstraint(tbl, row));
}}>Remove</Link
></svelte:fragment
>
</ObjectListControl>
{/if}
{#if !driver?.dialect?.omitForeignKeys}
<ForeignKeyObjectListControl
collection={foreignKeys}
onAddNew={isWritable && columns?.length > 0 ? addForeignKey : null}
title={`Foreign keys (${foreignKeys?.length || 0})`}
emptyMessage={isWritable ? 'No foreign key defined' : null}
clickable
onRemove={row => setTableInfo(tbl => editorDeleteConstraint(tbl, row))}
on:clickrow={e => showModal(ForeignKeyEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo, dbInfo })}
/>
<ForeignKeyObjectListControl collection={dependencies} title="Dependencies" />
{/if}
</div>
<style>

View File

@@ -71,10 +71,7 @@
changeSetToSql,
createChangeSet,
createGridCache,
createGridConfig,
getDeleteCascades,
TableFormViewDisplay,
TableGridDisplay,
} from 'dbgate-datalib';
import { findEngineDriver } from 'dbgate-tools';
import { reloadDataCacheFunc } from 'dbgate-datalib';
@@ -160,7 +157,11 @@
export function save() {
const driver = findEngineDriver($connection, $extensions);
const script = changeSetToSql($changeSetStore?.value, $dbinfo);
const script = driver.createSaveChangeSetScript($changeSetStore?.value, $dbinfo, () =>
changeSetToSql($changeSetStore?.value, $dbinfo)
);
const deleteCascades = getDeleteCascades($changeSetStore?.value, $dbinfo);
const sql = scriptToSql(driver, script);
const deleteCascadesScripts = _.map(deleteCascades, ({ title, commands }) => ({

View File

@@ -72,6 +72,7 @@
let domEditor;
let savedName;
let resetCounter = 0;
export const activator = createActivator('TableStructureTab', true);
@@ -157,7 +158,8 @@
export async function reset() {
await apiCall('database-connections/sync-model', { conid, database });
clearEditorData();
await clearEditorData();
resetCounter++;
}
// $: {
@@ -172,6 +174,7 @@
tableInfo={showTable}
dbInfo={$dbInfo}
{driver}
{resetCounter}
setTableInfo={objectTypeField == 'tables' && !$connection?.isReadOnly && hasPermission(`dbops/model/edit`)
? tableInfoUpdater =>
setEditorData(tbl =>
@@ -191,7 +194,7 @@
<ToolStripCommandButton command="tableStructure.save" />
<ToolStripCommandButton command="tableStructure.reset" />
<ToolStripCommandButton command="tableEditor.addColumn" />
<ToolStripCommandButton command="tableEditor.addIndex" />
<ToolStripCommandButton command="tableEditor.addIndex" hideDisabled />
{#if objectTypeField == 'tables'}
<ToolStripButton

View File

@@ -0,0 +1,6 @@
[![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
[![NPM version](https://img.shields.io/npm/v/dbgate-plugin-clickhouse.svg)](https://www.npmjs.com/package/dbgate-plugin-clickhouse)
# dbgate-plugin-clickhouse
Use DbGate for install of this plugin

View File

@@ -0,0 +1,35 @@
<?xml version="1.0" encoding="iso-8859-1"?>
<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 384 384" style="enable-background:new 0 0 384 384;" xml:space="preserve">
<polygon style="fill:#EFEEEE;" points="64,0 64,384 288,384 384,288 384,0 "/>
<polygon style="fill:#ABABAB;" points="288,288 288,384 384,288 "/>
<polygon style="fill:#DEDEDD;" points="192,384 288,384 288,288 "/>
<path style="fill:#448E47;" d="M0,96v112h256V96L0,96L0,96z"/>
<g>
<path style="fill:#FFFFFF;" d="M64.32,130.112c-1.184-2.288-3.344-3.424-6.48-3.424c-1.728,0-3.152,0.464-4.272,1.408
c-1.12,0.928-2,2.416-2.64,4.496s-1.088,4.8-1.344,8.176c-0.272,3.36-0.384,7.472-0.384,12.336c0,5.184,0.176,9.376,0.528,12.576
c0.336,3.2,0.896,5.664,1.632,7.44s1.664,2.96,2.784,3.552c1.12,0.608,2.416,0.928,3.888,0.928c1.216,0,2.352-0.208,3.408-0.624
s1.968-1.248,2.736-2.496c0.784-1.248,1.392-3.008,1.824-5.28c0.448-2.272,0.672-5.264,0.672-8.976H80.48
c0,3.696-0.288,7.232-0.864,10.56s-1.664,6.24-3.216,8.736c-1.584,2.48-3.776,4.432-6.624,5.84
c-2.848,1.408-6.544,2.128-11.088,2.128c-5.168,0-9.312-0.848-12.368-2.496c-3.072-1.664-5.424-4.064-7.056-7.2
s-2.688-6.88-3.168-11.232c-0.464-4.336-0.72-9.152-0.72-14.384c0-5.184,0.256-9.968,0.72-14.352
c0.48-4.368,1.552-8.144,3.168-11.28c1.648-3.12,3.984-5.584,7.056-7.344c3.056-1.744,7.2-2.64,12.368-2.64
c4.944,0,8.816,0.8,11.664,2.4c2.848,1.6,4.976,3.632,6.368,6.096s2.304,5.12,2.64,7.968c0.352,2.848,0.528,5.52,0.528,8.016H66.08
C66.08,136,65.488,132.368,64.32,130.112z"/>
<path style="fill:#FFFFFF;" d="M109.072,167.008c0,1.6,0.144,3.056,0.384,4.352c0.272,1.312,0.736,2.416,1.44,3.312
c0.704,0.912,1.664,1.616,2.848,2.128c1.168,0.496,2.672,0.768,4.448,0.768c2.128,0,4.016-0.688,5.712-2.064
c1.68-1.376,2.544-3.52,2.544-6.384c0-1.536-0.224-2.864-0.624-3.984c-0.416-1.12-1.104-2.128-2.064-3.008
c-0.976-0.912-2.24-1.712-3.792-2.448s-3.504-1.488-5.808-2.256c-3.056-1.024-5.712-2.16-7.968-3.376
c-2.24-1.2-4.112-2.624-5.616-4.272c-1.504-1.632-2.608-3.52-3.312-5.664c-0.704-2.16-1.056-4.624-1.056-7.456
c0-6.784,1.888-11.824,5.664-15.152c3.76-3.328,8.96-4.992,15.552-4.992c3.072,0,5.904,0.336,8.496,1.008s4.832,1.744,6.72,3.264
c1.888,1.504,3.36,3.424,4.416,5.744c1.04,2.336,1.584,5.136,1.584,8.4v1.92h-13.232c0-3.264-0.576-5.776-1.712-7.552
c-1.152-1.744-3.072-2.64-5.76-2.64c-1.536,0-2.816,0.24-3.84,0.672c-1.008,0.448-1.84,1.04-2.448,1.776s-1.04,1.616-1.264,2.576
c-0.24,0.96-0.336,1.952-0.336,2.976c0,2.128,0.448,3.888,1.344,5.328c0.896,1.456,2.816,2.784,5.76,3.984l10.656,4.608
c2.624,1.152,4.768,2.352,6.416,3.616c1.664,1.248,3.008,2.592,3.984,4.032c0.992,1.44,1.68,3.008,2.064,4.752
c0.384,1.712,0.576,3.648,0.576,5.744c0,7.232-2.096,12.496-6.288,15.792c-4.192,3.296-10.032,4.96-17.52,4.96
c-7.808,0-13.392-1.696-16.768-5.088c-3.36-3.392-5.024-8.256-5.024-14.592v-2.784h13.824L109.072,167.008L109.072,167.008z"/>
<path style="fill:#FFFFFF;" d="M177.344,168.544h0.304l10.176-50.688h14.32L186.4,186.4h-17.76l-15.728-68.544h14.784
L177.344,168.544z"/>
</g>
</svg>

After

Width:  |  Height:  |  Size: 3.1 KiB

View File

@@ -0,0 +1,38 @@
{
"name": "dbgate-plugin-clickhouse",
"main": "dist/backend.js",
"version": "5.0.0-alpha.1",
"license": "GPL-3.0",
"author": "Jan Prochazka",
"description": "Clickhouse connector for DbGate",
"keywords": [
"dbgate",
"dbgateplugin",
"clickhouse"
],
"files": [
"dist",
"icon.svg"
],
"scripts": {
"build:frontend": "webpack --config webpack-frontend.config",
"build:frontend:watch": "webpack --watch --config webpack-frontend.config",
"build:backend": "webpack --config webpack-backend.config.js",
"build": "yarn build:frontend && yarn build:backend",
"plugin": "yarn build && yarn pack && dbgate-plugin dbgate-plugin-clickhouse",
"plugout": "dbgate-plugout dbgate-plugin-clickhouse",
"copydist": "yarn build && yarn pack && dbgate-copydist ../dist/dbgate-plugin-clickhouse",
"prepublishOnly": "yarn build"
},
"devDependencies": {
"byline": "^5.0.0",
"dbgate-plugin-tools": "^1.0.8",
"dbgate-tools": "^5.0.0-alpha.1",
"json-stable-stringify": "^1.0.1",
"webpack": "^5.91.0",
"webpack-cli": "^5.1.4"
},
"dependencies": {
"@clickhouse/client": "^1.5.0"
}
}

View File

@@ -0,0 +1,8 @@
module.exports = {
trailingComma: 'es5',
tabWidth: 2,
semi: true,
singleQuote: true,
arrowParen: 'avoid',
printWidth: 120,
};

View File

@@ -0,0 +1,92 @@
const { DatabaseAnalyser } = require('dbgate-tools');
const sql = require('./sql');
function extractDataType(dataType) {
if (!dataType) return {};
if (dataType.startsWith('Nullable(')) {
const displayedDataType = dataType.substring('Nullable('.length, dataType.length - 1);
return {
dataType,
displayedDataType,
notNull: false,
};
}
return {
dataType,
notNull: true,
};
}
class Analyser extends DatabaseAnalyser {
constructor(connection, driver) {
super(connection, driver);
}
createQuery(resFileName, typeFields, replacements = {}) {
let res = sql[resFileName];
res = res.replace('#DATABASE#', this.pool._database_name);
return super.createQuery(res, typeFields, replacements);
}
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
const tables = await this.analyserQuery('tables', ['tables']);
this.feedback({ analysingMessage: 'Loading columns' });
const columns = await this.analyserQuery('columns', ['tables', 'views']);
this.feedback({ analysingMessage: 'Loading views' });
const views = await this.analyserQuery('views', ['views']);
const res = {
tables: tables.rows.map((table) => ({
...table,
primaryKeyColumns: undefined,
sortingKeyColumns: undefined,
columns: columns.rows
.filter((col) => col.pureName == table.pureName)
.map((col) => ({
...col,
...extractDataType(col.dataType),
})),
primaryKey: table.primaryKeyColumns
? { columns: (table.primaryKeyColumns || '').split(',').map((x) => ({ columnName: x.trim() })) }
: null,
sortingKey: table.sortingKeyColumns
? { columns: (table.sortingKeyColumns || '').split(',').map((x) => ({ columnName: x.trim() })) }
: null,
foreignKeys: [],
})),
views: views.rows.map((view) => ({
...view,
columns: columns.rows
.filter((col) => col.pureName == view.pureName)
.map((col) => ({
...col,
...extractDataType(col.dataType),
})),
createSql: `CREATE VIEW "${view.pureName}"\nAS\n${view.viewDefinition}`,
})),
};
this.feedback({ analysingMessage: null });
return res;
}
async _getFastSnapshot() {
const tableModificationsQueryData = await this.analyserQuery('tableModifications');
return {
tables: tableModificationsQueryData.rows.filter((x) => x.tableEngine != 'View'),
views: tableModificationsQueryData.rows.filter((x) => x.tableEngine == 'View'),
};
}
async _computeSingleObjectId() {
const { pureName } = this.singleObjectFilter;
const resId = await this.driver.query(
this.pool,
`SELECT uuid as id FROM system.tables WHERE database = '${this.pool._database_name}' AND name='${pureName}'`
);
this.singleObjectId = resId.rows[0]?.id;
}
}
module.exports = Analyser;

View File

@@ -0,0 +1,23 @@
const { createBulkInsertStreamBase } = global.DBGATE_PACKAGES['dbgate-tools'];
const _ = require('lodash');
/**
*
* @param {import('dbgate-types').EngineDriver} driver
*/
function createOracleBulkInsertStream(driver, stream, pool, name, options) {
const writable = createBulkInsertStreamBase(driver, stream, pool, name, options);
writable.send = async () => {
await pool.insert({
table: name.pureName,
values: writable.buffer,
format: 'JSONEachRow',
});
writable.buffer = [];
};
return writable;
}
module.exports = createOracleBulkInsertStream;

View File

@@ -0,0 +1,222 @@
const _ = require('lodash');
const stream = require('stream');
const driverBase = require('../frontend/driver');
const Analyser = require('./Analyser');
const { createClient } = require('@clickhouse/client');
const createBulkInsertStream = require('./createBulkInsertStream');
/** @type {import('dbgate-types').EngineDriver} */
const driver = {
...driverBase,
analyserClass: Analyser,
// creating connection
async connect({ server, port, user, password, database, useDatabaseUrl, databaseUrl }) {
const client = createClient({
url: databaseUrl,
username: user,
password: password,
database: database,
});
client._database_name = database;
return client;
},
// called for retrieve data (eg. browse in data grid) and for update database
async query(client, query, options) {
if (options?.discardResult) {
await client.command({
query,
});
return {
rows: [],
columns: [],
};
} else {
const resultSet = await client.query({
query,
format: 'JSONCompactEachRowWithNamesAndTypes',
});
const dataSet = await resultSet.json();
if (!dataSet?.[0]) {
return {
rows: [],
columns: [],
};
}
const columns = dataSet[0].map((columnName, i) => ({
columnName,
dataType: dataSet[1][i],
}));
return {
rows: dataSet.slice(2).map((row) => _.zipObject(dataSet[0], row)),
columns,
};
}
},
// called in query console
async stream(client, query, options) {
try {
if (!query.match(/^\s*SELECT/i)) {
const resp = await client.command({
query,
});
// console.log('RESP', resp);
// const { rowsAffected } = resp || {};
// if (rowsAffected) {
// options.info({
// message: `${rowsAffected} rows affected`,
// time: new Date(),
// severity: 'info',
// });
// }
options.done();
return;
}
const resultSet = await client.query({
query,
format: 'JSONCompactEachRowWithNamesAndTypes',
});
let columnNames = null;
let dataTypes = null;
const strm = resultSet.stream();
strm.on('data', (rows) => {
rows.forEach((row) => {
const json = row.json();
if (!columnNames) {
columnNames = json;
return;
}
if (!dataTypes) {
dataTypes = json;
const columns = columnNames.map((columnName, i) => ({
columnName,
dataType: dataTypes[i],
}));
options.recordset(columns);
return;
}
const data = _.zipObject(columnNames, json);
options.row(data);
});
});
strm.on('end', () => {
options.done();
});
strm.on('error', (err) => {
options.info({
message: err.toString(),
time: new Date(),
severity: 'error',
});
options.done();
});
} catch (err) {
const mLine = err.message.match(/\(line (\d+)\,/);
let line = undefined;
if (mLine) {
line = parseInt(mLine[1]) - 1;
}
options.info({
message: err.message,
time: new Date(),
severity: 'error',
line,
});
options.done();
}
},
// called when exporting table or view
async readQuery(client, query, structure) {
const pass = new stream.PassThrough({
objectMode: true,
highWaterMark: 100,
});
const resultSet = await client.query({
query,
format: 'JSONCompactEachRowWithNamesAndTypes',
});
let columnNames = null;
let dataTypes = null;
const strm = resultSet.stream();
strm.on('data', (rows) => {
rows.forEach((row) => {
const json = row.json();
if (!columnNames) {
columnNames = json;
return;
}
if (!dataTypes) {
dataTypes = json;
const columns = columnNames.map((columnName, i) => ({
columnName,
dataType: dataTypes[i],
}));
pass.write({
__isStreamHeader: true,
...(structure || { columns }),
});
return;
}
const data = _.zipObject(columnNames, json);
pass.write(data);
});
});
strm.on('end', () => {
pass.end();
});
strm.on('error', (err) => {
pass.end();
});
return pass;
},
async writeTable(pool, name, options) {
return createBulkInsertStream(this, stream, pool, name, options);
},
// detect server version
async getVersion(client) {
const resultSet = await client.query({
query: 'SELECT version() as version',
format: 'JSONEachRow',
});
const dataset = await resultSet.json();
return { version: dataset[0].version };
},
// list databases on server
async listDatabases(client) {
const resultSet = await client.query({
query: `SELECT name
FROM system.databases
WHERE name NOT IN ('system', 'information_schema', 'information_schema_ro', 'INFORMATION_SCHEMA')`,
format: 'JSONEachRow',
});
const dataset = await resultSet.json();
return dataset;
},
async close(client) {
return client.close();
},
};
module.exports = driver;

View File

@@ -0,0 +1,6 @@
const driver = require('./driver');
module.exports = {
packageName: 'dbgate-plugin-clickhouse',
drivers: [driver],
};

View File

@@ -0,0 +1,12 @@
module.exports = `
select
columns.table as "pureName",
tables.uuid as "objectId",
columns.name as "columnName",
columns.type as "dataType",
columns.comment as "columnComment"
from system.columns
inner join system.tables on columns.table = tables.name and columns.database = tables.database
where columns.database='#DATABASE#' and tables.uuid =OBJECT_ID_CONDITION
order by toInt32(columns.position)
`;

View File

@@ -0,0 +1,11 @@
const columns = require('./columns');
const tables = require('./tables');
const views = require('./views');
const tableModifications = require('./tableModifications');
module.exports = {
columns,
tables,
views,
tableModifications,
};

View File

@@ -0,0 +1,5 @@
module.exports = `
select metadata_modification_time as "contentHash", uuid as "objectId", engine as "tableEngine"
from system.tables
where database='#DATABASE#';
`;

View File

@@ -0,0 +1,6 @@
module.exports = `
select name as "pureName", metadata_modification_time as "contentHash", total_rows as "tableRowCount", uuid as "objectId", comment as "objectComment",
engine as "tableEngine", primary_key as "primaryKeyColumns", sorting_key as "sortingKeyColumns"
from system.tables
where database='#DATABASE#' and uuid =OBJECT_ID_CONDITION and engine != 'View';
`;

View File

@@ -0,0 +1,10 @@
module.exports = `
select
tables.name as "pureName",
tables.uuid as "objectId",
views.view_definition as "viewDefinition",
tables.metadata_modification_time as "contentHash"
from information_schema.views
inner join system.tables on views.table_name = tables.name and views.table_schema = tables.database
where views.table_schema='#DATABASE#' and tables.uuid =OBJECT_ID_CONDITION
`;

View File

@@ -0,0 +1,44 @@
const { SqlDumper } = require('dbgate-tools');
class Dumper extends SqlDumper {
setTableOptionCore(table, optionName, optionValue, formatString) {
this.put('^alter ^table %f ^modify ', table);
this.put(formatString, optionValue);
}
changeColumn(oldcol, newcol, constraints) {
if (oldcol.columnName != newcol.columnName) {
this.putCmd('^alter ^table %f ^rename ^column %i ^to %i', oldcol, oldcol.columnName, newcol.columnName);
}
this.put('^alter ^table %f ^modify ^column %i ', newcol, newcol.columnName);
this.columnDefinition(newcol);
this.endCommand();
}
columnType(dataType) {
this.putRaw(dataType || this.dialect.fallbackDataType);
}
renameColumn(column, newcol) {
this.putCmd('^alter ^table %f ^rename ^column %i ^to %i', column, column.columnName, newcol);
}
renameTable(obj, newName) {
this.putCmd('^rename ^table %f ^to %i', obj, newName);
}
tableOptions(table) {
super.tableOptions(table);
if (table.sortingKey) {
this.put(
'&n^order ^by (%,i)',
table.sortingKey.columns.map((x) => x.columnName)
);
}
}
autoIncrement() {}
}
module.exports = Dumper;

View File

@@ -0,0 +1,204 @@
const { driverBase } = require('dbgate-tools');
const Dumper = require('./Dumper');
const { mysqlSplitterOptions } = require('dbgate-query-splitter/lib/options');
const _cloneDeepWith = require('lodash/cloneDeepWith');
const clickhouseEngines = [
'MergeTree',
'ReplacingMergeTree',
'SummingMergeTree',
'AggregatingMergeTree',
'CollapsingMergeTree',
'VersionedCollapsingMergeTree',
'GraphiteMergeTree',
'Distributed',
'Log',
'TinyLog',
'StripeLog',
'Memory',
'File',
'URL',
'JDBC',
'ODBC',
'Buffer',
'Null',
'Kafka',
'HDFS',
'S3',
'Merge',
'Join',
'MaterializedView',
'Dictionary',
'MySQL',
'PostgreSQL',
'MongoDB',
'EmbeddedRocksDB',
'View',
'MaterializeMySQL',
'MaterializePostgreSQL',
'ReplicatedMergeTree',
'ReplicatedReplacingMergeTree',
'ReplicatedSummingMergeTree',
'ReplicatedAggregatingMergeTree',
'ReplicatedCollapsingMergeTree',
'ReplicatedVersionedCollapsingMergeTree',
'ReplicatedGraphiteMergeTree',
'ExternalDistributed',
'Iceberg',
'Parquet',
'ORC',
'DeltaLake',
];
const clickhouseDataTypes = [
'Int8',
'Int16',
'Int32',
'Int64',
'UInt8',
'UInt16',
'UInt32',
'UInt64',
'Float32',
'Float64',
'Decimal',
'String',
'FixedString',
'UUID',
'Date',
'DateTime',
'DateTime64',
"DateTime('UTC')",
'Date32',
'Enum8',
'Enum16',
'Array',
'Tuple',
'Nullable',
'LowCardinality',
'Map',
'JSON',
'IPv4',
'IPv6',
'Nested',
'AggregateFunction',
'SimpleAggregateFunction',
];
/** @type {import('dbgate-types').SqlDialect} */
const dialect = {
limitSelect: true,
rangeSelect: true,
stringEscapeChar: "'",
fallbackDataType: 'String',
dropColumnDependencies: ['primaryKey', 'sortingKey'],
changeColumnDependencies: ['primaryKey', 'sortingKey'],
renameColumnDependencies: ['primaryKey', 'sortingKey'],
createColumn: true,
dropColumn: true,
changeColumn: true,
createIndex: true,
dropIndex: true,
anonymousPrimaryKey: true,
createColumnWithColumnKeyword: true,
specificNullabilityImplementation: true,
omitForeignKeys: true,
omitUniqueConstraints: true,
omitIndexes: true,
sortingKeys: true,
columnProperties: {
columnComment: true,
},
quoteIdentifier(s) {
return `"${s}"`;
},
getTableFormOptions(intent) {
const isNewTable = intent == 'newTableForm' || intent == 'sqlCreateTable';
return [
{
type: isNewTable ? 'dropdowntext' : 'text',
options: clickhouseEngines,
label: 'Engine',
name: 'tableEngine',
sqlFormatString: '^engine = %s',
disabled: !isNewTable,
},
{
type: 'text',
label: 'Comment',
name: 'objectComment',
sqlFormatString: '^comment %v',
allowEmptyValue: true,
},
];
},
predefinedDataTypes: clickhouseDataTypes,
};
/** @type {import('dbgate-types').EngineDriver} */
const driver = {
...driverBase,
dumperClass: Dumper,
dialect,
engine: 'clickhouse@dbgate-plugin-clickhouse',
title: 'ClickHouse',
showConnectionField: (field, values) => {
return ['databaseUrl', 'defaultDatabase', 'singleDatabase', 'isReadOnly', 'user', 'password'].includes(field);
},
getQuerySplitterOptions: (usage) =>
usage == 'editor'
? { ...mysqlSplitterOptions, ignoreComments: true, preventSingleLineSplit: true }
: mysqlSplitterOptions,
createSaveChangeSetScript(changeSet, dbinfo, defaultCreator) {
function removeConditionSource(cmd) {
cmd.where = _cloneDeepWith(cmd.where, (expr) => {
if (expr.exprType == 'column') {
return {
...expr,
source: undefined,
};
}
});
}
const res = defaultCreator(changeSet, dbinfo);
for (const cmd of res) {
if (cmd.commandType == 'update') {
cmd.alterTableUpdateSyntax = true;
removeConditionSource(cmd);
}
if (cmd.commandType == 'delete') {
const table = dbinfo?.tables?.find((x) => x.pureName == cmd?.from?.name?.pureName);
if (table?.tableEngine != 'MergeTree') {
cmd.alterTableDeleteSyntax = true;
}
removeConditionSource(cmd);
}
}
return res;
},
beforeConnectionSave: (connection) => {
return {
...connection,
useDatabaseUrl: 1,
};
},
adaptTableInfo(table) {
if (!table.primaryKey && !table.sortingKey) {
return {
...table,
tableEngine: 'Log',
};
}
return table;
},
};
module.exports = driver;

View File

@@ -0,0 +1,6 @@
import driver from './driver';
export default {
packageName: 'dbgate-plugin-clickhouse',
drivers: [driver],
};

View File

@@ -0,0 +1,23 @@
var webpack = require('webpack');
var path = require('path');
var config = {
context: __dirname + '/src/backend',
entry: {
app: './index.js',
},
target: 'node',
output: {
path: path.resolve(__dirname, 'dist'),
filename: 'backend.js',
libraryTarget: 'commonjs2',
},
// uncomment for disable minimalization
// optimization: {
// minimize: false,
// },
};
module.exports = config;

View File

@@ -0,0 +1,24 @@
var webpack = require("webpack");
var path = require("path");
var config = {
context: __dirname + "/src/frontend",
entry: {
app: "./index.js",
},
target: "web",
output: {
path: path.resolve(__dirname, "dist"),
filename: "frontend.js",
libraryTarget: "var",
library: 'plugin',
},
// uncomment for disable minimalization
// optimization: {
// minimize: false,
// },
};
module.exports = config;

View File

@@ -142,6 +142,7 @@ const driver = {
title: 'Microsoft SQL Server',
defaultPort: 1433,
defaultAuthTypeName: 'tedious',
supportsTransactions: true,
// databaseUrlPlaceholder: 'e.g. server=localhost&authentication.type=default&authentication.type.user=myuser&authentication.type.password=pwd&options.database=mydb',
getNewObjectTemplates() {

View File

@@ -66,10 +66,10 @@ class Analyser extends DatabaseAnalyser {
super(pool, driver, version);
}
createQuery(resFileName, typeFields) {
createQuery(resFileName, typeFields, replacements = {}) {
let res = sql[resFileName];
res = res.replace('#DATABASE#', this.pool._database_name);
return super.createQuery(res, typeFields);
return super.createQuery(res, typeFields, replacements);
}
getRequestedViewNames(allViewNames) {

View File

@@ -2,6 +2,8 @@ module.exports = `
select
TABLE_NAME as pureName,
TABLE_ROWS as tableRowCount,
ENGINE as tableEngine,
TABLE_COMMENT as objectComment,
case when ENGINE='InnoDB' then CREATE_TIME else coalesce(UPDATE_TIME, CREATE_TIME) end as modifyDate
from information_schema.tables
where TABLE_SCHEMA = '#DATABASE#' and (TABLE_TYPE='BASE TABLE' or TABLE_TYPE='SYSTEM VERSIONED') and TABLE_NAME =OBJECT_ID_CONDITION;

View File

@@ -99,6 +99,53 @@ const dialect = {
};
}
},
getSupportedEngines() {
return [];
},
getTableFormOptions(intent) {
return [
{
type: 'dropdowntext',
options: this.getSupportedEngines(),
label: 'Engine',
name: 'tableEngine',
sqlFormatString: '^engine = %s',
},
{
type: 'text',
label: 'Comment',
name: 'objectComment',
sqlFormatString: '^comment = %v',
allowEmptyValue: true,
},
];
},
};
const mysqlDialect = {
...dialect,
getSupportedEngines() {
const mysqlEngines = [
'InnoDB', // Default and most commonly used engine with ACID transaction support and referential integrity.
'MyISAM', // Older engine without transaction or referential integrity support.
'MEMORY', // Tables stored in memory, very fast but volatile, used for temporary data.
'CSV', // Tables stored in CSV format, useful for import/export of data.
'ARCHIVE', // Engine for storing large amounts of historical data with compression.
'BLACKHOLE', // Engine that discards data, useful for replication.
'FEDERATED', // Access tables on remote MySQL servers.
'MRG_MYISAM', // Merges multiple MyISAM tables into one.
'NDB', // Cluster storage engine for MySQL Cluster.
'EXAMPLE', // Example engine for developers, has no real functionality.
'PERFORMANCE_SCHEMA', // Engine used for performance monitoring in MySQL.
'SEQUENCE', // Special engine for sequences, used in MariaDB.
'SPIDER', // Engine for horizontal partitioning, often used in MariaDB.
'ROCKSDB', // Engine optimized for read-heavy workloads, commonly used in Facebook MySQL.
'TokuDB', // Engine with high data compression and SSD optimization.
];
return mysqlEngines;
},
};
const mysqlDriverBase = {
@@ -108,7 +155,6 @@ const mysqlDriverBase = {
(values.authType == 'socket' && ['socketPath'].includes(field)) ||
(values.authType != 'socket' && ['server', 'port'].includes(field)),
dumperClass: Dumper,
dialect,
defaultPort: 3306,
getQuerySplitterOptions: usage =>
usage == 'editor'
@@ -120,6 +166,7 @@ const mysqlDriverBase = {
authTypeLabel: 'Connection mode',
defaultAuthTypeName: 'hostPort',
defaultSocketPath: '/var/run/mysqld/mysqld.sock',
supportsTransactions: true,
getNewObjectTemplates() {
return [
@@ -136,6 +183,7 @@ const mysqlDriverBase = {
/** @type {import('dbgate-types').EngineDriver} */
const mysqlDriver = {
...mysqlDriverBase,
dialect: mysqlDialect,
engine: 'mysql@dbgate-plugin-mysql',
title: 'MySQL',
__analyserInternals: {
@@ -143,9 +191,39 @@ const mysqlDriver = {
},
};
const mariaDbDialect = {
...dialect,
getSupportedEngines() {
const mariaDBEngines = [
'InnoDB', // Main transactional engine, similar to MySQL, supports ACID transactions and referential integrity.
'Aria', // Replacement for MyISAM, supports crash recovery and optimized for high speed.
'MyISAM', // Older engine without transaction support, still supported for compatibility.
'MEMORY', // Tables stored in memory, suitable for temporary data.
'CSV', // Stores data in CSV format, easy for export/import.
'ARCHIVE', // Stores compressed data, suitable for historical records.
'BLACKHOLE', // Engine that does not store data, often used for replication.
'FEDERATED', // Allows access to tables on remote MariaDB/MySQL servers.
'MRG_MyISAM', // Allows merging multiple MyISAM tables into one.
'SEQUENCE', // Special engine for generating sequences.
'SphinxSE', // Engine for full-text search using Sphinx.
'SPIDER', // Engine for sharding, supports horizontal partitioning.
'TokuDB', // High-compression engine optimized for large data sets and SSDs.
'RocksDB', // Read-optimized engine focused on performance with large data.
'CONNECT', // Engine for accessing external data sources (e.g., files, web services).
'OQGRAPH', // Graph engine, suitable for hierarchical and graph structures.
'ColumnStore', // Analytical engine for columnar data storage, suitable for Big Data.
'Mroonga', // Engine supporting full-text search in Japanese and other languages.
'S3', // Allows storing data in Amazon S3-compatible storage.
'XtraDB', // Enhanced InnoDB engine with optimizations from Percona (commonly used in older MariaDB versions).
];
return mariaDBEngines;
},
};
/** @type {import('dbgate-types').EngineDriver} */
const mariaDriver = {
...mysqlDriverBase,
dialect: mariaDbDialect,
engine: 'mariadb@dbgate-plugin-mysql',
title: 'MariaDB',
__analyserInternals: {

View File

@@ -14,7 +14,7 @@ const dialect = {
// stringEscapeChar: '\\',
stringEscapeChar: "'",
fallbackDataType: 'varchar',
anonymousPrimaryKey: true,
anonymousPrimaryKey: false,
enableConstraintsPerTable: true,
dropColumnDependencies: ['dependencies'],
quoteIdentifier(s) {
@@ -97,6 +97,7 @@ const oracleDriver = {
// ['server', 'port', 'user', 'password', 'defaultDatabase', 'singleDatabase'].includes(field),
getQuerySplitterOptions: () => oracleSplitterOptions,
readOnlySessions: true,
supportsTransactions: true,
databaseUrlPlaceholder: 'e.g. localhost:1521/orcl',

View File

@@ -12,7 +12,7 @@ const dialect = {
// stringEscapeChar: '\\',
stringEscapeChar: "'",
fallbackDataType: 'varchar',
anonymousPrimaryKey: true,
anonymousPrimaryKey: false,
enableConstraintsPerTable: true,
dropColumnDependencies: ['dependencies'],
quoteIdentifier(s) {

View File

@@ -44,6 +44,7 @@ const driver = {
engine: 'sqlite@dbgate-plugin-sqlite',
title: 'SQLite',
readOnlySessions: true,
supportsTransactions: true,
showConnectionField: (field, values) => field == 'databaseFile' || field == 'isReadOnly',
showConnectionTab: (field) => false,
beforeConnectionSave: (connection) => ({

View File

@@ -54,3 +54,4 @@ changePackageFile('plugins/dbgate-plugin-postgres', json.version);
changePackageFile('plugins/dbgate-plugin-sqlite', json.version);
changePackageFile('plugins/dbgate-plugin-redis', json.version);
changePackageFile('plugins/dbgate-plugin-oracle', json.version);
changePackageFile('plugins/dbgate-plugin-clickhouse', json.version);

View File

@@ -466,6 +466,18 @@
resolved "https://registry.yarnpkg.com/@changesets/types/-/types-0.4.0.tgz#3413badb2c3904357a36268cb9f8c7e0afc3a804"
integrity sha512-TclHHKDVYQ8rJGZgVeWiF7c91yWzTTWdPagltgutelGu/Psup5PQlUq6svx7S8suj+jXcaE34yEEsfIvzXXB2Q==
"@clickhouse/client-common@1.5.0":
version "1.5.0"
resolved "https://registry.yarnpkg.com/@clickhouse/client-common/-/client-common-1.5.0.tgz#fa621ee4fbdf8f4b44e5548fd5d9fe1e44b07e88"
integrity sha512-U3vDp+PDnNVEv6kia+Mq5ygnlMZzsYU+3TX+0da3XvL926jzYLMBlIvFUxe2+/5k47ySvnINRC/2QxVK7PC2/A==
"@clickhouse/client@^1.5.0":
version "1.5.0"
resolved "https://registry.yarnpkg.com/@clickhouse/client/-/client-1.5.0.tgz#ce6110a710396544a2435fe9ed8f61f20bd178b8"
integrity sha512-Udwyoec+AHHS1TiLxDiWiJWcm2BvhZEqGjmUnvzL54NyT8D8eh2mxn5RR/W5ie64JDnsKLeZFlPYKRRhZMhkxA==
dependencies:
"@clickhouse/client-common" "1.5.0"
"@cnakazawa/watch@^1.0.3":
version "1.0.4"
resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a"
@@ -4060,7 +4072,7 @@ extsprintf@^1.2.0:
resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07"
integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA==
fast-copy@^3.0.0:
fast-copy@^3.0.0, fast-copy@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/fast-copy/-/fast-copy-3.0.2.tgz#59c68f59ccbcac82050ba992e0d5c389097c9d35"
integrity sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==
@@ -4726,6 +4738,11 @@ help-me@^4.0.1:
glob "^8.0.0"
readable-stream "^3.6.0"
help-me@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/help-me/-/help-me-5.0.0.tgz#b1ebe63b967b74060027c2ac61f9be12d354a6f6"
integrity sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==
highlight.js@11.9.0:
version "11.9.0"
resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-11.9.0.tgz#04ab9ee43b52a41a047432c8103e2158a1b8b5b0"
@@ -8302,6 +8319,26 @@ pino-abstract-transport@^1.0.0:
readable-stream "^4.0.0"
split2 "^4.0.0"
pino-pretty@^11.2.2:
version "11.2.2"
resolved "https://registry.yarnpkg.com/pino-pretty/-/pino-pretty-11.2.2.tgz#5e8ec69b31e90eb187715af07b1d29a544e60d39"
integrity sha512-2FnyGir8nAJAqD3srROdrF1J5BIcMT4nwj7hHSc60El6Uxlym00UbCCd8pYIterstVBFlMyF1yFV8XdGIPbj4A==
dependencies:
colorette "^2.0.7"
dateformat "^4.6.3"
fast-copy "^3.0.2"
fast-safe-stringify "^2.1.1"
help-me "^5.0.0"
joycon "^3.1.1"
minimist "^1.2.6"
on-exit-leak-free "^2.1.0"
pino-abstract-transport "^1.0.0"
pump "^3.0.0"
readable-stream "^4.0.0"
secure-json-parse "^2.4.0"
sonic-boom "^4.0.1"
strip-json-comments "^3.1.1"
pino-pretty@^9.1.1:
version "9.4.1"
resolved "https://registry.yarnpkg.com/pino-pretty/-/pino-pretty-9.4.1.tgz#89121ef32d00a4d2e4b1c62850dcfff26f62a185"
@@ -9478,6 +9515,13 @@ sonic-boom@^3.0.0:
dependencies:
atomic-sleep "^1.0.0"
sonic-boom@^4.0.1:
version "4.1.0"
resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-4.1.0.tgz#4f039663ba191fac5cfe4f1dc330faac079e4342"
integrity sha512-NGipjjRicyJJ03rPiZCJYjwlsuP2d1/5QUviozRXC7S3WdVWNK5e3Ojieb9CCyfhq2UC+3+SRd9nG3I2lPRvUw==
dependencies:
atomic-sleep "^1.0.0"
sorcery@^0.10.0:
version "0.10.0"
resolved "https://registry.yarnpkg.com/sorcery/-/sorcery-0.10.0.tgz#8ae90ad7d7cb05fc59f1ab0c637845d5c15a52b7"