From 83f69d89ffb6458a10954a1a8af46681042610cf Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 1 Apr 2025 17:32:24 +0200 Subject: [PATCH 01/49] WIP --- common/volatilePackages.js | 1 + .../__tests__/alter-database.spec.js | 2 + .../__tests__/alter-table.spec.js | 140 +++---- .../__tests__/object-analyse.spec.js | 6 +- integration-tests/__tests__/query.spec.js | 6 +- integration-tests/engines.js | 21 +- integration-tests/package.json | 2 +- packages/api/testduck.db | Bin 0 -> 2371584 bytes packages/types/engines.d.ts | 1 + packages/types/query.d.ts | 1 + packages/types/test-engines.d.ts | 1 + plugins/dbgate-plugin-duckdb/.gitignore | 25 ++ plugins/dbgate-plugin-duckdb/README.md | 6 + plugins/dbgate-plugin-duckdb/icon.svg | 1 + plugins/dbgate-plugin-duckdb/package.json | 45 +++ .../dbgate-plugin-duckdb/prettier.config.js | 8 + .../src/backend/Analyser.helpers.js | 374 ++++++++++++++++++ .../src/backend/Analyser.js | 85 ++++ .../src/backend/driver.js | 165 ++++++++ .../src/backend/helpers.js | 65 +++ .../dbgate-plugin-duckdb/src/backend/index.js | 6 + .../src/backend/sql/columns.js | 1 + .../src/backend/sql/foreignKeys.js | 1 + .../src/backend/sql/index.js | 17 + .../src/backend/sql/indexes.js | 1 + .../src/backend/sql/primaryKeys.js | 1 + .../src/backend/sql/tables.js | 1 + .../src/backend/sql/uniques.js | 1 + .../src/backend/sql/views.js | 1 + .../src/frontend/Dumper.js | 7 + .../src/frontend/driver.js | 72 ++++ .../src/frontend/index.js | 6 + .../webpack-backend.config.js | 28 ++ .../webpack-frontend.config.js | 30 ++ yarn.lock | 43 ++ 35 files changed, 1096 insertions(+), 75 deletions(-) create mode 100644 packages/api/testduck.db create mode 100644 plugins/dbgate-plugin-duckdb/.gitignore create mode 100644 plugins/dbgate-plugin-duckdb/README.md create mode 100644 plugins/dbgate-plugin-duckdb/icon.svg create mode 100644 plugins/dbgate-plugin-duckdb/package.json create mode 100644 plugins/dbgate-plugin-duckdb/prettier.config.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/Analyser.helpers.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/Analyser.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/driver.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/helpers.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/index.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/sql/columns.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/sql/foreignKeys.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/sql/index.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/sql/indexes.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/sql/primaryKeys.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/sql/tables.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/sql/uniques.js create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/sql/views.js create mode 100644 plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js create mode 100644 plugins/dbgate-plugin-duckdb/src/frontend/driver.js create mode 100644 plugins/dbgate-plugin-duckdb/src/frontend/index.js create mode 100644 plugins/dbgate-plugin-duckdb/webpack-backend.config.js create mode 100644 plugins/dbgate-plugin-duckdb/webpack-frontend.config.js diff --git a/common/volatilePackages.js b/common/volatilePackages.js index 00ee25a33..f3ffe1285 100644 --- a/common/volatilePackages.js +++ b/common/volatilePackages.js @@ -21,6 +21,7 @@ const volatilePackages = [ 'axios', 'ssh2', 'wkx', + '@duckdb/node-api', ]; module.exports = volatilePackages; diff --git a/integration-tests/__tests__/alter-database.spec.js b/integration-tests/__tests__/alter-database.spec.js index 490de76a9..6ffaa35f7 100644 --- a/integration-tests/__tests__/alter-database.spec.js +++ b/integration-tests/__tests__/alter-database.spec.js @@ -36,6 +36,8 @@ async function testDatabaseDiff(conn, driver, mangle, createObject = null) { if (createObject) await driver.query(conn, createObject); const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn))); + console.log('str1'); + console.dir(structure1, { depth: 10 }); let structure2 = _.cloneDeep(structure1); mangle(structure2); structure2 = extendDatabaseInfo(structure2); diff --git a/integration-tests/__tests__/alter-table.spec.js b/integration-tests/__tests__/alter-table.spec.js index 06baeab95..31303a3c3 100644 --- a/integration-tests/__tests__/alter-table.spec.js +++ b/integration-tests/__tests__/alter-table.spec.js @@ -136,76 +136,76 @@ describe('Alter table', () => { ); } - const hasEnginesWithNullable = engines.filter(x => !x.skipNullable).length > 0; - - if (hasEnginesWithNullable) { - const source = create_engines_columns_source(engines.filter(x => !x.skipNullable)); - - test.each(source)( - 'Change nullability - %s - %s', - testWrapper(async (conn, driver, column, engine) => { - await testTableDiff( - engine, - conn, - driver, - tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, notNull: true } : x))) - ); - }) - ); - } - - test.each(columnsSource)( - 'Rename column - %s - %s', - testWrapper(async (conn, driver, column, engine) => { - await testTableDiff( - engine, - conn, - driver, - tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x))) - ); - }) - ); - - test.each(engines.map(engine => [engine.label, engine]))( - 'Drop index - %s', - testWrapper(async (conn, driver, engine) => { - await testTableDiff(engine, conn, driver, tbl => { - tbl.indexes = []; - }); - }) - ); - - const enginesWithDefault = engines.filter(x => !x.skipDefaultValue); - const hasEnginesWithDefault = enginesWithDefault.length > 0; - - if (hasEnginesWithDefault) { - test.each(enginesWithDefault.map(engine => [engine.label, engine]))( - 'Add default value - %s', - testWrapper(async (conn, driver, engine) => { - await testTableDiff(engine, conn, driver, tbl => { - tbl.columns.find(x => x.columnName == 'col_std').defaultValue = '123'; - }); - }) - ); - - test.each(enginesWithDefault.map(engine => [engine.label, engine]))( - 'Unset default value - %s', - testWrapper(async (conn, driver, engine) => { - await testTableDiff(engine, conn, driver, tbl => { - tbl.columns.find(x => x.columnName == 'col_def').defaultValue = undefined; - }); - }) - ); - - test.each(enginesWithDefault.map(engine => [engine.label, engine]))( - 'Change default value - %s', - testWrapper(async (conn, driver, engine) => { - await testTableDiff(engine, conn, driver, tbl => { - tbl.columns.find(x => x.columnName == 'col_def').defaultValue = '567'; - }); - }) - ); - } + // const hasEnginesWithNullable = engines.filter(x => !x.skipNullable).length > 0; + // + // if (hasEnginesWithNullable) { + // const source = create_engines_columns_source(engines.filter(x => !x.skipNullable)); + // + // test.each(source)( + // 'Change nullability - %s - %s', + // testWrapper(async (conn, driver, column, engine) => { + // await testTableDiff( + // engine, + // conn, + // driver, + // tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, notNull: true } : x))) + // ); + // }) + // ); + // } + // + // test.each(columnsSource)( + // 'Rename column - %s - %s', + // testWrapper(async (conn, driver, column, engine) => { + // await testTableDiff( + // engine, + // conn, + // driver, + // tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x))) + // ); + // }) + // ); + // + // test.each(engines.map(engine => [engine.label, engine]))( + // 'Drop index - %s', + // testWrapper(async (conn, driver, engine) => { + // await testTableDiff(engine, conn, driver, tbl => { + // tbl.indexes = []; + // }); + // }) + // ); + // + // const enginesWithDefault = engines.filter(x => !x.skipDefaultValue); + // const hasEnginesWithDefault = enginesWithDefault.length > 0; + // + // if (hasEnginesWithDefault) { + // test.each(enginesWithDefault.map(engine => [engine.label, engine]))( + // 'Add default value - %s', + // testWrapper(async (conn, driver, engine) => { + // await testTableDiff(engine, conn, driver, tbl => { + // tbl.columns.find(x => x.columnName == 'col_std').defaultValue = '123'; + // }); + // }) + // ); + // + // test.each(enginesWithDefault.map(engine => [engine.label, engine]))( + // 'Unset default value - %s', + // testWrapper(async (conn, driver, engine) => { + // await testTableDiff(engine, conn, driver, tbl => { + // tbl.columns.find(x => x.columnName == 'col_def').defaultValue = undefined; + // }); + // }) + // ); + // + // test.each(enginesWithDefault.map(engine => [engine.label, engine]))( + // 'Change default value - %s', + // testWrapper(async (conn, driver, engine) => { + // await testTableDiff(engine, conn, driver, tbl => { + // tbl.columns.find(x => x.columnName == 'col_def').defaultValue = '567'; + // }); + // }) + // ); + // } // test.each(engines.map(engine => [engine.label, engine]))( // 'Change autoincrement - %s', diff --git a/integration-tests/__tests__/object-analyse.spec.js b/integration-tests/__tests__/object-analyse.spec.js index 25055fd04..60bfbaf18 100644 --- a/integration-tests/__tests__/object-analyse.spec.js +++ b/integration-tests/__tests__/object-analyse.spec.js @@ -20,7 +20,11 @@ function flatSourceParameters() { } function flatSourceTriggers() { - return _.flatten(engines.map(engine => (engine.triggers || []).map(trigger => [engine.label, trigger, engine]))); + return _.flatten( + engines + .filter(engine => !engine.skipTriggers) + .map(engine => (engine.triggers || []).map(trigger => [engine.label, trigger, engine])) + ); } function flatSourceSchedulerEvents() { diff --git a/integration-tests/__tests__/query.spec.js b/integration-tests/__tests__/query.spec.js index 4f6896a54..e774594fd 100644 --- a/integration-tests/__tests__/query.spec.js +++ b/integration-tests/__tests__/query.spec.js @@ -147,6 +147,8 @@ describe('Query', () => { engine.skipOrderBy ? '' : 'ORDER BY ~id' }; ` ); + console.log('res'); + console.dir(results, { depth: 10 }); expect(results.length).toEqual(1); const res1 = results[0]; @@ -183,8 +185,8 @@ describe('Query', () => { { discardResult: true } ); const res = await runQueryOnDriver(conn, driver, dmp => dmp.put('SELECT COUNT(*) AS ~cnt FROM ~t1')); - // console.log(res); - expect(res.rows[0].cnt == 3).toBeTruthy(); + const cnt = parseInt(res.rows[0].cnt); + expect(cnt).toEqual(3); }) ); diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 75ead4348..0073f5fb5 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -654,6 +654,24 @@ const cassandraEngine = { objects: [], }; +/** @type {import('dbgate-types').TestEngineInfo} */ +const duckdbEngine = { + label: 'DuckDB', + generateDbFile: true, + connection: { + engine: 'duckdb@dbgate-plugin-duckdb', + }, + objects: [views], + skipOnCI: false, + skipChangeColumn: true, + skipIndexes: true, + skipStringLength: true, + skipTriggers: true, + skipDataDuplicator: true, + skipAutoIncrement: true, + supportRenameSqlObject: true, +}; + const enginesOnCi = [ // all engines, which would be run on GitHub actions mysqlEngine, @@ -680,8 +698,9 @@ const enginesOnLocal = [ // cockroachDbEngine, // clickhouseEngine, // libsqlFileEngine, - libsqlWsEngine, + // libsqlWsEngine, // oracleEngine, + duckdbEngine, ]; /** @type {import('dbgate-types').TestEngineInfo[] & Record} */ diff --git a/integration-tests/package.json b/integration-tests/package.json index 5e8bb394d..a8c3e670b 100644 --- a/integration-tests/package.json +++ b/integration-tests/package.json @@ -12,7 +12,7 @@ "wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js", "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js", "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest --testTimeout=5000", - "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-duplicator.spec.js", + "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/alter-database.spec.js", "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults --detectOpenHandles --forceExit --testTimeout=10000", "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local" }, diff --git a/packages/api/testduck.db b/packages/api/testduck.db new file mode 100644 index 0000000000000000000000000000000000000000..9b9f42b80c1d4d9e43b35563bbed9de7a252f6ad GIT binary patch literal 2371584 zcmeI*O^h5@LIB{`-Sg*=texMrlWgq89QLx2ldu9#I2&NSi%Ft{1?6N--0jQ=+hgnr zF^W(|KA=U>YS)LA&_a|05*LJ4$^}-dNJtCHj1zti2IYWIScH@t5{D=flAHIcx_YL2 zdd4%;Q$79jmE2QRudCkA_o}MvxvF0M?En7b=8u2+{#SnO+t2;x>9Dim51xAR=_jAs zdf8uYeC_GR`CokISH8COGQJ@|fB*pk1PBlyK!5-N0t5yp@W>Cp*Zj`hKYUW_o#ZbZ zYKKqdQ*-&K(hfI4fB*pk1PBlyK!5-N0t5&QyTA*deER)|U;Lv>)y|C0v}V7k{oe^= zjsO7y1PBlyK!5-N0t5(*qQKjaUj6D1|N8dx&DKy4v*$y#9xC-vy%44@%`aYF3?Y4a zK82-!jWD;|xVBt>{rvB(FR>a9U2eQ_Zh8LIi;cCAufJGt#Hst5&DTO$$RoKGsws-q zG#4jotD&-%#hxW!xfLqa#d?}fYp%40lwz}aA%rZRxY~m+zVyxCdiJGP&b5;~_tb?j zQx8X*&3ZVM{(L3<`O`}J`*SAq!B*7r@h!GGsR+isq*P>UyE+3zNg~m|9YJ1RoPR$E4dMO zk&Bz^zBd0_R`XpOe`j46No}s^;+t5@#bl2vj$3!!c2X4=3+?;zoz*(k2>)}aqgoFa z)tV7*SEp0$%G$C{WAyu0r!!fGo*qP{rXyN@AWJ)NQ5731wG;7JM2wrohQryB4#cO= zo!YNsS&s0`5F$L@2tT-g8-!=}5g|V;CWPsFm|IE*$zJ<+QE^?rS=;{t%>DlB^Ou(w zm;Z3{T6seC`Q_|DaJciJKh?3#v@^B+4qf>u-cHNe_1-W z&Et*mSMfKrIBv(EzxKA7D{i-k&)j_vpV{#q9#j5}7hd}HZ+_$XbFGx0+WPlm_U;b8S7=7F2Ae#j6vUlg3)H$?bAH#?QKM@!0<^fyQL< zvg^6*_{XFg6oUW(0t5&U7@$Dj2I+2B+_iP2i7~)Bl2j=IPR&bU73q))v^w(}c<##H z&&rN};9wFYK!CuQ2=s1ZY<)>Y+hR;Aeu9hPoMeJ4U(pE=7;1q@IzCSibxn+R=G;qh zu=Q8mLY3d=ROF@Op0i?C&z^IMw~oaJT+<^hX^3}s-H6>c)I*v{Lk##?cfh|qn(pYz z`Xear28;JuZN8%{YtU@E7i&+qk;Qwrdf&A6?8Vr-#EEY$#hcciTbRFkaj|itxM?ju zUY>?s+)dUlO(UG?RiL@~l-lXW^_bfEJhjtBY7h34+V`W<7Jd=Gb<^L;JcV7AC_Q1l z`EjOAb)tb;>?nI57kky-W3c8h3zS{IWye3vw(>@R009C7N)$L+V%!rGb>0f;Zdc6R ztsz!M6XT%n-#Dnc?s`4NL1ja_1PBlqIRVGdk*hKpCO}|91h!~e-0iAM8BL1`5#Mt$ zN8>VYamB12X7<>hf4kSha2Hf zvcAyUuN`Ukg`T}zKo|A-i!wTP=<>U$f1nZGJ-)t!>!Lor^W~f_>i2v;XGH=82oNAJ zHUe|m9msms%>El>7xzKyRdZmwfV(~iMtvX&m0iDO$A2Jb5+y)@009C7Mp|HtOZs)Uti=#Jhb)Gj_NE!1 zmYUnxnP!|{=$ikxtmj94X{Me#^`%*CV3R)fvpL>dK-u+McKqXAJ4#1jqy&C$q;e~> z1PBm_0=+L?=}U6}8;8E&27oLH0tCiV!0~e|t5u;0jEum&Ba_pE%jgzO41H=IT$OB< zvWvZ2$p|Mf`~psihrf>GkN^P!qbM+JcQy2GT4WF2Y4XVmL!xAkjz0tCtv*uz6NQ}wXY8sfWeetO@=r)*-tUwok};O613 zfMyJME!~h6U#-;rZsV*)lQK2iY^K*h#V3JMTG!k8yb$6lG4$rqGq0RITON#<2@oJK zsRCb^)PgHF0RjXFjH*E1R_Jb7#NGk&VJ*xRCA$^EhatW}Fnc&8UN+IVC}fNY5FkK+ z0D%b-I6OgRRa64I70|fYt(fHr5FkK+009C72oUHH=-sFo=)F)!Ix=L2009C72oNAZ zfB=CJ6v$f^+rCt#D3zx%3JGHZlOk~cq!d&+2@oJafB=EP3G}|U;c1M))qs=;5FkK+ zKwIE}_LK<%1PF|xfX2lr)|xyMAV7cs0RjXF3{K#|!6h~7DfMn#4D_jtQ74pt0t5&U zAV7cs0Rj^&u&bvr;!7@S7vd{0rqkOpDhqKcWL?yT*TQQ4W{gVv9T~Me=4P{zf0M2F zR$8&zv0|}UdiO><{zf=beiA3jPvX=LlBgB}dxJW?WfC8}vtAo(X*wTvk<4^=kHuLP zt;gD`<2}CJ@u3Oc?Y1nsw`pxjGZtTzUB6|=KNgjs5CjMiAV7e?Kn3zPNU80LftF|3 z6LUT|>^0P1{>rZ365@ZjzY<%U009C72oNBU3uspic1tMXyl}7uklMBaW!G=n@o!r= zUlJfNXn{utouKpx5FpTxz>eA#ty{SEcpp?h*|RQzkrF8Tyua-DM+#eJ2@oJafWQF= zjX)k{ac}~8n<8r| z#3$*CAw7X#Sq&@kiTY-ejsPuIgrYj~g$e)L#+3V%bu_a5&G_}}h;fBjO| z5`X(ddLTV$^%ee;H_o@i^(|noMNo{ZaelTHmZoPmOi#C_(|Bq_cyX*tVQHN-q;au9 zr|*mD^j#W$HEo7*$mjeeK!5-N0t5&UAV7cs0RjZ}QQ+7<#w98OV0(o1ZyJZnuA^ETtipOWdtq?v8m5=hLTjFICjf=kVML>W60RjXF5FkKc90ba2 zR6IToK*~UX009C72oNAZfWXcKdN(ez`<&8O?uFVJqooLplE9ZnDYNoQfB*pk1PF|Z zK;E+0wkPYND4xbBB#a3VAV7dX83Ip~5ve!{5FjvY0==(ocp78a>PdbG5FkK+009C7 z_FdrkzQ;J}5o%ma`WB%41PBlyK!5-N0tEUL=-sIBG)AASle_wf$t}8)6Cgl<009C7 zCR<=vPh-TFT+}YaS71!1w`Eio;#A1Gs12`$)%?vEmG(O_YI)4fW+VS5Tk);5Vzpz% zVzKl_j&}TwaHRYsPL!X-sU0LyEd=%kb$ZJrK6q!nHrCQ~KI|fy>FyqjvnpDTwN=O7 z?%4k^#9OrbHyq{|%C6rX#{cDEHj+021PBlyKwvxt@-~Qe#h_|;hv$WZ%8qpQS)lCt zEj#{w<`5kL0-FVXeshoz0t5(*hk$m)VC&I&;b6T7om$!TTXy_|_L3d}0t5&U7$Sj_ zLzI`1%*u}16|Gyi+y^z1RV>p36e#<=zwG!2NF^x(1PBlyKwwM+wrNlFd|qP`+7y#^ z&Q@l5^X;qI9FyHeF$fSKK!5;&E`eItyxGYX$j|+<7DIfJz8KOI_?6YL5}&AVHj7iw z?8*K3Nd9y^th9#XjqsC)(o^^w0{-mbu7Eq;0sr|(*AjpK#q>aWa0XiV8xtTvpj?6O zcK=)}x#EDI&K0Vyurw~t9j4-R`lKO+PiIs4c&0U-!e=*37jdWXBL3Fb;qT&s|C5-R zV&0el0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t9k_YyW-XGr1>+A`TCWGrOSCeev^1{G|Ek7Ur*BTx^{9`qia3E`NRL)e|pYT3lXu z_Tu9C#))q&EnL2~bU8c|m-|Ni+#f$D;wR0Y&(}NMxITaB%EiW$=a(+My7cPC+4&U5 z#(?ldT<=)?q;w4g2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7e?U*EoMZ3009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PGKMaO}P#4=um)#_v{Q+|~H0wbIz>d|ECIA3PtzYW%FlkAVOI0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oM-;f%iT?b^FGfAAfeI zX&^v=0D=AmDq*GbXQ9@btvSy2uPcVh5jb4zIF)L#52cXVN*+F2ZH@DhzXS*nAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+Kp6t>eSYfpjW<93>`v1_fB*pk z{R>pWO6S44)|#z3&i1b>hRG2)T#^O3&<2oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!89Q0`Gl(>h_H{KmP1a(?Ea#0RsICRKiN< z!MfI(tvSy2uPcVh5jb4zIF)L#52cXVN*+F2ZH@DhzXS*nAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly cK!5-N0t5&UAV7cs0RjXF5FkK+Kp6u65A8A|761SM literal 0 HcmV?d00001 diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 652592dd8..e2d6abfb0 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -33,6 +33,7 @@ export interface QueryOptions { discardResult?: boolean; importSqlDump?: boolean; range?: { offset: number; limit: number }; + readonly?: boolean; } export interface WriteTableOptions { diff --git a/packages/types/query.d.ts b/packages/types/query.d.ts index e3abbccfe..633c1b01d 100644 --- a/packages/types/query.d.ts +++ b/packages/types/query.d.ts @@ -7,6 +7,7 @@ export interface QueryResultColumn { columnName: string; notNull: boolean; autoIncrement?: boolean; + dataType?: string; } export interface QueryResult { diff --git a/packages/types/test-engines.d.ts b/packages/types/test-engines.d.ts index c08e07f56..d0aeed3ee 100644 --- a/packages/types/test-engines.d.ts +++ b/packages/types/test-engines.d.ts @@ -40,6 +40,7 @@ export type TestEngineInfo = { skipPkDrop?: boolean; skipOrderBy?: boolean; skipImportModel?: boolean; + skipTriggers?: boolean; forceSortResults?: boolean; forceSortStructureColumns?: boolean; diff --git a/plugins/dbgate-plugin-duckdb/.gitignore b/plugins/dbgate-plugin-duckdb/.gitignore new file mode 100644 index 000000000..1dad8826d --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/.gitignore @@ -0,0 +1,25 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# production +build +dist +lib + +# misc +.DS_Store +.env.local +.env.development.local +.env.test.local +.env.production.local + +npm-debug.log* +yarn-debug.log* +yarn-error.log* diff --git a/plugins/dbgate-plugin-duckdb/README.md b/plugins/dbgate-plugin-duckdb/README.md new file mode 100644 index 000000000..b47943413 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/README.md @@ -0,0 +1,6 @@ +[![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier) +[![NPM version](https://img.shields.io/npm/v/dbgate-plugin-sqlite.svg)](https://www.npmjs.com/package/dbgate-plugin-sqlite) + +# dbgate-plugin-sqlite + +Use DbGate for install of this plugin diff --git a/plugins/dbgate-plugin-duckdb/icon.svg b/plugins/dbgate-plugin-duckdb/icon.svg new file mode 100644 index 000000000..e5e0d0138 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/plugins/dbgate-plugin-duckdb/package.json b/plugins/dbgate-plugin-duckdb/package.json new file mode 100644 index 000000000..fd1b853c1 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/package.json @@ -0,0 +1,45 @@ +{ + "name": "dbgate-plugin-duckdb", + "main": "dist/backend.js", + "version": "6.0.0-alpha.1", + "homepage": "https://dbgate.org", + "description": "DuckDB connect plugin for DbGate", + "repository": { + "type": "git", + "url": "https://github.com/dbgate/dbgate" + }, + "author": "Jan Prochazka", + "license": "GPL-3.0", + "keywords": [ + "dbgate", + "duckdb", + "dbgatebuiltin" + ], + "files": [ + "dist", + "icon.svg" + ], + "scripts": { + "build:frontend": "webpack --config webpack-frontend.config", + "build:frontend:watch": "webpack --watch --config webpack-frontend.config", + "build:backend": "webpack --config webpack-backend.config.js", + "build": "yarn build:frontend && yarn build:backend", + "plugin": "yarn build && yarn pack && dbgate-plugin dbgate-plugin-duckdb", + "copydist": "yarn build && yarn pack && dbgate-copydist ../dist/dbgate-plugin-duckdb", + "plugout": "dbgate-plugout dbgate-plugin-duckdb", + "prepublishOnly": "yarn build" + }, + "devDependencies": { + "dbgate-plugin-tools": "^1.0.4", + "webpack": "^5.91.0", + "webpack-cli": "^5.1.4" + }, + "dependencies": { + "dbgate-tools": "^6.0.0-alpha.1", + "lodash": "^4.17.21", + "dbgate-query-splitter": "^4.11.3" + }, + "optionalDependencies": { + "@duckdb/node-api": "^1.2.1-alpha.16" + } +} diff --git a/plugins/dbgate-plugin-duckdb/prettier.config.js b/plugins/dbgate-plugin-duckdb/prettier.config.js new file mode 100644 index 000000000..406484074 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/prettier.config.js @@ -0,0 +1,8 @@ +module.exports = { + trailingComma: 'es5', + tabWidth: 2, + semi: true, + singleQuote: true, + arrowParen: 'avoid', + printWidth: 120, +}; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/Analyser.helpers.js b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.helpers.js new file mode 100644 index 000000000..99ab7ad55 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.helpers.js @@ -0,0 +1,374 @@ +/** + * @typedef {object} DuckDbStringList + * @property {string[]} items + */ + +/** + * @typedef {object} DuckDbColumnRow + * @property {number | null} numeric_scale + * @property {number | null} numeric_precision_radix + * @property {number | null} numeric_precision + * @property {number | null} character_maximum_length + * @property {string | null} data_type_id + * @property {string} data_type + * @property {boolean} is_nullable + * @property {string | null} column_default + * @property {boolean} internal + * @property {string | null} comment + * @property {number} column_index + * @property {string} column_name + * @property {string} table_oid + * @property {string} table_name + * @property {string} schema_oid + * @property {string} schema_name + * @property {string} database_oid + * @property {string} database_name + */ + +/** + * @typedef {object} DuckDbConstraintRow + * @property {DuckDbStringList} referenced_column_names + * @property {string | null} referenced_table + * @property {string | null} constraint_name + * @property {DuckDbStringList} constraint_column_names + * @property {DuckDbStringList} constraint_column_indexes + * @property {string | null} expression + * @property {string | null} constraint_text + * @property {string} constraint_type + * @property {string} constraint_index + * @property {string} table_oid + * @property {string} table_name + * @property {string} schema_oid + * @property {string} schema_name + * @property {string} database_oid + * @property {string} database_name + */ + +/** + * @typedef {object} DuckDbTableRow + * @property {string | null} sql + * @property {string} check_constraint_count + * @property {string} index_count + * @property {string} column_count + * @property {string} estimated_size + * @property {boolean} has_primary_key + * @property {boolean} temporary + * @property {boolean} internal + * @property {{ entries: Array }} tags + * @property {string | null} comment + * @property {string} table_oid + * @property {string} table_name + * @property {string} schema_oid + * @property {string} schema_name + * @property {string} database_oid + * @property {string} database_name + */ + +/** + * Represents a single row returned from the duckdb_views() function. + * Note: Assumes OIDs and counts are represented as strings based on previous examples. + * + * @typedef {object} DuckDbViewRow + * @property {string} database_name + * @property {string} database_oid + * @property {string} schema_name + * @property {string} schema_oid + * @property {string} view_name + * @property {string} view_oid + * @property {string | null} comment + * @property {{ [key: string]: string } | null} tags + * @property {boolean} internal + * @property {boolean} temporary + * @property {string} column_count + * @property {string | null} sql + */ + +/** + * @param {DuckDbViewRow} duckDbViewRow + * @returns {import("dbgate-types").ViewInfo} + */ +function mapViewRowToViewInfo(duckDbViewRow) { + const viewInfo = { + pureName: duckDbViewRow.view_name, + schemaName: duckDbViewRow.schema_name, + objectId: duckDbViewRow.view_oid, + objectTypeField: 'view', + columns: [], + }; + + if (duckDbViewRow.comment != null) { + viewInfo.objectComment = duckDbViewRow.comment; + } + + if (duckDbViewRow.sql != null) { + viewInfo.createSql = duckDbViewRow.sql; + } + + return /** @type {import("dbgate-types").ViewInfo} */ (viewInfo); +} + +/** + * @param {DuckDbTableRow} rawTableData + */ +function mapRawTableToTableInfo(rawTableData) { + const pureName = rawTableData.table_name; + const schemaName = rawTableData.schema_name; + const objectId = rawTableData.table_oid; + const objectTypeField = 'table'; + const objectComment = rawTableData.comment; + + return { + pureName: pureName, + schemaName: schemaName, + objectId: objectId, + objectTypeField: objectTypeField, + objectComment: objectComment, + }; +} + +/** + * @typedef {object} DuckDbColumnDataTypeInfo + * @property {string} data_type + * @property {number | null} numeric_precision + * @property {number | null} numeric_scale + * @property {number | null} character_maximum_length + */ + +/** + * @param {DuckDbColumnDataTypeInfo | null | undefined} columnInfo + * @returns {string} + */ +function extractDataType(columnInfo) { + const baseType = columnInfo.data_type.toUpperCase(); + const precision = columnInfo.numeric_precision; + const scale = columnInfo.numeric_scale; + const maxLength = columnInfo.character_maximum_length; + + switch (baseType) { + case 'DECIMAL': + case 'NUMERIC': + if (typeof precision === 'number' && precision > 0 && typeof scale === 'number' && scale >= 0) { + return `${baseType}(${precision}, ${scale})`; + } + return baseType; + + case 'VARCHAR': + case 'CHAR': + console.log('this', maxLength); + if (typeof maxLength === 'number' && maxLength > 0) { + return `${baseType}(${maxLength})`; + } + return baseType; + + default: + return baseType; + } +} + +/** + * @param {DuckDbColumnRow} duckDbColumnData + */ +function mapRawColumnToColumnInfo(duckDbColumnData) { + const columnInfo = { + pureName: duckDbColumnData.table_name, + schemaName: duckDbColumnData.schema_name, + columnName: duckDbColumnData.column_name, + dataType: extractDataType(duckDbColumnData), + }; + + columnInfo.notNull = !duckDbColumnData.is_nullable; + + if (duckDbColumnData.column_default != null) { + columnInfo.defaultValue = duckDbColumnData.column_default; + } + + if (duckDbColumnData.comment != null) { + columnInfo.columnComment = duckDbColumnData.comment; + } + + if (duckDbColumnData.numeric_precision != null) { + columnInfo.precision = duckDbColumnData.numeric_precision; + } + + if (duckDbColumnData.numeric_scale != null) { + columnInfo.scale = duckDbColumnData.numeric_scale; + } + + if (duckDbColumnData.character_maximum_length != null) { + columnInfo.length = duckDbColumnData.character_maximum_length; + } + + return columnInfo; +} + +/** + * @param {DuckDbConstraintRow} duckDbConstraintData + * @returns {import("dbgate-types").ForeignKeyInfo} + */ +function mapConstraintRowToForeignKeyInfo(duckDbConstraintData) { + if ( + !duckDbConstraintData || + duckDbConstraintData.constraint_type !== 'FOREIGN KEY' || + duckDbConstraintData.referenced_table == null + ) { + return null; + } + + const columns = []; + const constraintColumns = duckDbConstraintData.constraint_column_names?.items; + const referencedColumns = duckDbConstraintData.referenced_column_names?.items; + + for (let i = 0; i < constraintColumns.length; i++) { + columns.push({ + columnName: constraintColumns[i], + refColumnName: referencedColumns[i], + }); + } + + const foreignKeyInfo = { + pureName: duckDbConstraintData.table_name, + schemaName: duckDbConstraintData.schema_name, + constraintType: 'foreignKey', + columns: columns, + refTableName: duckDbConstraintData.referenced_table, + }; + + if (duckDbConstraintData.constraint_name != null) { + foreignKeyInfo.constraintName = duckDbConstraintData.constraint_name; + } + + return /** @type {import("dbgate-types").ForeignKeyInfo} */ (foreignKeyInfo); +} + +/** + * @param {DuckDbConstraintRow} duckDbConstraintData + * @returns {import("dbgate-types").PrimaryKeyInfo} + */ +function mapConstraintRowToPrimaryKeyInfo(duckDbConstraintData) { + const columns = []; + const constraintColumns = duckDbConstraintData.constraint_column_names?.items; + + for (let i = 0; i < constraintColumns.length; i++) { + columns.push({ + columnName: constraintColumns[i], + }); + } + + const primaryKeyInfo = { + pureName: duckDbConstraintData.table_name, + schemaName: duckDbConstraintData.schema_name, + constraintType: 'primaryKey', + columns: columns, + }; + + if (duckDbConstraintData.constraint_name != null) { + primaryKeyInfo.constraintName = duckDbConstraintData.constraint_name; + } + + return /** @type {import("dbgate-types").PrimaryKeyInfo} */ (primaryKeyInfo); +} + +/** + * @typedef {object} DuckDbConstraintRow + * @property {DuckDbStringList} referenced_column_names + * @property {string | null} referenced_table + * @property {string | null} constraint_name + * @property {DuckDbStringList} constraint_column_names + * @property {DuckDbStringList} constraint_column_indexes + * @property {string | null} expression + * @property {string | null} constraint_text + * @property {string} constraint_type + * @property {string} constraint_index + * @property {string} table_oid + * @property {string} table_name + * @property {string} schema_oid + * @property {string} schema_name + * @property {string} database_oid + * @property {string} database_name + */ + +/** + * Maps a single DuckDbConstraintRow object to a UniqueInfo object if it represents a UNIQUE constraint. + * Assumes UniqueInfo and DuckDbConstraintRow are defined types/interfaces. + * @param {DuckDbConstraintRow} duckDbConstraintData - A single object conforming to DuckDbConstraintRow. + * @returns {import("dbgate-types").UniqueInfo | null} An object structured like UniqueInfo, or null if the input is not a valid UNIQUE constraint. + */ +function mapConstraintRowToUniqueInfo(duckDbConstraintData) { + if (!duckDbConstraintData || duckDbConstraintData.constraint_type !== 'UNIQUE') { + return null; + } + + const columns = []; + const constraintColumns = duckDbConstraintData.constraint_column_names?.items; + + if (Array.isArray(constraintColumns) && constraintColumns.length > 0) { + for (let i = 0; i < constraintColumns.length; i++) { + columns.push({ + columnName: constraintColumns[i], + }); + } + } else { + return null; + } + + const uniqueInfo = { + pureName: duckDbConstraintData.table_name, + schemaName: duckDbConstraintData.schema_name, + constraintType: 'unique', + columns: columns, + }; + + if (duckDbConstraintData.constraint_name != null) { + uniqueInfo.constraintName = duckDbConstraintData.constraint_name; + } + + return /** @type {import("dbgate-types").UniqueInfo} */ (uniqueInfo); +} + +/** + * @typedef {object} DuckDbIndexRow + * @property {string} database_name + * @property {string} database_oid + * @property {string} schema_name + * @property {string} schema_oid + * @property {string} index_name + * @property {string} index_oid + * @property {string} table_name + * @property {string} table_oid + * @property {string | null} comment + * @property {{ [key: string]: string } | null} tags + * @property {boolean} is_unique + * @property {boolean} is_primary + * @property {string | null} expressions + * @property {string | null} sql + */ + +/** + * @param {DuckDbIndexRow} duckDbIndexRow + * @returns {import("dbgate-types").IndexInfo} + */ +function mapIndexRowToIndexInfo(duckDbIndexRow) { + const indexInfo = { + pureName: duckDbIndexRow.table_name, + schemaName: duckDbIndexRow.schema_name, + constraintType: 'index', + columns: [], + isUnique: duckDbIndexRow.is_unique, + }; + + if (duckDbIndexRow.index_name != null) { + indexInfo.constraintName = duckDbIndexRow.index_name; + } + + return /** @type {import("dbgate-types").IndexInfo} */ (indexInfo); +} + +module.exports = { + mapRawTableToTableInfo, + mapRawColumnToColumnInfo, + mapConstraintRowToForeignKeyInfo, + mapConstraintRowToPrimaryKeyInfo, + mapConstraintRowToUniqueInfo, + mapViewRowToViewInfo, + mapIndexRowToIndexInfo, +}; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js new file mode 100644 index 000000000..81cfb02f4 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js @@ -0,0 +1,85 @@ +const _ = require('lodash'); +const { DatabaseAnalyser } = require('dbgate-tools'); +const sql = require('./sql'); +const { + mapRawTableToTableInfo, + mapRawColumnToColumnInfo, + mapConstraintRowToForeignKeyInfo: mapDuckDbFkConstraintToForeignKeyInfo, + mapConstraintRowToPrimaryKeyInfo, + mapIndexRowToIndexInfo, + mapConstraintRowToUniqueInfo, + mapViewRowToViewInfo, +} = require('./Analyser.helpers'); + +class Analyser extends DatabaseAnalyser { + constructor(dbhan, driver, version) { + super(dbhan, driver, version); + } + + async _computeSingleObjectId() { + const { pureName } = this.singleObjectFilter; + this.singleObjectId = pureName; + } + + async _getFastSnapshot() { + const tablesResult = await this.driver.query(this.dbhan, sql.tables); + const columnsResult = await this.driver.query(this.dbhan, sql.columns); + const foreignKeysResult = await this.driver.query(this.dbhan, sql.foreignKeys); + const primaryKeysResult = await this.driver.query(this.dbhan, sql.primaryKeys); + const uniquesResults = await this.driver.query(this.dbhan, sql.uniques); + const indexesResult = await this.driver.query(this.dbhan, sql.indexes); + const viewsResult = await this.driver.query(this.dbhan, sql.views); + + /** + * @type {import('dbgate-types').ForeignKeyInfo[]} + */ + const foreignKeys = foreignKeysResult.rows?.map(mapDuckDbFkConstraintToForeignKeyInfo).filter(Boolean); + + /** + * @type {import('dbgate-types').PrimaryKeyInfo[]} + */ + const primaryKeys = primaryKeysResult.rows?.map(mapConstraintRowToPrimaryKeyInfo).filter(Boolean); + + /** + * @type {import('dbgate-types').UniqueInfo[]} + */ + const uniques = uniquesResults.rows?.map(mapConstraintRowToUniqueInfo).filter(Boolean); + + /** + * @type {import('dbgate-types').IndexInfo[]} + */ + const indexes = indexesResult.rows?.map(mapIndexRowToIndexInfo).filter(Boolean); + + const views = viewsResult.rows?.map(mapViewRowToViewInfo); + + const columns = columnsResult.rows?.map(mapRawColumnToColumnInfo); + const tables = tablesResult.rows?.map(mapRawTableToTableInfo); + const tablesExtended = tables.map((table) => ({ + ...table, + columns: columns.filter((x) => x.pureName == table.pureName && x.schemaName == table.schemaName), + foreignKeys: foreignKeys.filter((x) => x.pureName == table.pureName && x.schemaName == table.schemaName), + primaryKey: primaryKeys.find((x) => x.pureName == table.pureName && x.schemaName == table.schemaName), + indexes: indexes.filter((x) => x.pureName == table.pureName && x.schemaName == table.schemaName), + uniques: uniques.filter((x) => x.pureName == table.pureName && x.schemaName == table.schemaName), + })); + + const viewsExtended = views.map((view) => ({ + ...view, + columns: columns.filter((x) => x.pureName == view.pureName && x.schemaName == view.schemaName), + })); + + return { + tables: tablesExtended, + views: viewsExtended, + }; + } + + async _runAnalysis() { + const structure = await this._getFastSnapshot(); + return structure; + throw new Error('Not implemented'); + return this._getFastSnapshot(); + } +} + +module.exports = Analyser; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/driver.js b/plugins/dbgate-plugin-duckdb/src/backend/driver.js new file mode 100644 index 000000000..be7cc23c4 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/driver.js @@ -0,0 +1,165 @@ +const Analyser = require('./Analyser'); +const Dumper = require('../frontend/Dumper'); +const driverBase = require('../frontend/driver'); +const { getLogger, extractErrorLogData } = require('dbgate-tools'); +const { getColumnsInfo, serializeRow, normalizeRow } = require('./helpers'); + +const logger = getLogger('sqliteDriver'); + +/** + * @type {import('@duckdb/node-api')} + */ +let duckDb; + +function getDuckDb() { + if (!duckDb) { + duckDb = require('@duckdb/node-api'); + } + + return duckDb; +} + +let fileToCon = {}; +async function getConnection(file) { + if (fileToCon[file]) { + fileToCon[file].close(); + } + + const duckDb = getDuckDb(); + const instance = await duckDb.DuckDBInstance.create(file); + console.log('DuckDB instance created', instance); + const connection = await instance.connect(); + + fileToCon[file] = connection; + + return fileToCon[file]; +} + +/** @type {import('dbgate-types').EngineDriver} */ +const driver = { + ...driverBase, + analyserClass: Analyser, + async connect({ databaseFile, isReadOnly }) { + return { + client: await getConnection(databaseFile), + }; + }, + async close(dbhan) { + dbhan.client.disconnect(); + dbhan.client.close(); + }, + async query(dbhan, sql, { readonly } = {}) { + const res = await dbhan.client.runAndReadAll(sql); + const rowsObjects = res.getRowObjects(); + + const columnNames = res.columnNames(); + const columnTypes = res.columnTypes(); + + const columns = getColumnsInfo(columnNames, columnTypes).map(normalizeRow); + + const rows = rowsObjects.map(normalizeRow); + return { + rows, + columns, + }; + }, + async stream(dbhan, sql, options) { + const duckdb = getDuckDb(); + const statements = await dbhan.client.extractStatements(sql); + const count = statements.count; + + try { + for (let i = 0; i < count; i++) { + let hasSentColumns = false; + const stmt = await statements.prepare(i); + const res = await stmt.runAndReadAll(); + + const returningStatemetes = [ + duckdb.StatementType.SELECT, + duckdb.StatementType.EXPLAIN, + duckdb.StatementType.EXECUTE, + duckdb.StatementType.RELATION, + duckdb.StatementType.LOGICAL_PLAN, + ]; + + if (!returningStatemetes.includes(stmt.statementType)) { + continue; + } + + options.info({ + message: JSON.stringify(res), + time: new Date(), + severity: 'info', + }); + + if (!hasSentColumns) { + const columnNames = res.columnNames(); + const columnTypes = res.columnTypes(); + const columns = getColumnsInfo(columnNames, columnTypes); + + options.recordset(columns); + hasSentColumns = true; + } + + const rows = res.getRowObjects(); + + for (const row of rows) { + options.row(normalizeRow(row)); + } + } + + options.done(); + } catch (error) { + logger.error(extractErrorLogData(error), 'Stream error'); + const { message, procName } = error; + options.info({ + message, + line: 0, + procedure: procName, + time: new Date(), + severity: 'error', + }); + options.done(); + } + }, + async script(dbhan, sql) { + const dmp1 = driver.createDumper(); + dmp1.beginTransaction(); + + await dbhan.client.run(dmp1.s); + + const statements = await dbhan.client.extractStatements(sql); + const count = statements.count; + + for (let i = 0; i < count; i++) { + const stmt = await statements.prepare(i); + await stmt.run(); + } + + const dmp2 = driver.createDumper(); + dmp2.commitTransaction(); + + await dbhan.client.run(dmp2.s); + }, + + async readQueryTask(stmt, pass) { + throw new Error('Not implemented'); + }, + async readQuery(dbhan, sql, structure) { + throw new Error('Not implemented'); + }, + async writeTable(dbhan, name, options) { + return createBulkInsertStreamBase(this, stream, dbhan, name, options); + }, + async getVersion(dbhan) { + const { rows } = await this.query(dbhan, 'SELECT version() AS version;'); + const { version } = rows[0]; + + return { + version, + versionText: `DuchDB ${version}`, + }; + }, +}; + +module.exports = driver; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/helpers.js b/plugins/dbgate-plugin-duckdb/src/backend/helpers.js new file mode 100644 index 000000000..0319b428a --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/helpers.js @@ -0,0 +1,65 @@ +/** + * @param {string[} columnNames + * @param {import('@duckdb/node-api').DuckDBType[]} columnTypes + */ +function getColumnsInfo(columnNames, columnTypes) { + const columns = []; + + for (let i = columnNames.length - 1; i >= 0; i--) { + columns.push({ + columnName: columnNames[i], + dataType: columnTypes[i], + }); + } + + return columns; +} + +function _normalizeValue(value) { + if (value === null) { + return null; + } + + if (typeof value === 'bigint') { + return `${value}n`; + } + + if (Array.isArray(value)) { + return value.map((item) => _normalizeValue(item)); + } + + if (typeof value === 'object') { + const normalizedObj = {}; + for (const key in value) { + if (Object.hasOwnProperty.call(value, key)) { + normalizedObj[key] = _normalizeValue(value[key]); + } + } + return normalizedObj; + } + + return value; +} + +/** + * @param {Record} obj + * + */ +function normalizeRow(obj) { + if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) { + return _normalizeValue(obj); + } + + const normalized = {}; + for (const key in obj) { + if (Object.hasOwnProperty.call(obj, key)) { + normalized[key] = _normalizeValue(obj[key]); + } + } + return normalized; +} + +module.exports = { + normalizeRow, + getColumnsInfo, +}; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/index.js b/plugins/dbgate-plugin-duckdb/src/backend/index.js new file mode 100644 index 000000000..7e25f4c20 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/index.js @@ -0,0 +1,6 @@ +const driver = require('./driver'); + +module.exports = { + packageName: 'dbgate-plugin-duckdb', + drivers: [driver], +}; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/sql/columns.js b/plugins/dbgate-plugin-duckdb/src/backend/sql/columns.js new file mode 100644 index 000000000..02b187cdc --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/sql/columns.js @@ -0,0 +1 @@ +module.exports = `SELECT * from duckdb_columns() WHERE internal = false`; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/sql/foreignKeys.js b/plugins/dbgate-plugin-duckdb/src/backend/sql/foreignKeys.js new file mode 100644 index 000000000..5f36ecc85 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/sql/foreignKeys.js @@ -0,0 +1 @@ +module.exports = `SELECT * FROM duckdb_constraints() WHERE constraint_type = 'FOREIGN KEY'`; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/sql/index.js b/plugins/dbgate-plugin-duckdb/src/backend/sql/index.js new file mode 100644 index 000000000..ed1ee30bb --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/sql/index.js @@ -0,0 +1,17 @@ +const tables = require('./tables.js'); +const columns = require('./columns.js'); +const foreignKeys = require('./foreignKeys.js'); +const primaryKeys = require('./primaryKeys.js'); +const indexes = require('./indexes.js'); +const uniques = require('./uniques.js'); +const views = require('./views.js'); + +module.exports = { + tables, + columns, + foreignKeys, + primaryKeys, + indexes, + uniques, + views, +}; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/sql/indexes.js b/plugins/dbgate-plugin-duckdb/src/backend/sql/indexes.js new file mode 100644 index 000000000..888739a03 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/sql/indexes.js @@ -0,0 +1 @@ +module.exports = `SELECT * FROM duckdb_indexes()`; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/sql/primaryKeys.js b/plugins/dbgate-plugin-duckdb/src/backend/sql/primaryKeys.js new file mode 100644 index 000000000..f95f72444 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/sql/primaryKeys.js @@ -0,0 +1 @@ +module.exports = `SELECT * FROM duckdb_constraints() WHERE constraint_type = 'PRIMARY KEY'`; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/sql/tables.js b/plugins/dbgate-plugin-duckdb/src/backend/sql/tables.js new file mode 100644 index 000000000..aaaebe624 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/sql/tables.js @@ -0,0 +1 @@ +module.exports = `SELECT * from duckdb_tables() WHERE internal = false`; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/sql/uniques.js b/plugins/dbgate-plugin-duckdb/src/backend/sql/uniques.js new file mode 100644 index 000000000..69de129e5 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/sql/uniques.js @@ -0,0 +1 @@ +module.exports = `SELECT * FROM duckdb_constraints() WHERE constraint_type = 'UNIQUE'`; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/sql/views.js b/plugins/dbgate-plugin-duckdb/src/backend/sql/views.js new file mode 100644 index 000000000..cb1ff41d6 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/sql/views.js @@ -0,0 +1 @@ +module.exports = `SELECT * FROM duckdb_views() WHERE internal = false`; diff --git a/plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js b/plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js new file mode 100644 index 000000000..5e45dae5d --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js @@ -0,0 +1,7 @@ +const { SqlDumper, arrayToHexString } = require('dbgate-tools'); + +class Dumper extends SqlDumper { + autoIncrement() {} +} + +module.exports = Dumper; diff --git a/plugins/dbgate-plugin-duckdb/src/frontend/driver.js b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js new file mode 100644 index 000000000..70c39b282 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js @@ -0,0 +1,72 @@ +// @ts-check + +const { driverBase } = global.DBGATE_PACKAGES['dbgate-tools']; +const Dumper = require('./Dumper'); +const { sqliteSplitterOptions, noSplitSplitterOptions } = require('dbgate-query-splitter/lib/options'); + +/** + * @param {string} databaseFile + */ +function getDatabaseFileLabel(databaseFile) { + if (!databaseFile) return databaseFile; + const m = databaseFile.match(/[\/]([^\/]+)$/); + if (m) return m[1]; + return databaseFile; +} + +/** @type {import('dbgate-types').SqlDialect} */ +const dialect = { + limitSelect: true, + rangeSelect: true, + offsetFetchRangeSyntax: false, + explicitDropConstraint: true, + stringEscapeChar: "'", + fallbackDataType: 'nvarchar', + allowMultipleValuesInsert: true, + dropColumnDependencies: ['indexes', 'primaryKey', 'uniques'], + quoteIdentifier(s) { + return `"${s}"`; + }, + anonymousPrimaryKey: true, + requireStandaloneSelectForScopeIdentity: true, + + createColumn: true, + dropColumn: true, + createIndex: true, + dropIndex: true, + createForeignKey: false, + enableForeignKeyChecks: false, + dropForeignKey: false, + createPrimaryKey: false, + dropPrimaryKey: false, + dropReferencesWhenDropTable: false, + filteredIndexes: true, + anonymousForeignKey: true, +}; + +/** @type {import('dbgate-types').EngineDriver} */ +const driver = { + ...driverBase, + dumperClass: Dumper, + dialect, + engine: 'duckdb@dbgate-plugin-duckdb', + title: 'DuckDB', + readOnlySessions: true, + supportsTransactions: true, + + getQuerySplitterOptions: (usage) => + usage == 'editor' + ? { ...sqliteSplitterOptions, ignoreComments: true, preventSingleLineSplit: true } + : usage == 'stream' + ? noSplitSplitterOptions + : sqliteSplitterOptions, + showConnectionTab: (field) => false, + showConnectionField: (field) => ['databaseFile'].includes(field), + beforeConnectionSave: (connection) => ({ + ...connection, + singleDatabase: true, + defaultDatabase: getDatabaseFileLabel(connection.databaseFile), + }), +}; + +module.exports = driver; diff --git a/plugins/dbgate-plugin-duckdb/src/frontend/index.js b/plugins/dbgate-plugin-duckdb/src/frontend/index.js new file mode 100644 index 000000000..cb650c7ec --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/frontend/index.js @@ -0,0 +1,6 @@ +import driver from './driver'; + +export default { + packageName: 'dbgate-plugin-duckdb', + drivers: [driver], +}; diff --git a/plugins/dbgate-plugin-duckdb/webpack-backend.config.js b/plugins/dbgate-plugin-duckdb/webpack-backend.config.js new file mode 100644 index 000000000..ebecca263 --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/webpack-backend.config.js @@ -0,0 +1,28 @@ +var webpack = require('webpack'); +var path = require('path'); + +const packageJson = require('./package.json'); +const buildPluginExternals = require('../../common/buildPluginExternals'); +const externals = buildPluginExternals(packageJson); + +var config = { + context: __dirname + '/src/backend', + + entry: { + app: './index.js', + }, + target: 'node', + output: { + path: path.resolve(__dirname, 'dist'), + filename: 'backend.js', + libraryTarget: 'commonjs2', + }, + + // uncomment for disable minimalization + // optimization: { + // minimize: false, + // }, + externals, +}; + +module.exports = config; diff --git a/plugins/dbgate-plugin-duckdb/webpack-frontend.config.js b/plugins/dbgate-plugin-duckdb/webpack-frontend.config.js new file mode 100644 index 000000000..cbc4a0a5a --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/webpack-frontend.config.js @@ -0,0 +1,30 @@ +var webpack = require('webpack'); +var path = require('path'); + +var config = { + context: __dirname + '/src/frontend', + + entry: { + app: './index.js', + }, + target: 'web', + output: { + path: path.resolve(__dirname, 'dist'), + filename: 'frontend.js', + libraryTarget: 'var', + library: 'plugin', + }, + + plugins: [ + new webpack.DefinePlugin({ + 'global.DBGATE_PACKAGES': 'window.DBGATE_PACKAGES', + }), + ], + + // uncomment for disable minimalization + // optimization: { + // minimize: false, + // }, +}; + +module.exports = config; diff --git a/yarn.lock b/yarn.lock index ee2f4a95e..7df1f9368 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1100,6 +1100,49 @@ resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz#1d572bfbbe14b7704e0ba0f39b74815b84870d70" integrity sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw== +"@duckdb/node-api@^1.2.1-alpha.16": + version "1.2.1-alpha.16" + resolved "https://registry.yarnpkg.com/@duckdb/node-api/-/node-api-1.2.1-alpha.16.tgz#4e0d8a17f227eed336ab4d50922bfc7be0b1024d" + integrity sha512-r2wkrqcDl3IsmFffpTj7xgSUEkjVaqi7wV0uypQWw4wTM5bIYk5ABc/Hxn2xlwV9UBRnxhk0Ls0EJypYS7g8ZQ== + dependencies: + "@duckdb/node-bindings" "1.2.1-alpha.16" + +"@duckdb/node-bindings-darwin-arm64@1.2.1-alpha.16": + version "1.2.1-alpha.16" + resolved "https://registry.yarnpkg.com/@duckdb/node-bindings-darwin-arm64/-/node-bindings-darwin-arm64-1.2.1-alpha.16.tgz#a7061c8fc8d968bf9657211ccacd1139bfad96af" + integrity sha512-anfLXcxjo6S0Kx8Z+e6/ca7WayprJ8iI4cpTvzWQc9NT/vKFHcGjvhGAiosHvtjWGOvAYo+O/eyAcmzMzazlMg== + +"@duckdb/node-bindings-darwin-x64@1.2.1-alpha.16": + version "1.2.1-alpha.16" + resolved "https://registry.yarnpkg.com/@duckdb/node-bindings-darwin-x64/-/node-bindings-darwin-x64-1.2.1-alpha.16.tgz#7d21558a50384115ba8eb8c41a84d689e2c797e9" + integrity sha512-IA2bQ/f0qFYb7Sd+leSjNg/JMBpWVVBoCmqp/1zzlw6fwhtT0BMSAT3FL4306t5StA8biOznlHz3rN3jovdVxg== + +"@duckdb/node-bindings-linux-arm64@1.2.1-alpha.16": + version "1.2.1-alpha.16" + resolved "https://registry.yarnpkg.com/@duckdb/node-bindings-linux-arm64/-/node-bindings-linux-arm64-1.2.1-alpha.16.tgz#5915a71f3520b8a2cfbf63ad89d63198aec6db57" + integrity sha512-zy9jTrrhTXJAOrYRTbT/HtBLClAoyo8vNRAqojFHVBxXL1nr4o+5Je9AJwb9IfS1/e38zdykDWeGnY/gB3NpfA== + +"@duckdb/node-bindings-linux-x64@1.2.1-alpha.16": + version "1.2.1-alpha.16" + resolved "https://registry.yarnpkg.com/@duckdb/node-bindings-linux-x64/-/node-bindings-linux-x64-1.2.1-alpha.16.tgz#8cff0e412d6201b57069c311775c375268cd5707" + integrity sha512-tdDAhUKenBhUQiTN+qvKj6nBshoooBLPbxVuLas8v64KmphjxOHd9zQ2KMzw1tN+fXhV9dqUTbCqiUN9A6ZFSQ== + +"@duckdb/node-bindings-win32-x64@1.2.1-alpha.16": + version "1.2.1-alpha.16" + resolved "https://registry.yarnpkg.com/@duckdb/node-bindings-win32-x64/-/node-bindings-win32-x64-1.2.1-alpha.16.tgz#9fb46578f55d5b24524ea9a2d791f8fa6982e613" + integrity sha512-FE9bZV8+LIiy5jQIsLoEmwLFIyPPJD4SXjKNCaU48DNVf1q81ZkhnoT7PVgeyNJmkR6rG2+mq6LzTSmdCBX0ig== + +"@duckdb/node-bindings@1.2.1-alpha.16": + version "1.2.1-alpha.16" + resolved "https://registry.yarnpkg.com/@duckdb/node-bindings/-/node-bindings-1.2.1-alpha.16.tgz#700ce66c74772be7a870ae68dd91a968d736ed7b" + integrity sha512-6ITHy26o99zxUhCGOxwkQbfmi5I8VXNGanhnrOe3pqUYRDXvGAe6T2MmBymYwU+fMZB341UE8krw7hUkPLfIeA== + optionalDependencies: + "@duckdb/node-bindings-darwin-arm64" "1.2.1-alpha.16" + "@duckdb/node-bindings-darwin-x64" "1.2.1-alpha.16" + "@duckdb/node-bindings-linux-arm64" "1.2.1-alpha.16" + "@duckdb/node-bindings-linux-x64" "1.2.1-alpha.16" + "@duckdb/node-bindings-win32-x64" "1.2.1-alpha.16" + "@gar/promisify@^1.0.1": version "1.1.3" resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" From 750265cb7904ce00dd749a369a3e8aa83caeede4 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Wed, 2 Apr 2025 16:38:47 +0200 Subject: [PATCH 02/49] removed incorrect dataType of duckdb result --- plugins/dbgate-plugin-duckdb/src/backend/helpers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/dbgate-plugin-duckdb/src/backend/helpers.js b/plugins/dbgate-plugin-duckdb/src/backend/helpers.js index 0319b428a..8d6123893 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/helpers.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/helpers.js @@ -8,7 +8,7 @@ function getColumnsInfo(columnNames, columnTypes) { for (let i = columnNames.length - 1; i >= 0; i--) { columns.push({ columnName: columnNames[i], - dataType: columnTypes[i], + // dataType: columnTypes[i], }); } From 890461bcf827d2ba559a33745d50d98a758843db Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Wed, 2 Apr 2025 16:39:23 +0200 Subject: [PATCH 03/49] duckDb: singleConnection query --- packages/types/engines.d.ts | 1 + .../src/query/SimpleQueryResultTabs.svelte | 52 +++++++++++++++++++ packages/web/src/tabs/QueryTab.svelte | 43 ++++++++++----- .../src/frontend/driver.js | 1 + 4 files changed, 84 insertions(+), 13 deletions(-) create mode 100644 packages/web/src/query/SimpleQueryResultTabs.svelte diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index a6fc762b6..0e63245d6 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -310,6 +310,7 @@ export interface EngineDriver extends FilterBehaviourProvider { analyserClass?: any; dumperClass?: any; + singleConnectionOnly?: boolean; } export interface DatabaseModification { diff --git a/packages/web/src/query/SimpleQueryResultTabs.svelte b/packages/web/src/query/SimpleQueryResultTabs.svelte new file mode 100644 index 000000000..91b53f676 --- /dev/null +++ b/packages/web/src/query/SimpleQueryResultTabs.svelte @@ -0,0 +1,52 @@ + + + + + {#if grider && display} + + {/if} + + + {#if result?.errorMessage} + + {:else if result?.rows} + + {/if} + + diff --git a/packages/web/src/tabs/QueryTab.svelte b/packages/web/src/tabs/QueryTab.svelte index e564e9338..1f05a94ff 100644 --- a/packages/web/src/tabs/QueryTab.svelte +++ b/packages/web/src/tabs/QueryTab.svelte @@ -143,6 +143,7 @@ import { isProApp } from '../utility/proTools'; import HorizontalSplitter from '../elements/HorizontalSplitter.svelte'; import QueryAiAssistant from '../query/QueryAiAssistant.svelte'; + import SimpleQueryResultTabs from '../query/SimpleQueryResultTabs.svelte'; export let tabid; export let conid; @@ -196,6 +197,8 @@ let isInTransaction = false; let isAutocommit = false; + let simpleQueryDataResult = null; + onMount(() => { intervalId = setInterval(() => { if (sessionId) { @@ -323,6 +326,16 @@ return; } + if (driver?.singleConnectionOnly) { + simpleQueryDataResult = await apiCall('database-connections/query-data', { + conid, + database, + sql, + }); + visibleResultTabs = true; + return; + } + executeStartLine = startLine; executeNumber++; visibleResultTabs = true; @@ -642,19 +655,23 @@ {/if} - - - - - + {#if driver?.singleConnectionOnly} + + {:else} + + + + + + {/if} diff --git a/plugins/dbgate-plugin-duckdb/src/frontend/driver.js b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js index 70c39b282..71a966f7b 100644 --- a/plugins/dbgate-plugin-duckdb/src/frontend/driver.js +++ b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js @@ -53,6 +53,7 @@ const driver = { title: 'DuckDB', readOnlySessions: true, supportsTransactions: true, + singleConnectionOnly: true, getQuerySplitterOptions: (usage) => usage == 'editor' From dd3e38355c57f2d4e54e3ae13df556494a8f1287 Mon Sep 17 00:00:00 2001 From: Nybkox Date: Thu, 3 Apr 2025 18:58:33 +0200 Subject: [PATCH 04/49] chore: remove test logging --- integration-tests/__tests__/alter-database.spec.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration-tests/__tests__/alter-database.spec.js b/integration-tests/__tests__/alter-database.spec.js index 6ffaa35f7..490de76a9 100644 --- a/integration-tests/__tests__/alter-database.spec.js +++ b/integration-tests/__tests__/alter-database.spec.js @@ -36,8 +36,6 @@ async function testDatabaseDiff(conn, driver, mangle, createObject = null) { if (createObject) await driver.query(conn, createObject); const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn))); - console.log('str1'); - console.dir(structure1, { depth: 10 }); let structure2 = _.cloneDeep(structure1); mangle(structure2); structure2 = extendDatabaseInfo(structure2); From a68a1334fcfb5372dc2e471237bda9922eb8791e Mon Sep 17 00:00:00 2001 From: Nybkox Date: Thu, 3 Apr 2025 18:58:40 +0200 Subject: [PATCH 05/49] feat(duckdb): add rename sql object --- plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js b/plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js index 5e45dae5d..8af6e8e8b 100644 --- a/plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js +++ b/plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js @@ -2,6 +2,10 @@ const { SqlDumper, arrayToHexString } = require('dbgate-tools'); class Dumper extends SqlDumper { autoIncrement() {} + + renameSqlObject(obj, newname) { + this.putCmd('^alter %k %f ^rename ^to %i', this.getSqlObjectSqlName(obj.objectTypeField), obj, newname); + } } module.exports = Dumper; From 5af76150541700e7a9b2f195f6cda9181eeb3352 Mon Sep 17 00:00:00 2001 From: Nybkox Date: Thu, 3 Apr 2025 22:11:29 +0200 Subject: [PATCH 06/49] fix: convert dataType to stirng --- .../src/backend/driver.js | 42 +++++++++++++++---- .../src/backend/helpers.js | 2 +- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/plugins/dbgate-plugin-duckdb/src/backend/driver.js b/plugins/dbgate-plugin-duckdb/src/backend/driver.js index be7cc23c4..82fe275a1 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/driver.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/driver.js @@ -1,8 +1,10 @@ +// @ts-check +// +const stream = require('stream'); const Analyser = require('./Analyser'); -const Dumper = require('../frontend/Dumper'); const driverBase = require('../frontend/driver'); -const { getLogger, extractErrorLogData } = require('dbgate-tools'); -const { getColumnsInfo, serializeRow, normalizeRow } = require('./helpers'); +const { getLogger, extractErrorLogData, createBulkInsertStreamBase } = require('dbgate-tools'); +const { getColumnsInfo, normalizeRow } = require('./helpers'); const logger = getLogger('sqliteDriver'); @@ -142,11 +144,37 @@ const driver = { await dbhan.client.run(dmp2.s); }, - async readQueryTask(stmt, pass) { - throw new Error('Not implemented'); - }, async readQuery(dbhan, sql, structure) { - throw new Error('Not implemented'); + const pass = new stream.PassThrough({ + objectMode: true, + highWaterMark: 100, + }); + + const res = await dbhan.client.runAndReadAll(sql); + const rowsObjects = res.getRowObjects(); + + const columnNames = res.columnNames(); + const columnTypes = res.columnTypes(); + + const columns = getColumnsInfo(columnNames, columnTypes).map(normalizeRow); + + const rows = rowsObjects.map(normalizeRow); + + pass.write({ + __isStreamHeader: true, + ...(structure || { + columns: columns.map((col) => ({ + columnName: col.name, + dataType: col.type, + })), + }), + }); + + for (const row of rows) { + pass.write(row); + } + + return pass; }, async writeTable(dbhan, name, options) { return createBulkInsertStreamBase(this, stream, dbhan, name, options); diff --git a/plugins/dbgate-plugin-duckdb/src/backend/helpers.js b/plugins/dbgate-plugin-duckdb/src/backend/helpers.js index 8d6123893..4a2e3d47d 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/helpers.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/helpers.js @@ -8,7 +8,7 @@ function getColumnsInfo(columnNames, columnTypes) { for (let i = columnNames.length - 1; i >= 0; i--) { columns.push({ columnName: columnNames[i], - // dataType: columnTypes[i], + dataType: columnTypes[i].toString(), }); } From c4a4cd0957ccb69b36d651d1adf0c6579819eb89 Mon Sep 17 00:00:00 2001 From: Nybkox Date: Thu, 3 Apr 2025 22:11:57 +0200 Subject: [PATCH 07/49] fix: parse cnt as int --- .../__tests__/db-import-export.spec.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/integration-tests/__tests__/db-import-export.spec.js b/integration-tests/__tests__/db-import-export.spec.js index 37a4134ca..aa9ce9e32 100644 --- a/integration-tests/__tests__/db-import-export.spec.js +++ b/integration-tests/__tests__/db-import-export.spec.js @@ -51,7 +51,8 @@ describe('DB Import/export', () => { await copyStream(reader, writer); const res = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t1`)); - expect(res.rows[0].cnt.toString()).toEqual('6'); + const cnt = parseInt(res.rows[0].cnt.toString()); + expect(cnt).toEqual(6); }) ); @@ -75,7 +76,8 @@ describe('DB Import/export', () => { await copyStream(reader, writer); const res = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t1`)); - expect(res.rows[0].cnt.toString()).toEqual('6'); + const cnt = parseInt(res.rows[0].cnt.toString()); + expect(cnt).toEqual(6); }) ); @@ -103,10 +105,12 @@ describe('DB Import/export', () => { await copyStream(reader2, writer2); const res1 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t1`)); - expect(res1.rows[0].cnt.toString()).toEqual('6'); + const cnt = parseInt(res1.rows[0].cnt.toString()); + expect(cnt).toEqual(6); const res2 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t2`)); - expect(res2.rows[0].cnt.toString()).toEqual('6'); + const cnt2 = parseInt(res2.rows[0].cnt.toString()); + expect(cnt2).toEqual(6); }) ); const enginesWithDumpFile = engines.filter(x => x.dumpFile); @@ -128,7 +132,8 @@ describe('DB Import/export', () => { for (const check of engine.dumpChecks || []) { const res = await driver.query(conn, check.sql); - expect(res.rows[0].res.toString()).toEqual(check.res); + const cnt = parseInt(res.rows[0].res.toString()); + expect(cnt).toEqual(check.res); } // const res1 = await driver.query(conn, `select count(*) as cnt from t1`); @@ -192,7 +197,8 @@ describe('DB Import/export', () => { }); const res1 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~categories`)); - expect(res1.rows[0].cnt.toString()).toEqual('4'); + const cnt1 = parseInt(res1.rows[0].cnt.toString()); + expect(cnt1).toEqual(4); }) ); }); From 9d85a58634d051d65f75b6048946a2680b6a0676 Mon Sep 17 00:00:00 2001 From: Nybkox Date: Thu, 3 Apr 2025 22:37:18 +0200 Subject: [PATCH 08/49] fix: skipDeploy tests for duckdb --- integration-tests/engines.js | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 0073f5fb5..839751b27 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -669,6 +669,7 @@ const duckdbEngine = { skipTriggers: true, skipDataDuplicator: true, skipAutoIncrement: true, + skipDeploy: true, supportRenameSqlObject: true, }; From d68cf4e44da00a05f3ac0ec8d7d794c3481bae16 Mon Sep 17 00:00:00 2001 From: Nybkox Date: Thu, 3 Apr 2025 22:58:50 +0200 Subject: [PATCH 09/49] fix: remove test throw --- plugins/dbgate-plugin-duckdb/src/backend/Analyser.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js index 81cfb02f4..62059ef9f 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js @@ -77,8 +77,6 @@ class Analyser extends DatabaseAnalyser { async _runAnalysis() { const structure = await this._getFastSnapshot(); return structure; - throw new Error('Not implemented'); - return this._getFastSnapshot(); } } From edece02c1306b54c3df85aa9bf12305d3cf9547e Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 09:08:00 +0200 Subject: [PATCH 10/49] feat: enable duckdb test on ci --- integration-tests/engines.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 839751b27..83b0dc41a 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -686,6 +686,7 @@ const enginesOnCi = [ clickhouseEngine, oracleEngine, cassandraEngine, + duckdbEngine, ]; const enginesOnLocal = [ @@ -716,3 +717,6 @@ module.exports.cockroachDbEngine = cockroachDbEngine; module.exports.clickhouseEngine = clickhouseEngine; module.exports.oracleEngine = oracleEngine; module.exports.cassandraEngine = cassandraEngine; +module.exports.libsqlFileEngine = libsqlFileEngine; +module.exports.libsqlWsEngine = libsqlWsEngine; +module.exports.duckdbEngine = duckdbEngine; From d1a6be6ca69afd39b0ef8777de82c2cfe284ee6e Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 09:34:56 +0200 Subject: [PATCH 11/49] fix: parse res to int in tests --- integration-tests/__tests__/db-import-export.spec.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/integration-tests/__tests__/db-import-export.spec.js b/integration-tests/__tests__/db-import-export.spec.js index aa9ce9e32..3dff4282c 100644 --- a/integration-tests/__tests__/db-import-export.spec.js +++ b/integration-tests/__tests__/db-import-export.spec.js @@ -133,14 +133,16 @@ describe('DB Import/export', () => { for (const check of engine.dumpChecks || []) { const res = await driver.query(conn, check.sql); const cnt = parseInt(res.rows[0].res.toString()); - expect(cnt).toEqual(check.res); + expect(cnt).toEqual(parseInt(check.res)); } - // const res1 = await driver.query(conn, `select count(*) as cnt from t1`); - // expect(res1.rows[0].cnt.toString()).toEqual('6'); + const res1 = await driver.query(conn, `select count(*) as cnt from t1`); + const cnt1 = parseInt(res1.rows[0].cnt.toString()); + expect(cnt1).toEqual(6); - // const res2 = await driver.query(conn, `select count(*) as cnt from t2`); - // expect(res2.rows[0].cnt.toString()).toEqual('6'); + const res2 = await driver.query(conn, `select count(*) as cnt from t2`); + const cnt2 = parseInt(res2.rows[0].cnt.toString()); + expect(cnt2).toEqual(6); }) ); } From 18faf89b8978121a0e74640c98afe8442d5beed5 Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 18:06:09 +0200 Subject: [PATCH 12/49] fix: update listSchemas typing --- packages/types/engines.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 0e63245d6..af5fd29df 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -287,7 +287,7 @@ export interface EngineDriver extends FilterBehaviourProvider { adaptTableInfo(table: TableInfo): TableInfo; // simple data type adapter adaptDataType(dataType: string): string; - listSchemas(dbhan: DatabaseHandle): SchemaInfo[]; + listSchemas(dbhan: DatabaseHandle): Promise; backupDatabaseCommand( connection: any, settings: BackupDatabaseSettings, From e7c42f3623c2af474e4f6234df04c665919fb68b Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 18:06:46 +0200 Subject: [PATCH 13/49] fix: remove temp duckdb connection cache --- .../src/backend/driver.js | 21 ++++--------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/plugins/dbgate-plugin-duckdb/src/backend/driver.js b/plugins/dbgate-plugin-duckdb/src/backend/driver.js index 82fe275a1..81be19869 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/driver.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/driver.js @@ -21,29 +21,16 @@ function getDuckDb() { return duckDb; } -let fileToCon = {}; -async function getConnection(file) { - if (fileToCon[file]) { - fileToCon[file].close(); - } - - const duckDb = getDuckDb(); - const instance = await duckDb.DuckDBInstance.create(file); - console.log('DuckDB instance created', instance); - const connection = await instance.connect(); - - fileToCon[file] = connection; - - return fileToCon[file]; -} - /** @type {import('dbgate-types').EngineDriver} */ const driver = { ...driverBase, analyserClass: Analyser, async connect({ databaseFile, isReadOnly }) { + const instance = await getDuckDb().DuckDBInstance.create(databaseFile); + const connection = await instance.connect(); + return { - client: await getConnection(databaseFile), + client: connection, }; }, async close(dbhan) { From 0ece8c7decb22cd068c647651863c2a87bfd64f4 Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 18:07:09 +0200 Subject: [PATCH 14/49] feat: add listSchemas to duckdb --- .../src/backend/Analyser.helpers.js | 27 +++++++++++++++++ .../src/backend/driver.js | 29 ++++++++++++++----- .../src/backend/sql/index.js | 2 ++ .../src/backend/sql/schemas.js | 1 + 4 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 plugins/dbgate-plugin-duckdb/src/backend/sql/schemas.js diff --git a/plugins/dbgate-plugin-duckdb/src/backend/Analyser.helpers.js b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.helpers.js index 99ab7ad55..ace6e464e 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/Analyser.helpers.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.helpers.js @@ -231,6 +231,7 @@ function mapConstraintRowToForeignKeyInfo(duckDbConstraintData) { constraintType: 'foreignKey', columns: columns, refTableName: duckDbConstraintData.referenced_table, + refSchemaName: duckDbConstraintData.schema_name, }; if (duckDbConstraintData.constraint_name != null) { @@ -363,6 +364,31 @@ function mapIndexRowToIndexInfo(duckDbIndexRow) { return /** @type {import("dbgate-types").IndexInfo} */ (indexInfo); } +/** + * @typedef {object} DuckDbSchemaRow + * @property {string} oid + * @property {string} database_name + * @property {string} database_oid + * @property {string} schema_name + * @property {string | null} comment + * @property {{ [key: string]: string } | null} tags + * @property {boolean} internal + * @property {string | null} sql + */ + +/** + * @param {DuckDbSchemaRow} duckDbSchemaRow + * @returns {import("dbgate-types").SchemaInfo} + */ +function mapSchemaRowToSchemaInfo(duckDbSchemaRow) { + const schemaInfo = { + schemaName: duckDbSchemaRow.schema_name, + objectId: duckDbSchemaRow.oid, + }; + + return /** @type {import("dbgate-types").SchemaInfo} */ (schemaInfo); +} + module.exports = { mapRawTableToTableInfo, mapRawColumnToColumnInfo, @@ -371,4 +397,5 @@ module.exports = { mapConstraintRowToUniqueInfo, mapViewRowToViewInfo, mapIndexRowToIndexInfo, + mapSchemaRowToSchemaInfo, }; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/driver.js b/plugins/dbgate-plugin-duckdb/src/backend/driver.js index 81be19869..c1fc43bcc 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/driver.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/driver.js @@ -5,6 +5,8 @@ const Analyser = require('./Analyser'); const driverBase = require('../frontend/driver'); const { getLogger, extractErrorLogData, createBulkInsertStreamBase } = require('dbgate-tools'); const { getColumnsInfo, normalizeRow } = require('./helpers'); +const sql = require('./sql'); +const { mapSchemaRowToSchemaInfo } = require('./Analyser.helpers'); const logger = getLogger('sqliteDriver'); @@ -111,11 +113,13 @@ const driver = { options.done(); } }, - async script(dbhan, sql) { - const dmp1 = driver.createDumper(); - dmp1.beginTransaction(); + async script(dbhan, sql, { useTransaction } = { useTransaction: false }) { + if (useTransaction) { + const dmp1 = driver.createDumper(); + dmp1.beginTransaction(); - await dbhan.client.run(dmp1.s); + await dbhan.client.run(dmp1.s); + } const statements = await dbhan.client.extractStatements(sql); const count = statements.count; @@ -125,10 +129,12 @@ const driver = { await stmt.run(); } - const dmp2 = driver.createDumper(); - dmp2.commitTransaction(); + if (useTransaction) { + const dmp2 = driver.createDumper(); + dmp2.commitTransaction(); - await dbhan.client.run(dmp2.s); + await dbhan.client.run(dmp2.s); + } }, async readQuery(dbhan, sql, structure) { @@ -168,13 +174,20 @@ const driver = { }, async getVersion(dbhan) { const { rows } = await this.query(dbhan, 'SELECT version() AS version;'); - const { version } = rows[0]; + const { version } = rows?.[0]; return { version, versionText: `DuchDB ${version}`, }; }, + + async listSchemas(dbhan) { + const schemasResult = await this.query(dbhan, sql.schemas); + const schemas = schemasResult.rows?.map(mapSchemaRowToSchemaInfo); + + return schemas ?? null; + }, }; module.exports = driver; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/sql/index.js b/plugins/dbgate-plugin-duckdb/src/backend/sql/index.js index ed1ee30bb..d867c203d 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/sql/index.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/sql/index.js @@ -5,6 +5,7 @@ const primaryKeys = require('./primaryKeys.js'); const indexes = require('./indexes.js'); const uniques = require('./uniques.js'); const views = require('./views.js'); +const schemas = require('./schemas.js'); module.exports = { tables, @@ -14,4 +15,5 @@ module.exports = { indexes, uniques, views, + schemas, }; diff --git a/plugins/dbgate-plugin-duckdb/src/backend/sql/schemas.js b/plugins/dbgate-plugin-duckdb/src/backend/sql/schemas.js new file mode 100644 index 000000000..ff810b57a --- /dev/null +++ b/plugins/dbgate-plugin-duckdb/src/backend/sql/schemas.js @@ -0,0 +1 @@ +module.exports = `SELECT * FROM duckdb_schemas() WHERE internal = false`; From caefc438b9995bab787cf05ad76374d15821a4b9 Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 18:07:14 +0200 Subject: [PATCH 15/49] chore: remove unused imports --- plugins/dbgate-plugin-duckdb/src/backend/Analyser.js | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js index 62059ef9f..bc7e9fdd0 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js @@ -1,4 +1,3 @@ -const _ = require('lodash'); const { DatabaseAnalyser } = require('dbgate-tools'); const sql = require('./sql'); const { From ca1899409292050181ac5f7981c943f4e6f0a40e Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 18:07:22 +0200 Subject: [PATCH 16/49] fix: update duckdb dialects --- plugins/dbgate-plugin-duckdb/src/frontend/driver.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/dbgate-plugin-duckdb/src/frontend/driver.js b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js index 71a966f7b..57146584f 100644 --- a/plugins/dbgate-plugin-duckdb/src/frontend/driver.js +++ b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js @@ -36,10 +36,13 @@ const dialect = { dropIndex: true, createForeignKey: false, enableForeignKeyChecks: false, - dropForeignKey: false, + dropCheck: true, + dropUnique: true, + dropForeignKey: true, createPrimaryKey: false, - dropPrimaryKey: false, - dropReferencesWhenDropTable: false, + dropPrimaryKey: true, + dropReferencesWhenDropTable: true, + dropIndexContainsTableSpec: true, filteredIndexes: true, anonymousForeignKey: true, }; From b8d86518e7af87093ea1dc8e3c48a4ce8025d317 Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 18:56:17 +0200 Subject: [PATCH 17/49] feat: add new duck db command --- app/src/mainMenuDefinition.js | 1 + packages/api/src/controllers/connections.js | 16 ++++++++++++ packages/api/testduck.db | Bin 2371584 -> 2371584 bytes packages/web/src/commands/stdCommands.ts | 27 +++++++++++++++++--- translations/en.json | 6 ++++- 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/app/src/mainMenuDefinition.js b/app/src/mainMenuDefinition.js index b74a94136..8e676db79 100644 --- a/app/src/mainMenuDefinition.js +++ b/app/src/mainMenuDefinition.js @@ -4,6 +4,7 @@ module.exports = ({ editMenu, isMac }) => [ submenu: [ { command: 'new.connection', hideDisabled: true }, { command: 'new.sqliteDatabase', hideDisabled: true }, + { command: 'new.duckdbDatabase', hideDisabled: true }, { divider: true }, { command: 'new.query', hideDisabled: true }, { command: 'new.queryDesign', hideDisabled: true }, diff --git a/packages/api/src/controllers/connections.js b/packages/api/src/controllers/connections.js index bf6b86544..8662f5e3e 100644 --- a/packages/api/src/controllers/connections.js +++ b/packages/api/src/controllers/connections.js @@ -435,6 +435,22 @@ module.exports = { return res; }, + newDuckdbDatabase_meta: true, + async newDuckdbDatabase({ file }) { + const duckdbDir = path.join(filesdir(), 'duckdb'); + if (!(await fs.exists(duckdbDir))) { + await fs.mkdir(duckdbDir); + } + const databaseFile = path.join(duckdbDir, `${file}.duckdb`); + const res = await this.save({ + engine: 'duckdb@dbgate-plugin-duckdb', + databaseFile, + singleDatabase: true, + defaultDatabase: `${file}.duckdb`, + }); + return res; + }, + dbloginWeb_meta: { raw: true, method: 'get', diff --git a/packages/api/testduck.db b/packages/api/testduck.db index 9b9f42b80c1d4d9e43b35563bbed9de7a252f6ad..90a8e2dd2acd8082b1e2e1f432d6bc13c88d5bfe 100644 GIT binary patch delta 3712 zcmdT`Yiv|S6rQ=ecOTuB-L~`%eb|;33hk6y%0q-ko<`fHKnO_OHr?BHwe6N}2^vCL zOc*f&-7;;L1eGR6A)+a8jaOr|HAbL3EJck0O^QE=Bq*)^A;HM?%x!nKyS9K4W7uSq zIp55kIrsa{H}|w%^Wrx6_t?xmn>H`H^=V70*byC%7s~0jYu5BDpPjn8^UL?k^(f?L z7?1L`Rk4w2QR}ve<$K1_5+|~-Sh6sodb_8^-5@o429OgO038EZM`ye-qQ=$iX>lAZ zjx(6+tF+PFEU~H1-6%=0I!RS)5voH-Rh=lhS*mq;TAHg`T<%8ZC`q&%u@mXeB0PZ5 zIfTdem6{zcr>epfhAIC*&<45zq=S{B)KKNB_fP5k1Cimbq)aR!ld9Grq@64{QLMdU z?X#s7n;n4{VN1Glnmy$lO;5mDt~uyITlkT zG%T>Rt`t_>d0otq&>jOsf576D?!B!*f*wdPjUVb9m@qzv&;>Lh9nVFC(bou%QT9+C zTl8SZsVdjHZTy!dp~F$~N8LLytL#iqotPmnAkm4UCU7`fJe{$wWOll1fpuC_v#VNi zGy%Wc8(l4|sxU(`g_OvIC#NnFD4Re@=(XCo2O0PRGPv1^qIl-&PQy*?weH+m^Cfi0 z94sN76UF&UD44m27f~WittzirxpsA#W5Sf980u)&RFex}a!x)JCcK(ZgUk&tJ3Q5O zyrj8LRJ9t7&y5=qB7<_mjw>Qr^TpuhlIo*c_WZvXZPz0zKQn*`;BQ@BUe@{`}%znHU= ztmEJ=L3wo_XcGAMNUZuy`5X77O51uwE<-ItCEBB6WQXze-`|yh} z9dlnbmks_SHK8*JRTF?vH39IeSu#9@ngt{GQ8PzXAwn9L2~!BEsAWPaFM@3m<6>roiHQpUcuB8-Z$?B|=oRs- zzacq*j0y^TiFfcSxuSiP6`adTZT&&VD9i1_COw}0cB##zc()3b+#@~_&w-~m2>$QC z4hQN#Wj);6$DwZtx&VOJBf@}N;lPk3Y{iF>nhQ88@-9HI;5;~V9rhRP(nt=M33x8R zVuT<5Uc~bNs2y`foO-6WCp*w&?Ed)D@D^27+E(DrW@YqwVMsGep1kDH8=Qqot<~5k zq_xR~hu-DLc_SvU-6TvEvdNg5AI?+}JUs(@qd$z=Gx3FE;itZ{y3r)GZDbZ6RQevRQ zNQsFOGbIs}SSX34B#M%0lvpXTQ4&o_3?;FYOs6D{l6XoIC`qIwiIQYWQYe{0$xKR8 QDM_OwostYn9-}1lFI);Dr2qf` delta 1785 zcmaKsZD<@t7{~XS-Mzi-^j_SzSe( zsEU)duWTNl`o7&>9bJ+}D{zWdHaq@mRT8Hq)$58tf- zzCE%nK@U7KLjTwvP?e07*R4ZMt)SNMsekf93e>ucG*(70q@whvZ8=crO!EnPE)`JY z8R=ZLWp6(W?4uQQ1zAR7T2@e(1T6WY+%DEDy4iUQJ#3x1{i5Ch*PUi;A++b+4x zb+<5I%UWZw2T>z&p0!dmsawAdJ`UoLu=!04lIEE=&~A45A^2M6W%eLuZ-HByIkAGg za+l=cE;iAH#(n&@ZCi(3?r9n6JSw=Sth(3fxTj~N8=-=G>-n_OaW9yW-mG53z4DB7 zI$rADf&yM^{I*}`CGA(7V(&P$1JSwcI}9H}o$~+v`cbtm&zwSHyA`eF+Jn4iZ?S27 z-Q!wcgCftC4xk2)H8VE|yq0G9{>HtoSZs0Qs=H0~kYv}I;1i8Mc|$gNrV$4&i#-O< zBP*LLQ25z9kW;nQ zyX%x-Jf@nbgK(42Ql%SHUJ-Z9G~+PKcguaEoXxEJDEuw6;RbkFH1aLABw1c&$6Mfc znHq_Ol8v`Cb1)4HTsIicdR(Bqf*8+u7_t5isNyD5Wcql(7mQm&5$o%NeI6e63$Xc7 zG^w%fLHJIbc^w#TmiLIe{OV!6HS2i9XJ}a%+a@rF&|T-E2JfeAwAd0Z4A;4YKi$F) zF;~Rr^|swDNw#h6O#BPtg$S7+2jnBx)ek(P7kUFMorS0@UYH4$8i|XgguhkICud-< z5br~ne`n@A#V2zitGR^tOFEWEoYV#3h;FvbIUqaM0vV>gyPhzc79E(6`UHh#nKk4K z9RE`9((TdMCYv+lmqLZ!z0)c$Xl6W!1jPOlxKmz{?J2|}aip&kQ1dDaHXsLI2nl@s z8q->kgD>C+5g)VL(aw7)|Iq}*P9%U{6_}%{8zuj+epy*;1{dC|atSML;pea`;;YC# zlrO0Tluu8~18fvz&Ciym<>f&&Bx2@Vll tMsPX76$DojTt#r0;0VFh1lJH;OK_CnI)Y;a*ApBkI6-id;0Fla@E;WXsa^m8 diff --git a/packages/web/src/commands/stdCommands.ts b/packages/web/src/commands/stdCommands.ts index 8b99b6584..2be460669 100644 --- a/packages/web/src/commands/stdCommands.ts +++ b/packages/web/src/commands/stdCommands.ts @@ -46,6 +46,7 @@ import { openImportExportTab } from '../utility/importExportTools'; import newTable from '../tableeditor/newTable'; import { isProApp } from '../utility/proTools'; import { openWebLink } from '../utility/simpleTools'; +import { _t } from '../translations'; // function themeCommand(theme: ThemeDefinition) { // return { @@ -389,12 +390,12 @@ registerCommand({ category: 'New', icon: 'img sqlite-database', name: 'SQLite database', - menuName: 'New SQLite database', + menuName: _t('command.new.sqliteDatabase', { defaultMessage: 'New SQLite database' }), onClick: () => { showModal(InputTextModal, { value: 'newdb', - label: 'New database name', - header: 'Create SQLite database', + label: _t('command.new.sqliteDatabase', { defaultMessage: 'New SQLite database' }), + header: _t('command.new.sqliteDatabase.header', { defaultMessage: 'Create SQLite database' }), onConfirm: async file => { const resp = await apiCall('connections/new-sqlite-database', { file }); const connection = resp; @@ -404,6 +405,26 @@ registerCommand({ }, }); +registerCommand({ + id: 'new.duckdbDatabase', + category: 'New', + icon: 'img sqlite-database', + name: 'DuckDB database', + menuName: _t('command.new.duckdbDatabase', { defaultMessage: 'New DuckDB database' }), + onClick: () => { + showModal(InputTextModal, { + value: 'newdb', + label: _t('command.new.duckdbDatabase', { defaultMessage: 'New DuckDB database' }), + header: _t('command.new.duckdbDatabase.header', { defaultMessage: 'Create DuckDB database' }), + onConfirm: async file => { + const resp = await apiCall('connections/new-duckdb-database', { file }); + const connection = resp; + switchCurrentDatabase({ connection, name: `${file}.duckdb` }); + }, + }); + }, +}); + registerCommand({ id: 'tabs.changelog', category: 'Tabs', diff --git a/translations/en.json b/translations/en.json index d47f7e144..3eadc7994 100644 --- a/translations/en.json +++ b/translations/en.json @@ -26,6 +26,10 @@ "command.datagrid.undo": "Undo", "command.datagrid.viewJsonDocument": "View row as JSON document", "command.datagrid.viewJsonValue": "View cell as JSON document", + "command.new.duckdbDatabase": "New DuckDB database", + "command.new.duckdbDatabase.header": "Create DuckDB database", + "command.new.sqliteDatabase": "New SQLite database", + "command.new.sqliteDatabase.header": "Create SQLite database", "command.tabs.addToFavorites": "Add current tab to favorites", "command.tabs.closeAll": "Close all tabs", "command.tabs.closeTab": "Close tab", @@ -51,7 +55,6 @@ "connection.newQuery": "New Query (server)", "connection.refresh": "Refresh", "connection.serverSummary": "Server summary", - "connection.sqlRestore": "Restore/import SQL dump", "connection.viewDetails": "View details", "error.driverNotFound": "Invalid database connection, driver not found", "importExport.sourceFiles": "Source files", @@ -61,6 +64,7 @@ "schema.delete": "Delete schema", "schema.resetToDefault": "Reset to default", "schema.schemaName": "Schema name", + "settings.behaviour.jsonPreviewWrap": "Wrap json in preview", "settings.localization": "Localization", "tab.administration": "Administration", "widget.databaseContent": "Database content", From 0e30cb1439ff7b9b85aa72757663bc6e1093394d Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 19:19:23 +0200 Subject: [PATCH 18/49] fix: set multipleSchema to true for duck db --- plugins/dbgate-plugin-duckdb/src/frontend/driver.js | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/dbgate-plugin-duckdb/src/frontend/driver.js b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js index 57146584f..404eb4c2f 100644 --- a/plugins/dbgate-plugin-duckdb/src/frontend/driver.js +++ b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js @@ -32,6 +32,7 @@ const dialect = { createColumn: true, dropColumn: true, + multipleSchema: true, createIndex: true, dropIndex: true, createForeignKey: false, From 5e9366fa9255d3e0c299a1f5c31979ec64f1f0db Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 20:22:41 +0200 Subject: [PATCH 19/49] feat: cli open duckdb --- packages/api/src/controllers/connections.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/api/src/controllers/connections.js b/packages/api/src/controllers/connections.js index 8662f5e3e..f006fb7b5 100644 --- a/packages/api/src/controllers/connections.js +++ b/packages/api/src/controllers/connections.js @@ -38,6 +38,11 @@ function getNamedArgs() { res.databaseFile = name; res.engine = 'sqlite@dbgate-plugin-sqlite'; } + + if (name.endsWith('.duckdb')) { + res.databaseFile = name; + res.engine = 'duckdb@dbgate-plugin-duckdb'; + } } } return res; From ccf78285b6c1c0eaee800d24baac4c5b99daabfa Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 20:23:16 +0200 Subject: [PATCH 20/49] feat: file open duckdb --- packages/web/src/utility/openElectronFile.ts | 40 ++++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/web/src/utility/openElectronFile.ts b/packages/web/src/utility/openElectronFile.ts index cbfa8d501..f771cdcdf 100644 --- a/packages/web/src/utility/openElectronFile.ts +++ b/packages/web/src/utility/openElectronFile.ts @@ -2,7 +2,7 @@ import { showModal } from '../modals/modalTools'; import { get } from 'svelte/store'; import newQuery from '../query/newQuery'; import getElectron from './getElectron'; -import { currentDatabase, extensions, getCurrentDatabase } from '../stores'; +import { extensions, getCurrentDatabase } from '../stores'; import { getUploadListener } from './uploadFiles'; import { getConnectionLabel, getDatabaseFileLabel } from 'dbgate-tools'; import { apiCall } from './api'; @@ -13,6 +13,7 @@ import _ from 'lodash'; import ErrorMessageModal from '../modals/ErrorMessageModal.svelte'; import { openImportExportTab } from './importExportTools'; import { switchCurrentDatabase } from './common'; +import { _t } from '../translations'; export function canOpenByElectron(file, extensions) { if (!file) return false; @@ -22,7 +23,13 @@ export function canOpenByElectron(file, extensions) { if (nameLower.endsWith('.qdesign')) return true; if (nameLower.endsWith('.perspective')) return true; if (nameLower.endsWith('.json')) return true; - if (nameLower.endsWith('.db') || nameLower.endsWith('.sqlite') || nameLower.endsWith('.sqlite3')) return true; + if ( + nameLower.endsWith('.db') || + nameLower.endsWith('.sqlite') || + nameLower.endsWith('.sqlite3') || + nameLower.endsWith('.duckdb') + ) + return true; for (const format of extensions.fileFormats) { if (nameLower.endsWith(`.${format.extension}`)) return true; if (format.extensions?.find(ext => nameLower.endsWith(`.${ext}`))) return true; @@ -30,12 +37,12 @@ export function canOpenByElectron(file, extensions) { return false; } -export async function openSqliteFile(filePath) { +export async function openDatabaseFile(filePath, engine) { const defaultDatabase = getDatabaseFileLabel(filePath); const resp = await apiCall('connections/save', { _id: undefined, databaseFile: filePath, - engine: 'sqlite@dbgate-plugin-sqlite', + engine, singleDatabase: true, defaultDatabase, }); @@ -136,7 +143,11 @@ export function openElectronFileCore(filePath, extensions) { return; } if (nameLower.endsWith('.db') || nameLower.endsWith('.sqlite') || nameLower.endsWith('.sqlite')) { - openSqliteFile(filePath); + openDatabaseFile(filePath, 'sqlite@dbgate-plugin-sqlite'); + return; + } + if (nameLower.endsWith('.duckdb')) { + openDatabaseFile(filePath, 'duckdb@dbgate-plugin-duckdb'); return; } if (nameLower.endsWith('.jsonl') || nameLower.endsWith('.ndjson')) { @@ -225,7 +236,7 @@ export async function openElectronFile() { const filePaths = await electron.showOpenDialog({ filters: [ { - name: `All supported files`, + name: _t('file.allSupported', { defaultMessage: 'All supported files' }), extensions: [ 'sql', 'sqlite', @@ -235,15 +246,20 @@ export async function openElectronFile() { 'qdesign', 'perspective', 'json', + 'duckdb', ...getFileFormatExtensions(ext), ], }, - { name: `SQL files`, extensions: ['sql'] }, - { name: `JSON files`, extensions: ['json'] }, - { name: `Diagram files`, extensions: ['diagram'] }, - { name: `Query designer files`, extensions: ['qdesign'] }, - { name: `Perspective files`, extensions: ['perspective'] }, - { name: `SQLite database`, extensions: ['sqlite', 'db', 'sqlite3'] }, + { name: _t('file.sqlFiles', { defaultMessage: 'SQL files' }), extensions: ['sql'] }, + { name: _t('file.jsonFiles', { defaultMessage: 'JSON files' }), extensions: ['json'] }, + { name: _t('file.diagramFiles', { defaultMessage: 'Diagram files' }), extensions: ['diagram'] }, + { name: _t('file.queryDesignerFiles', { defaultMessage: 'Query designer files' }), extensions: ['qdesign'] }, + { name: _t('file.perspectiveFiles', { defaultMessage: 'Perspective files' }), extensions: ['perspective'] }, + { + name: _t('file.sqliteDatabase', { defaultMessage: 'SQLite database' }), + extensions: ['sqlite', 'db', 'sqlite3'], + }, + { name: _t('file.duckdb', { defaultMessage: 'DuckDB database' }), extensions: ['duckdb', 'db'] }, ...getFileFormatFilters(ext), ], properties: ['showHiddenFiles', 'openFile'], From 6548c286a6a5e419a247c934b356fe8bba9d8c9e Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 20:23:23 +0200 Subject: [PATCH 21/49] chore: remove unused code --- plugins/dbgate-plugin-sqlite/src/backend/driver.libsql.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/dbgate-plugin-sqlite/src/backend/driver.libsql.js b/plugins/dbgate-plugin-sqlite/src/backend/driver.libsql.js index 6cca49394..7bc669297 100644 --- a/plugins/dbgate-plugin-sqlite/src/backend/driver.libsql.js +++ b/plugins/dbgate-plugin-sqlite/src/backend/driver.libsql.js @@ -1,7 +1,6 @@ // @ts-check const _ = require('lodash'); const stream = require('stream'); -const sqliteDriver = require('./driver.sqlite'); const driverBases = require('../frontend/drivers'); const Analyser = require('./Analyser'); const { splitQuery, sqliteSplitterOptions } = require('dbgate-query-splitter'); @@ -30,7 +29,6 @@ const libsqlDriver = { ...driverBases[1], analyserClass: Analyser, async connect({ databaseFile, isReadOnly, authToken, databaseUrl, ...rest }) { - console.log('connect', databaseFile, isReadOnly, authToken, databaseUrl, rest); const Database = getLibsql(); const client = databaseFile ? new Database(databaseFile, { readonly: !!isReadOnly }) From 041397b137563cd8030241d32ac4644c0c57a529 Mon Sep 17 00:00:00 2001 From: Nybkox Date: Tue, 8 Apr 2025 20:24:24 +0200 Subject: [PATCH 22/49] chore: remove test db file --- packages/api/testduck.db | Bin 2371584 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 packages/api/testduck.db diff --git a/packages/api/testduck.db b/packages/api/testduck.db deleted file mode 100644 index 90a8e2dd2acd8082b1e2e1f432d6bc13c88d5bfe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2371584 zcmeI*O^hU0eE{&+-Sf5UUGLh)#5Q0rF-{-}&f3O&oU+CtERJIXj)Tl)HJ8L!?Ln*{9?{Buf?tB4i1PgLi=>fP8SGD4d6@4sIB=kLGv{y%>5;XgeUh8zCszI*S# z_r9&y{b=LC`x^@%df;OZZoQ5r1PBlyK!5-N0t5&UAV7e?!3n(lxu-t%tM9JgIXyVb zj~;5zpUS4@vQecy-vj{y1PBlyK!5-N0t5&UATa6z|Nh#m|MA@S|LO5obe|V@rZsV| z_&+BxCP07y0RjXF5FkK+009E!6*zL&(a(Hg`tdKfYVeVIxM3kw>!DH))x|J%etvmn zIfP~$7Ls|%zebo_Z(LljKeOmCXs=wAMJgd~tnk zetmg$27Cl?dB+Y+IZ$)EQpfBr5O zpdS7vrX2qsPuij$ZhrEyPkrXePjqeiz;K%m(U2>RXV0w9KYgx|G-M>GZfd{8t(xJ) zYPJEBqNPJ6`oyi0>287PM)<`$Is|dMHbFbjGxzORJ?1{Wwilx%#a-?~_I}Xr zcqCsoaEF|}u(rIA?mM?OQjp8x@RbnvhTc8ta2q1ole)6&+9l#DQ*n>#T6m@B-48ppAVKu4OndSAy`Q27-51U?_=HLu__vhOh;UDAixEN~b zAsyTH=KPS(_q{VG8=VtCwr9s(C@iMCQL_6~DodgAi^I!tW^!C-M}23Nc*GoTulDnq ze6`J|gq@G;=_fC?PR>*HF#Ak=Gw4W|8hiu{KJV7%&!&e(*QtH;DK*b(^HGp5iHATw zIbi?KPQS_ipRZK9^?P@lI~w8N?(ErZj^(>e+Cqr;o0gp-)3G=RbM;v!D3XCm%c0%K5&nKQw1| z&UtnZIp4o^&hg@Xr#jCKo%7CTo$fuRb<`bg9d+^gY0yV^dpFvA6y=w zcVl|ld*(vCPW|VcM))D z-zAakrDRreDv9gGP2$`aZ#(_eqmP#N|3H+S009C72oNAZfB=C#31l%se?$=@=xn$W zT8|MY-?@b!h8WqkLNP(bp4@FrU^fD_-K=a~0t5&UAV7e?^%4+VTrYX~5+Fc;K*h=szm=GA2NPKv4qOsXOUW`cPLtfco>_$Nv0d zH=`5YpZ^W9yVEi5&%d`n)2=Q?^Uh2=9shPMjD7fay&KS}e(yWKO*Vo3-hW_>`n_*9iwdgClkS}U{L9l=N=kqL z0RjXF5FkKc#00Xh6Z`M*(4YT^HIw2@xWJ(aufOUOAV7cs0Rklv5L}d`p%jw<0RjX@ zP2ljT6-apq5Fl{f1q2t@y$KY6009C72oNAZVBZ98*te7}8G2oNAZ zfB*pk9f83?h5r0ILZ*%&AV7cs0RjXF5FoIlKo(i(&wocKD-a++VB`djj$DZphyVcs z1PJUbu)7GP5o(Jee(}+4o(+|yI2B^-vKYeIa5s)#5n*mf7Yk=1w=}>lc<*SFVJ)C(h5Wt;gy3dmcXwl^=z}c}+UW44>Pn zLAj*~z4VPvm0tSlOkUcg|Np$SJ7iPq@1!f;vXg2HDh&c_sZ-n_rv2h=r=NQC(Vbh+ z8f}3}d&EIA!>#sz_7G-K(WgR5o+GLQ5c?1X$ zxc&mp&)2^V%3Ok4nF~J#WzS-YG*XE7H|0Zehgjuu*ob$0HJkY%p59p&ZyBrA!$xa3 z(g?pkn%pJUHRGo@_RaX-vE)v!(0wymyuWJmon(XW!MdiK)#5!{gKt)Q=-l!`<^ZB*QN5BFk?!i@BX1RH3=~B)3zIOEI^FEVom6Znx|ux36z~ zv)}FQoBi(2a@bXkrux6X8r&u$5?CJ3ch@qH^P{WGbx`)L1&VIpqSN2HW|j~jK!5-N z0t5yNWKl?eTrsy(AXY?(F*Ue^V>`Dxwl!0+Nfx+ik}I$72QJ_|df?=R0m%a(hTY;O)TXZESKwx-*QM;>QaEOuK9kuC> zD{;l}?QBf~1PDx;z^Hw`?bd(N{vW68?X}zXwLXE75x8Y!O0w?-$>NGMVi@-Eo2kyz zHa9lHEAQ%l+9uBU+PnK^{HA}#FW=LC9%u0L9!a5Q+s~0CkM>+@mvb@b33=c50tpfz zK!5-N0t5&UD4#$UBW!xoBb%Ls6LWc;@`vDKp8#o31A>ZN#<)x19bMBV2@oJafB*pk z1SU&Ba4}i!t+oUROs2rClUZ%GCP07yfrAzp99*PNcO;A43w6*9B0mBI2oNAZU{VC$ zIVlxY&q)!;B8zRG@5m|Y!P6MIg)spF1PBlyK!5-N0=p6z{GG;JHhZ+5#`vvWRm{o+ z2oNAZfB*pk1PC0gK#78jgN2bB0Rlq`+%x2QRwF=w009C7#$I4>aN%i;v2S=4BtU=w zf$bvL)$;&nFvJdp68hOMdC4oqi)6AC$%1>BfBh!qVzW?0@-*^Yd%#aXRmR`EXv7 zPBO#ic4|;=X+kf3qf@1qzB-eab{CbZ_4gt@7%FXoS_1r(;NF{ zeD7FtCs*jcnJnIjR_M)Y@t&>0H>*8#Zh4_`;*r&Ov)aQ;^K0jp8z=Ic)nYx8VHbCi z<+sMg+)fXw(A<2I+o{H-nA<{@+o?RaTlSLM*SEge@Amf1es^a%?5akCdkF8`ZP&Vi za6|&jcjvonnaBCjRpvS<`_=+Qw{OwuZ(TD>2oNAZfB*pkg9WlEWYc|dI|X7zgcwtU zOE|W3t7BU;6`N#%_e^r-)&0N)oJS9wT!sV)5Ex2eix6X{Z(oWCF@`Gd1h3@l-U)7e zRWH4OOX1QtVRNfQ{g&$r$B zPulP~)NK1XlH}2zOYL$l20bC~`(7YH0t5&UAV7cs0RrU{$YO*|PkLmt zlW<}#uT%aIeC!h-?P)+zk;@o&37qJfHc5a00RjXF5FjvF0)mUlYHzhAKwvTj-aDDq zR%-$T2oN}Efx*E=`gBLK$h}Yp-5~NKK!5-N0t6;S;I}8GqUt#*0$F6S?eiTuMLl>L zBeyUnK!5-N0t5&UAV6SO0)xNPn9F95*3%ew@2X-}CP07y0RjXF5FkL{U<2oM-=fj=DY>Zu(80;4F9MHWLo zjS+v3qqZ38VLJJ>jLK4+3USND5YC3n@l9M@Y5zt>Ex)a*ku6C#d_LK*+OuK4TJjSe z?erVrc=1`BC_ammLu66S1+ES1)RtMi{>@HnTuG+0VHep<_xD^pxQ)}jr83I{oX4CvyS>2oNAZpp*hb#TBhbwmkP#$|hFm0~9FwdcWxO_lfEQ`;?Ak2oNAZ zfB=CK32YNr4D6?25@L&zT)XYtUN&j<-8VQ15gZ3XY2oNAZfB*pk!w85hhEcK(0RjZZ zLEwYqPziM)K!5;&krmi3vUvTKm+xsmU-6~y{P<^|&(@9)hIqbWWLs3h2oNAZfB=Eg z2xOO5ogc|AO~DnEz#xHktdRcPd%i}`L*)5PV*&&S5FkK+009C72uy-Nf80OUs&qcC z#`m+WdCBxl*YtF2IvG!O&CmODOXelpBttUJJMBuA=hMluWcZ`l)z)1b{{JD<@RtAq z0t5&UAV7cs0RjXF?5n`ZeN9bb1V&sS`~R+i-7Z$zHyhWwZXT#LLhN5&ckb8NuKoYd zca6!|Y(xc$Zr`HQA5m7tB0zuu0RjX@SfHfw#C;=7I#khI7Y@Z#+LelK-=fnmExdvg zAV7cs0Rq=eV5qpF>U)ao)<7~Yl|a$g`$ea}PuTbGQ#z6%K!5-N0t8AVuuWX?`Zqgw zKwU|ulOcAtSPx@<4^&C6-S%xSo3#4w8=Qm)5Fk)Kfd|T0WF;PTfxMrGQ7@Jx*@U275X3vbT9PXR(tNs~3xuI{3Zy#wk&&JL~ zVnWjcAM;2`Iop2JBgy<9V^tsRu7TyK2ci`AV7cs0RjXF5FkLHyaIXLzZed!G@iY&rmXZ@7K!5-N0t5&UAV7cs0RmAV3-CK3{#;8X?^rwOlfBxSmyGj1+Na`l@=Ate z{&YH(jb~cZ$^6-_={)UZex817arj;=!q1-%SM&GA1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB=D=1TOy1iMv1ew_ka!5?8FoSFN?d9dUebe9gqy@%T#K zpKi?8FD$LDG%h}IetvB|oqTlu!uslk5MKK0B!v)GZ_37ZJ(j-phX4Tr1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5GbL*t8brtA@Nfn)iBQ>o_1P%>w>lFgs3w#M1WUjhUO5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZpa_9K|A$|^_r?D_cV8tYUX8C> zE0eq9_#^RkG``*$U&;H2m*&^bEjLbld~NmFm5;AJed3Ap%j-)Iom*aLoOon)Y31VT zN_Zfy_sRIWDZWm`SMq)#+wN53(){@g=Nk7ete$^*_37@n^GO=rGs2y5yW8U{$=5)D z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5>iY>p zpZw~jzxeLMSDOX`1PBn=yFevugiz@^lh<0)HRsvA>q=pA1dingPNkY3L&==k*1h zmBQo*9Lo=!N;N-*k~y=LZ2oMuHO@x<5+Fc;009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ T009C72oNAZfB*pkMF{*q9FDYk From ed583d80a3a9fefd403aa4afc7873c30789aed01 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 24 Apr 2025 09:33:45 +0200 Subject: [PATCH 23/49] test fix --- integration-tests/engines.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 27522248a..9214a88bd 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -667,7 +667,7 @@ const duckdbEngine = { skipIndexes: true, skipStringLength: true, skipTriggers: true, - skipDataDuplicator: true, + skipDataReplicator: true, skipAutoIncrement: true, skipDeploy: true, supportRenameSqlObject: true, @@ -693,7 +693,7 @@ const enginesOnLocal = [ // all engines, which would be run on local test // cassandraEngine, // mysqlEngine, - mariaDbEngine, + // mariaDbEngine, // postgreSqlEngine, // sqlServerEngine, // sqliteEngine, From 065eb9b878dec307eb7c62db41c2025a46a54819 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 24 Apr 2025 09:52:30 +0200 Subject: [PATCH 24/49] duckdb fixes --- integration-tests/engines.js | 2 ++ plugins/dbgate-plugin-duckdb/src/frontend/driver.js | 1 + 2 files changed, 3 insertions(+) diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 9214a88bd..ae4fe2b16 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -658,6 +658,7 @@ const cassandraEngine = { const duckdbEngine = { label: 'DuckDB', generateDbFile: true, + defaultSchemaName: 'main', connection: { engine: 'duckdb@dbgate-plugin-duckdb', }, @@ -671,6 +672,7 @@ const duckdbEngine = { skipAutoIncrement: true, skipDeploy: true, supportRenameSqlObject: true, + skipIncrementalAnalysis: true, }; const enginesOnCi = [ diff --git a/plugins/dbgate-plugin-duckdb/src/frontend/driver.js b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js index 404eb4c2f..e3fbd5dfa 100644 --- a/plugins/dbgate-plugin-duckdb/src/frontend/driver.js +++ b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js @@ -18,6 +18,7 @@ function getDatabaseFileLabel(databaseFile) { const dialect = { limitSelect: true, rangeSelect: true, + defaultSchemaName: 'main', offsetFetchRangeSyntax: false, explicitDropConstraint: true, stringEscapeChar: "'", From 7b4d408733aced4682984d9c416cd11f47e2087b Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Thu, 24 Apr 2025 10:31:39 +0200 Subject: [PATCH 25/49] duckdb - removed dummy fast snapshot --- plugins/dbgate-plugin-duckdb/src/backend/Analyser.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js index bc7e9fdd0..f5e146adc 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/Analyser.js @@ -20,7 +20,7 @@ class Analyser extends DatabaseAnalyser { this.singleObjectId = pureName; } - async _getFastSnapshot() { + async _runAnalysis() { const tablesResult = await this.driver.query(this.dbhan, sql.tables); const columnsResult = await this.driver.query(this.dbhan, sql.columns); const foreignKeysResult = await this.driver.query(this.dbhan, sql.foreignKeys); @@ -72,11 +72,6 @@ class Analyser extends DatabaseAnalyser { views: viewsExtended, }; } - - async _runAnalysis() { - const structure = await this._getFastSnapshot(); - return structure; - } } module.exports = Analyser; From de071b37eb67b6ac8b72f0d4eaea45f347555b4b Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Thu, 24 Apr 2025 10:54:14 +0200 Subject: [PATCH 26/49] duckdb fixes --- plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js | 4 ++++ plugins/dbgate-plugin-duckdb/src/frontend/driver.js | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js b/plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js index 8af6e8e8b..63c15cf39 100644 --- a/plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js +++ b/plugins/dbgate-plugin-duckdb/src/frontend/Dumper.js @@ -6,6 +6,10 @@ class Dumper extends SqlDumper { renameSqlObject(obj, newname) { this.putCmd('^alter %k %f ^rename ^to %i', this.getSqlObjectSqlName(obj.objectTypeField), obj, newname); } + + renameTable(obj, newname) { + this.putCmd('^alter ^table %f ^rename ^to %i', obj, newname); + } } module.exports = Dumper; diff --git a/plugins/dbgate-plugin-duckdb/src/frontend/driver.js b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js index e3fbd5dfa..f1e57f94e 100644 --- a/plugins/dbgate-plugin-duckdb/src/frontend/driver.js +++ b/plugins/dbgate-plugin-duckdb/src/frontend/driver.js @@ -38,14 +38,14 @@ const dialect = { dropIndex: true, createForeignKey: false, enableForeignKeyChecks: false, - dropCheck: true, - dropUnique: true, - dropForeignKey: true, + // dropCheck: true, + // dropUnique: true, + // dropForeignKey: true, createPrimaryKey: false, - dropPrimaryKey: true, + // dropPrimaryKey: true, dropReferencesWhenDropTable: true, - dropIndexContainsTableSpec: true, - filteredIndexes: true, + // dropIndexContainsTableSpec: true, + // filteredIndexes: true, anonymousForeignKey: true, }; From a566fb3988a4c44da7c3b2e6d4f0bbc78aa913ff Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Thu, 24 Apr 2025 11:02:44 +0200 Subject: [PATCH 27/49] typo + bigint converted to numbers --- plugins/dbgate-plugin-duckdb/src/backend/driver.js | 2 +- plugins/dbgate-plugin-duckdb/src/backend/helpers.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/dbgate-plugin-duckdb/src/backend/driver.js b/plugins/dbgate-plugin-duckdb/src/backend/driver.js index c1fc43bcc..c7e5d54d5 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/driver.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/driver.js @@ -178,7 +178,7 @@ const driver = { return { version, - versionText: `DuchDB ${version}`, + versionText: `DuckDB ${version}`, }; }, diff --git a/plugins/dbgate-plugin-duckdb/src/backend/helpers.js b/plugins/dbgate-plugin-duckdb/src/backend/helpers.js index 4a2e3d47d..3364cfd93 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/helpers.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/helpers.js @@ -21,7 +21,7 @@ function _normalizeValue(value) { } if (typeof value === 'bigint') { - return `${value}n`; + return parseInt(value); } if (Array.isArray(value)) { From e8b11bd42a8b2e1f829979026665ffa815a45268 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Thu, 24 Apr 2025 13:04:16 +0200 Subject: [PATCH 28/49] copied from master --- .../__tests__/db-import-export.spec.js | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/integration-tests/__tests__/db-import-export.spec.js b/integration-tests/__tests__/db-import-export.spec.js index 3dff4282c..7ec7a56f1 100644 --- a/integration-tests/__tests__/db-import-export.spec.js +++ b/integration-tests/__tests__/db-import-export.spec.js @@ -132,17 +132,14 @@ describe('DB Import/export', () => { for (const check of engine.dumpChecks || []) { const res = await driver.query(conn, check.sql); - const cnt = parseInt(res.rows[0].res.toString()); - expect(cnt).toEqual(parseInt(check.res)); + expect(res.rows[0].res.toString()).toEqual(check.res); } - const res1 = await driver.query(conn, `select count(*) as cnt from t1`); - const cnt1 = parseInt(res1.rows[0].cnt.toString()); - expect(cnt1).toEqual(6); + // const res1 = await driver.query(conn, `select count(*) as cnt from t1`); + // expect(res1.rows[0].cnt.toString()).toEqual('6'); - const res2 = await driver.query(conn, `select count(*) as cnt from t2`); - const cnt2 = parseInt(res2.rows[0].cnt.toString()); - expect(cnt2).toEqual(6); + // const res2 = await driver.query(conn, `select count(*) as cnt from t2`); + // expect(res2.rows[0].cnt.toString()).toEqual('6'); }) ); } From c4f17e42e11acf0d8f695b52deaa9383252e4e3f Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Thu, 24 Apr 2025 13:53:41 +0200 Subject: [PATCH 29/49] handleQueryStream refactor --- packages/api/src/proc/sessionProcess.js | 165 +---------------- packages/api/src/utility/handleQueryStream.js | 169 ++++++++++++++++++ 2 files changed, 173 insertions(+), 161 deletions(-) create mode 100644 packages/api/src/utility/handleQueryStream.js diff --git a/packages/api/src/proc/sessionProcess.js b/packages/api/src/proc/sessionProcess.js index e81abda1c..0caad1720 100644 --- a/packages/api/src/proc/sessionProcess.js +++ b/packages/api/src/proc/sessionProcess.js @@ -11,6 +11,7 @@ const { decryptConnection } = require('../utility/crypting'); const { connectUtility } = require('../utility/connectUtility'); const { handleProcessCommunication } = require('../utility/processComm'); const { getLogger, extractIntSettingsValue, extractBoolSettingsValue } = require('dbgate-tools'); +const { handleQueryStream, QueryStreamTableWriter } = require('../utility/handleQueryStream'); const logger = getLogger('sessionProcess'); @@ -23,164 +24,6 @@ let lastActivity = null; let currentProfiler = null; let executingScripts = 0; -class TableWriter { - constructor() { - this.currentRowCount = 0; - this.currentChangeIndex = 1; - this.initializedFile = false; - } - - initializeFromQuery(structure, resultIndex) { - this.jslid = crypto.randomUUID(); - this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); - fs.writeFileSync( - this.currentFile, - JSON.stringify({ - ...structure, - __isStreamHeader: true, - }) + '\n' - ); - this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' }); - this.writeCurrentStats(false, false); - this.resultIndex = resultIndex; - this.initializedFile = true; - process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex }); - } - - initializeFromReader(jslid) { - this.jslid = jslid; - this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); - this.writeCurrentStats(false, false); - } - - row(row) { - // console.log('ACCEPT ROW', row); - this.currentStream.write(JSON.stringify(row) + '\n'); - this.currentRowCount += 1; - - if (!this.plannedStats) { - this.plannedStats = true; - process.nextTick(() => { - if (this.currentStream) this.currentStream.uncork(); - process.nextTick(() => this.writeCurrentStats(false, true)); - this.plannedStats = false; - }); - } - } - - rowFromReader(row) { - if (!this.initializedFile) { - process.send({ msgtype: 'initializeFile', jslid: this.jslid }); - this.initializedFile = true; - - fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n'); - this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' }); - this.writeCurrentStats(false, false); - this.initializedFile = true; - return; - } - - this.row(row); - } - - writeCurrentStats(isFinished = false, emitEvent = false) { - const stats = { - rowCount: this.currentRowCount, - changeIndex: this.currentChangeIndex, - isFinished, - jslid: this.jslid, - }; - fs.writeFileSync(`${this.currentFile}.stats`, JSON.stringify(stats)); - this.currentChangeIndex += 1; - if (emitEvent) { - process.send({ msgtype: 'stats', ...stats }); - } - } - - close(afterClose) { - if (this.currentStream) { - this.currentStream.end(() => { - this.writeCurrentStats(true, true); - if (afterClose) afterClose(); - }); - } - } -} - -class StreamHandler { - constructor(resultIndexHolder, resolve, startLine) { - this.recordset = this.recordset.bind(this); - this.startLine = startLine; - this.row = this.row.bind(this); - // this.error = this.error.bind(this); - this.done = this.done.bind(this); - this.info = this.info.bind(this); - - // use this for cancelling - not implemented - // this.stream = null; - - this.plannedStats = false; - this.resultIndexHolder = resultIndexHolder; - this.resolve = resolve; - // currentHandlers = [...currentHandlers, this]; - } - - closeCurrentWriter() { - if (this.currentWriter) { - this.currentWriter.close(); - this.currentWriter = null; - } - } - - recordset(columns) { - this.closeCurrentWriter(); - this.currentWriter = new TableWriter(); - this.currentWriter.initializeFromQuery( - Array.isArray(columns) ? { columns } : columns, - this.resultIndexHolder.value - ); - this.resultIndexHolder.value += 1; - - // this.writeCurrentStats(); - - // this.onRow = _.throttle((jslid) => { - // if (jslid == this.jslid) { - // this.writeCurrentStats(false, true); - // } - // }, 500); - } - row(row) { - if (this.currentWriter) this.currentWriter.row(row); - else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } }); - // this.onRow(this.jslid); - } - // error(error) { - // process.send({ msgtype: 'error', error }); - // } - done(result) { - this.closeCurrentWriter(); - // currentHandlers = currentHandlers.filter((x) => x != this); - this.resolve(); - } - info(info) { - if (info && info.line != null) { - info = { - ...info, - line: this.startLine + info.line, - }; - } - process.send({ msgtype: 'info', info }); - } -} - -function handleStream(driver, resultIndexHolder, sqlItem) { - return new Promise((resolve, reject) => { - const start = sqlItem.trimStart || sqlItem.start; - const handler = new StreamHandler(resultIndexHolder, resolve, start && start.line); - driver.stream(dbhan, sqlItem.text, handler); - }); -} - function allowExecuteCustomScript(driver) { if (driver.readOnlySessions) { return true; @@ -227,7 +70,7 @@ async function handleStartProfiler({ jslid }) { return; } - const writer = new TableWriter(); + const writer = new QueryStreamTableWriter(); writer.initializeFromReader(jslid); currentProfiler = await driver.startProfiler(dbhan, { @@ -313,7 +156,7 @@ async function handleExecuteQuery({ sql, autoCommit }) { ...driver.getQuerySplitterOptions('stream'), returnRichInfo: true, })) { - await handleStream(driver, resultIndexHolder, sqlItem); + await handleQueryStream(dbhan, driver, resultIndexHolder, sqlItem); // const handler = new StreamHandler(resultIndex); // const stream = await driver.stream(systemConnection, sqlItem, handler); // handler.stream = stream; @@ -341,7 +184,7 @@ async function handleExecuteReader({ jslid, sql, fileName }) { } } - const writer = new TableWriter(); + const writer = new QueryStreamTableWriter(); writer.initializeFromReader(jslid); const reader = await driver.readQuery(dbhan, sql); diff --git a/packages/api/src/utility/handleQueryStream.js b/packages/api/src/utility/handleQueryStream.js new file mode 100644 index 000000000..8201feebc --- /dev/null +++ b/packages/api/src/utility/handleQueryStream.js @@ -0,0 +1,169 @@ +const crypto = require('crypto'); +const path = require('path'); +const fs = require('fs'); +const _ = require('lodash'); + +const { jsldir } = require('../utility/directories'); + +class QueryStreamTableWriter { + constructor() { + this.currentRowCount = 0; + this.currentChangeIndex = 1; + this.initializedFile = false; + } + + initializeFromQuery(structure, resultIndex) { + this.jslid = crypto.randomUUID(); + this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); + fs.writeFileSync( + this.currentFile, + JSON.stringify({ + ...structure, + __isStreamHeader: true, + }) + '\n' + ); + this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' }); + this.writeCurrentStats(false, false); + this.resultIndex = resultIndex; + this.initializedFile = true; + process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex }); + } + + initializeFromReader(jslid) { + this.jslid = jslid; + this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); + this.writeCurrentStats(false, false); + } + + row(row) { + // console.log('ACCEPT ROW', row); + this.currentStream.write(JSON.stringify(row) + '\n'); + this.currentRowCount += 1; + + if (!this.plannedStats) { + this.plannedStats = true; + process.nextTick(() => { + if (this.currentStream) this.currentStream.uncork(); + process.nextTick(() => this.writeCurrentStats(false, true)); + this.plannedStats = false; + }); + } + } + + rowFromReader(row) { + if (!this.initializedFile) { + process.send({ msgtype: 'initializeFile', jslid: this.jslid }); + this.initializedFile = true; + + fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n'); + this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' }); + this.writeCurrentStats(false, false); + this.initializedFile = true; + return; + } + + this.row(row); + } + + writeCurrentStats(isFinished = false, emitEvent = false) { + const stats = { + rowCount: this.currentRowCount, + changeIndex: this.currentChangeIndex, + isFinished, + jslid: this.jslid, + }; + fs.writeFileSync(`${this.currentFile}.stats`, JSON.stringify(stats)); + this.currentChangeIndex += 1; + if (emitEvent) { + process.send({ msgtype: 'stats', ...stats }); + } + } + + close(afterClose) { + if (this.currentStream) { + this.currentStream.end(() => { + this.writeCurrentStats(true, true); + if (afterClose) afterClose(); + }); + } + } +} + +class StreamHandler { + constructor(resultIndexHolder, resolve, startLine) { + this.recordset = this.recordset.bind(this); + this.startLine = startLine; + this.row = this.row.bind(this); + // this.error = this.error.bind(this); + this.done = this.done.bind(this); + this.info = this.info.bind(this); + + // use this for cancelling - not implemented + // this.stream = null; + + this.plannedStats = false; + this.resultIndexHolder = resultIndexHolder; + this.resolve = resolve; + // currentHandlers = [...currentHandlers, this]; + } + + closeCurrentWriter() { + if (this.currentWriter) { + this.currentWriter.close(); + this.currentWriter = null; + } + } + + recordset(columns) { + this.closeCurrentWriter(); + this.currentWriter = new QueryStreamTableWriter(); + this.currentWriter.initializeFromQuery( + Array.isArray(columns) ? { columns } : columns, + this.resultIndexHolder.value + ); + this.resultIndexHolder.value += 1; + + // this.writeCurrentStats(); + + // this.onRow = _.throttle((jslid) => { + // if (jslid == this.jslid) { + // this.writeCurrentStats(false, true); + // } + // }, 500); + } + row(row) { + if (this.currentWriter) this.currentWriter.row(row); + else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } }); + // this.onRow(this.jslid); + } + // error(error) { + // process.send({ msgtype: 'error', error }); + // } + done(result) { + this.closeCurrentWriter(); + // currentHandlers = currentHandlers.filter((x) => x != this); + this.resolve(); + } + info(info) { + if (info && info.line != null) { + info = { + ...info, + line: this.startLine + info.line, + }; + } + process.send({ msgtype: 'info', info }); + } +} + +function handleQueryStream(dbhan, driver, resultIndexHolder, sqlItem) { + return new Promise((resolve, reject) => { + const start = sqlItem.trimStart || sqlItem.start; + const handler = new StreamHandler(resultIndexHolder, resolve, start && start.line); + driver.stream(dbhan, sqlItem.text, handler); + }); +} + +module.exports = { + handleQueryStream, + QueryStreamTableWriter, +}; From 55896be69482b208c2834c3c2ffd58e967ee09a8 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Thu, 24 Apr 2025 14:56:25 +0200 Subject: [PATCH 30/49] changed query workflow for duckdb --- .../src/controllers/databaseConnections.js | 40 +++++++++ .../api/src/proc/databaseConnectionProcess.js | 32 +++++++ packages/api/src/proc/sessionProcess.js | 21 ++--- packages/api/src/utility/handleQueryStream.js | 34 +++++--- .../src/query/SimpleQueryResultTabs.svelte | 52 ------------ packages/web/src/tabs/QueryTab.svelte | 85 +++++++++---------- 6 files changed, 141 insertions(+), 123 deletions(-) delete mode 100644 packages/web/src/query/SimpleQueryResultTabs.svelte diff --git a/packages/api/src/controllers/databaseConnections.js b/packages/api/src/controllers/databaseConnections.js index 7b36da519..9d415701d 100644 --- a/packages/api/src/controllers/databaseConnections.js +++ b/packages/api/src/controllers/databaseConnections.js @@ -39,6 +39,8 @@ const axios = require('axios'); const { callTextToSqlApi, callCompleteOnCursorApi, callRefactorSqlQueryApi } = require('../utility/authProxy'); const { decryptConnection } = require('../utility/crypting'); const { getSshTunnel } = require('../utility/sshTunnel'); +const sessions = require('./sessions'); +const jsldata = require('./jsldata'); const logger = getLogger('databaseConnections'); @@ -96,6 +98,33 @@ module.exports = { handle_ping() {}, + // session event handlers + + handle_info(conid, database, props) { + const { sesid, info } = props; + sessions.dispatchMessage(sesid, info); + }, + + handle_done(conid, database, props) { + const { sesid } = props; + socket.emit(`session-done-${sesid}`); + sessions.dispatchMessage(sesid, 'Query execution finished'); + }, + + handle_recordset(conid, database, props) { + const { jslid, resultIndex } = props; + socket.emit(`session-recordset-${props.sesid}`, { jslid, resultIndex }); + }, + + handle_stats(conid, database, stats) { + jsldata.notifyChangedStats(stats); + }, + + handle_initializeFile(conid, database, props) { + const { jslid } = props; + socket.emit(`session-initialize-file-${jslid}`); + }, + async ensureOpened(conid, database) { const existing = this.opened.find(x => x.conid == conid && x.database == database); if (existing) return existing; @@ -763,4 +792,15 @@ module.exports = { commandLine: this.commandArgsToCommandLine(commandArgs), }; }, + + executeSessionQuery_meta: true, + async executeSessionQuery({ sesid, conid, database, sql }) { + logger.info({ sesid, sql }, 'Processing query'); + sessions.dispatchMessage(sesid, 'Query execution started'); + + const opened = await this.ensureOpened(conid, database); + opened.subprocess.send({ msgtype: 'executeSessionQuery', sql, sesid }); + + return { state: 'ok' }; + }, }; diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index e795af414..f75af1054 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -16,6 +16,7 @@ const { handleProcessCommunication } = require('../utility/processComm'); const { SqlGenerator } = require('dbgate-tools'); const generateDeploySql = require('../shell/generateDeploySql'); const { dumpSqlSelect } = require('dbgate-sqltree'); +const { allowExecuteCustomScript, handleQueryStream } = require('../utility/handleQueryStream'); const logger = getLogger('dbconnProcess'); @@ -375,6 +376,36 @@ async function handleGenerateDeploySql({ msgid, modelFolder }) { } } +async function handleExecuteSessionQuery({ sesid, sql }) { + await waitConnected(); + const driver = requireEngineDriver(storedConnection); + + if (!allowExecuteCustomScript(storedConnection, driver)) { + process.send({ + msgtype: 'info', + info: { + message: 'Connection without read-only sessions is read only', + severity: 'error', + }, + sesid, + }); + process.send({ msgtype: 'done', sesid, skipFinishedMessage: true }); + return; + //process.send({ msgtype: 'error', error: e.message }); + } + + const resultIndexHolder = { + value: 0, + }; + for (const sqlItem of splitQuery(sql, { + ...driver.getQuerySplitterOptions('stream'), + returnRichInfo: true, + })) { + await handleQueryStream(dbhan, driver, resultIndexHolder, sqlItem, sesid); + } + process.send({ msgtype: 'done', sesid }); +} + // async function handleRunCommand({ msgid, sql }) { // await waitConnected(); // const driver = engines(storedConnection); @@ -405,6 +436,7 @@ const messageHandlers = { sqlSelect: handleSqlSelect, exportKeys: handleExportKeys, schemaList: handleSchemaList, + executeSessionQuery: handleExecuteSessionQuery, // runCommand: handleRunCommand, }; diff --git a/packages/api/src/proc/sessionProcess.js b/packages/api/src/proc/sessionProcess.js index 0caad1720..7c6c3b4a4 100644 --- a/packages/api/src/proc/sessionProcess.js +++ b/packages/api/src/proc/sessionProcess.js @@ -11,7 +11,7 @@ const { decryptConnection } = require('../utility/crypting'); const { connectUtility } = require('../utility/connectUtility'); const { handleProcessCommunication } = require('../utility/processComm'); const { getLogger, extractIntSettingsValue, extractBoolSettingsValue } = require('dbgate-tools'); -const { handleQueryStream, QueryStreamTableWriter } = require('../utility/handleQueryStream'); +const { handleQueryStream, QueryStreamTableWriter, allowExecuteCustomScript } = require('../utility/handleQueryStream'); const logger = getLogger('sessionProcess'); @@ -24,17 +24,6 @@ let lastActivity = null; let currentProfiler = null; let executingScripts = 0; -function allowExecuteCustomScript(driver) { - if (driver.readOnlySessions) { - return true; - } - if (storedConnection.isReadOnly) { - return false; - // throw new Error('Connection is read only'); - } - return true; -} - async function handleConnect(connection) { storedConnection = connection; @@ -65,7 +54,7 @@ async function handleStartProfiler({ jslid }) { await waitConnected(); const driver = requireEngineDriver(storedConnection); - if (!allowExecuteCustomScript(driver)) { + if (!allowExecuteCustomScript(storedConnection, driver)) { process.send({ msgtype: 'done' }); return; } @@ -94,7 +83,7 @@ async function handleExecuteControlCommand({ command }) { await waitConnected(); const driver = requireEngineDriver(storedConnection); - if (command == 'commitTransaction' && !allowExecuteCustomScript(driver)) { + if (command == 'commitTransaction' && !allowExecuteCustomScript(storedConnection, driver)) { process.send({ msgtype: 'info', info: { @@ -134,7 +123,7 @@ async function handleExecuteQuery({ sql, autoCommit }) { await waitConnected(); const driver = requireEngineDriver(storedConnection); - if (!allowExecuteCustomScript(driver)) { + if (!allowExecuteCustomScript(storedConnection, driver)) { process.send({ msgtype: 'info', info: { @@ -178,7 +167,7 @@ async function handleExecuteReader({ jslid, sql, fileName }) { if (fileName) { sql = fs.readFileSync(fileName, 'utf-8'); } else { - if (!allowExecuteCustomScript(driver)) { + if (!allowExecuteCustomScript(storedConnection, driver)) { process.send({ msgtype: 'done' }); return; } diff --git a/packages/api/src/utility/handleQueryStream.js b/packages/api/src/utility/handleQueryStream.js index 8201feebc..2ba193395 100644 --- a/packages/api/src/utility/handleQueryStream.js +++ b/packages/api/src/utility/handleQueryStream.js @@ -6,10 +6,11 @@ const _ = require('lodash'); const { jsldir } = require('../utility/directories'); class QueryStreamTableWriter { - constructor() { + constructor(sesid = undefined) { this.currentRowCount = 0; this.currentChangeIndex = 1; this.initializedFile = false; + this.sesid = sesid; } initializeFromQuery(structure, resultIndex) { @@ -26,7 +27,7 @@ class QueryStreamTableWriter { this.writeCurrentStats(false, false); this.resultIndex = resultIndex; this.initializedFile = true; - process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex }); + process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex, sesid: this.sesid }); } initializeFromReader(jslid) { @@ -52,7 +53,7 @@ class QueryStreamTableWriter { rowFromReader(row) { if (!this.initializedFile) { - process.send({ msgtype: 'initializeFile', jslid: this.jslid }); + process.send({ msgtype: 'initializeFile', jslid: this.jslid, sesid: this.sesid }); this.initializedFile = true; fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n'); @@ -75,7 +76,7 @@ class QueryStreamTableWriter { fs.writeFileSync(`${this.currentFile}.stats`, JSON.stringify(stats)); this.currentChangeIndex += 1; if (emitEvent) { - process.send({ msgtype: 'stats', ...stats }); + process.send({ msgtype: 'stats', sesid: this.sesid, ...stats }); } } @@ -90,9 +91,10 @@ class QueryStreamTableWriter { } class StreamHandler { - constructor(resultIndexHolder, resolve, startLine) { + constructor(resultIndexHolder, resolve, startLine, sesid = undefined) { this.recordset = this.recordset.bind(this); this.startLine = startLine; + this.sesid = sesid; this.row = this.row.bind(this); // this.error = this.error.bind(this); this.done = this.done.bind(this); @@ -116,7 +118,7 @@ class StreamHandler { recordset(columns) { this.closeCurrentWriter(); - this.currentWriter = new QueryStreamTableWriter(); + this.currentWriter = new QueryStreamTableWriter(this.sesid); this.currentWriter.initializeFromQuery( Array.isArray(columns) ? { columns } : columns, this.resultIndexHolder.value @@ -133,7 +135,7 @@ class StreamHandler { } row(row) { if (this.currentWriter) this.currentWriter.row(row); - else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } }); + else if (row.message) process.send({ msgtype: 'info', info: { message: row.message }, sesid: this.sesid }); // this.onRow(this.jslid); } // error(error) { @@ -151,19 +153,31 @@ class StreamHandler { line: this.startLine + info.line, }; } - process.send({ msgtype: 'info', info }); + process.send({ msgtype: 'info', info, sesid: this.sesid }); } } -function handleQueryStream(dbhan, driver, resultIndexHolder, sqlItem) { +function handleQueryStream(dbhan, driver, resultIndexHolder, sqlItem, sesid = undefined) { return new Promise((resolve, reject) => { const start = sqlItem.trimStart || sqlItem.start; - const handler = new StreamHandler(resultIndexHolder, resolve, start && start.line); + const handler = new StreamHandler(resultIndexHolder, resolve, start && start.line, sesid); driver.stream(dbhan, sqlItem.text, handler); }); } +function allowExecuteCustomScript(storedConnection, driver) { + if (driver.readOnlySessions) { + return true; + } + if (storedConnection.isReadOnly) { + return false; + // throw new Error('Connection is read only'); + } + return true; +} + module.exports = { handleQueryStream, QueryStreamTableWriter, + allowExecuteCustomScript, }; diff --git a/packages/web/src/query/SimpleQueryResultTabs.svelte b/packages/web/src/query/SimpleQueryResultTabs.svelte deleted file mode 100644 index 91b53f676..000000000 --- a/packages/web/src/query/SimpleQueryResultTabs.svelte +++ /dev/null @@ -1,52 +0,0 @@ - - - - - {#if grider && display} - - {/if} - - - {#if result?.errorMessage} - - {:else if result?.rows} - - {/if} - - diff --git a/packages/web/src/tabs/QueryTab.svelte b/packages/web/src/tabs/QueryTab.svelte index 1f05a94ff..992ccdddc 100644 --- a/packages/web/src/tabs/QueryTab.svelte +++ b/packages/web/src/tabs/QueryTab.svelte @@ -107,7 +107,7 @@