From 0c12dcaf162fa8fee8ba42c5b05a948be84e5277 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 21 Jun 2024 11:50:29 +0200 Subject: [PATCH 01/81] storage controller --- packages/api/src/controllers/connections.js | 5 +++++ packages/api/src/controllers/storage.js | 6 ++++++ 2 files changed, 11 insertions(+) create mode 100644 packages/api/src/controllers/storage.js diff --git a/packages/api/src/controllers/connections.js b/packages/api/src/controllers/connections.js index 2cda2e876..41035023a 100644 --- a/packages/api/src/controllers/connections.js +++ b/packages/api/src/controllers/connections.js @@ -16,6 +16,7 @@ const { safeJsonParse, getLogger } = require('dbgate-tools'); const platformInfo = require('../utility/platformInfo'); const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission'); const pipeForkLogs = require('../utility/pipeForkLogs'); +const storage = require('./storage'); const logger = getLogger('connections'); @@ -199,6 +200,10 @@ module.exports = { list_meta: true, async list(_params, req) { + const storageConnectionst = await storage.connections(); + if (storageConnectionst) { + return storageConnectionst; + } if (portalConnections) { if (platformInfo.allowShellConnection) return portalConnections; return portalConnections.map(maskConnection).filter(x => connectionHasPermission(x, req)); diff --git a/packages/api/src/controllers/storage.js b/packages/api/src/controllers/storage.js new file mode 100644 index 000000000..3013eaf21 --- /dev/null +++ b/packages/api/src/controllers/storage.js @@ -0,0 +1,6 @@ +module.exports = { + connections_meta: true, + async connections() { + return null; + }, +}; From a8999855bfa073997d82be8e2de31efce5a0206f Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 21 Jun 2024 15:22:32 +0200 Subject: [PATCH 02/81] --version in dbmodel --- packages/api/src/index.js | 2 ++ packages/dbmodel/bin/dbmodel.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/api/src/index.js b/packages/api/src/index.js index dd2b0113d..e75101beb 100644 --- a/packages/api/src/index.js +++ b/packages/api/src/index.js @@ -98,6 +98,7 @@ if (processArgs.listenApi) { const shell = require('./shell/index'); const dbgateTools = require('dbgate-tools'); +const currentVersion = require('./currentVersion'); global['DBGATE_TOOLS'] = dbgateTools; @@ -116,6 +117,7 @@ module.exports = { ...shell, getLogger, configureLogger, + currentVersion, // loadLogsContent, getMainModule: () => require('./main'), }; diff --git a/packages/dbmodel/bin/dbmodel.js b/packages/dbmodel/bin/dbmodel.js index 8c666380c..b04acbc1c 100755 --- a/packages/dbmodel/bin/dbmodel.js +++ b/packages/dbmodel/bin/dbmodel.js @@ -26,6 +26,8 @@ async function runAndExit(promise) { } } +program.version(dbgateApi.currentVersion.version); + program .option('-s, --server ', 'server host') .option('-u, --user ', 'user name') From 297b321bc876839cfc65fffd1b04cf613bf9e309 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 21 Jun 2024 16:15:56 +0200 Subject: [PATCH 03/81] convert dbmodel to json --- package.json | 1 + packages/api/src/shell/dbModelToJson.js | 16 ++++++++++++++ packages/api/src/shell/index.js | 4 ++++ packages/api/src/shell/jsonToDbModel.js | 9 ++++++++ packages/dbmodel/bin/dbmodel.js | 29 ++++++++++++++++++++++++- 5 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 packages/api/src/shell/dbModelToJson.js create mode 100644 packages/api/src/shell/jsonToDbModel.js diff --git a/package.json b/package.json index de6bf204e..9c14b04e4 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "build:web:docker": "yarn workspace dbgate-web build", "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend", "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch", + "storage-json": "dbmodel model-to-json storage-db packages/api/src/storageModel.json", "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist", "build:app:local": "yarn plugins:copydist && cd app && yarn build:local", "start:app:local": "cd app && yarn start:local", diff --git a/packages/api/src/shell/dbModelToJson.js b/packages/api/src/shell/dbModelToJson.js new file mode 100644 index 000000000..4b24d637c --- /dev/null +++ b/packages/api/src/shell/dbModelToJson.js @@ -0,0 +1,16 @@ +const importDbModel = require('../utility/importDbModel'); +const fs = require('fs'); + +async function dbModelToJson({ modelFolder, outputFile, commonjs }) { + const dbInfo = await importDbModel(modelFolder); + + const json = JSON.stringify(dbInfo, null, 2); + if (commonjs) { + fs.writeFileSync(outputFile, `module.exports = ${json};`); + return; + } else { + fs.writeFileSync(outputFile, json); + } +} + +module.exports = dbModelToJson; diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index 2293a26e8..253bb053a 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -27,6 +27,8 @@ const loadDatabase = require('./loadDatabase'); const generateModelSql = require('./generateModelSql'); const modifyJsonLinesReader = require('./modifyJsonLinesReader'); const dataDuplicator = require('./dataDuplicator'); +const dbModelToJson = require('./dbModelToJson'); +const jsonToDbModel = require('./jsonToDbModel'); const dbgateApi = { queryReader, @@ -57,6 +59,8 @@ const dbgateApi = { generateModelSql, modifyJsonLinesReader, dataDuplicator, + dbModelToJson, + jsonToDbModel, }; requirePlugin.initializeDbgateApi(dbgateApi); diff --git a/packages/api/src/shell/jsonToDbModel.js b/packages/api/src/shell/jsonToDbModel.js new file mode 100644 index 000000000..dfa4bdc46 --- /dev/null +++ b/packages/api/src/shell/jsonToDbModel.js @@ -0,0 +1,9 @@ +const exportDbModel = require('../utility/exportDbModel'); +const fs = require('fs'); + +async function jsonToDbModel({ modelFile, outputDir }) { + const dbInfo = JSON.parse(fs.readFileSync(modelFile, 'utf-8')); + await exportDbModel(dbInfo, outputDir); +} + +module.exports = jsonToDbModel; diff --git a/packages/dbmodel/bin/dbmodel.js b/packages/dbmodel/bin/dbmodel.js index b04acbc1c..34855902c 100755 --- a/packages/dbmodel/bin/dbmodel.js +++ b/packages/dbmodel/bin/dbmodel.js @@ -38,7 +38,8 @@ program '--load-data-condition ', 'regex, which table data will be loaded and stored in model (in load command)' ) - .requiredOption('-e, --engine ', 'engine name, eg. mysql@dbgate-plugin-mysql'); + .option('-e, --engine ', 'engine name, eg. mysql@dbgate-plugin-mysql') + .option('--commonjs', 'Creates CommonJS module'); program .command('deploy ') @@ -117,4 +118,30 @@ program ); }); +program + .command('json-to-model ') + .description('Converts JSON file to model') + .action((jsonFile, modelFolder) => { + runAndExit( + dbgateApi.jsonToDbModel({ + modelFile: jsonFile, + outputDir: modelFolder, + }) + ); + }); + +program + .command('model-to-json ') + .description('Converts model to JSON file') + .action((modelFolder, jsonFile) => { + const { commonjs } = program.opts(); + runAndExit( + dbgateApi.dbModelToJson({ + modelFolder, + outputFile: jsonFile, + commonjs, + }) + ); + }); + program.parse(process.argv); From 9f9c4d82dab5340a6ecb19b23aaa1633a83b6b58 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 22 Jun 2024 10:57:23 +0200 Subject: [PATCH 04/81] storage DB --- package.json | 2 ++ packages/api/package.json | 1 + packages/api/src/controllers/connections.js | 3 ++- packages/api/src/main.js | 2 ++ packages/tools/src/yamlModelConv.ts | 8 ++++++-- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 9c14b04e4..be475f7a1 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,8 @@ "start:api:singledb": "yarn workspace dbgate-api start:singledb | pino-pretty", "start:api:auth": "yarn workspace dbgate-api start:auth | pino-pretty", "start:api:dblogin": "yarn workspace dbgate-api start:dblogin | pino-pretty", + "start:api:storage": "yarn workspace dbgate-api start:storage | pino-pretty", + "sync:pro": "cd sync && yarn start", "start:web": "yarn workspace dbgate-web dev", "start:sqltree": "yarn workspace dbgate-sqltree start", "start:tools": "yarn workspace dbgate-tools start", diff --git a/packages/api/package.json b/packages/api/package.json index 2ee30f825..172bb983b 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -66,6 +66,7 @@ "start:auth": "env-cmd -f env/auth/.env node src/index.js --listen-api", "start:dblogin": "env-cmd -f env/dblogin/.env node src/index.js --listen-api", "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db --listen-api", + "start:storage": "env-cmd -f env/storage/.env node src/index.js --listen-api", "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api", "ts": "tsc", "build": "webpack" diff --git a/packages/api/src/controllers/connections.js b/packages/api/src/controllers/connections.js index 41035023a..f0c82097e 100644 --- a/packages/api/src/controllers/connections.js +++ b/packages/api/src/controllers/connections.js @@ -16,7 +16,6 @@ const { safeJsonParse, getLogger } = require('dbgate-tools'); const platformInfo = require('../utility/platformInfo'); const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission'); const pipeForkLogs = require('../utility/pipeForkLogs'); -const storage = require('./storage'); const logger = getLogger('connections'); @@ -200,6 +199,8 @@ module.exports = { list_meta: true, async list(_params, req) { + const storage = require('./storage'); + const storageConnectionst = await storage.connections(); if (storageConnectionst) { return storageConnectionst; diff --git a/packages/api/src/main.js b/packages/api/src/main.js index 1164ada6d..6bed33e2f 100644 --- a/packages/api/src/main.js +++ b/packages/api/src/main.js @@ -18,6 +18,7 @@ const sessions = require('./controllers/sessions'); const runners = require('./controllers/runners'); const jsldata = require('./controllers/jsldata'); const config = require('./controllers/config'); +const storage = require('./controllers/storage'); const archive = require('./controllers/archive'); const apps = require('./controllers/apps'); const auth = require('./controllers/auth'); @@ -162,6 +163,7 @@ function useAllControllers(app, electron) { useController(app, electron, '/runners', runners); useController(app, electron, '/jsldata', jsldata); useController(app, electron, '/config', config); + useController(app, electron, '/storage', storage); useController(app, electron, '/archive', archive); useController(app, electron, '/uploads', uploads); useController(app, electron, '/plugins', plugins); diff --git a/packages/tools/src/yamlModelConv.ts b/packages/tools/src/yamlModelConv.ts index bf0c57982..a8a986559 100644 --- a/packages/tools/src/yamlModelConv.ts +++ b/packages/tools/src/yamlModelConv.ts @@ -134,11 +134,15 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml return res; } -export function databaseInfoFromYamlModel(files: DatabaseModelFile[]): DatabaseInfo { +export function databaseInfoFromYamlModel(filesOrDbInfo: DatabaseModelFile[] | DatabaseInfo): DatabaseInfo { + if (!Array.isArray(filesOrDbInfo)) { + return filesOrDbInfo; + } + const model = DatabaseAnalyser.createEmptyStructure(); const tablesYaml = []; - for (const file of files) { + for (const file of filesOrDbInfo) { if (file.name.endsWith('.table.yaml') || file.name.endsWith('.sql')) { if (file.name.endsWith('.table.yaml')) { tablesYaml.push(file.json); From 80438693325e4dcd38b107954c597e7447f8a6e4 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 24 Jun 2024 14:51:23 +0200 Subject: [PATCH 05/81] cherri pick file --- packages/api/src/controllers/connections.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/api/src/controllers/connections.js b/packages/api/src/controllers/connections.js index f0c82097e..b6d9d0bbc 100644 --- a/packages/api/src/controllers/connections.js +++ b/packages/api/src/controllers/connections.js @@ -201,9 +201,9 @@ module.exports = { async list(_params, req) { const storage = require('./storage'); - const storageConnectionst = await storage.connections(); - if (storageConnectionst) { - return storageConnectionst; + const storageConnections = await storage.connections(); + if (storageConnections) { + return storageConnections; } if (portalConnections) { if (platformInfo.allowShellConnection) return portalConnections; @@ -342,6 +342,14 @@ module.exports = { if (volatile) { return volatile; } + + const storage = require('./storage'); + + const storageConnection = await storage.getConnection({conid}); + if (storageConnection) { + return storageConnection; + } + if (portalConnections) { const res = portalConnections.find(x => x._id == conid) || null; return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res; From 49c90b9be9185d0ab57a3adce9bb301229f1a951 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 24 Jun 2024 14:52:38 +0200 Subject: [PATCH 06/81] icons from former develop --- packages/web/src/icons/FontIcon.svelte | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index bec34ccdd..708a5f769 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -24,6 +24,9 @@ 'icon sql-generator': 'mdi mdi-cog-transfer', 'icon keyboard': 'mdi mdi-keyboard-settings', 'icon settings': 'mdi mdi-cog', + 'icon users': 'mdi mdi-account-multiple', + 'icon role': 'mdi mdi-account-group', + 'icon admin': 'mdi mdi-security', 'icon version': 'mdi mdi-ticket-confirmation', 'icon pin': 'mdi mdi-pin', 'icon arrange': 'mdi mdi-arrange-send-to-back', From 6cdbfd1a89fce31cce6902b27a30f4c9968e5f9e Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 24 Jun 2024 14:59:36 +0200 Subject: [PATCH 07/81] admin menu widget --- packages/web/src/widgets/AdminMenuWidget.svelte | 1 + packages/web/src/widgets/WidgetContainer.svelte | 4 ++++ packages/web/src/widgets/WidgetIconPanel.svelte | 5 +++++ 3 files changed, 10 insertions(+) create mode 100644 packages/web/src/widgets/AdminMenuWidget.svelte diff --git a/packages/web/src/widgets/AdminMenuWidget.svelte b/packages/web/src/widgets/AdminMenuWidget.svelte new file mode 100644 index 000000000..8ad0da5dd --- /dev/null +++ b/packages/web/src/widgets/AdminMenuWidget.svelte @@ -0,0 +1 @@ +Sorry, administration is not available \ No newline at end of file diff --git a/packages/web/src/widgets/WidgetContainer.svelte b/packages/web/src/widgets/WidgetContainer.svelte index 2d41c11dc..096c920c8 100644 --- a/packages/web/src/widgets/WidgetContainer.svelte +++ b/packages/web/src/widgets/WidgetContainer.svelte @@ -7,6 +7,7 @@ import CellDataWidget from './CellDataWidget.svelte'; import HistoryWidget from './HistoryWidget.svelte'; import AppWidget from './AppWidget.svelte'; + import AdminMenuWidget from './AdminMenuWidget.svelte';