diff --git a/app/src/mainMenuDefinition.js b/app/src/mainMenuDefinition.js index 3bb581434..6d0ab52a8 100644 --- a/app/src/mainMenuDefinition.js +++ b/app/src/mainMenuDefinition.js @@ -4,6 +4,7 @@ module.exports = ({ editMenu, isMac }) => [ submenu: [ { command: 'new.connection', hideDisabled: true }, { command: 'new.sqliteDatabase', hideDisabled: true }, + { command: 'new.duckdbDatabase', hideDisabled: true }, { divider: true }, { command: 'new.query', hideDisabled: true }, { command: 'new.queryDesign', hideDisabled: true }, diff --git a/common/volatilePackages.js b/common/volatilePackages.js index 00ee25a33..f3ffe1285 100644 --- a/common/volatilePackages.js +++ b/common/volatilePackages.js @@ -21,6 +21,7 @@ const volatilePackages = [ 'axios', 'ssh2', 'wkx', + '@duckdb/node-api', ]; module.exports = volatilePackages; diff --git a/integration-tests/__tests__/alter-database.spec.js b/integration-tests/__tests__/alter-database.spec.js index 490de76a9..ad3b11694 100644 --- a/integration-tests/__tests__/alter-database.spec.js +++ b/integration-tests/__tests__/alter-database.spec.js @@ -52,7 +52,7 @@ async function testDatabaseDiff(conn, driver, mangle, createObject = null) { } describe('Alter database', () => { - test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))( + test.each(engines.filter(x => !x.skipReferences && !x.skipDropReferences).map(engine => [engine.label, engine]))( 'Drop referenced table - %s', testWrapper(async (conn, driver, engine) => { await testDatabaseDiff(conn, driver, db => { diff --git a/integration-tests/__tests__/alter-table.spec.js b/integration-tests/__tests__/alter-table.spec.js index 06baeab95..bcbe583ef 100644 --- a/integration-tests/__tests__/alter-table.spec.js +++ b/integration-tests/__tests__/alter-table.spec.js @@ -90,7 +90,7 @@ const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'co // const TESTED_COLUMNS = ['col_std']; // const TESTED_COLUMNS = ['col_ref']; -function create_engines_columns_source(engines) { +function createEnginesColumnsSource(engines) { return _.flatten( engines.map(engine => TESTED_COLUMNS.filter(col => col.endsWith('_pk') || !engine.skipNonPkRename) @@ -116,45 +116,30 @@ describe('Alter table', () => { }) ); - const columnsSource = create_engines_columns_source(engines); - const dropableColumnsSrouce = columnsSource.filter( - ([_label, col, engine]) => !engine.skipPkDrop || !col.endsWith('_pk') + test.each( + createEnginesColumnsSource(engines.filter(x => !x.skipDropColumn)).filter( + ([_label, col, engine]) => !engine.skipPkDrop || !col.endsWith('_pk') + ) + )( + 'Drop column - %s - %s', + testWrapper(async (conn, driver, column, engine) => { + await testTableDiff(engine, conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column))); + }) ); - const hasDropableColumns = dropableColumnsSrouce.length > 0; - if (hasDropableColumns) { - test.each(dropableColumnsSrouce)( - 'Drop column - %s - %s', - testWrapper(async (conn, driver, column, engine) => { - await testTableDiff( - engine, - conn, - driver, - tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)) - ); - }) - ); - } + test.each(createEnginesColumnsSource(engines.filter(x => !x.skipNullable && !x.skipChangeNullability)))( + 'Change nullability - %s - %s', + testWrapper(async (conn, driver, column, engine) => { + await testTableDiff( + engine, + conn, + driver, + tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, notNull: true } : x))) + ); + }) + ); - const hasEnginesWithNullable = engines.filter(x => !x.skipNullable).length > 0; - - if (hasEnginesWithNullable) { - const source = create_engines_columns_source(engines.filter(x => !x.skipNullable)); - - test.each(source)( - 'Change nullability - %s - %s', - testWrapper(async (conn, driver, column, engine) => { - await testTableDiff( - engine, - conn, - driver, - tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, notNull: true } : x))) - ); - }) - ); - } - - test.each(columnsSource)( + test.each(createEnginesColumnsSource(engines.filter(x => !x.skipRenameColumn)))( 'Rename column - %s - %s', testWrapper(async (conn, driver, column, engine) => { await testTableDiff( @@ -175,37 +160,32 @@ describe('Alter table', () => { }) ); - const enginesWithDefault = engines.filter(x => !x.skipDefaultValue); - const hasEnginesWithDefault = enginesWithDefault.length > 0; + test.each(engines.filter(x => !x.skipDefaultValue).map(engine => [engine.label, engine]))( + 'Add default value - %s', + testWrapper(async (conn, driver, engine) => { + await testTableDiff(engine, conn, driver, tbl => { + tbl.columns.find(x => x.columnName == 'col_std').defaultValue = '123'; + }); + }) + ); - if (hasEnginesWithDefault) { - test.each(enginesWithDefault.map(engine => [engine.label, engine]))( - 'Add default value - %s', - testWrapper(async (conn, driver, engine) => { - await testTableDiff(engine, conn, driver, tbl => { - tbl.columns.find(x => x.columnName == 'col_std').defaultValue = '123'; - }); - }) - ); + test.each(engines.filter(x => !x.skipDefaultValue).map(engine => [engine.label, engine]))( + 'Unset default value - %s', + testWrapper(async (conn, driver, engine) => { + await testTableDiff(engine, conn, driver, tbl => { + tbl.columns.find(x => x.columnName == 'col_def').defaultValue = undefined; + }); + }) + ); - test.each(enginesWithDefault.map(engine => [engine.label, engine]))( - 'Unset default value - %s', - testWrapper(async (conn, driver, engine) => { - await testTableDiff(engine, conn, driver, tbl => { - tbl.columns.find(x => x.columnName == 'col_def').defaultValue = undefined; - }); - }) - ); - - test.each(enginesWithDefault.map(engine => [engine.label, engine]))( - 'Change default value - %s', - testWrapper(async (conn, driver, engine) => { - await testTableDiff(engine, conn, driver, tbl => { - tbl.columns.find(x => x.columnName == 'col_def').defaultValue = '567'; - }); - }) - ); - } + test.each(engines.filter(x => !x.skipDefaultValue).map(engine => [engine.label, engine]))( + 'Change default value - %s', + testWrapper(async (conn, driver, engine) => { + await testTableDiff(engine, conn, driver, tbl => { + tbl.columns.find(x => x.columnName == 'col_def').defaultValue = '567'; + }); + }) + ); // test.each(engines.map(engine => [engine.label, engine]))( // 'Change autoincrement - %s', diff --git a/integration-tests/__tests__/db-import-export.spec.js b/integration-tests/__tests__/db-import-export.spec.js index 37a4134ca..7ec7a56f1 100644 --- a/integration-tests/__tests__/db-import-export.spec.js +++ b/integration-tests/__tests__/db-import-export.spec.js @@ -51,7 +51,8 @@ describe('DB Import/export', () => { await copyStream(reader, writer); const res = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t1`)); - expect(res.rows[0].cnt.toString()).toEqual('6'); + const cnt = parseInt(res.rows[0].cnt.toString()); + expect(cnt).toEqual(6); }) ); @@ -75,7 +76,8 @@ describe('DB Import/export', () => { await copyStream(reader, writer); const res = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t1`)); - expect(res.rows[0].cnt.toString()).toEqual('6'); + const cnt = parseInt(res.rows[0].cnt.toString()); + expect(cnt).toEqual(6); }) ); @@ -103,10 +105,12 @@ describe('DB Import/export', () => { await copyStream(reader2, writer2); const res1 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t1`)); - expect(res1.rows[0].cnt.toString()).toEqual('6'); + const cnt = parseInt(res1.rows[0].cnt.toString()); + expect(cnt).toEqual(6); const res2 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t2`)); - expect(res2.rows[0].cnt.toString()).toEqual('6'); + const cnt2 = parseInt(res2.rows[0].cnt.toString()); + expect(cnt2).toEqual(6); }) ); const enginesWithDumpFile = engines.filter(x => x.dumpFile); @@ -192,7 +196,8 @@ describe('DB Import/export', () => { }); const res1 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~categories`)); - expect(res1.rows[0].cnt.toString()).toEqual('4'); + const cnt1 = parseInt(res1.rows[0].cnt.toString()); + expect(cnt1).toEqual(4); }) ); }); diff --git a/integration-tests/__tests__/object-analyse.spec.js b/integration-tests/__tests__/object-analyse.spec.js index 25055fd04..60bfbaf18 100644 --- a/integration-tests/__tests__/object-analyse.spec.js +++ b/integration-tests/__tests__/object-analyse.spec.js @@ -20,7 +20,11 @@ function flatSourceParameters() { } function flatSourceTriggers() { - return _.flatten(engines.map(engine => (engine.triggers || []).map(trigger => [engine.label, trigger, engine]))); + return _.flatten( + engines + .filter(engine => !engine.skipTriggers) + .map(engine => (engine.triggers || []).map(trigger => [engine.label, trigger, engine])) + ); } function flatSourceSchedulerEvents() { diff --git a/integration-tests/__tests__/query.spec.js b/integration-tests/__tests__/query.spec.js index 2a2e662b2..b8afeadd4 100644 --- a/integration-tests/__tests__/query.spec.js +++ b/integration-tests/__tests__/query.spec.js @@ -183,8 +183,8 @@ describe('Query', () => { { discardResult: true } ); const res = await runQueryOnDriver(conn, driver, dmp => dmp.put('SELECT COUNT(*) AS ~cnt FROM ~t1')); - // console.log(res); - expect(res.rows[0].cnt == 3).toBeTruthy(); + const cnt = parseInt(res.rows[0].cnt); + expect(cnt).toEqual(3); }) ); diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 5b7f1eb02..51ba9705e 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -654,6 +654,32 @@ const cassandraEngine = { objects: [], }; +/** @type {import('dbgate-types').TestEngineInfo} */ +const duckdbEngine = { + label: 'DuckDB', + generateDbFile: true, + defaultSchemaName: 'main', + connection: { + engine: 'duckdb@dbgate-plugin-duckdb', + }, + objects: [views], + skipOnCI: false, + skipChangeColumn: true, + // skipIndexes: true, + skipStringLength: true, + skipTriggers: true, + skipDataReplicator: true, + skipAutoIncrement: true, + skipDropColumn: true, + skipRenameColumn: true, + skipChangeNullability: true, + skipDeploy: true, + supportRenameSqlObject: true, + skipIncrementalAnalysis: true, + skipDefaultValue: true, + skipDropReferences: true, +}; + const enginesOnCi = [ // all engines, which would be run on GitHub actions mysqlEngine, @@ -667,13 +693,14 @@ const enginesOnCi = [ clickhouseEngine, oracleEngine, cassandraEngine, + duckdbEngine, ]; const enginesOnLocal = [ // all engines, which would be run on local test // cassandraEngine, // mysqlEngine, - mariaDbEngine, + // mariaDbEngine, // postgreSqlEngine, // sqlServerEngine, // sqliteEngine, @@ -682,6 +709,7 @@ const enginesOnLocal = [ // libsqlFileEngine, // libsqlWsEngine, // oracleEngine, + duckdbEngine, ]; /** @type {import('dbgate-types').TestEngineInfo[] & Record} */ @@ -696,3 +724,6 @@ module.exports.cockroachDbEngine = cockroachDbEngine; module.exports.clickhouseEngine = clickhouseEngine; module.exports.oracleEngine = oracleEngine; module.exports.cassandraEngine = cassandraEngine; +module.exports.libsqlFileEngine = libsqlFileEngine; +module.exports.libsqlWsEngine = libsqlWsEngine; +module.exports.duckdbEngine = duckdbEngine; diff --git a/integration-tests/package.json b/integration-tests/package.json index 84c6f466c..a8c3e670b 100644 --- a/integration-tests/package.json +++ b/integration-tests/package.json @@ -12,7 +12,7 @@ "wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js", "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js", "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest --testTimeout=5000", - "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-replicator.spec.js", + "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/alter-database.spec.js", "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults --detectOpenHandles --forceExit --testTimeout=10000", "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local" }, diff --git a/packages/api/src/controllers/connections.js b/packages/api/src/controllers/connections.js index 43690a646..13a452a64 100644 --- a/packages/api/src/controllers/connections.js +++ b/packages/api/src/controllers/connections.js @@ -38,6 +38,11 @@ function getNamedArgs() { res.databaseFile = name; res.engine = 'sqlite@dbgate-plugin-sqlite'; } + + if (name.endsWith('.duckdb')) { + res.databaseFile = name; + res.engine = 'duckdb@dbgate-plugin-duckdb'; + } } } return res; @@ -447,6 +452,22 @@ module.exports = { return res; }, + newDuckdbDatabase_meta: true, + async newDuckdbDatabase({ file }) { + const duckdbDir = path.join(filesdir(), 'duckdb'); + if (!(await fs.exists(duckdbDir))) { + await fs.mkdir(duckdbDir); + } + const databaseFile = path.join(duckdbDir, `${file}.duckdb`); + const res = await this.save({ + engine: 'duckdb@dbgate-plugin-duckdb', + databaseFile, + singleDatabase: true, + defaultDatabase: `${file}.duckdb`, + }); + return res; + }, + dbloginWeb_meta: { raw: true, method: 'get', diff --git a/packages/api/src/controllers/databaseConnections.js b/packages/api/src/controllers/databaseConnections.js index 7b36da519..c4f77ce74 100644 --- a/packages/api/src/controllers/databaseConnections.js +++ b/packages/api/src/controllers/databaseConnections.js @@ -39,6 +39,8 @@ const axios = require('axios'); const { callTextToSqlApi, callCompleteOnCursorApi, callRefactorSqlQueryApi } = require('../utility/authProxy'); const { decryptConnection } = require('../utility/crypting'); const { getSshTunnel } = require('../utility/sshTunnel'); +const sessions = require('./sessions'); +const jsldata = require('./jsldata'); const logger = getLogger('databaseConnections'); @@ -96,6 +98,52 @@ module.exports = { handle_ping() {}, + // session event handlers + + handle_info(conid, database, props) { + const { sesid, info } = props; + sessions.dispatchMessage(sesid, info); + }, + + handle_done(conid, database, props) { + const { sesid } = props; + socket.emit(`session-done-${sesid}`); + sessions.dispatchMessage(sesid, 'Query execution finished'); + }, + + handle_recordset(conid, database, props) { + const { jslid, resultIndex } = props; + socket.emit(`session-recordset-${props.sesid}`, { jslid, resultIndex }); + }, + + handle_stats(conid, database, stats) { + jsldata.notifyChangedStats(stats); + }, + + handle_initializeFile(conid, database, props) { + const { jslid } = props; + socket.emit(`session-initialize-file-${jslid}`); + }, + + // eval event handler + handle_runnerDone(conid, database, props) { + const { runid } = props; + socket.emit(`runner-done-${runid}`); + }, + + handle_progress(conid, database, progressData) { + const { progressName } = progressData; + const { name, runid } = progressName; + socket.emit(`runner-progress-${runid}`, { ...progressData, progressName: name }); + }, + + handle_copyStreamError(conid, database, { copyStreamError }) { + const { progressName } = copyStreamError; + const { runid } = progressName; + logger.error(`Error in database connection ${conid}, database ${database}: ${copyStreamError}`); + socket.emit(`runner-done-${runid}`); + }, + async ensureOpened(conid, database) { const existing = this.opened.find(x => x.conid == conid && x.database == database); if (existing) return existing; @@ -136,7 +184,13 @@ module.exports = { const { msgtype } = message; if (handleProcessCommunication(message, subprocess)) return; if (newOpened.disconnected) return; - this[`handle_${msgtype}`](conid, database, message); + const funcName = `handle_${msgtype}`; + if (!this[funcName]) { + logger.error(`Unknown message type ${msgtype} from subprocess databaseConnectionProcess`); + return; + } + + this[funcName](conid, database, message); }); subprocess.on('exit', () => { if (newOpened.disconnected) return; @@ -763,4 +817,25 @@ module.exports = { commandLine: this.commandArgsToCommandLine(commandArgs), }; }, + + executeSessionQuery_meta: true, + async executeSessionQuery({ sesid, conid, database, sql }, req) { + testConnectionPermission(conid, req); + logger.info({ sesid, sql }, 'Processing query'); + sessions.dispatchMessage(sesid, 'Query execution started'); + + const opened = await this.ensureOpened(conid, database); + opened.subprocess.send({ msgtype: 'executeSessionQuery', sql, sesid }); + + return { state: 'ok' }; + }, + + evalJsonScript_meta: true, + async evalJsonScript({ conid, database, script, runid }, req) { + testConnectionPermission(conid, req); + const opened = await this.ensureOpened(conid, database); + + opened.subprocess.send({ msgtype: 'evalJsonScript', script, runid }); + return { state: 'ok' }; + }, }; diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js index 1a11319a3..5ed9bbea1 100644 --- a/packages/api/src/controllers/runners.js +++ b/packages/api/src/controllers/runners.js @@ -8,7 +8,7 @@ const { fork, spawn } = require('child_process'); const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories'); const { extractShellApiPlugins, - extractShellApiFunctionName, + compileShellApiFunctionName, jsonScriptToJavascript, getLogger, safeJsonParse, @@ -58,7 +58,7 @@ dbgateApi.initializeApiEnvironment(); ${requirePluginsTemplate(extractShellApiPlugins(functionName, props))} require=null; async function run() { -const reader=await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)}); +const reader=await ${compileShellApiFunctionName(functionName)}(${JSON.stringify(props)}); const writer=await dbgateApi.collectorWriter({runid: '${runid}'}); await dbgateApi.copyStream(reader, writer); } @@ -273,7 +273,7 @@ module.exports = { const runid = crypto.randomUUID(); if (script.type == 'json') { - const js = jsonScriptToJavascript(script); + const js = await jsonScriptToJavascript(script); return this.startCore(runid, scriptTemplate(js, false)); } @@ -335,7 +335,7 @@ module.exports = { return { errorMessage: 'Only JSON scripts are allowed' }; } - const promise = new Promise((resolve, reject) => { + const promise = new Promise(async (resolve, reject) => { const runid = crypto.randomUUID(); this.requests[runid] = { resolve, reject, exitOnStreamError: true }; const cloned = _.cloneDeepWith(script, node => { @@ -343,7 +343,7 @@ module.exports = { return runid; } }); - const js = jsonScriptToJavascript(cloned); + const js = await jsonScriptToJavascript(cloned); this.startCore(runid, scriptTemplate(js, false)); }); return promise; diff --git a/packages/api/src/controllers/serverConnections.js b/packages/api/src/controllers/serverConnections.js index 61dfe6a82..d9a065bd9 100644 --- a/packages/api/src/controllers/serverConnections.js +++ b/packages/api/src/controllers/serverConnections.js @@ -54,6 +54,9 @@ module.exports = { if (!connection) { throw new Error(`Connection with conid="${conid}" not found`); } + if (connection.singleDatabase) { + return null; + } if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') { throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode }); } @@ -142,14 +145,14 @@ module.exports = { if (conid == '__model') return []; testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid); - return opened.databases; + return opened?.databases ?? []; }, version_meta: true, async version({ conid }, req) { testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid); - return opened.version; + return opened?.version ?? null; }, serverStatus_meta: true, @@ -170,6 +173,9 @@ module.exports = { } this.lastPinged[conid] = new Date().getTime(); const opened = await this.ensureOpened(conid); + if (!opened) { + return Promise.resolve(); + } try { opened.subprocess.send({ msgtype: 'ping' }); } catch (err) { @@ -194,6 +200,9 @@ module.exports = { async sendDatabaseOp({ conid, msgtype, name }, req) { testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid); + if (!opened) { + return null; + } if (opened.connection.isReadOnly) return false; const res = await this.sendRequest(opened, { msgtype, name }); if (res.errorMessage) { @@ -233,6 +242,9 @@ module.exports = { async loadDataCore(msgtype, { conid, ...args }, req) { testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid); + if (!opened) { + return null; + } const res = await this.sendRequest(opened, { msgtype, ...args }); if (res.errorMessage) { console.error(res.errorMessage); @@ -254,6 +266,9 @@ module.exports = { async summaryCommand({ conid, command, row }, req) { testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid); + if (!opened) { + return null; + } if (opened.connection.isReadOnly) return false; return this.loadDataCore('summaryCommand', { conid, command, row }); }, diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index e795af414..95cf19fc6 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -9,13 +9,21 @@ const { dbNameLogCategory, extractErrorMessage, extractErrorLogData, + ScriptWriterEval, + SqlGenerator, + playJsonScriptWriter, } = require('dbgate-tools'); const requireEngineDriver = require('../utility/requireEngineDriver'); const { connectUtility } = require('../utility/connectUtility'); const { handleProcessCommunication } = require('../utility/processComm'); -const { SqlGenerator } = require('dbgate-tools'); const generateDeploySql = require('../shell/generateDeploySql'); const { dumpSqlSelect } = require('dbgate-sqltree'); +const { allowExecuteCustomScript, handleQueryStream } = require('../utility/handleQueryStream'); +const dbgateApi = require('../shell'); +const requirePlugin = require('../shell/requirePlugin'); +const path = require('path'); +const { rundir } = require('../utility/directories'); +const fs = require('fs-extra'); const logger = getLogger('dbconnProcess'); @@ -375,6 +383,52 @@ async function handleGenerateDeploySql({ msgid, modelFolder }) { } } +async function handleExecuteSessionQuery({ sesid, sql }) { + await waitConnected(); + const driver = requireEngineDriver(storedConnection); + + if (!allowExecuteCustomScript(storedConnection, driver)) { + process.send({ + msgtype: 'info', + info: { + message: 'Connection without read-only sessions is read only', + severity: 'error', + }, + sesid, + }); + process.send({ msgtype: 'done', sesid, skipFinishedMessage: true }); + return; + //process.send({ msgtype: 'error', error: e.message }); + } + + const resultIndexHolder = { + value: 0, + }; + for (const sqlItem of splitQuery(sql, { + ...driver.getQuerySplitterOptions('stream'), + returnRichInfo: true, + })) { + await handleQueryStream(dbhan, driver, resultIndexHolder, sqlItem, sesid); + } + process.send({ msgtype: 'done', sesid }); +} + +async function handleEvalJsonScript({ script, runid }) { + const directory = path.join(rundir(), runid); + fs.mkdirSync(directory); + const originalCwd = process.cwd(); + + try { + process.chdir(directory); + + const evalWriter = new ScriptWriterEval(dbgateApi, requirePlugin, dbhan, runid); + await playJsonScriptWriter(script, evalWriter); + process.send({ msgtype: 'runnerDone', runid }); + } finally { + process.chdir(originalCwd); + } +} + // async function handleRunCommand({ msgid, sql }) { // await waitConnected(); // const driver = engines(storedConnection); @@ -405,6 +459,8 @@ const messageHandlers = { sqlSelect: handleSqlSelect, exportKeys: handleExportKeys, schemaList: handleSchemaList, + executeSessionQuery: handleExecuteSessionQuery, + evalJsonScript: handleEvalJsonScript, // runCommand: handleRunCommand, }; diff --git a/packages/api/src/proc/sessionProcess.js b/packages/api/src/proc/sessionProcess.js index e81abda1c..7c6c3b4a4 100644 --- a/packages/api/src/proc/sessionProcess.js +++ b/packages/api/src/proc/sessionProcess.js @@ -11,6 +11,7 @@ const { decryptConnection } = require('../utility/crypting'); const { connectUtility } = require('../utility/connectUtility'); const { handleProcessCommunication } = require('../utility/processComm'); const { getLogger, extractIntSettingsValue, extractBoolSettingsValue } = require('dbgate-tools'); +const { handleQueryStream, QueryStreamTableWriter, allowExecuteCustomScript } = require('../utility/handleQueryStream'); const logger = getLogger('sessionProcess'); @@ -23,175 +24,6 @@ let lastActivity = null; let currentProfiler = null; let executingScripts = 0; -class TableWriter { - constructor() { - this.currentRowCount = 0; - this.currentChangeIndex = 1; - this.initializedFile = false; - } - - initializeFromQuery(structure, resultIndex) { - this.jslid = crypto.randomUUID(); - this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); - fs.writeFileSync( - this.currentFile, - JSON.stringify({ - ...structure, - __isStreamHeader: true, - }) + '\n' - ); - this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' }); - this.writeCurrentStats(false, false); - this.resultIndex = resultIndex; - this.initializedFile = true; - process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex }); - } - - initializeFromReader(jslid) { - this.jslid = jslid; - this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); - this.writeCurrentStats(false, false); - } - - row(row) { - // console.log('ACCEPT ROW', row); - this.currentStream.write(JSON.stringify(row) + '\n'); - this.currentRowCount += 1; - - if (!this.plannedStats) { - this.plannedStats = true; - process.nextTick(() => { - if (this.currentStream) this.currentStream.uncork(); - process.nextTick(() => this.writeCurrentStats(false, true)); - this.plannedStats = false; - }); - } - } - - rowFromReader(row) { - if (!this.initializedFile) { - process.send({ msgtype: 'initializeFile', jslid: this.jslid }); - this.initializedFile = true; - - fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n'); - this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' }); - this.writeCurrentStats(false, false); - this.initializedFile = true; - return; - } - - this.row(row); - } - - writeCurrentStats(isFinished = false, emitEvent = false) { - const stats = { - rowCount: this.currentRowCount, - changeIndex: this.currentChangeIndex, - isFinished, - jslid: this.jslid, - }; - fs.writeFileSync(`${this.currentFile}.stats`, JSON.stringify(stats)); - this.currentChangeIndex += 1; - if (emitEvent) { - process.send({ msgtype: 'stats', ...stats }); - } - } - - close(afterClose) { - if (this.currentStream) { - this.currentStream.end(() => { - this.writeCurrentStats(true, true); - if (afterClose) afterClose(); - }); - } - } -} - -class StreamHandler { - constructor(resultIndexHolder, resolve, startLine) { - this.recordset = this.recordset.bind(this); - this.startLine = startLine; - this.row = this.row.bind(this); - // this.error = this.error.bind(this); - this.done = this.done.bind(this); - this.info = this.info.bind(this); - - // use this for cancelling - not implemented - // this.stream = null; - - this.plannedStats = false; - this.resultIndexHolder = resultIndexHolder; - this.resolve = resolve; - // currentHandlers = [...currentHandlers, this]; - } - - closeCurrentWriter() { - if (this.currentWriter) { - this.currentWriter.close(); - this.currentWriter = null; - } - } - - recordset(columns) { - this.closeCurrentWriter(); - this.currentWriter = new TableWriter(); - this.currentWriter.initializeFromQuery( - Array.isArray(columns) ? { columns } : columns, - this.resultIndexHolder.value - ); - this.resultIndexHolder.value += 1; - - // this.writeCurrentStats(); - - // this.onRow = _.throttle((jslid) => { - // if (jslid == this.jslid) { - // this.writeCurrentStats(false, true); - // } - // }, 500); - } - row(row) { - if (this.currentWriter) this.currentWriter.row(row); - else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } }); - // this.onRow(this.jslid); - } - // error(error) { - // process.send({ msgtype: 'error', error }); - // } - done(result) { - this.closeCurrentWriter(); - // currentHandlers = currentHandlers.filter((x) => x != this); - this.resolve(); - } - info(info) { - if (info && info.line != null) { - info = { - ...info, - line: this.startLine + info.line, - }; - } - process.send({ msgtype: 'info', info }); - } -} - -function handleStream(driver, resultIndexHolder, sqlItem) { - return new Promise((resolve, reject) => { - const start = sqlItem.trimStart || sqlItem.start; - const handler = new StreamHandler(resultIndexHolder, resolve, start && start.line); - driver.stream(dbhan, sqlItem.text, handler); - }); -} - -function allowExecuteCustomScript(driver) { - if (driver.readOnlySessions) { - return true; - } - if (storedConnection.isReadOnly) { - return false; - // throw new Error('Connection is read only'); - } - return true; -} - async function handleConnect(connection) { storedConnection = connection; @@ -222,12 +54,12 @@ async function handleStartProfiler({ jslid }) { await waitConnected(); const driver = requireEngineDriver(storedConnection); - if (!allowExecuteCustomScript(driver)) { + if (!allowExecuteCustomScript(storedConnection, driver)) { process.send({ msgtype: 'done' }); return; } - const writer = new TableWriter(); + const writer = new QueryStreamTableWriter(); writer.initializeFromReader(jslid); currentProfiler = await driver.startProfiler(dbhan, { @@ -251,7 +83,7 @@ async function handleExecuteControlCommand({ command }) { await waitConnected(); const driver = requireEngineDriver(storedConnection); - if (command == 'commitTransaction' && !allowExecuteCustomScript(driver)) { + if (command == 'commitTransaction' && !allowExecuteCustomScript(storedConnection, driver)) { process.send({ msgtype: 'info', info: { @@ -291,7 +123,7 @@ async function handleExecuteQuery({ sql, autoCommit }) { await waitConnected(); const driver = requireEngineDriver(storedConnection); - if (!allowExecuteCustomScript(driver)) { + if (!allowExecuteCustomScript(storedConnection, driver)) { process.send({ msgtype: 'info', info: { @@ -313,7 +145,7 @@ async function handleExecuteQuery({ sql, autoCommit }) { ...driver.getQuerySplitterOptions('stream'), returnRichInfo: true, })) { - await handleStream(driver, resultIndexHolder, sqlItem); + await handleQueryStream(dbhan, driver, resultIndexHolder, sqlItem); // const handler = new StreamHandler(resultIndex); // const stream = await driver.stream(systemConnection, sqlItem, handler); // handler.stream = stream; @@ -335,13 +167,13 @@ async function handleExecuteReader({ jslid, sql, fileName }) { if (fileName) { sql = fs.readFileSync(fileName, 'utf-8'); } else { - if (!allowExecuteCustomScript(driver)) { + if (!allowExecuteCustomScript(storedConnection, driver)) { process.send({ msgtype: 'done' }); return; } } - const writer = new TableWriter(); + const writer = new QueryStreamTableWriter(); writer.initializeFromReader(jslid); const reader = await driver.readQuery(dbhan, sql); diff --git a/packages/api/src/shell/copyStream.js b/packages/api/src/shell/copyStream.js index 6e3c1aa59..152d45b80 100644 --- a/packages/api/src/shell/copyStream.js +++ b/packages/api/src/shell/copyStream.js @@ -69,6 +69,7 @@ async function copyStream(input, output, options) { msgtype: 'copyStreamError', copyStreamError: { message: extractErrorMessage(err), + progressName, ...err, }, }); diff --git a/packages/api/src/utility/handleQueryStream.js b/packages/api/src/utility/handleQueryStream.js new file mode 100644 index 000000000..2ba193395 --- /dev/null +++ b/packages/api/src/utility/handleQueryStream.js @@ -0,0 +1,183 @@ +const crypto = require('crypto'); +const path = require('path'); +const fs = require('fs'); +const _ = require('lodash'); + +const { jsldir } = require('../utility/directories'); + +class QueryStreamTableWriter { + constructor(sesid = undefined) { + this.currentRowCount = 0; + this.currentChangeIndex = 1; + this.initializedFile = false; + this.sesid = sesid; + } + + initializeFromQuery(structure, resultIndex) { + this.jslid = crypto.randomUUID(); + this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); + fs.writeFileSync( + this.currentFile, + JSON.stringify({ + ...structure, + __isStreamHeader: true, + }) + '\n' + ); + this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' }); + this.writeCurrentStats(false, false); + this.resultIndex = resultIndex; + this.initializedFile = true; + process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex, sesid: this.sesid }); + } + + initializeFromReader(jslid) { + this.jslid = jslid; + this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); + this.writeCurrentStats(false, false); + } + + row(row) { + // console.log('ACCEPT ROW', row); + this.currentStream.write(JSON.stringify(row) + '\n'); + this.currentRowCount += 1; + + if (!this.plannedStats) { + this.plannedStats = true; + process.nextTick(() => { + if (this.currentStream) this.currentStream.uncork(); + process.nextTick(() => this.writeCurrentStats(false, true)); + this.plannedStats = false; + }); + } + } + + rowFromReader(row) { + if (!this.initializedFile) { + process.send({ msgtype: 'initializeFile', jslid: this.jslid, sesid: this.sesid }); + this.initializedFile = true; + + fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n'); + this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' }); + this.writeCurrentStats(false, false); + this.initializedFile = true; + return; + } + + this.row(row); + } + + writeCurrentStats(isFinished = false, emitEvent = false) { + const stats = { + rowCount: this.currentRowCount, + changeIndex: this.currentChangeIndex, + isFinished, + jslid: this.jslid, + }; + fs.writeFileSync(`${this.currentFile}.stats`, JSON.stringify(stats)); + this.currentChangeIndex += 1; + if (emitEvent) { + process.send({ msgtype: 'stats', sesid: this.sesid, ...stats }); + } + } + + close(afterClose) { + if (this.currentStream) { + this.currentStream.end(() => { + this.writeCurrentStats(true, true); + if (afterClose) afterClose(); + }); + } + } +} + +class StreamHandler { + constructor(resultIndexHolder, resolve, startLine, sesid = undefined) { + this.recordset = this.recordset.bind(this); + this.startLine = startLine; + this.sesid = sesid; + this.row = this.row.bind(this); + // this.error = this.error.bind(this); + this.done = this.done.bind(this); + this.info = this.info.bind(this); + + // use this for cancelling - not implemented + // this.stream = null; + + this.plannedStats = false; + this.resultIndexHolder = resultIndexHolder; + this.resolve = resolve; + // currentHandlers = [...currentHandlers, this]; + } + + closeCurrentWriter() { + if (this.currentWriter) { + this.currentWriter.close(); + this.currentWriter = null; + } + } + + recordset(columns) { + this.closeCurrentWriter(); + this.currentWriter = new QueryStreamTableWriter(this.sesid); + this.currentWriter.initializeFromQuery( + Array.isArray(columns) ? { columns } : columns, + this.resultIndexHolder.value + ); + this.resultIndexHolder.value += 1; + + // this.writeCurrentStats(); + + // this.onRow = _.throttle((jslid) => { + // if (jslid == this.jslid) { + // this.writeCurrentStats(false, true); + // } + // }, 500); + } + row(row) { + if (this.currentWriter) this.currentWriter.row(row); + else if (row.message) process.send({ msgtype: 'info', info: { message: row.message }, sesid: this.sesid }); + // this.onRow(this.jslid); + } + // error(error) { + // process.send({ msgtype: 'error', error }); + // } + done(result) { + this.closeCurrentWriter(); + // currentHandlers = currentHandlers.filter((x) => x != this); + this.resolve(); + } + info(info) { + if (info && info.line != null) { + info = { + ...info, + line: this.startLine + info.line, + }; + } + process.send({ msgtype: 'info', info, sesid: this.sesid }); + } +} + +function handleQueryStream(dbhan, driver, resultIndexHolder, sqlItem, sesid = undefined) { + return new Promise((resolve, reject) => { + const start = sqlItem.trimStart || sqlItem.start; + const handler = new StreamHandler(resultIndexHolder, resolve, start && start.line, sesid); + driver.stream(dbhan, sqlItem.text, handler); + }); +} + +function allowExecuteCustomScript(storedConnection, driver) { + if (driver.readOnlySessions) { + return true; + } + if (storedConnection.isReadOnly) { + return false; + // throw new Error('Connection is read only'); + } + return true; +} + +module.exports = { + handleQueryStream, + QueryStreamTableWriter, + allowExecuteCustomScript, +}; diff --git a/packages/tools/src/DatabaseAnalyser.ts b/packages/tools/src/DatabaseAnalyser.ts index 0ebb69472..a7750241d 100644 --- a/packages/tools/src/DatabaseAnalyser.ts +++ b/packages/tools/src/DatabaseAnalyser.ts @@ -92,10 +92,10 @@ export class DatabaseAnalyser { this.singleObjectFilter = { ...name, typeField }; await this._computeSingleObjectId(); const res = this.addEngineField(await this._runAnalysis()); - // console.log('SINGLE OBJECT RES', res); + // console.log('SINGLE OBJECT RES', JSON.stringify(res, null, 2)); const obj = res[typeField]?.length == 1 - ? res[typeField][0] + ? res[typeField]?.find(x => x.pureName.toLowerCase() == name.pureName.toLowerCase()) : res[typeField]?.find(x => x.pureName == name.pureName && x.schemaName == name.schemaName); // console.log('SINGLE OBJECT', obj); return obj; diff --git a/packages/tools/src/ScriptWriter.ts b/packages/tools/src/ScriptWriter.ts index 72c67e664..35aae6548 100644 --- a/packages/tools/src/ScriptWriter.ts +++ b/packages/tools/src/ScriptWriter.ts @@ -1,7 +1,22 @@ import _uniq from 'lodash/uniq'; -import { extractShellApiFunctionName, extractShellApiPlugins } from './packageTools'; +import _cloneDeepWith from 'lodash/cloneDeepWith'; +import { evalShellApiFunctionName, compileShellApiFunctionName, extractShellApiPlugins } from './packageTools'; -export class ScriptWriter { +export interface ScriptWriterGeneric { + allocVariable(prefix?: string); + endLine(); + assign(variableName: string, functionName: string, props: any); + assignValue(variableName: string, jsonValue: any); + requirePackage(packageName: string); + copyStream(sourceVar: string, targetVar: string, colmapVar?: string, progressName?: string); + importDatabase(options: any); + dataReplicator(options: any); + comment(s: string); + zipDirectory(inputDirectory: string, outputFile: string); + getScript(schedule?: any): any; +} + +export class ScriptWriterJavaScript implements ScriptWriterGeneric { s = ''; packageNames: string[] = []; varCount = 0; @@ -29,7 +44,7 @@ export class ScriptWriter { } assign(variableName, functionName, props) { - this.assignCore(variableName, extractShellApiFunctionName(functionName), props); + this.assignCore(variableName, compileShellApiFunctionName(functionName), props); this.packageNames.push(...extractShellApiPlugins(functionName, props)); } @@ -41,10 +56,10 @@ export class ScriptWriter { this.packageNames.push(packageName); } - copyStream(sourceVar, targetVar, colmapVar = null, progressName?: string) { + copyStream(sourceVar, targetVar, colmapVar = null, progressName?: string | { name: string; runid: string }) { let opts = '{'; if (colmapVar) opts += `columns: ${colmapVar}, `; - if (progressName) opts += `progressName: "${progressName}", `; + if (progressName) opts += `progressName: ${JSON.stringify(progressName)}, `; opts += '}'; this._put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar}, ${opts});`); @@ -78,7 +93,7 @@ export class ScriptWriter { } } -export class ScriptWriterJson { +export class ScriptWriterJson implements ScriptWriterGeneric { s = ''; packageNames: string[] = []; varCount = 0; @@ -103,13 +118,17 @@ export class ScriptWriterJson { this.commands.push({ type: 'assign', variableName, - functionName: extractShellApiFunctionName(functionName), + functionName, props, }); this.packageNames.push(...extractShellApiPlugins(functionName, props)); } + requirePackage(packageName) { + this.packageNames.push(packageName); + } + assignValue(variableName, jsonValue) { this.commands.push({ type: 'assignValue', @@ -118,7 +137,7 @@ export class ScriptWriterJson { }); } - copyStream(sourceVar, targetVar, colmapVar = null, progressName?: string) { + copyStream(sourceVar, targetVar, colmapVar = null, progressName?: string | { name: string; runid: string }) { this.commands.push({ type: 'copyStream', sourceVar, @@ -167,9 +186,119 @@ export class ScriptWriterJson { } } -export function jsonScriptToJavascript(json) { - const { schedule, commands, packageNames } = json; - const script = new ScriptWriter(); +export class ScriptWriterEval implements ScriptWriterGeneric { + s = ''; + varCount = 0; + commands = []; + dbgateApi: any; + requirePlugin: (name: string) => any; + variables: { [name: string]: any } = {}; + hostConnection: any; + runid: string; + + constructor(dbgateApi, requirePlugin, hostConnection, runid, varCount = '0') { + this.varCount = parseInt(varCount) || 0; + this.dbgateApi = dbgateApi; + this.requirePlugin = requirePlugin; + this.hostConnection = hostConnection; + this.runid = runid; + } + + allocVariable(prefix = 'var') { + this.varCount += 1; + return `${prefix}${this.varCount}`; + } + + endLine() {} + + requirePackage(packageName) {} + + async assign(variableName, functionName, props) { + const func = evalShellApiFunctionName(functionName, this.dbgateApi, this.requirePlugin); + + this.variables[variableName] = await func( + _cloneDeepWith(props, node => { + if (node?.$hostConnection) { + return this.hostConnection; + } + }) + ); + } + + assignValue(variableName, jsonValue) { + this.variables[variableName] = jsonValue; + } + + async copyStream(sourceVar, targetVar, colmapVar = null, progressName?: string | { name: string; runid: string }) { + await this.dbgateApi.copyStream(this.variables[sourceVar], this.variables[targetVar], { + progressName: _cloneDeepWith(progressName, node => { + if (node?.$runid) { + if (node?.$runid) { + return this.runid; + } + } + }), + columns: colmapVar ? this.variables[colmapVar] : null, + }); + } + + comment(text) {} + + async importDatabase(options) { + await this.dbgateApi.importDatabase(options); + } + + async dataReplicator(options) { + await this.dbgateApi.dataReplicator(options); + } + + async zipDirectory(inputDirectory, outputFile) { + await this.dbgateApi.zipDirectory(inputDirectory, outputFile); + } + + getScript(schedule?: any) { + throw new Error('Not implemented'); + } +} + +async function playJsonCommand(cmd, script: ScriptWriterGeneric) { + switch (cmd.type) { + case 'assign': + await script.assign(cmd.variableName, cmd.functionName, cmd.props); + break; + case 'assignValue': + await script.assignValue(cmd.variableName, cmd.jsonValue); + break; + case 'copyStream': + await script.copyStream(cmd.sourceVar, cmd.targetVar, cmd.colmapVar, cmd.progressName); + break; + case 'endLine': + await script.endLine(); + break; + case 'comment': + await script.comment(cmd.text); + break; + case 'importDatabase': + await script.importDatabase(cmd.options); + break; + case 'dataReplicator': + await script.dataReplicator(cmd.options); + break; + case 'zipDirectory': + await script.zipDirectory(cmd.inputDirectory, cmd.outputFile); + break; + } +} + +export async function playJsonScriptWriter(json, script: ScriptWriterGeneric) { + for (const cmd of json.commands) { + await playJsonCommand(cmd, script); + } +} + +export async function jsonScriptToJavascript(json) { + const { schedule, packageNames } = json; + const script = new ScriptWriterJavaScript(); for (const packageName of packageNames) { if (!/^dbgate-plugin-.*$/.test(packageName)) { throw new Error('Unallowed package name:' + packageName); @@ -177,34 +306,7 @@ export function jsonScriptToJavascript(json) { script.packageNames.push(packageName); } - for (const cmd of commands) { - switch (cmd.type) { - case 'assign': - script.assignCore(cmd.variableName, cmd.functionName, cmd.props); - break; - case 'assignValue': - script.assignValue(cmd.variableName, cmd.jsonValue); - break; - case 'copyStream': - script.copyStream(cmd.sourceVar, cmd.targetVar, cmd.colmapVar, cmd.progressName); - break; - case 'endLine': - script.endLine(); - break; - case 'comment': - script.comment(cmd.text); - break; - case 'importDatabase': - script.importDatabase(cmd.options); - break; - case 'dataReplicator': - script.dataReplicator(cmd.options); - break; - case 'zipDirectory': - script.zipDirectory(cmd.inputDirectory, cmd.outputFile); - break; - } - } + await playJsonScriptWriter(json, script); return script.getScript(schedule); } diff --git a/packages/tools/src/packageTools.ts b/packages/tools/src/packageTools.ts index b0c506261..fdd8cc32d 100644 --- a/packages/tools/src/packageTools.ts +++ b/packages/tools/src/packageTools.ts @@ -27,7 +27,7 @@ export function extractPackageName(name): string { return null; } -export function extractShellApiFunctionName(functionName) { +export function compileShellApiFunctionName(functionName) { const nsMatch = functionName.match(/^([^@]+)@([^@]+)/); if (nsMatch) { return `${_camelCase(nsMatch[2])}.shellApi.${nsMatch[1]}`; @@ -35,6 +35,14 @@ export function extractShellApiFunctionName(functionName) { return `dbgateApi.${functionName}`; } +export function evalShellApiFunctionName(functionName, dbgateApi, requirePlugin) { + const nsMatch = functionName.match(/^([^@]+)@([^@]+)/); + if (nsMatch) { + return requirePlugin(nsMatch[2]).shellApi[nsMatch[1]]; + } + return dbgateApi[functionName]; +} + export function findEngineDriver(connection, extensions: ExtensionsDirectory): EngineDriver { if (!extensions) { return null; diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 6b6c7b705..af5fd29df 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -33,6 +33,7 @@ export interface QueryOptions { discardResult?: boolean; importSqlDump?: boolean; range?: { offset: number; limit: number }; + readonly?: boolean; } export interface WriteTableOptions { @@ -286,7 +287,7 @@ export interface EngineDriver extends FilterBehaviourProvider { adaptTableInfo(table: TableInfo): TableInfo; // simple data type adapter adaptDataType(dataType: string): string; - listSchemas(dbhan: DatabaseHandle): SchemaInfo[]; + listSchemas(dbhan: DatabaseHandle): Promise; backupDatabaseCommand( connection: any, settings: BackupDatabaseSettings, @@ -309,6 +310,7 @@ export interface EngineDriver extends FilterBehaviourProvider { analyserClass?: any; dumperClass?: any; + singleConnectionOnly?: boolean; } export interface DatabaseModification { diff --git a/packages/types/query.d.ts b/packages/types/query.d.ts index e3abbccfe..633c1b01d 100644 --- a/packages/types/query.d.ts +++ b/packages/types/query.d.ts @@ -7,6 +7,7 @@ export interface QueryResultColumn { columnName: string; notNull: boolean; autoIncrement?: boolean; + dataType?: string; } export interface QueryResult { diff --git a/packages/types/test-engines.d.ts b/packages/types/test-engines.d.ts index 7a426349a..9cea205c4 100644 --- a/packages/types/test-engines.d.ts +++ b/packages/types/test-engines.d.ts @@ -40,6 +40,11 @@ export type TestEngineInfo = { skipPkDrop?: boolean; skipOrderBy?: boolean; skipImportModel?: boolean; + skipTriggers?: boolean; + skipDropColumn?: boolean; + skipChangeNullability?: boolean; + skipRenameColumn?: boolean; + skipDropReferences?: boolean; forceSortResults?: boolean; forceSortStructureColumns?: boolean; diff --git a/packages/web/src/appobj/DatabaseObjectAppObject.svelte b/packages/web/src/appobj/DatabaseObjectAppObject.svelte index 41d61a460..484e50269 100644 --- a/packages/web/src/appobj/DatabaseObjectAppObject.svelte +++ b/packages/web/src/appobj/DatabaseObjectAppObject.svelte @@ -893,9 +893,10 @@ { functionName: menu.functionName, props: { - connection: extractShellConnection(coninfo, data.database), + ...extractShellConnectionHostable(coninfo, data.database), ..._.pick(data, ['pureName', 'schemaName']), }, + hostConnection: extractShellHostConnection(coninfo, data.database), }, fmt ); @@ -1031,7 +1032,7 @@ import { alterDatabaseDialog, renameDatabaseObjectDialog } from '../utility/alterDatabaseTools'; import ConfirmModal from '../modals/ConfirmModal.svelte'; import InputTextModal from '../modals/InputTextModal.svelte'; - import { extractShellConnection } from '../impexp/createImpExpScript'; + import { extractShellConnectionHostable, extractShellHostConnection } from '../impexp/createImpExpScript'; import { format as dateFormat } from 'date-fns'; import { getDefaultFileFormat } from '../plugins/fileformats'; import hasPermission from '../utility/hasPermission'; diff --git a/packages/web/src/commands/stdCommands.ts b/packages/web/src/commands/stdCommands.ts index cef4daacd..fbe314a26 100644 --- a/packages/web/src/commands/stdCommands.ts +++ b/packages/web/src/commands/stdCommands.ts @@ -46,6 +46,7 @@ import { openImportExportTab } from '../utility/importExportTools'; import newTable from '../tableeditor/newTable'; import { isProApp } from '../utility/proTools'; import { openWebLink } from '../utility/simpleTools'; +import { _t } from '../translations'; import ExportImportConnectionsModal from '../modals/ExportImportConnectionsModal.svelte'; // function themeCommand(theme: ThemeDefinition) { @@ -390,12 +391,12 @@ registerCommand({ category: 'New', icon: 'img sqlite-database', name: 'SQLite database', - menuName: 'New SQLite database', + menuName: _t('command.new.sqliteDatabase', { defaultMessage: 'New SQLite database' }), onClick: () => { showModal(InputTextModal, { value: 'newdb', - label: 'New database name', - header: 'Create SQLite database', + label: _t('app.databaseName', { defaultMessage: 'Database name' }), + header: _t('command.new.sqliteDatabase', { defaultMessage: 'New SQLite database' }), onConfirm: async file => { const resp = await apiCall('connections/new-sqlite-database', { file }); const connection = resp; @@ -405,6 +406,26 @@ registerCommand({ }, }); +registerCommand({ + id: 'new.duckdbDatabase', + category: 'New', + icon: 'img sqlite-database', + name: 'DuckDB database', + menuName: _t('command.new.duckdbDatabase', { defaultMessage: 'New DuckDB database' }), + onClick: () => { + showModal(InputTextModal, { + value: 'newdb', + label: _t('app.databaseName', { defaultMessage: 'Database name' }), + header: _t('command.new.duckdbDatabase', { defaultMessage: 'New DuckDB database' }), + onConfirm: async file => { + const resp = await apiCall('connections/new-duckdb-database', { file }); + const connection = resp; + switchCurrentDatabase({ connection, name: `${file}.duckdb` }); + }, + }); + }, +}); + registerCommand({ id: 'tabs.changelog', category: 'Tabs', diff --git a/packages/web/src/datagrid/CollectionDataGridCore.svelte b/packages/web/src/datagrid/CollectionDataGridCore.svelte index 8dda67cd3..144d69e4f 100644 --- a/packages/web/src/datagrid/CollectionDataGridCore.svelte +++ b/packages/web/src/datagrid/CollectionDataGridCore.svelte @@ -121,7 +121,7 @@ import _ from 'lodash'; import { registerQuickExportHandler } from '../buttons/ToolStripExportButton.svelte'; import registerCommand from '../commands/registerCommand'; - import { extractShellConnection } from '../impexp/createImpExpScript'; + import { extractShellConnection, extractShellConnectionHostable, extractShellHostConnection } from '../impexp/createImpExpScript'; import { apiCall } from '../utility/api'; import { registerMenu } from '../utility/contextMenu'; @@ -235,10 +235,11 @@ { functionName: 'queryReader', props: { - connection: extractShellConnection(coninfo, database), + ...extractShellConnectionHostable(coninfo, database), queryType: coninfo.isReadOnly ? 'json' : 'native', query: coninfo.isReadOnly ? getExportQueryJson() : getExportQuery(), }, + hostConnection: extractShellHostConnection(coninfo, database), }, fmt, display.getExportColumnMap() diff --git a/packages/web/src/datagrid/SqlDataGridCore.svelte b/packages/web/src/datagrid/SqlDataGridCore.svelte index 3241c92b2..0e192b38b 100644 --- a/packages/web/src/datagrid/SqlDataGridCore.svelte +++ b/packages/web/src/datagrid/SqlDataGridCore.svelte @@ -68,7 +68,11 @@ import { registerQuickExportHandler } from '../buttons/ToolStripExportButton.svelte'; import registerCommand from '../commands/registerCommand'; - import { extractShellConnection } from '../impexp/createImpExpScript'; + import { + extractShellConnection, + extractShellConnectionHostable, + extractShellHostConnection, + } from '../impexp/createImpExpScript'; import { apiCall } from '../utility/api'; import { registerMenu } from '../utility/contextMenu'; @@ -215,10 +219,11 @@ { functionName: 'queryReader', props: { - connection: extractShellConnection(coninfo, database), + ...extractShellConnectionHostable(coninfo, database), queryType: coninfo.isReadOnly ? 'json' : 'native', query: coninfo.isReadOnly ? display.getExportQueryJson() : display.getExportQuery(), }, + hostConnection: extractShellHostConnection(coninfo, database), }, fmt, display.getExportColumnMap() diff --git a/packages/web/src/impexp/createImpExpScript.ts b/packages/web/src/impexp/createImpExpScript.ts index 3414a306b..23638f6fe 100644 --- a/packages/web/src/impexp/createImpExpScript.ts +++ b/packages/web/src/impexp/createImpExpScript.ts @@ -1,11 +1,11 @@ import _ from 'lodash'; import moment from 'moment'; -import { ScriptWriter, ScriptWriterJson } from 'dbgate-tools'; +import { ScriptWriterGeneric, ScriptWriterJavaScript, ScriptWriterJson } from 'dbgate-tools'; import getAsArray from '../utility/getAsArray'; import { getConnectionInfo } from '../utility/metadataLoaders'; import { findEngineDriver, findObjectLike } from 'dbgate-tools'; import { findFileFormat } from '../plugins/fileformats'; -import { getCurrentConfig } from '../stores'; +import { getCurrentConfig, getExtensions } from '../stores'; export function getTargetName(extensions, source, values) { const key = `targetName_${source}`; @@ -53,6 +53,32 @@ export function extractShellConnection(connection, database) { }; } +export function extractShellConnectionHostable(connection, database) { + const driver = findEngineDriver(connection, getExtensions()); + if (driver?.singleConnectionOnly) { + return { + systemConnection: { $hostConnection: true }, + connection: driver.engine, + }; + } + + return { + connection: extractShellConnection(connection, database), + }; +} + +export function extractShellHostConnection(connection, database) { + const driver = findEngineDriver(connection, getExtensions()); + if (driver?.singleConnectionOnly) { + return { + conid: connection._id, + database, + }; + } + + return undefined; +} + async function getConnection(extensions, storageType, conid, database) { if (storageType == 'database' || storageType == 'query') { const conn = await getConnectionInfo({ conid }); @@ -63,14 +89,23 @@ async function getConnection(extensions, storageType, conid, database) { return [null, null]; } -function getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver) { +function getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver, hostConnection) { const { sourceStorageType } = values; + const connectionParams = + sourceDriver?.singleConnectionOnly && hostConnection + ? { + systemConnection: { $hostConnection: true }, + connection: sourceDriver?.engine, + } + : { + connection: sourceConnection, + }; if (sourceStorageType == 'database') { const fullName = { schemaName: values.sourceSchemaName, pureName: sourceName }; return [ 'tableReader', { - connection: sourceConnection, + ...connectionParams, ...extractDriverApiParameters(values, 'source', sourceDriver), ...fullName, }, @@ -80,7 +115,7 @@ function getSourceExpr(extensions, sourceName, values, sourceConnection, sourceD return [ 'queryReader', { - connection: sourceConnection, + ...connectionParams, ...extractDriverApiParameters(values, 'source', sourceDriver), queryType: values.sourceQueryType, query: values.sourceQueryType == 'json' ? JSON.parse(values.sourceQuery) : values.sourceQuery, @@ -145,8 +180,17 @@ function getFlagsFroAction(action) { }; } -function getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver) { +function getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver, hostConnection) { const { targetStorageType } = values; + const connectionParams = + targetDriver?.singleConnectionOnly && hostConnection + ? { + systemConnection: { $hostConnection: true }, + connection: targetDriver?.engine, + } + : { + connection: targetConnection, + }; const format = findFileFormat(extensions, targetStorageType); if (format && format.writerFunc) { const outputParams = format.getOutputParams && format.getOutputParams(sourceName, values); @@ -166,7 +210,7 @@ function getTargetExpr(extensions, sourceName, values, targetConnection, targetD return [ 'tableWriter', { - connection: targetConnection, + ...connectionParams, schemaName: values.targetSchemaName, pureName: getTargetName(extensions, sourceName, values), ...extractDriverApiParameters(values, 'target', targetDriver), @@ -203,12 +247,12 @@ export function normalizeExportColumnMap(colmap) { return null; } -export default async function createImpExpScript(extensions, values, forceScript = false) { +export default async function createImpExpScript(extensions, values, format = undefined, detectHostConnection = false) { const config = getCurrentConfig(); - const script = - config.allowShellScripting || forceScript - ? new ScriptWriter(values.startVariableIndex || 0) - : new ScriptWriterJson(values.startVariableIndex || 0); + let script: ScriptWriterGeneric = new ScriptWriterJson(values.startVariableIndex || 0); + if (format == 'script' && config.allowShellScripting) { + script = new ScriptWriterJavaScript(values.startVariableIndex || 0); + } const [sourceConnection, sourceDriver] = await getConnection( extensions, @@ -223,15 +267,39 @@ export default async function createImpExpScript(extensions, values, forceScript values.targetDatabaseName ); + let hostConnection = null; + if (detectHostConnection) { + // @ts-ignore + if (sourceDriver?.singleConnectionOnly) { + hostConnection = { conid: values.sourceConnectionId, database: values.sourceDatabaseName }; + } + // @ts-ignore + if (targetDriver?.singleConnectionOnly) { + if ( + hostConnection && + (hostConnection.conid != values.targetConnectionId || hostConnection.database != values.targetDatabaseName) + ) { + throw new Error('Cannot use two different single-connections in the same script'); + } + hostConnection = { conid: values.targetConnectionId, database: values.targetDatabaseName }; + } + } + const sourceList = getAsArray(values.sourceList); for (const sourceName of sourceList) { const sourceVar = script.allocVariable(); - // @ts-ignore - script.assign(sourceVar, ...getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver)); + script.assign( + sourceVar, + // @ts-ignore + ...getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver, hostConnection) + ); const targetVar = script.allocVariable(); - // @ts-ignore - script.assign(targetVar, ...getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver)); + script.assign( + targetVar, + // @ts-ignore + ...getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver, hostConnection) + ); const colmap = normalizeExportColumnMap(values[`columns_${sourceName}`]); @@ -241,7 +309,12 @@ export default async function createImpExpScript(extensions, values, forceScript script.assignValue(colmapVar, colmap); } - script.copyStream(sourceVar, targetVar, colmapVar, sourceName); + script.copyStream( + sourceVar, + targetVar, + colmapVar, + hostConnection ? { name: sourceName, runid: { $runid: true } } : sourceName + ); script.endLine(); } @@ -251,7 +324,11 @@ export default async function createImpExpScript(extensions, values, forceScript script.zipDirectory('.', values.createZipFileInArchive ? 'archive:' + zipFileName : zipFileName); } - return script.getScript(values.schedule); + const res = script.getScript(values.schedule); + if (format == 'json') { + res.hostConnection = hostConnection; + } + return res; } export function getActionOptions(extensions, source, values, targetDbinfo) { @@ -289,7 +366,7 @@ export async function createPreviewReader(extensions, values, sourceName) { values.sourceConnectionId, values.sourceDatabaseName ); - const [functionName, props] = getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver); + const [functionName, props] = getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver, null); return { functionName, props: { diff --git a/packages/web/src/tabs/ImportExportTab.svelte b/packages/web/src/tabs/ImportExportTab.svelte index 04935cd8b..1b167a93b 100644 --- a/packages/web/src/tabs/ImportExportTab.svelte +++ b/packages/web/src/tabs/ImportExportTab.svelte @@ -50,6 +50,8 @@ import { registerFileCommands } from '../commands/stdCommands'; import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte'; import ToolStripSaveButton from '../buttons/ToolStripSaveButton.svelte'; + import uuidv1 from 'uuid/v1'; + import { tick } from 'svelte'; let busy = false; let executeNumber = 0; @@ -167,7 +169,7 @@ const handleGenerateScript = async e => { const values = $formValues as any; - const code = await createImpExpScript($extensions, values, true); + const code = await createImpExpScript($extensions, values, 'script', false); openNewTab( { title: 'Shell #', @@ -183,12 +185,24 @@ progressHolder = {}; const values = $formValues as any; busy = true; - const script = await createImpExpScript($extensions, values); + const script = await createImpExpScript($extensions, values, 'json', true); executeNumber += 1; - let runid = runnerId; - const resp = await apiCall('runners/start', { script }); - runid = resp.runid; - runnerId = runid; + + if (script.hostConnection) { + runnerId = uuidv1(); + await tick(); + await apiCall('database-connections/eval-json-script', { + runid: runnerId, + conid: script.hostConnection.conid, + database: script.hostConnection.database, + script, + }); + } else { + let runid = runnerId; + const resp = await apiCall('runners/start', { script }); + runid = resp.runid; + runnerId = runid; + } if (values.targetStorageType == 'archive') { refreshArchiveFolderRef.set(values.targetArchiveFolder); diff --git a/packages/web/src/tabs/QueryTab.svelte b/packages/web/src/tabs/QueryTab.svelte index e564e9338..992ccdddc 100644 --- a/packages/web/src/tabs/QueryTab.svelte +++ b/packages/web/src/tabs/QueryTab.svelte @@ -107,7 +107,7 @@