diff --git a/packages/api/src/controllers/databaseConnections.js b/packages/api/src/controllers/databaseConnections.js index 74974f0ed..8e913e50b 100644 --- a/packages/api/src/controllers/databaseConnections.js +++ b/packages/api/src/controllers/databaseConnections.js @@ -131,6 +131,12 @@ module.exports = { socket.emit(`runner-done-${runid}`); }, + handle_progress(conid, database, progressData) { + const { progressName } = progressData; + const { name, runid } = progressName; + socket.emit(`runner-progress-${runid}`, { ...progressData, progressName: name }); + }, + async ensureOpened(conid, database) { const existing = this.opened.find(x => x.conid == conid && x.database == database); if (existing) return existing; diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js index 8394dadcc..5ed9bbea1 100644 --- a/packages/api/src/controllers/runners.js +++ b/packages/api/src/controllers/runners.js @@ -8,7 +8,7 @@ const { fork, spawn } = require('child_process'); const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories'); const { extractShellApiPlugins, - extractShellApiFunctionName, + compileShellApiFunctionName, jsonScriptToJavascript, getLogger, safeJsonParse, @@ -58,7 +58,7 @@ dbgateApi.initializeApiEnvironment(); ${requirePluginsTemplate(extractShellApiPlugins(functionName, props))} require=null; async function run() { -const reader=await ${extractShellApiFunctionName(functionName, true)}(${JSON.stringify(props)}); +const reader=await ${compileShellApiFunctionName(functionName)}(${JSON.stringify(props)}); const writer=await dbgateApi.collectorWriter({runid: '${runid}'}); await dbgateApi.copyStream(reader, writer); } @@ -273,7 +273,7 @@ module.exports = { const runid = crypto.randomUUID(); if (script.type == 'json') { - const js = jsonScriptToJavascript(script); + const js = await jsonScriptToJavascript(script); return this.startCore(runid, scriptTemplate(js, false)); } @@ -335,7 +335,7 @@ module.exports = { return { errorMessage: 'Only JSON scripts are allowed' }; } - const promise = new Promise((resolve, reject) => { + const promise = new Promise(async (resolve, reject) => { const runid = crypto.randomUUID(); this.requests[runid] = { resolve, reject, exitOnStreamError: true }; const cloned = _.cloneDeepWith(script, node => { @@ -343,7 +343,7 @@ module.exports = { return runid; } }); - const js = jsonScriptToJavascript(cloned); + const js = await jsonScriptToJavascript(cloned); this.startCore(runid, scriptTemplate(js, false)); }); return promise; diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index 35ca5747b..95cf19fc6 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -21,6 +21,9 @@ const { dumpSqlSelect } = require('dbgate-sqltree'); const { allowExecuteCustomScript, handleQueryStream } = require('../utility/handleQueryStream'); const dbgateApi = require('../shell'); const requirePlugin = require('../shell/requirePlugin'); +const path = require('path'); +const { rundir } = require('../utility/directories'); +const fs = require('fs-extra'); const logger = getLogger('dbconnProcess'); @@ -411,9 +414,19 @@ async function handleExecuteSessionQuery({ sesid, sql }) { } async function handleEvalJsonScript({ script, runid }) { - const evalWriter = new ScriptWriterEval(dbgateApi, requirePlugin, dbhan); - await playJsonScriptWriter(script, evalWriter); - process.send({ msgtype: 'runnerDone', runid }); + const directory = path.join(rundir(), runid); + fs.mkdirSync(directory); + const originalCwd = process.cwd(); + + try { + process.chdir(directory); + + const evalWriter = new ScriptWriterEval(dbgateApi, requirePlugin, dbhan, runid); + await playJsonScriptWriter(script, evalWriter); + process.send({ msgtype: 'runnerDone', runid }); + } finally { + process.chdir(originalCwd); + } } // async function handleRunCommand({ msgid, sql }) { diff --git a/packages/tools/src/ScriptWriter.ts b/packages/tools/src/ScriptWriter.ts index bfbb293c0..35aae6548 100644 --- a/packages/tools/src/ScriptWriter.ts +++ b/packages/tools/src/ScriptWriter.ts @@ -1,6 +1,6 @@ import _uniq from 'lodash/uniq'; import _cloneDeepWith from 'lodash/cloneDeepWith'; -import { evalShellApiFunctionName, extractShellApiFunctionName, extractShellApiPlugins } from './packageTools'; +import { evalShellApiFunctionName, compileShellApiFunctionName, extractShellApiPlugins } from './packageTools'; export interface ScriptWriterGeneric { allocVariable(prefix?: string); @@ -44,7 +44,7 @@ export class ScriptWriterJavaScript implements ScriptWriterGeneric { } assign(variableName, functionName, props) { - this.assignCore(variableName, extractShellApiFunctionName(functionName, true), props); + this.assignCore(variableName, compileShellApiFunctionName(functionName), props); this.packageNames.push(...extractShellApiPlugins(functionName, props)); } @@ -56,10 +56,10 @@ export class ScriptWriterJavaScript implements ScriptWriterGeneric { this.packageNames.push(packageName); } - copyStream(sourceVar, targetVar, colmapVar = null, progressName?: string) { + copyStream(sourceVar, targetVar, colmapVar = null, progressName?: string | { name: string; runid: string }) { let opts = '{'; if (colmapVar) opts += `columns: ${colmapVar}, `; - if (progressName) opts += `progressName: "${progressName}", `; + if (progressName) opts += `progressName: ${JSON.stringify(progressName)}, `; opts += '}'; this._put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar}, ${opts});`); @@ -118,7 +118,7 @@ export class ScriptWriterJson implements ScriptWriterGeneric { this.commands.push({ type: 'assign', variableName, - functionName: extractShellApiFunctionName(functionName, false), + functionName, props, }); @@ -137,7 +137,7 @@ export class ScriptWriterJson implements ScriptWriterGeneric { }); } - copyStream(sourceVar, targetVar, colmapVar = null, progressName?: string) { + copyStream(sourceVar, targetVar, colmapVar = null, progressName?: string | { name: string; runid: string }) { this.commands.push({ type: 'copyStream', sourceVar, @@ -194,12 +194,14 @@ export class ScriptWriterEval implements ScriptWriterGeneric { requirePlugin: (name: string) => any; variables: { [name: string]: any } = {}; hostConnection: any; + runid: string; - constructor(dbgateApi, requirePlugin, hostConnection, varCount = '0') { + constructor(dbgateApi, requirePlugin, hostConnection, runid, varCount = '0') { this.varCount = parseInt(varCount) || 0; this.dbgateApi = dbgateApi; this.requirePlugin = requirePlugin; this.hostConnection = hostConnection; + this.runid = runid; } allocVariable(prefix = 'var') { @@ -227,9 +229,15 @@ export class ScriptWriterEval implements ScriptWriterGeneric { this.variables[variableName] = jsonValue; } - async copyStream(sourceVar, targetVar, colmapVar = null, progressName?: string) { + async copyStream(sourceVar, targetVar, colmapVar = null, progressName?: string | { name: string; runid: string }) { await this.dbgateApi.copyStream(this.variables[sourceVar], this.variables[targetVar], { - progressName, + progressName: _cloneDeepWith(progressName, node => { + if (node?.$runid) { + if (node?.$runid) { + return this.runid; + } + } + }), columns: colmapVar ? this.variables[colmapVar] : null, }); } @@ -253,38 +261,42 @@ export class ScriptWriterEval implements ScriptWriterGeneric { } } -export async function playJsonScriptWriter(json, script: ScriptWriterGeneric) { - for (const cmd of json.commands) { - switch (cmd.type) { - case 'assign': - await script.assign(cmd.variableName, cmd.functionName, cmd.props); - break; - case 'assignValue': - await script.assignValue(cmd.variableName, cmd.jsonValue); - break; - case 'copyStream': - await script.copyStream(cmd.sourceVar, cmd.targetVar, cmd.colmapVar, cmd.progressName); - break; - case 'endLine': - await script.endLine(); - break; - case 'comment': - await script.comment(cmd.text); - break; - case 'importDatabase': - await script.importDatabase(cmd.options); - break; - case 'dataReplicator': - await script.dataReplicator(cmd.options); - break; - case 'zipDirectory': - await script.zipDirectory(cmd.inputDirectory, cmd.outputFile); - break; - } +async function playJsonCommand(cmd, script: ScriptWriterGeneric) { + switch (cmd.type) { + case 'assign': + await script.assign(cmd.variableName, cmd.functionName, cmd.props); + break; + case 'assignValue': + await script.assignValue(cmd.variableName, cmd.jsonValue); + break; + case 'copyStream': + await script.copyStream(cmd.sourceVar, cmd.targetVar, cmd.colmapVar, cmd.progressName); + break; + case 'endLine': + await script.endLine(); + break; + case 'comment': + await script.comment(cmd.text); + break; + case 'importDatabase': + await script.importDatabase(cmd.options); + break; + case 'dataReplicator': + await script.dataReplicator(cmd.options); + break; + case 'zipDirectory': + await script.zipDirectory(cmd.inputDirectory, cmd.outputFile); + break; } } -export function jsonScriptToJavascript(json) { +export async function playJsonScriptWriter(json, script: ScriptWriterGeneric) { + for (const cmd of json.commands) { + await playJsonCommand(cmd, script); + } +} + +export async function jsonScriptToJavascript(json) { const { schedule, packageNames } = json; const script = new ScriptWriterJavaScript(); for (const packageName of packageNames) { @@ -294,7 +306,7 @@ export function jsonScriptToJavascript(json) { script.packageNames.push(packageName); } - playJsonScriptWriter(json, script); + await playJsonScriptWriter(json, script); return script.getScript(schedule); } diff --git a/packages/tools/src/packageTools.ts b/packages/tools/src/packageTools.ts index 36f1dd4db..fdd8cc32d 100644 --- a/packages/tools/src/packageTools.ts +++ b/packages/tools/src/packageTools.ts @@ -27,15 +27,12 @@ export function extractPackageName(name): string { return null; } -export function extractShellApiFunctionName(functionName, usePrefixForDbGateApi) { +export function compileShellApiFunctionName(functionName) { const nsMatch = functionName.match(/^([^@]+)@([^@]+)/); if (nsMatch) { return `${_camelCase(nsMatch[2])}.shellApi.${nsMatch[1]}`; } - if (usePrefixForDbGateApi) { - return `dbgateApi.${functionName}`; - } - return functionName; + return `dbgateApi.${functionName}`; } export function evalShellApiFunctionName(functionName, dbgateApi, requirePlugin) { diff --git a/packages/web/src/impexp/createImpExpScript.ts b/packages/web/src/impexp/createImpExpScript.ts index ef4160e14..eac4852d0 100644 --- a/packages/web/src/impexp/createImpExpScript.ts +++ b/packages/web/src/impexp/createImpExpScript.ts @@ -69,6 +69,7 @@ function getSourceExpr(extensions, sourceName, values, sourceConnection, sourceD sourceDriver?.singleConnectionOnly && hostConnection ? { systemConnection: { $hostConnection: true }, + connection: sourceDriver?.engine, } : { connection: sourceConnection, @@ -159,6 +160,7 @@ function getTargetExpr(extensions, sourceName, values, targetConnection, targetD targetDriver?.singleConnectionOnly && hostConnection ? { systemConnection: { $hostConnection: true }, + connection: targetDriver?.engine, } : { connection: targetConnection, @@ -281,7 +283,12 @@ export default async function createImpExpScript(extensions, values, format = un script.assignValue(colmapVar, colmap); } - script.copyStream(sourceVar, targetVar, colmapVar, sourceName); + script.copyStream( + sourceVar, + targetVar, + colmapVar, + hostConnection ? { name: sourceName, runid: { $runid: true } } : sourceName + ); script.endLine(); } diff --git a/packages/web/src/tabs/ImportExportTab.svelte b/packages/web/src/tabs/ImportExportTab.svelte index 49aa635bd..00e2077c8 100644 --- a/packages/web/src/tabs/ImportExportTab.svelte +++ b/packages/web/src/tabs/ImportExportTab.svelte @@ -169,7 +169,7 @@ const handleGenerateScript = async e => { const values = $formValues as any; - const code = await createImpExpScript($extensions, values, 'script'); + const code = await createImpExpScript($extensions, values, 'script', true); openNewTab( { title: 'Shell #', diff --git a/plugins/dbgate-plugin-duckdb/src/backend/driver.js b/plugins/dbgate-plugin-duckdb/src/backend/driver.js index 08aa46bce..ede90e4cd 100644 --- a/plugins/dbgate-plugin-duckdb/src/backend/driver.js +++ b/plugins/dbgate-plugin-duckdb/src/backend/driver.js @@ -168,6 +168,8 @@ const driver = { pass.write(row); } + pass.end(); + return pass; }, async writeTable(dbhan, name, options) {