diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js index b87ae7776..e8f573d8e 100644 --- a/packages/api/src/controllers/runners.js +++ b/packages/api/src/controllers/runners.js @@ -38,7 +38,16 @@ const dbgateApi = require(process.env.DBGATE_API); ${requirePluginsTemplate(extractShellApiPlugins(functionName, props))} require=null; async function run() { -const reader=await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)}); +${ + props.downloadUrl + ? ` + const downloaded=await dbgateApi.download(${JSON.stringify(props.downloadUrl)}); + const reader=await ${extractShellApiFunctionName(functionName)}(Object.assign(${JSON.stringify( + props + )}, { fileName: downloaded, downloadUrl: undefined })); + ` + : `const reader=await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});` +} const writer=await dbgateApi.collectorWriter({runid: '${runid}'}); await dbgateApi.copyStream(reader, writer); } diff --git a/packages/api/src/shell/download.js b/packages/api/src/shell/download.js new file mode 100644 index 000000000..070f68213 --- /dev/null +++ b/packages/api/src/shell/download.js @@ -0,0 +1,12 @@ +const path = require('path'); +const uuidv1 = require('uuid/v1'); +const { uploadsdir } = require('../utility/directories'); +const { downloadFile } = require('../utility/downloader'); + +async function download(url) { + const tmpFile = path.join(uploadsdir(), uuidv1() + '.tgz'); + await downloadFile(url, tmpFile); + return tmpFile; +} + +module.exports = download; diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index 521ca41c2..135af5ab8 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -14,6 +14,7 @@ const collectorWriter = require('./collectorWriter'); const finalizer = require('./finalizer'); const registerPlugins = require('./registerPlugins'); const requirePlugin = require('./requirePlugin'); +const download = require('./download'); const dbgateApi = { queryReader, @@ -31,6 +32,7 @@ const dbgateApi = { collectorWriter, finalizer, registerPlugins, + download, }; requirePlugin.initialize(dbgateApi); diff --git a/packages/api/src/utility/downloadPackage.js b/packages/api/src/utility/downloadPackage.js index 37a5ea661..581e62f05 100644 --- a/packages/api/src/utility/downloadPackage.js +++ b/packages/api/src/utility/downloadPackage.js @@ -8,6 +8,7 @@ const zlib = require('zlib'); const tar = require('tar'); const ncp = require('ncp').ncp; const { uploadsdir } = require('./directories'); +const { downloadFile } = require('./downloader'); function extractTarball(tmpFile, destination) { return new Promise((resolve, reject) => { @@ -19,13 +20,6 @@ function extractTarball(tmpFile, destination) { }); } -function saveStreamToFile(pipedStream, fileName) { - return new Promise((resolve, reject) => { - const fileStream = fs.createWriteStream(fileName); - fileStream.on('close', () => resolve()); - pipedStream.pipe(fileStream); - }); -} function copyDirectory(source, target) { return new Promise((resolve, reject) => { @@ -46,13 +40,7 @@ async function downloadPackage(packageName, directory) { const tarball = infoResp.data.versions[latest].dist.tarball; const tmpFile = path.join(uploadsdir(), uuidv1() + '.tgz'); - console.log(`Downloading tarball ${tarball} into ${tmpFile}`); - const tarballResp = await axios.default({ - method: 'get', - url: tarball, - responseType: 'stream', - }); - await saveStreamToFile(tarballResp.data, tmpFile); + downloadFile(tarball, tmpFile); const tmpDir = path.join(uploadsdir(), uuidv1()); fs.mkdirSync(tmpDir); await extractTarball(tmpFile, tmpDir); diff --git a/packages/api/src/utility/downloader.js b/packages/api/src/utility/downloader.js new file mode 100644 index 000000000..8a4dec76e --- /dev/null +++ b/packages/api/src/utility/downloader.js @@ -0,0 +1,25 @@ +const axios = require('axios'); +const fs = require('fs'); + +function saveStreamToFile(pipedStream, fileName) { + return new Promise((resolve, reject) => { + const fileStream = fs.createWriteStream(fileName); + fileStream.on('close', () => resolve()); + pipedStream.pipe(fileStream); + }); +} + +async function downloadFile(url, file) { + console.log(`Downloading ${url} into ${file}`); + const tarballResp = await axios.default({ + method: 'get', + url, + responseType: 'stream', + }); + await saveStreamToFile(tarballResp.data, file); +} + +module.exports = { + saveStreamToFile, + downloadFile, +}; diff --git a/packages/web/src/impexp/ImportExportConfigurator.js b/packages/web/src/impexp/ImportExportConfigurator.js index 9d735f068..e2bf92134 100644 --- a/packages/web/src/impexp/ImportExportConfigurator.js +++ b/packages/web/src/impexp/ImportExportConfigurator.js @@ -12,14 +12,13 @@ import { FormArchiveFolderSelect, FormArchiveFilesSelect, } from '../utility/forms'; -import { useArchiveFiles, useConnectionInfo, useDatabaseInfo, useInstalledPlugins } from '../utility/metadataLoaders'; +import { useArchiveFiles, useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders'; import TableControl, { TableColumn } from '../utility/TableControl'; import { TextField, SelectField, CheckboxField } from '../utility/inputs'; import { createPreviewReader, getActionOptions, getTargetName } from './createImpExpScript'; import getElectron from '../utility/getElectron'; import ErrorInfo from '../widgets/ErrorInfo'; import getAsArray from '../utility/getAsArray'; -import axios from '../utility/axios'; import LoadingInfo from '../widgets/LoadingInfo'; import SqlEditor from '../sqleditor/SqlEditor'; import { useUploadsProvider } from '../utility/UploadsProvider'; @@ -29,6 +28,8 @@ import { findFileFormat, getFileFormatDirections } from '../utility/fileformats' import FormArgumentList from '../utility/FormArgumentList'; import useExtensions from '../utility/useExtensions'; import UploadButton from '../utility/UploadButton'; +import useShowModal from '../modals/showModal'; +import ChangeDownloadUrlModal from '../modals/ChangeDownloadUrlModal'; const Container = styled.div` // max-height: 50vh; @@ -91,6 +92,10 @@ const Title = styled.div` margin: 10px 0px; `; +const ButtonsLine = styled.div` + display: flex; +`; + function getFileFilters(extensions, storageType) { const res = []; const format = findFileFormat(extensions, storageType); @@ -104,6 +109,7 @@ async function addFilesToSourceListDefault(file, newSources, newValues) { newSources.push(sourceName); newValues[`sourceFile_${sourceName}`] = { fileName: file.full, + downloadUrl: file.url, }; } @@ -168,12 +174,47 @@ function ElectronFilesInput() { ); } -function FilesInput() { +function extractUrlName(url) { + const match = url.match(/\/([^/]+)($|\?)/); + if (match) { + const res = match[1]; + if (res.includes('.')) { + return res.slice(0, res.indexOf('.')); + } + return res; + } + return 'url'; +} + +function FilesInput({ setPreviewSource = undefined }) { const theme = useTheme(); const electron = getElectron(); + const showModal = useShowModal(); + const { values, setValues } = useFormikContext(); + const extensions = useExtensions(); + const doAddUrl = (url) => { + addFilesToSourceList( + extensions, + [ + { + url, + name: extractUrlName(url), + }, + ], + values, + setValues, + null, + setPreviewSource + ); + }; + const handleAddUrl = () => + showModal((modalState) => ); return ( <> - {electron ? : } + + {electron ? : } + + Drag & drop imported files here ); @@ -188,6 +229,7 @@ function SourceTargetConfig({ schemaNameField, tablesField = undefined, engine = undefined, + setPreviewSource = undefined, }) { const extensions = useExtensions(); const theme = useTheme(); @@ -314,7 +356,7 @@ function SourceTargetConfig({ )} - {!!format && direction == 'source' && } + {!!format && direction == 'source' && } {format && format.args && ( { - console.log('UPLOAD', extensions); addFilesToSourceList( extensions, [ @@ -430,6 +471,7 @@ export default function ImportExportConfigurator({ uploadedFile = undefined, onC schemaNameField="sourceSchemaName" tablesField="sourceList" engine={sourceEngine} + setPreviewSource={setPreviewSource} /> diff --git a/packages/web/src/impexp/ScriptWriter.js b/packages/web/src/impexp/ScriptWriter.js index acd751ff3..832a7f7f3 100644 --- a/packages/web/src/impexp/ScriptWriter.js +++ b/packages/web/src/impexp/ScriptWriter.js @@ -20,7 +20,20 @@ export default class ScriptWriter { } assign(variableName, functionName, props) { - this.put(`const ${variableName} = await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});`); + if (props.downloadUrl) { + const fileNameVar = this.allocVariable(); + this.put(`const ${fileNameVar} = await dbgateApi.download(${JSON.stringify(props.downloadUrl)});`); + this.put( + `const ${variableName} = await ${extractShellApiFunctionName(functionName)}( + Object.assign(${JSON.stringify(props)}, { + fileName: ${fileNameVar}, + downloadUrl: undefined + }) + );` + ); + } else { + this.put(`const ${variableName} = await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});`); + } this.packageNames.push(...extractShellApiPlugins(functionName, props)); } @@ -42,6 +55,12 @@ export default class ScriptWriter { // this.comment(JSON.stringify(this.engines)); // } const packageNames = this.packageNames; - return _.uniq(packageNames).map((packageName) => `// @require ${packageName}\n`).join('') + '\n' + this.s; + return ( + _.uniq(packageNames) + .map((packageName) => `// @require ${packageName}\n`) + .join('') + + '\n' + + this.s + ); } } diff --git a/packages/web/src/modals/ChangeDownloadUrlModal.js b/packages/web/src/modals/ChangeDownloadUrlModal.js new file mode 100644 index 000000000..0ef4fe1d7 --- /dev/null +++ b/packages/web/src/modals/ChangeDownloadUrlModal.js @@ -0,0 +1,39 @@ +import React from 'react'; +import axios from '../utility/axios'; +import ModalBase from './ModalBase'; +import { FormButtonRow, FormButton, FormTextField, FormSelectField, FormSubmit } from '../utility/forms'; +import { TextField } from '../utility/inputs'; +import { Formik, Form } from 'formik'; +import { useSetSavedSqlFiles } from '../utility/globalState'; +import ModalHeader from './ModalHeader'; +import ModalContent from './ModalContent'; +import ModalFooter from './ModalFooter'; +import FormStyledButton from '../widgets/FormStyledButton'; +// import FormikForm from '../utility/FormikForm'; + +export default function ChangeDownloadUrlModal({ modalState, url = '', onConfirm = undefined }) { + const textFieldRef = React.useRef(null); + React.useEffect(() => { + if (textFieldRef.current) textFieldRef.current.focus(); + }, [textFieldRef.current]); + const handleSubmit = async (values) => { + onConfirm(values.url); + modalState.close(); + }; + return ( + + Dwonload imported file from web + +
+ + + + + + modalState.close()} /> + +
+
+
+ ); +}