diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js
index b87ae7776..e8f573d8e 100644
--- a/packages/api/src/controllers/runners.js
+++ b/packages/api/src/controllers/runners.js
@@ -38,7 +38,16 @@ const dbgateApi = require(process.env.DBGATE_API);
${requirePluginsTemplate(extractShellApiPlugins(functionName, props))}
require=null;
async function run() {
-const reader=await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});
+${
+ props.downloadUrl
+ ? `
+ const downloaded=await dbgateApi.download(${JSON.stringify(props.downloadUrl)});
+ const reader=await ${extractShellApiFunctionName(functionName)}(Object.assign(${JSON.stringify(
+ props
+ )}, { fileName: downloaded, downloadUrl: undefined }));
+ `
+ : `const reader=await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});`
+}
const writer=await dbgateApi.collectorWriter({runid: '${runid}'});
await dbgateApi.copyStream(reader, writer);
}
diff --git a/packages/api/src/shell/download.js b/packages/api/src/shell/download.js
new file mode 100644
index 000000000..070f68213
--- /dev/null
+++ b/packages/api/src/shell/download.js
@@ -0,0 +1,12 @@
+const path = require('path');
+const uuidv1 = require('uuid/v1');
+const { uploadsdir } = require('../utility/directories');
+const { downloadFile } = require('../utility/downloader');
+
+async function download(url) {
+ const tmpFile = path.join(uploadsdir(), uuidv1() + '.tgz');
+ await downloadFile(url, tmpFile);
+ return tmpFile;
+}
+
+module.exports = download;
diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js
index 521ca41c2..135af5ab8 100644
--- a/packages/api/src/shell/index.js
+++ b/packages/api/src/shell/index.js
@@ -14,6 +14,7 @@ const collectorWriter = require('./collectorWriter');
const finalizer = require('./finalizer');
const registerPlugins = require('./registerPlugins');
const requirePlugin = require('./requirePlugin');
+const download = require('./download');
const dbgateApi = {
queryReader,
@@ -31,6 +32,7 @@ const dbgateApi = {
collectorWriter,
finalizer,
registerPlugins,
+ download,
};
requirePlugin.initialize(dbgateApi);
diff --git a/packages/api/src/utility/downloadPackage.js b/packages/api/src/utility/downloadPackage.js
index 37a5ea661..581e62f05 100644
--- a/packages/api/src/utility/downloadPackage.js
+++ b/packages/api/src/utility/downloadPackage.js
@@ -8,6 +8,7 @@ const zlib = require('zlib');
const tar = require('tar');
const ncp = require('ncp').ncp;
const { uploadsdir } = require('./directories');
+const { downloadFile } = require('./downloader');
function extractTarball(tmpFile, destination) {
return new Promise((resolve, reject) => {
@@ -19,13 +20,6 @@ function extractTarball(tmpFile, destination) {
});
}
-function saveStreamToFile(pipedStream, fileName) {
- return new Promise((resolve, reject) => {
- const fileStream = fs.createWriteStream(fileName);
- fileStream.on('close', () => resolve());
- pipedStream.pipe(fileStream);
- });
-}
function copyDirectory(source, target) {
return new Promise((resolve, reject) => {
@@ -46,13 +40,7 @@ async function downloadPackage(packageName, directory) {
const tarball = infoResp.data.versions[latest].dist.tarball;
const tmpFile = path.join(uploadsdir(), uuidv1() + '.tgz');
- console.log(`Downloading tarball ${tarball} into ${tmpFile}`);
- const tarballResp = await axios.default({
- method: 'get',
- url: tarball,
- responseType: 'stream',
- });
- await saveStreamToFile(tarballResp.data, tmpFile);
+ downloadFile(tarball, tmpFile);
const tmpDir = path.join(uploadsdir(), uuidv1());
fs.mkdirSync(tmpDir);
await extractTarball(tmpFile, tmpDir);
diff --git a/packages/api/src/utility/downloader.js b/packages/api/src/utility/downloader.js
new file mode 100644
index 000000000..8a4dec76e
--- /dev/null
+++ b/packages/api/src/utility/downloader.js
@@ -0,0 +1,25 @@
+const axios = require('axios');
+const fs = require('fs');
+
+function saveStreamToFile(pipedStream, fileName) {
+ return new Promise((resolve, reject) => {
+ const fileStream = fs.createWriteStream(fileName);
+ fileStream.on('close', () => resolve());
+ pipedStream.pipe(fileStream);
+ });
+}
+
+async function downloadFile(url, file) {
+ console.log(`Downloading ${url} into ${file}`);
+ const tarballResp = await axios.default({
+ method: 'get',
+ url,
+ responseType: 'stream',
+ });
+ await saveStreamToFile(tarballResp.data, file);
+}
+
+module.exports = {
+ saveStreamToFile,
+ downloadFile,
+};
diff --git a/packages/web/src/impexp/ImportExportConfigurator.js b/packages/web/src/impexp/ImportExportConfigurator.js
index 9d735f068..e2bf92134 100644
--- a/packages/web/src/impexp/ImportExportConfigurator.js
+++ b/packages/web/src/impexp/ImportExportConfigurator.js
@@ -12,14 +12,13 @@ import {
FormArchiveFolderSelect,
FormArchiveFilesSelect,
} from '../utility/forms';
-import { useArchiveFiles, useConnectionInfo, useDatabaseInfo, useInstalledPlugins } from '../utility/metadataLoaders';
+import { useArchiveFiles, useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders';
import TableControl, { TableColumn } from '../utility/TableControl';
import { TextField, SelectField, CheckboxField } from '../utility/inputs';
import { createPreviewReader, getActionOptions, getTargetName } from './createImpExpScript';
import getElectron from '../utility/getElectron';
import ErrorInfo from '../widgets/ErrorInfo';
import getAsArray from '../utility/getAsArray';
-import axios from '../utility/axios';
import LoadingInfo from '../widgets/LoadingInfo';
import SqlEditor from '../sqleditor/SqlEditor';
import { useUploadsProvider } from '../utility/UploadsProvider';
@@ -29,6 +28,8 @@ import { findFileFormat, getFileFormatDirections } from '../utility/fileformats'
import FormArgumentList from '../utility/FormArgumentList';
import useExtensions from '../utility/useExtensions';
import UploadButton from '../utility/UploadButton';
+import useShowModal from '../modals/showModal';
+import ChangeDownloadUrlModal from '../modals/ChangeDownloadUrlModal';
const Container = styled.div`
// max-height: 50vh;
@@ -91,6 +92,10 @@ const Title = styled.div`
margin: 10px 0px;
`;
+const ButtonsLine = styled.div`
+ display: flex;
+`;
+
function getFileFilters(extensions, storageType) {
const res = [];
const format = findFileFormat(extensions, storageType);
@@ -104,6 +109,7 @@ async function addFilesToSourceListDefault(file, newSources, newValues) {
newSources.push(sourceName);
newValues[`sourceFile_${sourceName}`] = {
fileName: file.full,
+ downloadUrl: file.url,
};
}
@@ -168,12 +174,47 @@ function ElectronFilesInput() {
);
}
-function FilesInput() {
+function extractUrlName(url) {
+ const match = url.match(/\/([^/]+)($|\?)/);
+ if (match) {
+ const res = match[1];
+ if (res.includes('.')) {
+ return res.slice(0, res.indexOf('.'));
+ }
+ return res;
+ }
+ return 'url';
+}
+
+function FilesInput({ setPreviewSource = undefined }) {
const theme = useTheme();
const electron = getElectron();
+ const showModal = useShowModal();
+ const { values, setValues } = useFormikContext();
+ const extensions = useExtensions();
+ const doAddUrl = (url) => {
+ addFilesToSourceList(
+ extensions,
+ [
+ {
+ url,
+ name: extractUrlName(url),
+ },
+ ],
+ values,
+ setValues,
+ null,
+ setPreviewSource
+ );
+ };
+ const handleAddUrl = () =>
+ showModal((modalState) => );
return (
<>
- {electron ? : }
+
+ {electron ? : }
+
+
Drag & drop imported files here
>
);
@@ -188,6 +229,7 @@ function SourceTargetConfig({
schemaNameField,
tablesField = undefined,
engine = undefined,
+ setPreviewSource = undefined,
}) {
const extensions = useExtensions();
const theme = useTheme();
@@ -314,7 +356,7 @@ function SourceTargetConfig({
>
)}
- {!!format && direction == 'source' && }
+ {!!format && direction == 'source' && }
{format && format.args && (
{
- console.log('UPLOAD', extensions);
addFilesToSourceList(
extensions,
[
@@ -430,6 +471,7 @@ export default function ImportExportConfigurator({ uploadedFile = undefined, onC
schemaNameField="sourceSchemaName"
tablesField="sourceList"
engine={sourceEngine}
+ setPreviewSource={setPreviewSource}
/>
diff --git a/packages/web/src/impexp/ScriptWriter.js b/packages/web/src/impexp/ScriptWriter.js
index acd751ff3..832a7f7f3 100644
--- a/packages/web/src/impexp/ScriptWriter.js
+++ b/packages/web/src/impexp/ScriptWriter.js
@@ -20,7 +20,20 @@ export default class ScriptWriter {
}
assign(variableName, functionName, props) {
- this.put(`const ${variableName} = await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});`);
+ if (props.downloadUrl) {
+ const fileNameVar = this.allocVariable();
+ this.put(`const ${fileNameVar} = await dbgateApi.download(${JSON.stringify(props.downloadUrl)});`);
+ this.put(
+ `const ${variableName} = await ${extractShellApiFunctionName(functionName)}(
+ Object.assign(${JSON.stringify(props)}, {
+ fileName: ${fileNameVar},
+ downloadUrl: undefined
+ })
+ );`
+ );
+ } else {
+ this.put(`const ${variableName} = await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});`);
+ }
this.packageNames.push(...extractShellApiPlugins(functionName, props));
}
@@ -42,6 +55,12 @@ export default class ScriptWriter {
// this.comment(JSON.stringify(this.engines));
// }
const packageNames = this.packageNames;
- return _.uniq(packageNames).map((packageName) => `// @require ${packageName}\n`).join('') + '\n' + this.s;
+ return (
+ _.uniq(packageNames)
+ .map((packageName) => `// @require ${packageName}\n`)
+ .join('') +
+ '\n' +
+ this.s
+ );
}
}
diff --git a/packages/web/src/modals/ChangeDownloadUrlModal.js b/packages/web/src/modals/ChangeDownloadUrlModal.js
new file mode 100644
index 000000000..0ef4fe1d7
--- /dev/null
+++ b/packages/web/src/modals/ChangeDownloadUrlModal.js
@@ -0,0 +1,39 @@
+import React from 'react';
+import axios from '../utility/axios';
+import ModalBase from './ModalBase';
+import { FormButtonRow, FormButton, FormTextField, FormSelectField, FormSubmit } from '../utility/forms';
+import { TextField } from '../utility/inputs';
+import { Formik, Form } from 'formik';
+import { useSetSavedSqlFiles } from '../utility/globalState';
+import ModalHeader from './ModalHeader';
+import ModalContent from './ModalContent';
+import ModalFooter from './ModalFooter';
+import FormStyledButton from '../widgets/FormStyledButton';
+// import FormikForm from '../utility/FormikForm';
+
+export default function ChangeDownloadUrlModal({ modalState, url = '', onConfirm = undefined }) {
+ const textFieldRef = React.useRef(null);
+ React.useEffect(() => {
+ if (textFieldRef.current) textFieldRef.current.focus();
+ }, [textFieldRef.current]);
+ const handleSubmit = async (values) => {
+ onConfirm(values.url);
+ modalState.close();
+ };
+ return (
+
+ Dwonload imported file from web
+
+
+
+
+ );
+}