import - import files from URL

packages/api/src/controllers/runners.js

@@ -38,7 +38,16 @@ const dbgateApi = require(process.env.DBGATE_API);
 ${requirePluginsTemplate(extractShellApiPlugins(functionName, props))}
 require=null;
 async function run() {
-const reader=await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});
+${
+  props.downloadUrl
+    ? `
+    const downloaded=await dbgateApi.download(${JSON.stringify(props.downloadUrl)});
+    const reader=await ${extractShellApiFunctionName(functionName)}(Object.assign(${JSON.stringify(
+        props
+      )}, { fileName: downloaded, downloadUrl: undefined }));
+    `
+    : `const reader=await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});`
+}
 const writer=await dbgateApi.collectorWriter({runid: '${runid}'});
 await dbgateApi.copyStream(reader, writer);
 }

packages/api/src/shell/download.js

@@ -0,0 +1,12 @@
+const path = require('path');
+const uuidv1 = require('uuid/v1');
+const { uploadsdir } = require('../utility/directories');
+const { downloadFile } = require('../utility/downloader');
+
+async function download(url) {
+  const tmpFile = path.join(uploadsdir(), uuidv1() + '.tgz');
+  await downloadFile(url, tmpFile);
+  return tmpFile;
+}
+
+module.exports = download;

packages/api/src/shell/index.js

@@ -14,6 +14,7 @@ const collectorWriter = require('./collectorWriter');
 const finalizer = require('./finalizer');
 const registerPlugins = require('./registerPlugins');
 const requirePlugin = require('./requirePlugin');
+const download = require('./download');
 
 const dbgateApi = {
   queryReader,
@@ -31,6 +32,7 @@ const dbgateApi = {
   collectorWriter,
   finalizer,
   registerPlugins,
+  download,
 };
 
 requirePlugin.initialize(dbgateApi);

packages/api/src/utility/downloadPackage.js

@@ -8,6 +8,7 @@ const zlib = require('zlib');
 const tar = require('tar');
 const ncp = require('ncp').ncp;
 const { uploadsdir } = require('./directories');
+const { downloadFile } = require('./downloader');
 
 function extractTarball(tmpFile, destination) {
   return new Promise((resolve, reject) => {
@@ -19,13 +20,6 @@ function extractTarball(tmpFile, destination) {
   });
 }
 
-function saveStreamToFile(pipedStream, fileName) {
-  return new Promise((resolve, reject) => {
-    const fileStream = fs.createWriteStream(fileName);
-    fileStream.on('close', () => resolve());
-    pipedStream.pipe(fileStream);
-  });
-}
 
 function copyDirectory(source, target) {
   return new Promise((resolve, reject) => {
@@ -46,13 +40,7 @@ async function downloadPackage(packageName, directory) {
   const tarball = infoResp.data.versions[latest].dist.tarball;
 
   const tmpFile = path.join(uploadsdir(), uuidv1() + '.tgz');
-  console.log(`Downloading tarball ${tarball} into ${tmpFile}`);
-  const tarballResp = await axios.default({
-    method: 'get',
-    url: tarball,
-    responseType: 'stream',
-  });
-  await saveStreamToFile(tarballResp.data, tmpFile);
+  downloadFile(tarball, tmpFile);
   const tmpDir = path.join(uploadsdir(), uuidv1());
   fs.mkdirSync(tmpDir);
   await extractTarball(tmpFile, tmpDir);

packages/api/src/utility/downloader.js

@@ -0,0 +1,25 @@
+const axios = require('axios');
+const fs = require('fs');
+
+function saveStreamToFile(pipedStream, fileName) {
+  return new Promise((resolve, reject) => {
+    const fileStream = fs.createWriteStream(fileName);
+    fileStream.on('close', () => resolve());
+    pipedStream.pipe(fileStream);
+  });
+}
+
+async function downloadFile(url, file) {
+  console.log(`Downloading ${url} into ${file}`);
+  const tarballResp = await axios.default({
+    method: 'get',
+    url,
+    responseType: 'stream',
+  });
+  await saveStreamToFile(tarballResp.data, file);
+}
+
+module.exports = {
+  saveStreamToFile,
+  downloadFile,
+};

packages/web/src/impexp/ImportExportConfigurator.js

@@ -12,14 +12,13 @@ import {
   FormArchiveFolderSelect,
   FormArchiveFilesSelect,
 } from '../utility/forms';
-import { useArchiveFiles, useConnectionInfo, useDatabaseInfo, useInstalledPlugins } from '../utility/metadataLoaders';
+import { useArchiveFiles, useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders';
 import TableControl, { TableColumn } from '../utility/TableControl';
 import { TextField, SelectField, CheckboxField } from '../utility/inputs';
 import { createPreviewReader, getActionOptions, getTargetName } from './createImpExpScript';
 import getElectron from '../utility/getElectron';
 import ErrorInfo from '../widgets/ErrorInfo';
 import getAsArray from '../utility/getAsArray';
-import axios from '../utility/axios';
 import LoadingInfo from '../widgets/LoadingInfo';
 import SqlEditor from '../sqleditor/SqlEditor';
 import { useUploadsProvider } from '../utility/UploadsProvider';
@@ -29,6 +28,8 @@ import { findFileFormat, getFileFormatDirections } from '../utility/fileformats'
 import FormArgumentList from '../utility/FormArgumentList';
 import useExtensions from '../utility/useExtensions';
 import UploadButton from '../utility/UploadButton';
+import useShowModal from '../modals/showModal';
+import ChangeDownloadUrlModal from '../modals/ChangeDownloadUrlModal';
 
 const Container = styled.div`
   // max-height: 50vh;
@@ -91,6 +92,10 @@ const Title = styled.div`
   margin: 10px 0px;
 `;
 
+const ButtonsLine = styled.div`
+  display: flex;
+`;
+
 function getFileFilters(extensions, storageType) {
   const res = [];
   const format = findFileFormat(extensions, storageType);
@@ -104,6 +109,7 @@ async function addFilesToSourceListDefault(file, newSources, newValues) {
   newSources.push(sourceName);
   newValues[`sourceFile_${sourceName}`] = {
     fileName: file.full,
+    downloadUrl: file.url,
   };
 }
 
@@ -168,12 +174,47 @@ function ElectronFilesInput() {
   );
 }
 
-function FilesInput() {
+function extractUrlName(url) {
+  const match = url.match(/\/([^/]+)($|\?)/);
+  if (match) {
+    const res = match[1];
+    if (res.includes('.')) {
+      return res.slice(0, res.indexOf('.'));
+    }
+    return res;
+  }
+  return 'url';
+}
+
+function FilesInput({ setPreviewSource = undefined }) {
   const theme = useTheme();
   const electron = getElectron();
+  const showModal = useShowModal();
+  const { values, setValues } = useFormikContext();
+  const extensions = useExtensions();
+  const doAddUrl = (url) => {
+    addFilesToSourceList(
+      extensions,
+      [
+        {
+          url,
+          name: extractUrlName(url),
+        },
+      ],
+      values,
+      setValues,
+      null,
+      setPreviewSource
+    );
+  };
+  const handleAddUrl = () =>
+    showModal((modalState) => <ChangeDownloadUrlModal modalState={modalState} onConfirm={doAddUrl} />);
   return (
     <>
-      {electron ? <ElectronFilesInput /> : <UploadButton />}
+      <ButtonsLine>
+        {electron ? <ElectronFilesInput /> : <UploadButton />}
+        <FormStyledButton value="Add web URL" onClick={handleAddUrl} />
+      </ButtonsLine>
       <DragWrapper theme={theme}>Drag &amp; drop imported files here</DragWrapper>
     </>
   );
@@ -188,6 +229,7 @@ function SourceTargetConfig({
   schemaNameField,
   tablesField = undefined,
   engine = undefined,
+  setPreviewSource = undefined,
 }) {
   const extensions = useExtensions();
   const theme = useTheme();
@@ -314,7 +356,7 @@ function SourceTargetConfig({
         </>
       )}
 
-      {!!format && direction == 'source' && <FilesInput />}
+      {!!format && direction == 'source' && <FilesInput setPreviewSource={setPreviewSource} />}
 
       {format && format.args && (
         <FormArgumentList
@@ -359,7 +401,6 @@ export default function ImportExportConfigurator({ uploadedFile = undefined, onC
 
   const handleUpload = React.useCallback(
     (file) => {
-      console.log('UPLOAD', extensions);
       addFilesToSourceList(
         extensions,
         [
@@ -430,6 +471,7 @@ export default function ImportExportConfigurator({ uploadedFile = undefined, onC
           schemaNameField="sourceSchemaName"
           tablesField="sourceList"
           engine={sourceEngine}
+          setPreviewSource={setPreviewSource}
         />
         <ArrowWrapper theme={theme}>
           <FontIcon icon="icon arrow-right" />

packages/web/src/impexp/ScriptWriter.js

@@ -20,7 +20,20 @@ export default class ScriptWriter {
   }
 
   assign(variableName, functionName, props) {
-    this.put(`const ${variableName} = await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});`);
+    if (props.downloadUrl) {
+      const fileNameVar = this.allocVariable();
+      this.put(`const ${fileNameVar} = await dbgateApi.download(${JSON.stringify(props.downloadUrl)});`);
+      this.put(
+        `const ${variableName} = await ${extractShellApiFunctionName(functionName)}(
+          Object.assign(${JSON.stringify(props)}, {
+            fileName: ${fileNameVar},
+            downloadUrl: undefined
+          })
+        );`
+      );
+    } else {
+      this.put(`const ${variableName} = await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});`);
+    }
     this.packageNames.push(...extractShellApiPlugins(functionName, props));
   }
 
@@ -42,6 +55,12 @@ export default class ScriptWriter {
     //   this.comment(JSON.stringify(this.engines));
     // }
     const packageNames = this.packageNames;
-    return _.uniq(packageNames).map((packageName) => `// @require ${packageName}\n`).join('') + '\n' + this.s;
+    return (
+      _.uniq(packageNames)
+        .map((packageName) => `// @require ${packageName}\n`)
+        .join('') +
+      '\n' +
+      this.s
+    );
   }
 }

packages/web/src/modals/ChangeDownloadUrlModal.js

@@ -0,0 +1,39 @@
+import React from 'react';
+import axios from '../utility/axios';
+import ModalBase from './ModalBase';
+import { FormButtonRow, FormButton, FormTextField, FormSelectField, FormSubmit } from '../utility/forms';
+import { TextField } from '../utility/inputs';
+import { Formik, Form } from 'formik';
+import { useSetSavedSqlFiles } from '../utility/globalState';
+import ModalHeader from './ModalHeader';
+import ModalContent from './ModalContent';
+import ModalFooter from './ModalFooter';
+import FormStyledButton from '../widgets/FormStyledButton';
+// import FormikForm from '../utility/FormikForm';
+
+export default function ChangeDownloadUrlModal({ modalState, url = '', onConfirm = undefined }) {
+  const textFieldRef = React.useRef(null);
+  React.useEffect(() => {
+    if (textFieldRef.current) textFieldRef.current.focus();
+  }, [textFieldRef.current]);
+  const handleSubmit = async (values) => {
+    onConfirm(values.url);
+    modalState.close();
+  };
+  return (
+    <ModalBase modalState={modalState}>
+      <ModalHeader modalState={modalState}>Dwonload imported file from web</ModalHeader>
+      <Formik onSubmit={handleSubmit} initialValues={{ url }}>
+        <Form>
+          <ModalContent>
+            <FormTextField label="URL" name="url" editorRef={textFieldRef} style={{ width: '30vw' }} />
+          </ModalContent>
+          <ModalFooter>
+            <FormSubmit text="OK" />
+            <FormStyledButton value="Cancel" onClick={() => modalState.close()} />
+          </ModalFooter>
+        </Form>
+      </Formik>
+    </ModalBase>
+  );
+}