import - import files from URL

packages/api/src/controllers/runners.js

@@ -38,7 +38,16 @@ const dbgateApi = require(process.env.DBGATE_API);
 ${requirePluginsTemplate(extractShellApiPlugins(functionName, props))}
 require=null;
 async function run() {
-const reader=await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});
+${
+  props.downloadUrl
+    ? `
+    const downloaded=await dbgateApi.download(${JSON.stringify(props.downloadUrl)});
+    const reader=await ${extractShellApiFunctionName(functionName)}(Object.assign(${JSON.stringify(
+        props
+      )}, { fileName: downloaded, downloadUrl: undefined }));
+    `
+    : `const reader=await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});`
+}
 const writer=await dbgateApi.collectorWriter({runid: '${runid}'});
 await dbgateApi.copyStream(reader, writer);
 }

packages/api/src/shell/download.js

@@ -0,0 +1,12 @@
+const path = require('path');
+const uuidv1 = require('uuid/v1');
+const { uploadsdir } = require('../utility/directories');
+const { downloadFile } = require('../utility/downloader');
+
+async function download(url) {
+  const tmpFile = path.join(uploadsdir(), uuidv1() + '.tgz');
+  await downloadFile(url, tmpFile);
+  return tmpFile;
+}
+
+module.exports = download;

packages/api/src/shell/index.js

@@ -14,6 +14,7 @@ const collectorWriter = require('./collectorWriter');
 const finalizer = require('./finalizer');
 const registerPlugins = require('./registerPlugins');
 const requirePlugin = require('./requirePlugin');
+const download = require('./download');
 
 const dbgateApi = {
   queryReader,
@@ -31,6 +32,7 @@ const dbgateApi = {
   collectorWriter,
   finalizer,
   registerPlugins,
+  download,
 };
 
 requirePlugin.initialize(dbgateApi);

packages/api/src/utility/downloadPackage.js

@@ -8,6 +8,7 @@ const zlib = require('zlib');
 const tar = require('tar');
 const ncp = require('ncp').ncp;
 const { uploadsdir } = require('./directories');
+const { downloadFile } = require('./downloader');
 
 function extractTarball(tmpFile, destination) {
   return new Promise((resolve, reject) => {
@@ -19,13 +20,6 @@ function extractTarball(tmpFile, destination) {
   });
 }
 
-function saveStreamToFile(pipedStream, fileName) {
-  return new Promise((resolve, reject) => {
-    const fileStream = fs.createWriteStream(fileName);
-    fileStream.on('close', () => resolve());
-    pipedStream.pipe(fileStream);
-  });
-}
 
 function copyDirectory(source, target) {
   return new Promise((resolve, reject) => {
@@ -46,13 +40,7 @@ async function downloadPackage(packageName, directory) {
   const tarball = infoResp.data.versions[latest].dist.tarball;
 
   const tmpFile = path.join(uploadsdir(), uuidv1() + '.tgz');
-  console.log(`Downloading tarball ${tarball} into ${tmpFile}`);
-  const tarballResp = await axios.default({
-    method: 'get',
-    url: tarball,
-    responseType: 'stream',
-  });
-  await saveStreamToFile(tarballResp.data, tmpFile);
+  downloadFile(tarball, tmpFile);
   const tmpDir = path.join(uploadsdir(), uuidv1());
   fs.mkdirSync(tmpDir);
   await extractTarball(tmpFile, tmpDir);

packages/api/src/utility/downloader.js

@@ -0,0 +1,25 @@
+const axios = require('axios');
+const fs = require('fs');
+
+function saveStreamToFile(pipedStream, fileName) {
+  return new Promise((resolve, reject) => {
+    const fileStream = fs.createWriteStream(fileName);
+    fileStream.on('close', () => resolve());
+    pipedStream.pipe(fileStream);
+  });
+}
+
+async function downloadFile(url, file) {
+  console.log(`Downloading ${url} into ${file}`);
+  const tarballResp = await axios.default({
+    method: 'get',
+    url,
+    responseType: 'stream',
+  });
+  await saveStreamToFile(tarballResp.data, file);
+}
+
+module.exports = {
+  saveStreamToFile,
+  downloadFile,
+};