download fileat first in imports

packages/api/src/shell/jsonLinesReader.js

@@ -2,6 +2,7 @@ const fs = require('fs');
 const stream = require('stream');
 const byline = require('byline');
 const { getLogger } = require('dbgate-tools');
+const download = require('./download');
 const logger = getLogger('jsonLinesReader');
 
 class ParseStream extends stream.Transform {
@@ -35,8 +36,10 @@ class ParseStream extends stream.Transform {
 async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
   logger.info(`Reading file ${fileName}`);
 
+  const downloadedFile = await download(fileName);
+
   const fileStream = fs.createReadStream(
-    fileName,
+    downloadedFile,
     // @ts-ignore
     encoding
   );

packages/api/src/shell/jsonReader.js

@@ -6,16 +6,18 @@ const { parser } = require('stream-json');
 const { pick } = require('stream-json/filters/Pick');
 const { streamArray } = require('stream-json/streamers/StreamArray');
 const { streamObject } = require('stream-json/streamers/StreamObject');
+const download = require('./download');
 
 const logger = getLogger('jsonReader');
 
+
 class ParseStream extends stream.Transform {
   constructor({ limitRows, jsonStyle, keyField }) {
     super({ objectMode: true });
     this.wasHeader = false;
     this.limitRows = limitRows;
     this.jsonStyle = jsonStyle;
-    this.keyField = keyField;
+    this.keyField = keyField || '_key';
     this.rowsWritten = 0;
   }
   _transform(chunk, encoding, done) {
@@ -53,8 +55,9 @@ async function jsonReader({
 }) {
   logger.info(`Reading file ${fileName}`);
 
+  const downloadedFile = await download(fileName);
   const fileStream = fs.createReadStream(
-    fileName,
+    downloadedFile,
     // @ts-ignore
     encoding
   );

packages/api/src/shell/jsonWriter.js

@@ -11,12 +11,11 @@ class StringifyStream extends stream.Transform {
     this.wasHeader = false;
     this.wasRecord = false;
     this.jsonStyle = jsonStyle;
-    this.keyField = keyField;
+    this.keyField = keyField || '_key';
     this.rootField = rootField;
   }
   _transform(chunk, encoding, done) {
     let skip = false;
-    const keyField = this.keyField || '_key';
 
     if (!this.wasHeader) {
       skip = chunk.__isStreamHeader;
@@ -43,8 +42,8 @@ class StringifyStream extends stream.Transform {
       this.wasRecord = true;
 
       if (this.jsonStyle === 'object') {
-        const key = chunk[keyField] ?? chunk[Object.keys(chunk)[0]];
+        const key = chunk[this.keyField] ?? chunk[Object.keys(chunk)[0]];
-        this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [keyField]))}`);
+        this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [this.keyField]))}`);
       } else {
         this.push(JSON.stringify(chunk));
       }
@@ -86,7 +85,7 @@ class StringifyStream extends stream.Transform {
   }
 }
 
-async function jsonWriter({ fileName, jsonStyle, keyField, rootField, encoding = 'utf-8' }) {
+async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, encoding = 'utf-8' }) {
   logger.info(`Writing file ${fileName}`);
   const stringify = new StringifyStream({ jsonStyle, keyField, rootField });
   const fileStream = fs.createWriteStream(fileName, encoding);

packages/web/src/impexp/FilesInput.svelte

@@ -1,6 +1,6 @@
 <script lang="ts" context="module">
   function extractUrlName(url, values) {
-    const match = url.match(/\/([^/]+)($|\?)/);
+    const match = url.match(/\/([^/\?]+)($|\?)/);
     if (match) {
       const res = match[1];
       if (res.includes('.')) {

packages/web/src/impexp/ImportExportConfigurator.svelte

@@ -46,7 +46,6 @@
 </script>
 
 <script lang="ts">
-  import { onMount } from 'svelte';
   import { writable } from 'svelte/store';
   import Link from '../elements/Link.svelte';
   import TableControl from '../elements/TableControl.svelte';

packages/web/src/modals/ChangeDownloadUrlModal.svelte

@@ -8,6 +8,7 @@
   import { closeCurrentModal } from './modalTools';
 
   export let onConfirm;
+  export let url;
 
   const handleSubmit = e => {
     onConfirm(e.detail.url);
@@ -15,7 +16,7 @@
   };
 </script>
 
-<FormProvider>
+<FormProvider initialValues={{ url }}>
   <ModalBase {...$$restProps}>
     <svelte:fragment slot="header">Download imported file from web</svelte:fragment>