export/import column map support

packages/api/src/shell/copyStream.js

@@ -1,18 +1,47 @@
 const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
+const Stream = require('stream');
+const ColumnMapTransformStream = require('../utility/ColumnMapTransformStream');
+
+function copyStream(input, output, options) {
+  const { columns } = options || {};
+
+  const transforms = [];
+  if (columns) {
+    transforms.push(new ColumnMapTransformStream(columns));
+  }
+  if (output.requireFixedStructure) {
+    transforms.push(new EnsureStreamHeaderStream());
+  }
+
+  // return new Promise((resolve, reject) => {
+  //   Stream.pipeline(input, ...transforms, output, err => {
+  //     if (err) {
+  //       reject(err);
+  //     } else {
+  //       resolve();
+  //     }
+  //   });
+  // });
 
-function copyStream(input, output) {
   return new Promise((resolve, reject) => {
     const finisher = output['finisher'] || output;
     finisher.on('finish', resolve);
     finisher.on('error', reject);
 
-    if (output.requireFixedStructure) {
-      const ensureHeader = new EnsureStreamHeaderStream();
-      input.pipe(ensureHeader);
-      ensureHeader.pipe(output);
-    } else {
-      input.pipe(output);
+    let lastStream = input;
+    for (const tran of transforms) {
+      lastStream.pipe(tran);
+      lastStream = tran;
     }
+    lastStream.pipe(output);
+
+    //  if (output.requireFixedStructure) {
+    //   const ensureHeader = new EnsureStreamHeaderStream();
+    //   input.pipe(ensureHeader);
+    //   ensureHeader.pipe(output);
+    // } else {
+    //   input.pipe(output);
+    // }
   });
 }
 

packages/api/src/utility/ColumnMapTransformStream.js

@@ -0,0 +1,21 @@
+const stream = require('stream');
+const { transformRowUsingColumnMap } = require('dbgate-tools');
+
+class ColumnMapTransformStream extends stream.Transform {
+  constructor(columns) {
+    super({ objectMode: true });
+    this.columns = columns;
+  }
+  _transform(chunk, encoding, done) {
+    if (chunk.__isStreamHeader) {
+      // skip stream header
+      done();
+      return;
+    }
+
+    this.push(transformRowUsingColumnMap(chunk, this.columns));
+    done();
+  }
+}
+
+module.exports = ColumnMapTransformStream;

packages/tools/src/tableTransforms.ts

@@ -1,5 +1,7 @@
 import { TableInfo } from 'dbgate-types';
 import _cloneDeep from 'lodash/cloneDeep';
+import _fromPairs from 'lodash/fromPairs';
+import _get from 'lodash/get';
 
 export function prepareTableForImport(table: TableInfo): TableInfo {
   const res = _cloneDeep(table);
@@ -10,3 +12,12 @@ export function prepareTableForImport(table: TableInfo): TableInfo {
   if (res.primaryKey) res.primaryKey.constraintName = null;
   return res;
 }
+
+interface TransformColumnDefinition {
+  src: string;
+  dst: string;
+}
+
+export function transformRowUsingColumnMap(row, columns: TransformColumnDefinition[]) {
+  return _fromPairs(columns.map(col => [col.dst, _get(row, col.src)]));
+}

packages/web/src/elements/TableControl.svelte

@@ -24,6 +24,7 @@
   export let selectedIndex = 0;
   export let clickable = false;
   export let disableFocusOutline = false;
+  export let emptyMessage = null;
 
   export let domTable = undefined;
 
@@ -99,6 +100,11 @@
         {/each}
       </tr>
     {/each}
+    {#if emptyMessage && rows.length == 0}
+      <tr>
+        <td colspan={columnList.length}>{emptyMessage}</td>
+      </tr>
+    {/if}
   </tbody>
 </table>
 

packages/web/src/forms/TextField.svelte

@@ -2,8 +2,8 @@
   import { onMount } from 'svelte';
 
   export let value;
-  export let focused;
-  export let domEditor;
+  export let focused = false;
+  export let domEditor = undefined;
 
   if (focused) onMount(() => domEditor.focus());
 </script>

packages/web/src/impexp/ImportExportConfigurator.svelte

@@ -42,11 +42,14 @@
 <script lang="ts">
   import { onMount } from 'svelte';
   import { writable } from 'svelte/store';
+  import Link from '../elements/Link.svelte';
   import TableControl from '../elements/TableControl.svelte';
   import CheckboxField from '../forms/CheckboxField.svelte';
   import { getFormContext } from '../forms/FormProviderCore.svelte';
   import TextField from '../forms/TextField.svelte';
   import FontIcon from '../icons/FontIcon.svelte';
+  import ColumnMapModal from '../modals/ColumnMapModal.svelte';
+  import { showModal } from '../modals/modalTools';
   import { findFileFormat } from '../plugins/fileformats';
   import { extensions } from '../stores';
   import getAsArray from '../utility/getAsArray';
@@ -189,6 +192,11 @@
           header: 'Preview',
           slot: 0,
         },
+        {
+          fieldName: 'columns',
+          header: 'Columns',
+          slot: 2,
+        },
       ]}
     >
       <svelte:fragment slot="0" let:row>
@@ -214,6 +222,18 @@
             )}
         />
       </svelte:fragment>
+      <svelte:fragment slot="2" let:row>
+        {@const columnCount = ($values[`columns_${row}`] || []).filter(x => !x.skip).length}
+        <Link
+          onClick={() => {
+            showModal(ColumnMapModal, {
+              value: $values[`columns_${row}`],
+              onConfirm: value => setFieldValue(`columns_${row}`, value),
+            });
+          }}
+          >{columnCount > 0 ? `(${columnCount} columns)` : '(copy from source)'}
+        </Link>
+      </svelte:fragment>
     </TableControl>
   </div>
 </div>

packages/web/src/impexp/ScriptWriter.ts

@@ -25,12 +25,20 @@ export default class ScriptWriter {
     this.packageNames.push(...extractShellApiPlugins(functionName, props));
   }
 
+  assignValue(variableName, jsonValue) {
+    this.put(`const ${variableName} = ${JSON.stringify(jsonValue)};`);
+  }
+
   requirePackage(packageName) {
     this.packageNames.push(packageName);
   }
 
-  copyStream(sourceVar, targetVar) {
-    this.put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar});`);
+  copyStream(sourceVar, targetVar, colmapVar) {
+    if (colmapVar) {
+      this.put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar}, {columns: ${colmapVar}});`);
+    } else {
+      this.put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar});`);
+    }
   }
 
   comment(s) {

packages/web/src/impexp/createImpExpScript.ts

@@ -186,7 +186,14 @@ export default async function createImpExpScript(extensions, values, addEditorIn
     // @ts-ignore
     script.assign(targetVar, ...getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver));
 
-    script.copyStream(sourceVar, targetVar);
+    const colmap = (values[`columns_${sourceName}`] || []).filter(x => !x.skip);
+    let colmapVar = null;
+    if (colmap.length > 0) {
+      colmapVar = script.allocVariable();
+      script.assignValue(colmapVar, colmap);
+    }
+
+    script.copyStream(sourceVar, targetVar, colmapVar);
     script.put();
   }
   if (addEditorInfo) {

packages/web/src/modals/ColumnMapModal.svelte

@@ -0,0 +1,88 @@
+<script lang="ts">
+  import FormStyledButton from '../buttons/FormStyledButton.svelte';
+  import Link from '../elements/Link.svelte';
+  import TableControl from '../elements/TableControl.svelte';
+  import CheckboxField from '../forms/CheckboxField.svelte';
+
+  import FormProvider from '../forms/FormProvider.svelte';
+  import FormSubmit from '../forms/FormSubmit.svelte';
+  import TextField from '../forms/TextField.svelte';
+  import ModalBase from './ModalBase.svelte';
+  import { closeCurrentModal } from './modalTools';
+
+  export let header = 'Configure columns';
+  export let onConfirm;
+  export let value = [];
+</script>
+
+<FormProvider>
+  <ModalBase {...$$restProps}>
+    <div slot="header">{header}</div>
+
+    <div class="m-3">
+      When no columns are defined in this mapping, source row is copied to target without any modifications
+    </div>
+
+    <TableControl
+      columns={[
+        { fieldName: 'use', header: 'Use', slot: 4 },
+        { fieldName: 'src', header: 'Source column', slot: 1 },
+        { fieldName: 'dst', header: 'Target column', slot: 2 },
+        { fieldName: 'actions', header: '', slot: 3 },
+      ]}
+      rows={value || []}
+      emptyMessage="No transform defined"
+    >
+      <svelte:fragment slot="4" let:row let:index>
+        <CheckboxField
+          checked={!row['skip']}
+          on:change={e => (value = (value || []).map((x, i) => (i == index ? { ...x, skip: !e.target.checked } : x)))}
+        />
+      </svelte:fragment>
+      <svelte:fragment slot="1" let:row let:index>
+        <TextField
+          value={row['src']}
+          on:change={e => (value = (value || []).map((x, i) => (i == index ? { ...x, src: e.target.value } : x)))}
+        />
+      </svelte:fragment>
+      <svelte:fragment slot="2" let:row let:index>
+        <TextField
+          value={row['dst']}
+          on:change={e => (value = (value || []).map((x, i) => (i == index ? { ...x, dst: e.target.value } : x)))}
+        />
+      </svelte:fragment>
+      <svelte:fragment slot="3" let:index>
+        <Link
+          onClick={() => {
+            value = value.filter((x, i) => i != index);
+          }}>Remove</Link
+        >
+      </svelte:fragment>
+    </TableControl>
+
+    <svelte:fragment slot="footer">
+      <FormSubmit
+        value="OK"
+        on:click={() => {
+          closeCurrentModal();
+          onConfirm(!value || value.length == 0 ? null : value);
+        }}
+      />
+      <FormStyledButton type="button" value="Close" on:click={closeCurrentModal} />
+      <FormStyledButton
+        type="button"
+        value="Add column"
+        on:click={() => {
+          value = [...(value || []), {}];
+        }}
+      />
+      <FormStyledButton
+        type="button"
+        value="Reset"
+        on:click={() => {
+          value = [];
+        }}
+      />
+    </svelte:fragment>
+  </ModalBase>
+</FormProvider>