import export

packages/web/src/impexp/FormConnectionSelect.svelte

@@ -0,0 +1,17 @@
+<script lang="ts">
+import FormSelectField from '../forms/FormSelectField.svelte';
+
+  import { useConnectionList } from '../utility/metadataLoaders';
+
+  $: connections = useConnectionList();
+  $: connectionOptions = ($connections || []).map(conn => ({
+    value: conn._id,
+    label: conn.displayName || conn.server,
+  }));
+</script>
+
+{#if connectionOptions.length == 0}
+  <div>Not available</div>
+{:else}
+  <FormSelectField {...$$restProps} options={connectionOptions} />
+{/if}

packages/web/src/impexp/FormDatabaseSelect.svelte

@@ -0,0 +1,21 @@
+<script lang="ts">
+  import { getFormContext } from '../forms/FormProviderCore.svelte';
+  import FormSelectField from '../forms/FormSelectField.svelte';
+  import { useDatabaseList } from '../utility/metadataLoaders';
+
+  export let conidName;
+
+  const { values } = getFormContext();
+  $: databases = useDatabaseList({ conid: $values[conidName] });
+
+  $: databaseOptions = ($databases || []).map(db => ({
+    value: db.name,
+    label: db.name,
+  }));
+</script>
+
+{#if databaseOptions.length == 0}
+  <div>Not available</div>
+{:else}
+  <FormSelectField {...$$restProps} options={databaseOptions} />
+{/if}

packages/web/src/impexp/ImportExportConfigurator.svelte

@@ -0,0 +1,41 @@
+<script lang="ts">
+  import FontIcon from '../icons/FontIcon.svelte';
+
+  import SourceTargetConfig from './SourceTargetConfig.svelte';
+
+  //   engine={sourceEngine}
+  //       {setPreviewSource}
+</script>
+
+<div class="flex1">
+  <div class="flex">
+    <SourceTargetConfig
+      direction="source"
+      storageTypeField="sourceStorageType"
+      connectionIdField="sourceConnectionId"
+      databaseNameField="sourceDatabaseName"
+      archiveFolderField="sourceArchiveFolder"
+      schemaNameField="sourceSchemaName"
+      tablesField="sourceList"
+    />
+    <div class="arrow">
+      <FontIcon icon="icon arrow-right" />
+    </div>
+    <SourceTargetConfig
+      direction="target"
+      storageTypeField="targetStorageType"
+      connectionIdField="targetConnectionId"
+      databaseNameField="targetDatabaseName"
+      archiveFolderField="targetArchiveFolder"
+      schemaNameField="targetSchemaName"
+    />
+  </div>
+</div>
+
+<style>
+  .arrow {
+    font-size: 30px;
+    color: var(--theme-icon-blue);
+    align-self: center;
+  }
+</style>

packages/web/src/impexp/ScriptWriter.ts

@@ -0,0 +1,50 @@
+import _ from 'lodash';
+import { extractShellApiFunctionName, extractShellApiPlugins } from 'dbgate-tools';
+
+export default class ScriptWriter {
+  s = '';
+  packageNames: string[] = [];
+  varCount = 0;
+
+  constructor(varCount = '0') {
+    this.varCount = parseInt(varCount) || 0;
+  }
+
+  allocVariable(prefix = 'var') {
+    this.varCount += 1;
+    return `${prefix}${this.varCount}`;
+  }
+
+  put(s = '') {
+    this.s += s;
+    this.s += '\n';
+  }
+
+  assign(variableName, functionName, props) {
+    this.put(`const ${variableName} = await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});`);
+    this.packageNames.push(...extractShellApiPlugins(functionName, props));
+  }
+
+  requirePackage(packageName) {
+    this.packageNames.push(packageName);
+  }
+
+  copyStream(sourceVar, targetVar) {
+    this.put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar});`);
+  }
+
+  comment(s) {
+    this.put(`// ${s}`);
+  }
+
+  getScript(schedule = null) {
+    const packageNames = this.packageNames;
+    let prefix = _.uniq(packageNames)
+      .map(packageName => `// @require ${packageName}\n`)
+      .join('');
+    if (schedule) prefix += `// @schedule ${schedule}`;
+    if (prefix) prefix += '\n';
+
+    return prefix + this.s;
+  }
+}

packages/web/src/impexp/SourceTargetConfig.svelte

@@ -0,0 +1,79 @@
+<script lang="ts">
+  import { getFormContext } from '../forms/FormProviderCore.svelte';
+  import FormSelectField from '../forms/FormSelectField.svelte';
+
+  import FontIcon from '../icons/FontIcon.svelte';
+  import { findFileFormat, getFileFormatDirections } from '../plugins/fileformats';
+  import { extensions } from '../stores';
+  import { useArchiveFiles, useDatabaseInfo } from '../utility/metadataLoaders';
+  import FormConnectionSelect from './FormConnectionSelect.svelte';
+  import FormDatabaseSelect from './FormDatabaseSelect.svelte';
+
+  export let direction;
+  export let storageTypeField;
+
+  export let connectionIdField;
+  export let databaseNameField;
+  export let archiveFolderField;
+  export let schemaNameField;
+  export let tablesField = undefined;
+  export let engine = undefined;
+
+  const { values, setFieldValue } = getFormContext();
+
+  $: types =
+    $values[storageTypeField] == 'jsldata'
+      ? [{ value: 'jsldata', label: 'Query result data', directions: ['source'] }]
+      : [
+          { value: 'database', label: 'Database', directions: ['source', 'target'] },
+          ...$extensions.fileFormats.map(format => ({
+            value: format.storageType,
+            label: `${format.name} files(s)`,
+            directions: getFileFormatDirections(format),
+          })),
+          { value: 'query', label: 'SQL Query', directions: ['source'] },
+          { value: 'archive', label: 'Archive', directions: ['source', 'target'] },
+        ];
+
+  $: storageType = $values[storageTypeField];
+  $: dbinfo = useDatabaseInfo({ conid: $values[connectionIdField], database: $values[databaseNameField] });
+  $: archiveFiles = useArchiveFiles({ folder: $values[archiveFolderField] });
+  $: format = findFileFormat($extensions, storageType);
+</script>
+
+<div class="column">
+  {#if direction == 'source'}
+    <div class="title">
+      <FontIcon icon="icon import" /> Source configuration
+    </div>
+  {/if}
+  {#if direction == 'target'}
+    <div class="title">
+      <FontIcon icon="icon export" /> Target configuration
+    </div>
+  {/if}
+
+  <FormSelectField
+    options={types.filter(x => x.directions.includes(direction))}
+    name={storageTypeField}
+    label="Storage type"
+  />
+
+  {#if storageType == 'database' || storageType == 'query'}
+    <FormConnectionSelect name={connectionIdField} label="Server" />
+    <FormDatabaseSelect conidName={connectionIdField} name={databaseNameField} label="Database" />
+  {/if}
+</div>
+
+<style>
+  .title {
+    font-size: 20px;
+    text-align: center;
+    margin: 10px 0px;
+  }
+
+  .column {
+    margin: 10px;
+    flex: 1;
+  }
+</style>

packages/web/src/impexp/createImpExpScript.ts

@@ -0,0 +1,240 @@
+import _ from 'lodash';
+import ScriptWriter from './ScriptWriter';
+import getAsArray from '../utility/getAsArray';
+import { getConnectionInfo } from '../utility/metadataLoaders';
+import { findEngineDriver, findObjectLike } from 'dbgate-tools';
+import { findFileFormat } from '../plugins/fileformats';
+
+export function getTargetName(extensions, source, values) {
+  const key = `targetName_${source}`;
+  if (values[key]) return values[key];
+  const format = findFileFormat(extensions, values.targetStorageType);
+  if (format) {
+    const res = format.getDefaultOutputName ? format.getDefaultOutputName(source, values) : null;
+    if (res) return res;
+    return `${source}.${format.extension}`;
+  }
+  return source;
+}
+
+function extractApiParameters(values, direction, format) {
+  const pairs = (format.args || [])
+    .filter(arg => arg.apiName)
+    .map(arg => [arg.apiName, values[`${direction}_${format.storageType}_${arg.name}`]])
+    .filter(x => x[1] != null);
+  return _.fromPairs(pairs);
+}
+
+async function getConnection(extensions, storageType, conid, database) {
+  if (storageType == 'database' || storageType == 'query') {
+    const conn = await getConnectionInfo({ conid });
+    const driver = findEngineDriver(conn, extensions);
+    return [
+      {
+        ..._.omit(conn, ['_id', 'displayName']),
+        database,
+      },
+      driver,
+    ];
+  }
+  return [null, null];
+}
+
+function getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver) {
+  const { sourceStorageType } = values;
+  if (sourceStorageType == 'database') {
+    const fullName = { schemaName: values.sourceSchemaName, pureName: sourceName };
+    return [
+      'tableReader',
+      {
+        connection: sourceConnection,
+        ...fullName,
+      },
+    ];
+  }
+  if (sourceStorageType == 'query') {
+    return [
+      'queryReader',
+      {
+        connection: sourceConnection,
+        sql: values.sourceSql,
+      },
+    ];
+  }
+  if (findFileFormat(extensions, sourceStorageType)) {
+    const sourceFile = values[`sourceFile_${sourceName}`];
+    const format = findFileFormat(extensions, sourceStorageType);
+    if (format && format.readerFunc) {
+      return [
+        format.readerFunc,
+        {
+          ..._.omit(sourceFile, ['isDownload']),
+          ...extractApiParameters(values, 'source', format),
+        },
+      ];
+    }
+  }
+  if (sourceStorageType == 'jsldata') {
+    return ['jslDataReader', { jslid: values.sourceJslId }];
+  }
+  if (sourceStorageType == 'archive') {
+    return [
+      'archiveReader',
+      {
+        folderName: values.sourceArchiveFolder,
+        fileName: sourceName,
+      },
+    ];
+  }
+  throw new Error(`Unknown source storage type: ${sourceStorageType}`);
+}
+
+function getFlagsFroAction(action) {
+  switch (action) {
+    case 'dropCreateTable':
+      return {
+        createIfNotExists: true,
+        dropIfExists: true,
+      };
+    case 'truncate':
+      return {
+        createIfNotExists: true,
+        truncate: true,
+      };
+  }
+
+  return {
+    createIfNotExists: true,
+  };
+}
+
+function getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver) {
+  const { targetStorageType } = values;
+  const format = findFileFormat(extensions, targetStorageType);
+  if (format && format.writerFunc) {
+    const outputParams = format.getOutputParams && format.getOutputParams(sourceName, values);
+    return [
+      format.writerFunc,
+      {
+        ...(outputParams
+          ? outputParams
+          : {
+              fileName: getTargetName(extensions, sourceName, values),
+            }),
+        ...extractApiParameters(values, 'target', format),
+      },
+    ];
+  }
+  if (targetStorageType == 'database') {
+    return [
+      'tableWriter',
+      {
+        connection: targetConnection,
+        schemaName: values.targetSchemaName,
+        pureName: getTargetName(extensions, sourceName, values),
+        ...getFlagsFroAction(values[`actionType_${sourceName}`]),
+      },
+    ];
+  }
+  if (targetStorageType == 'archive') {
+    return [
+      'archiveWriter',
+      {
+        folderName: values.targetArchiveFolder,
+        fileName: getTargetName(extensions, sourceName, values),
+      },
+    ];
+  }
+
+  throw new Error(`Unknown target storage type: ${targetStorageType}`);
+}
+
+export default async function createImpExpScript(extensions, values, addEditorInfo = true) {
+  const script = new ScriptWriter(values.startVariableIndex || 0);
+
+  const [sourceConnection, sourceDriver] = await getConnection(
+    extensions,
+    values.sourceStorageType,
+    values.sourceConnectionId,
+    values.sourceDatabaseName
+  );
+  const [targetConnection, targetDriver] = await getConnection(
+    extensions,
+    values.targetStorageType,
+    values.targetConnectionId,
+    values.targetDatabaseName
+  );
+
+  const sourceList = getAsArray(values.sourceList);
+  for (const sourceName of sourceList) {
+    const sourceVar = script.allocVariable();
+    // @ts-ignore
+    script.assign(sourceVar, ...getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver));
+
+    const targetVar = script.allocVariable();
+    // @ts-ignore
+    script.assign(targetVar, ...getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver));
+
+    script.copyStream(sourceVar, targetVar);
+    script.put();
+  }
+  if (addEditorInfo) {
+    script.comment('@ImportExportConfigurator');
+    script.comment(JSON.stringify(values));
+  }
+  return script.getScript(values.schedule);
+}
+
+export function getActionOptions(extensions, source, values, targetDbinfo) {
+  const res = [];
+  const targetName = getTargetName(extensions, source, values);
+  if (values.targetStorageType == 'database') {
+    let existing = findObjectLike(
+      { schemaName: values.targetSchemaName, pureName: targetName },
+      targetDbinfo,
+      'tables'
+    );
+    if (existing) {
+      res.push({
+        label: 'Append data',
+        value: 'appendData',
+      });
+      res.push({
+        label: 'Truncate and import',
+        value: 'truncate',
+      });
+      res.push({
+        label: 'Drop and create table',
+        value: 'dropCreateTable',
+      });
+    } else {
+      res.push({
+        label: 'Create table',
+        value: 'createTable',
+      });
+    }
+  } else {
+    res.push({
+      label: 'Create file',
+      value: 'createFile',
+    });
+  }
+  return res;
+}
+
+export async function createPreviewReader(extensions, values, sourceName) {
+  const [sourceConnection, sourceDriver] = await getConnection(
+    extensions,
+    values.sourceStorageType,
+    values.sourceConnectionId,
+    values.sourceDatabaseName
+  );
+  const [functionName, props] = getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver);
+  return {
+    functionName,
+    props: {
+      ...props,
+      limitRows: 100,
+    },
+  };
+}