dbgate/packages/web/src/impexp/createImpExpScript.ts

import _ from 'lodash';
import moment from 'moment';
import { ScriptWriterGeneric, ScriptWriterJavaScript, ScriptWriterJson } from 'dbgate-tools';
import getAsArray from '../utility/getAsArray';
import { getConnectionInfo } from '../utility/metadataLoaders';
import { findEngineDriver, findObjectLike } from 'dbgate-tools';
import { findFileFormat } from '../plugins/fileformats';
import { getCurrentConfig, getExtensions } from '../stores';

export function getTargetName(extensions, source, values) {
  const key = `targetName_${source}`;
  if (values[key]) return values[key];
  const format = findFileFormat(extensions, values.targetStorageType);
  if (format) {
    const res = format.getDefaultOutputName ? format.getDefaultOutputName(source, values) : null;
    if (res) return res;
    return `${source}.${format.extension}`;
  }
  return source;
}

function extractFormatApiParameters(values, direction, format) {
  const pairs = (format.args || [])
    .filter(arg => arg.apiName)
    .map(arg => [arg.apiName, values[`${direction}_${format.storageType}_${arg.name}`]])
    .filter(x => x[1] != null);
  return _.fromPairs(pairs);
}

function extractDriverApiParameters(values, direction, driver) {
  const pairs = (driver.importExportArgs || [])
    .filter(arg => arg.apiName)
    .map(arg => [arg.apiName, values[`${direction}_${driver.engine}_${arg.name}`]])
    .filter(x => x[1] != null);
  return _.fromPairs(pairs);
}

export function extractShellConnection(connection, database) {
  const config = getCurrentConfig();

  return config.allowShellConnection
    ? {
        ..._.omitBy(
          _.omit(connection, ['_id', 'displayName', 'databases', 'connectionColor', 'status', 'unsaved']),
          v => !v
        ),
        database,
      }
    : {
        _id: connection._id,
        engine: connection.engine,
        database,
      };
}

export function extractShellConnectionHostable(connection, database) {
  const driver = findEngineDriver(connection, getExtensions());
  if (driver?.singleConnectionOnly) {
    return {
      systemConnection: { $hostConnection: true },
      connection: driver.engine,
    };
  }

  return {
    connection: extractShellConnection(connection, database),
  };
}

export function extractShellHostConnection(connection, database) {
  const driver = findEngineDriver(connection, getExtensions());
  if (driver?.singleConnectionOnly) {
    return {
      conid: connection._id,
      database,
    };
  }

  return undefined;
}

async function getConnection(extensions, storageType, conid, database) {
  if (storageType == 'database' || storageType == 'query') {
    const conn = await getConnectionInfo({ conid });
    const driver = findEngineDriver(conn, extensions);
    const connection = extractShellConnection(conn, database);
    return [connection, driver];
  }
  return [null, null];
}

function getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver, hostConnection) {
  const { sourceStorageType } = values;
  const connectionParams =
    sourceDriver?.singleConnectionOnly && hostConnection
      ? {
          systemConnection: { $hostConnection: true },
          connection: sourceDriver?.engine,
        }
      : {
          connection: sourceConnection,
        };
  if (sourceStorageType == 'database') {
    const fullName = { schemaName: values.sourceSchemaName, pureName: sourceName };
    return [
      'tableReader',
      {
        ...connectionParams,
        ...extractDriverApiParameters(values, 'source', sourceDriver),
        ...fullName,
      },
    ];
  }
  if (sourceStorageType == 'query') {
    return [
      'queryReader',
      {
        ...connectionParams,
        ...extractDriverApiParameters(values, 'source', sourceDriver),
        queryType: values.sourceQueryType,
        query: values.sourceQueryType == 'json' ? JSON.parse(values.sourceQuery) : values.sourceQuery,
      },
    ];
  }
  if (findFileFormat(extensions, sourceStorageType)) {
    const sourceFile = values[`sourceFile_${sourceName}`];
    const format = findFileFormat(extensions, sourceStorageType);
    if (format && format.readerFunc) {
      return [
        format.readerFunc,
        {
          ...(sourceFile
            ? _.omit(sourceFile, ['isDownload'])
            : {
                fileName: values.importFromZipFile
                  ? `zip://archive:${values.sourceArchiveFolder}//${sourceName}`
                  : sourceName,
              }),
          ...extractFormatApiParameters(values, 'source', format),
        },
      ];
    }
  }
  if (sourceStorageType == 'jsldata') {
    return ['jslDataReader', { jslid: values.sourceJslId }];
  }
  if (sourceStorageType == 'archive') {
    return [
      'archiveReader',
      {
        folderName: values.sourceArchiveFolder,
        fileName: sourceName,
      },
    ];
  }
  throw new Error(`Unknown source storage type: ${sourceStorageType}`);
}

function getFlagsFroAction(action) {
  switch (action) {
    case 'dropCreateTable':
      return {
        createIfNotExists: true,
        dropIfExists: true,
      };
    case 'truncate':
      return {
        createIfNotExists: true,
        truncate: true,
      };
    case 'appendData':
      return {
        createIfNotExists: false,
        truncate: false,
      };
  }

  return {
    createIfNotExists: true,
  };
}

function getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver, hostConnection) {
  const { targetStorageType } = values;
  const connectionParams =
    targetDriver?.singleConnectionOnly && hostConnection
      ? {
          systemConnection: { $hostConnection: true },
          connection: targetDriver?.engine,
        }
      : {
          connection: targetConnection,
        };
  const format = findFileFormat(extensions, targetStorageType);
  if (format && format.writerFunc) {
    const outputParams = format.getOutputParams && format.getOutputParams(sourceName, values);
    return [
      format.writerFunc,
      {
        ...(outputParams
          ? outputParams
          : {
              fileName: getTargetName(extensions, sourceName, values),
            }),
        ...extractFormatApiParameters(values, 'target', format),
      },
    ];
  }
  if (targetStorageType == 'database') {
    return [
      'tableWriter',
      {
        ...connectionParams,
        schemaName: values.targetSchemaName,
        pureName: getTargetName(extensions, sourceName, values),
        ...extractDriverApiParameters(values, 'target', targetDriver),
        ...getFlagsFroAction(values[`actionType_${sourceName}`]),
        progressName: sourceName,
      },
    ];
  }
  if (targetStorageType == 'archive') {
    return [
      'archiveWriter',
      {
        folderName: values.targetArchiveFolder,
        fileName: getTargetName(extensions, sourceName, values),
      },
    ];
  }

  throw new Error(`Unknown target storage type: ${targetStorageType}`);
}

export function normalizeExportColumnMap(colmap) {
  if (!colmap) {
    return null;
  }
  if (!colmap.find(x => !x.ignore)) {
    // all values are ignored, ignore column map
    return null;
  }
  colmap = colmap.filter(x => !x.skip);
  if (colmap.length > 0) {
    return colmap.map(x => _.omit(x, ['ignore']));
  }
  return null;
}

export default async function createImpExpScript(extensions, values, format = undefined, detectHostConnection = false) {
  const config = getCurrentConfig();
  let script: ScriptWriterGeneric = new ScriptWriterJson(values.startVariableIndex || 0);
  if (format == 'script' && config.allowShellScripting) {
    script = new ScriptWriterJavaScript(values.startVariableIndex || 0);
  }

  const [sourceConnection, sourceDriver] = await getConnection(
    extensions,
    values.sourceStorageType,
    values.sourceConnectionId,
    values.sourceDatabaseName
  );
  const [targetConnection, targetDriver] = await getConnection(
    extensions,
    values.targetStorageType,
    values.targetConnectionId,
    values.targetDatabaseName
  );

  let hostConnection = null;
  if (detectHostConnection) {
    // @ts-ignore
    if (sourceDriver?.singleConnectionOnly) {
      hostConnection = { conid: values.sourceConnectionId, database: values.sourceDatabaseName };
    }
    // @ts-ignore
    if (targetDriver?.singleConnectionOnly) {
      if (
        hostConnection &&
        (hostConnection.conid != values.targetConnectionId || hostConnection.database != values.targetDatabaseName)
      ) {
        throw new Error('Cannot use two different single-connections in the same script');
      }
      hostConnection = { conid: values.targetConnectionId, database: values.targetDatabaseName };
    }
  }

  const sourceList = getAsArray(values.sourceList);
  for (const sourceName of sourceList) {
    const sourceVar = script.allocVariable();
    script.assign(
      sourceVar,
      // @ts-ignore
      ...getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver, hostConnection)
    );

    const targetVar = script.allocVariable();
    script.assign(
      targetVar,
      // @ts-ignore
      ...getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver, hostConnection)
    );

    const colmap = normalizeExportColumnMap(values[`columns_${sourceName}`]);

    let colmapVar = null;
    if (colmap) {
      colmapVar = script.allocVariable();
      script.assignValue(colmapVar, colmap);
    }

    script.copyStream(
      sourceVar,
      targetVar,
      colmapVar,
      hostConnection ? { name: sourceName, runid: { $runid: true } } : sourceName
    );
    script.endLine();
  }

  if (values.exportToZipFile) {
    let zipFileName = values.exportToZipFileName || `zip-archive-${moment().format('YYYY-MM-DD-HH-mm-ss')}.zip`;
    if (!zipFileName.endsWith('.zip')) zipFileName += '.zip';
    script.zipDirectory('.', values.createZipFileInArchive ? 'archive:' + zipFileName : zipFileName);
  }

  const res = script.getScript(values.schedule);
  if (format == 'json') {
    res.hostConnection = hostConnection;
  }
  return res;
}

export function getActionOptions(extensions, source, values, targetDbinfo) {
  const res = [];
  if (values.targetStorageType == 'database') {
    res.push({
      label: 'Create table/append',
      value: 'createTable',
    });
    res.push({
      label: 'Append data',
      value: 'appendData',
    });
    res.push({
      label: 'Truncate and import',
      value: 'truncate',
    });
    res.push({
      label: 'Drop and create table',
      value: 'dropCreateTable',
    });
  } else {
    res.push({
      label: 'Create file',
      value: 'createFile',
    });
  }
  return res;
}

export async function createPreviewReader(extensions, values, sourceName) {
  const [sourceConnection, sourceDriver] = await getConnection(
    extensions,
    values.sourceStorageType,
    values.sourceConnectionId,
    values.sourceDatabaseName
  );
  const [functionName, props] = getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver, null);
  return {
    functionName,
    props: {
      ...props,
      limitRows: 100,
    },
  };
}