file formats refactor

packages/web/src/appobj/databaseAppObject.js

@@ -3,6 +3,7 @@ import _ from 'lodash';
 import { DropDownMenuItem } from '../modals/DropDownMenu';
 import { openNewTab } from '../utility/common';
 import ImportExportModal from '../modals/ImportExportModal';
+import { defaultFileFormat } from '../fileformats';
 
 function Menu({ data, setOpenedTabs, showModal }) {
   const { connection, name } = data;
@@ -26,7 +27,7 @@ function Menu({ data, setOpenedTabs, showModal }) {
       <ImportExportModal
         modalState={modalState}
         initialValues={{
-          sourceStorageType: 'csv',
+          sourceStorageType: defaultFileFormat.storageType,
           targetStorageType: 'database',
           targetConnectionId: data.connection._id,
           targetDatabaseName: data.name,
@@ -40,7 +41,7 @@ function Menu({ data, setOpenedTabs, showModal }) {
       <ImportExportModal
         modalState={modalState}
         initialValues={{
-          targetStorageType: 'csv',
+          targetStorageType: defaultFileFormat.storageType,
           sourceStorageType: 'database',
           sourceConnectionId: data.connection._id,
           sourceDatabaseName: data.name,

packages/web/src/fileformats/csv.ts

@@ -0,0 +1,7 @@
+export default {
+  storageType: 'csv',
+  extension: 'csv',
+  readerFunc: 'csvReader',
+  writerFunc: 'csvWriter',
+  filesTitle: 'CSV files',
+};

packages/web/src/fileformats/excel.ts

@@ -0,0 +1,6 @@
+export default {
+  storageType: 'excel',
+  extension: 'xlsx',
+  readerFunc: 'excelSheetReader',
+  filesTitle: 'MS Excel files',
+};

packages/web/src/fileformats/index.ts

@@ -0,0 +1,20 @@
+import csv from './csv';
+import jsonl from './jsonl';
+import excel from './excel';
+import { FileFormatDefinition } from './types';
+
+export const fileformats = [csv, jsonl, excel];
+
+export function findFileFormat(storageType): FileFormatDefinition {
+  return fileformats.find((x) => x.storageType == storageType);
+}
+
+export function getFileFormatDirections(format: FileFormatDefinition) {
+  if (!format) return [];
+  const res = [];
+  if (format.readerFunc) res.push('source');
+  if (format.writerFunc) res.push('target');
+  return res;
+}
+
+export const defaultFileFormat = csv;

packages/web/src/fileformats/jsonl.ts

@@ -0,0 +1,7 @@
+export default {
+  storageType: 'jsonl',
+  extension: 'jsonl',
+  readerFunc: 'jsonLinesReader',
+  writerFunc: 'jsonLinesWriter',
+  filesTitle: 'JSON lines',
+};

packages/web/src/fileformats/types.ts

@@ -0,0 +1,7 @@
+export interface FileFormatDefinition {
+  storageType: string;
+  extension: string;
+  readerFunc?: string;
+  writerFunc?: string;
+  filesTitle: string;
+}

packages/web/src/impexp/ImportExportConfigurator.js

@@ -25,6 +25,7 @@ import SqlEditor from '../sqleditor/SqlEditor';
 import { useUploadsProvider } from '../utility/UploadsProvider';
 import { FontIcon } from '../icons';
 import useTheme from '../theme/useTheme';
+import { fileformats, findFileFormat, getFileFormatDirections } from '../fileformats';
 
 const Container = styled.div`
   // max-height: 50vh;
@@ -89,9 +90,8 @@ const Title = styled.div`
 
 function getFileFilters(storageType) {
   const res = [];
-  if (storageType == 'csv') res.push({ name: 'CSV files', extensions: ['csv'] });
+  const format = findFileFormat(storageType);
-  if (storageType == 'jsonl') res.push({ name: 'JSON lines', extensions: ['jsonl'] });
+  if (format) res.push({ name: format.filesTitle, extensions: [format.extension] });
-  if (storageType == 'excel') res.push({ name: 'MS Excel files', extensions: ['xlsx'] });
   res.push({ name: 'All Files', extensions: ['*'] });
   return res;
 }
@@ -193,9 +193,11 @@ function SourceTargetConfig({
       ? [{ value: 'jsldata', label: 'Query result data', directions: ['source'] }]
       : [
           { value: 'database', label: 'Database', directions: ['source', 'target'] },
-          { value: 'csv', label: 'CSV file(s)', directions: ['source', 'target'] },
+          ...fileformats.map((format) => ({
-          { value: 'jsonl', label: 'JSON lines file(s)', directions: ['source', 'target'] },
+            value: format.storageType,
-          { value: 'excel', label: 'MS Excel file(s)', directions: ['source'] },
+            label: format.filesTitle,
+            directions: getFileFormatDirections(format),
+          })),
           { value: 'query', label: 'SQL Query', directions: ['source'] },
           { value: 'archive', label: 'Archive', directions: ['source', 'target'] },
         ];
@@ -374,7 +376,7 @@ export default function ImportExportConfigurator({ uploadedFile = undefined, onC
     }
   }, []);
 
-  const supportsPreview = ['csv', 'jsonl', 'excel'].includes(values.sourceStorageType);
+  const supportsPreview = !!findFileFormat(values.sourceStorageType);
 
   const handleChangePreviewSource = async () => {
     if (previewSource && supportsPreview) {

packages/web/src/impexp/createImpExpScript.js

@@ -4,17 +4,18 @@ import getAsArray from '../utility/getAsArray';
 import { getConnectionInfo } from '../utility/metadataLoaders';
 import engines from 'dbgate-engines';
 import { findObjectLike } from 'dbgate-tools';
+import { findFileFormat } from '../fileformats';
 
 export function getTargetName(source, values) {
   const key = `targetName_${source}`;
   if (values[key]) return values[key];
-  if (values.targetStorageType == 'csv') return `${source}.csv`;
+  const format = findFileFormat(values.targetStorageType);
-  if (values.targetStorageType == 'jsonl') return `${source}.jsonl`;
+  if (format) return `${source}.${format.extension}`;
   return source;
 }
 
 export function isFileStorage(storageType) {
-  return storageType == 'csv' || storageType == 'jsonl' || storageType == 'excel';
+  return !!findFileFormat(storageType);
 }
 
 async function getConnection(storageType, conid, database) {
@@ -55,14 +56,9 @@ function getSourceExpr(sourceName, values, sourceConnection, sourceDriver) {
   }
   if (isFileStorage(sourceStorageType)) {
     const sourceFile = values[`sourceFile_${sourceName}`];
-    if (sourceStorageType == 'excel') {
+    const format = findFileFormat(sourceStorageType);
-      return ['excelSheetReader', sourceFile];
+    if (format && format.readerFunc) {
-    }
+      return [format.readerFunc, sourceFile];
-    if (sourceStorageType == 'jsonl') {
-      return ['jsonLinesReader', sourceFile];
-    }
-    if (sourceStorageType == 'csv') {
-      return ['csvReader', sourceFile];
     }
   }
   if (sourceStorageType == 'jsldata') {
@@ -101,21 +97,15 @@ function getFlagsFroAction(action) {
 
 function getTargetExpr(sourceName, values, targetConnection, targetDriver) {
   const { targetStorageType } = values;
-  if (targetStorageType == 'csv') {
+  const format = findFileFormat(targetStorageType);
+  if (format && format.writerFunc) {
     return [
-      'csvWriter',
+      format.writerFunc,
-      {
-        fileName: getTargetName(sourceName, values),
-      },
-    ];
-  }
-  if (targetStorageType == 'jsonl') {
-    return [
-      'jsonLinesWriter',
       {
         fileName: getTargetName(sourceName, values),
       },
     ];
+
   }
   if (targetStorageType == 'database') {
     return [

packages/web/src/modals/ImportExportModal.js

@@ -22,6 +22,7 @@ import useSocket from '../utility/SocketProvider';
 import LoadingInfo from '../widgets/LoadingInfo';
 import { FontIcon } from '../icons';
 import LargeButton from '../widgets/LargeButton';
+import { defaultFileFormat } from '../fileformats';
 
 const headerHeight = '60px';
 const footerHeight = '100px';
@@ -193,7 +194,7 @@ export default function ImportExportModal({
         onSubmit={handleExecute}
         initialValues={{
           sourceStorageType: 'database',
-          targetStorageType: importToArchive ? 'archive' : 'csv',
+          targetStorageType: importToArchive ? 'archive' : defaultFileFormat.storageType,
           sourceArchiveFolder: archive,
           targetArchiveFolder,
           ...initialValues,

packages/web/src/utility/UploadsProvider.js

@@ -1,5 +1,6 @@
 import React from 'react';
 import { useDropzone } from 'react-dropzone';
+import { findFileFormat } from '../fileformats';
 import ImportExportModal from '../modals/ImportExportModal';
 import useShowModal from '../modals/showModal';
 import resolveApi from './resolveApi';
@@ -42,7 +43,7 @@ export function useUploadsZone() {
         if (uploadListener) {
           uploadListener(fileData);
         } else {
-          if (['csv', 'excel', 'jsonl'].includes(fileData.storageType)) {
+          if (findFileFormat(fileData.storageType)) {
             showModal((modalState) => (
               <ImportExportModal
                 uploadedFile={fileData}

packages/web/src/widgets/Toolbar.js

@@ -10,6 +10,7 @@ import { openNewTab } from '../utility/common';
 import useNewFreeTable from '../freetable/useNewFreeTable';
 import ImportExportModal from '../modals/ImportExportModal';
 import useShowModal from '../modals/showModal';
+import { defaultFileFormat } from '../fileformats';
 
 const ToolbarContainer = styled.div`
   display: flex;
@@ -40,7 +41,7 @@ export default function ToolBar({ toolbarPortalRef }) {
         modalState={modalState}
         importToArchive
         initialValues={{
-          sourceStorageType: 'csv',
+          sourceStorageType: defaultFileFormat.storageType,
           // sourceConnectionId: data.conid,
           // sourceDatabaseName: data.database,
           // sourceSchemaName: data.schemaName,