csv delimiter configuration

packages/api/src/shell/csvReader.js

@@ -43,6 +43,7 @@ async function csvReader({ fileName, encoding = 'utf-8', header = true, delimite
     // @ts-ignore
     delimiter,
     quoted,
+    skip_lines_with_error: true,
     to_line: limitRows ? limitRows + 1 : -1,
   });
   const fileStream = fs.createReadStream(fileName, encoding);

packages/web/src/fileformats/csv.ts

@@ -8,6 +8,20 @@ const csvFormat: FileFormatDefinition = {
   name: 'CSV',
   readerFunc: 'csvReader',
   writerFunc: 'csvWriter',
+  args: [
+    {
+      type: 'select',
+      name: 'delimiter',
+      label: 'Delimiter',
+      options: [
+        { name: 'Comma (,)', value: ',' },
+        { name: 'Semicolon (;)', value: ';' },
+        { name: 'Tab', value: '\t' },
+        { name: 'Pipe (|)', value: '|' },
+      ],
+      apiName: 'delimiter',
+    },
+  ],
 };
 
 export default csvFormat;

packages/web/src/fileformats/types.ts

@@ -4,6 +4,7 @@ export interface FileFormatDefinition {
   name: string;
   readerFunc?: string;
   writerFunc?: string;
+  args?: any[];
   addFilesToSourceList: (
     file: {
       full: string;

packages/web/src/freetable/MacroParameters.js

@@ -1,54 +1,8 @@
 import React from 'react';
 import _ from 'lodash';
-import styled from 'styled-components';
-import {
-  FormTextField,
-  FormSubmit,
-  FormArchiveFolderSelect,
-  FormRow,
-  FormLabel,
-  FormSelectField,
-  FormCheckboxField,
-} from '../utility/forms';
 import { Formik, Form, useFormikContext } from 'formik';
+import FormArgumentList from '../utility/FormArgumentList';
 
-const MacroArgumentsWrapper = styled.div`
-`;
-
-function MacroArgument({ arg, namePrefix }) {
-  const name = `${namePrefix}${arg.name}`;
-  if (arg.type == 'text') {
-    return <FormTextField label={arg.label} name={name} />;
-  }
-  if (arg.type == 'checkbox') {
-    return <FormCheckboxField label={arg.label} name={name} />;
-  }
-  if (arg.type == 'select') {
-    return (
-      <FormSelectField label={arg.label} name={name}>
-        {arg.options.map((opt) =>
-          _.isString(opt) ? <option value={opt}>{opt}</option> : <option value={opt.value}>{opt.name}</option>
-        )}
-      </FormSelectField>
-    );
-  }
-  return null;
-}
-
-function MacroArgumentList({ args, onChangeValues, namePrefix }) {
-  const { values } = useFormikContext();
-  React.useEffect(() => {
-    if (onChangeValues) onChangeValues(values);
-  }, [values]);
-  return (
-    <MacroArgumentsWrapper>
-      {' '}
-      {args.map((arg) => (
-        <MacroArgument arg={arg} key={arg.name} namePrefix={namePrefix} />
-      ))}
-    </MacroArgumentsWrapper>
-  );
-}
 
 export default function MacroParameters({ args, onChangeValues, macroValues, namePrefix }) {
   if (!args || args.length == 0) return null;
@@ -59,7 +13,7 @@ export default function MacroParameters({ args, onChangeValues, macroValues, nam
   return (
     <Formik initialValues={initialValues} onSubmit={() => {}}>
       <Form>
-        <MacroArgumentList args={args} onChangeValues={onChangeValues} namePrefix={namePrefix} />
+        <FormArgumentList args={args} onChangeValues={onChangeValues} namePrefix={namePrefix} />
       </Form>
     </Formik>
   );

packages/web/src/impexp/ImportExportConfigurator.js

@@ -26,6 +26,7 @@ import { useUploadsProvider } from '../utility/UploadsProvider';
 import { FontIcon } from '../icons';
 import useTheme from '../theme/useTheme';
 import { fileformats, findFileFormat, getFileFormatDirections } from '../fileformats';
+import FormArgumentList from '../utility/FormArgumentList';
 
 const Container = styled.div`
   // max-height: 50vh;
@@ -192,6 +193,7 @@ function SourceTargetConfig({
   const storageType = values[storageTypeField];
   const dbinfo = useDatabaseInfo({ conid: values[connectionIdField], database: values[databaseNameField] });
   const archiveFiles = useArchiveFiles({ folder: values[archiveFolderField] });
+  const format = findFileFormat(storageType);
   return (
     <Column>
       {direction == 'source' && (
@@ -297,7 +299,14 @@ function SourceTargetConfig({
         </>
       )}
 
-      {isFileStorage(storageType) && direction == 'source' && <FilesInput />}
+      {!!format && direction == 'source' && <FilesInput />}
+
+      {format && format.args && (
+        <FormArgumentList
+          args={format.args.filter((arg) => !arg.direction || arg.direction == direction)}
+          namePrefix={`${direction}_${format.storageType}_`}
+        />
+      )}
     </Column>
   );
 }
@@ -379,6 +388,18 @@ export default function ImportExportConfigurator({ uploadedFile = undefined, onC
     handleChangePreviewSource();
   }, [previewSource, supportsPreview]);
 
+  const oldValues = React.useRef({});
+  React.useEffect(() => {
+    const changed = _.pickBy(
+      values,
+      (v, k) => k.startsWith(`source_${values.sourceStorageType}_`) && oldValues.current[k] != v
+    );
+    if (!_.isEmpty(changed)) {
+      handleChangePreviewSource();
+    }
+    oldValues.current = values;
+  }, [values]);
+
   return (
     <Container>
       <Wrapper>

packages/web/src/impexp/PreviewDataGrid.js

@@ -21,6 +21,7 @@ export default function PreviewDataGrid({ reader, ...other }) {
         setGrider(null);
         return;
       }
+      setErrorMessage(null);
       setIsLoading(true);
       const resp = await axios.post('runners/load-reader', reader);
       // @ts-ignore

packages/web/src/impexp/createImpExpScript.js

@@ -18,6 +18,14 @@ export function isFileStorage(storageType) {
   return !!findFileFormat(storageType);
 }
 
+function extractApiParameters(values, direction, format) {
+  const pairs = (format.args || [])
+    .filter((arg) => arg.apiName)
+    .map((arg) => [arg.apiName, values[`${direction}_${format.storageType}_${arg.name}`]])
+    .filter((x) => x[1]);
+  return _.fromPairs(pairs);
+}
+
 async function getConnection(storageType, conid, database) {
   if (storageType == 'database' || storageType == 'query') {
     const conn = await getConnectionInfo({ conid });
@@ -58,7 +66,13 @@ function getSourceExpr(sourceName, values, sourceConnection, sourceDriver) {
     const sourceFile = values[`sourceFile_${sourceName}`];
     const format = findFileFormat(sourceStorageType);
     if (format && format.readerFunc) {
-      return [format.readerFunc, sourceFile];
+      return [
+        format.readerFunc,
+        {
+          ...sourceFile,
+          ...extractApiParameters(values, 'source', format),
+        },
+      ];
     }
   }
   if (sourceStorageType == 'jsldata') {
@@ -103,9 +117,9 @@ function getTargetExpr(sourceName, values, targetConnection, targetDriver) {
       format.writerFunc,
       {
         fileName: getTargetName(sourceName, values),
+        ...extractApiParameters(values, 'target', format),
       },
     ];
-
   }
   if (targetStorageType == 'database') {
     return [

packages/web/src/utility/FormArgumentList.js

@@ -0,0 +1,50 @@
+import React from 'react';
+import _ from 'lodash';
+import styled from 'styled-components';
+import {
+  FormTextField,
+  FormSubmit,
+  FormArchiveFolderSelect,
+  FormRow,
+  FormLabel,
+  FormSelectField,
+  FormCheckboxField,
+} from './forms';
+import { Formik, Form, useFormikContext } from 'formik';
+
+const FormArgumentsWrapper = styled.div``;
+
+function FormArgument({ arg, namePrefix }) {
+  const name = `${namePrefix}${arg.name}`;
+  if (arg.type == 'text') {
+    return <FormTextField label={arg.label} name={name} />;
+  }
+  if (arg.type == 'checkbox') {
+    return <FormCheckboxField label={arg.label} name={name} />;
+  }
+  if (arg.type == 'select') {
+    return (
+      <FormSelectField label={arg.label} name={name}>
+        {arg.options.map((opt) =>
+          _.isString(opt) ? <option value={opt}>{opt}</option> : <option value={opt.value}>{opt.name}</option>
+        )}
+      </FormSelectField>
+    );
+  }
+  return null;
+}
+
+export default function FormArgumentList({ args, onChangeValues = undefined, namePrefix }) {
+  const { values } = useFormikContext();
+  React.useEffect(() => {
+    if (onChangeValues) onChangeValues(values);
+  }, [values]);
+  return (
+    <FormArgumentsWrapper>
+      {' '}
+      {args.map((arg) => (
+        <FormArgument arg={arg} key={arg.name} namePrefix={namePrefix} />
+      ))}
+    </FormArgumentsWrapper>
+  );
+}