diff --git a/packages/api/src/shell/csvReader.js b/packages/api/src/shell/csvReader.js
index d4016376a..deb987e36 100644
--- a/packages/api/src/shell/csvReader.js
+++ b/packages/api/src/shell/csvReader.js
@@ -43,6 +43,7 @@ async function csvReader({ fileName, encoding = 'utf-8', header = true, delimite
// @ts-ignore
delimiter,
quoted,
+ skip_lines_with_error: true,
to_line: limitRows ? limitRows + 1 : -1,
});
const fileStream = fs.createReadStream(fileName, encoding);
diff --git a/packages/web/src/fileformats/csv.ts b/packages/web/src/fileformats/csv.ts
index 410a70d38..4bec4d46b 100644
--- a/packages/web/src/fileformats/csv.ts
+++ b/packages/web/src/fileformats/csv.ts
@@ -8,6 +8,20 @@ const csvFormat: FileFormatDefinition = {
name: 'CSV',
readerFunc: 'csvReader',
writerFunc: 'csvWriter',
+ args: [
+ {
+ type: 'select',
+ name: 'delimiter',
+ label: 'Delimiter',
+ options: [
+ { name: 'Comma (,)', value: ',' },
+ { name: 'Semicolon (;)', value: ';' },
+ { name: 'Tab', value: '\t' },
+ { name: 'Pipe (|)', value: '|' },
+ ],
+ apiName: 'delimiter',
+ },
+ ],
};
export default csvFormat;
diff --git a/packages/web/src/fileformats/types.ts b/packages/web/src/fileformats/types.ts
index b44e65f19..0f21e710f 100644
--- a/packages/web/src/fileformats/types.ts
+++ b/packages/web/src/fileformats/types.ts
@@ -4,6 +4,7 @@ export interface FileFormatDefinition {
name: string;
readerFunc?: string;
writerFunc?: string;
+ args?: any[];
addFilesToSourceList: (
file: {
full: string;
diff --git a/packages/web/src/freetable/MacroParameters.js b/packages/web/src/freetable/MacroParameters.js
index a0fb251ce..5b2abdabd 100644
--- a/packages/web/src/freetable/MacroParameters.js
+++ b/packages/web/src/freetable/MacroParameters.js
@@ -1,54 +1,8 @@
import React from 'react';
import _ from 'lodash';
-import styled from 'styled-components';
-import {
- FormTextField,
- FormSubmit,
- FormArchiveFolderSelect,
- FormRow,
- FormLabel,
- FormSelectField,
- FormCheckboxField,
-} from '../utility/forms';
import { Formik, Form, useFormikContext } from 'formik';
+import FormArgumentList from '../utility/FormArgumentList';
-const MacroArgumentsWrapper = styled.div`
-`;
-
-function MacroArgument({ arg, namePrefix }) {
- const name = `${namePrefix}${arg.name}`;
- if (arg.type == 'text') {
- return ;
- }
- if (arg.type == 'checkbox') {
- return ;
- }
- if (arg.type == 'select') {
- return (
-
- {arg.options.map((opt) =>
- _.isString(opt) ? :
- )}
-
- );
- }
- return null;
-}
-
-function MacroArgumentList({ args, onChangeValues, namePrefix }) {
- const { values } = useFormikContext();
- React.useEffect(() => {
- if (onChangeValues) onChangeValues(values);
- }, [values]);
- return (
-
- {' '}
- {args.map((arg) => (
-
- ))}
-
- );
-}
export default function MacroParameters({ args, onChangeValues, macroValues, namePrefix }) {
if (!args || args.length == 0) return null;
@@ -59,7 +13,7 @@ export default function MacroParameters({ args, onChangeValues, macroValues, nam
return (
{}}>
);
diff --git a/packages/web/src/impexp/ImportExportConfigurator.js b/packages/web/src/impexp/ImportExportConfigurator.js
index f90172c4b..b416f3ca5 100644
--- a/packages/web/src/impexp/ImportExportConfigurator.js
+++ b/packages/web/src/impexp/ImportExportConfigurator.js
@@ -26,6 +26,7 @@ import { useUploadsProvider } from '../utility/UploadsProvider';
import { FontIcon } from '../icons';
import useTheme from '../theme/useTheme';
import { fileformats, findFileFormat, getFileFormatDirections } from '../fileformats';
+import FormArgumentList from '../utility/FormArgumentList';
const Container = styled.div`
// max-height: 50vh;
@@ -192,6 +193,7 @@ function SourceTargetConfig({
const storageType = values[storageTypeField];
const dbinfo = useDatabaseInfo({ conid: values[connectionIdField], database: values[databaseNameField] });
const archiveFiles = useArchiveFiles({ folder: values[archiveFolderField] });
+ const format = findFileFormat(storageType);
return (
{direction == 'source' && (
@@ -297,7 +299,14 @@ function SourceTargetConfig({
>
)}
- {isFileStorage(storageType) && direction == 'source' && }
+ {!!format && direction == 'source' && }
+
+ {format && format.args && (
+ !arg.direction || arg.direction == direction)}
+ namePrefix={`${direction}_${format.storageType}_`}
+ />
+ )}
);
}
@@ -379,6 +388,18 @@ export default function ImportExportConfigurator({ uploadedFile = undefined, onC
handleChangePreviewSource();
}, [previewSource, supportsPreview]);
+ const oldValues = React.useRef({});
+ React.useEffect(() => {
+ const changed = _.pickBy(
+ values,
+ (v, k) => k.startsWith(`source_${values.sourceStorageType}_`) && oldValues.current[k] != v
+ );
+ if (!_.isEmpty(changed)) {
+ handleChangePreviewSource();
+ }
+ oldValues.current = values;
+ }, [values]);
+
return (
diff --git a/packages/web/src/impexp/PreviewDataGrid.js b/packages/web/src/impexp/PreviewDataGrid.js
index ca1849765..301348093 100644
--- a/packages/web/src/impexp/PreviewDataGrid.js
+++ b/packages/web/src/impexp/PreviewDataGrid.js
@@ -21,6 +21,7 @@ export default function PreviewDataGrid({ reader, ...other }) {
setGrider(null);
return;
}
+ setErrorMessage(null);
setIsLoading(true);
const resp = await axios.post('runners/load-reader', reader);
// @ts-ignore
diff --git a/packages/web/src/impexp/createImpExpScript.js b/packages/web/src/impexp/createImpExpScript.js
index 8afde4819..393a4493c 100644
--- a/packages/web/src/impexp/createImpExpScript.js
+++ b/packages/web/src/impexp/createImpExpScript.js
@@ -18,6 +18,14 @@ export function isFileStorage(storageType) {
return !!findFileFormat(storageType);
}
+function extractApiParameters(values, direction, format) {
+ const pairs = (format.args || [])
+ .filter((arg) => arg.apiName)
+ .map((arg) => [arg.apiName, values[`${direction}_${format.storageType}_${arg.name}`]])
+ .filter((x) => x[1]);
+ return _.fromPairs(pairs);
+}
+
async function getConnection(storageType, conid, database) {
if (storageType == 'database' || storageType == 'query') {
const conn = await getConnectionInfo({ conid });
@@ -58,7 +66,13 @@ function getSourceExpr(sourceName, values, sourceConnection, sourceDriver) {
const sourceFile = values[`sourceFile_${sourceName}`];
const format = findFileFormat(sourceStorageType);
if (format && format.readerFunc) {
- return [format.readerFunc, sourceFile];
+ return [
+ format.readerFunc,
+ {
+ ...sourceFile,
+ ...extractApiParameters(values, 'source', format),
+ },
+ ];
}
}
if (sourceStorageType == 'jsldata') {
@@ -103,9 +117,9 @@ function getTargetExpr(sourceName, values, targetConnection, targetDriver) {
format.writerFunc,
{
fileName: getTargetName(sourceName, values),
+ ...extractApiParameters(values, 'target', format),
},
];
-
}
if (targetStorageType == 'database') {
return [
diff --git a/packages/web/src/utility/FormArgumentList.js b/packages/web/src/utility/FormArgumentList.js
new file mode 100644
index 000000000..fd9041763
--- /dev/null
+++ b/packages/web/src/utility/FormArgumentList.js
@@ -0,0 +1,50 @@
+import React from 'react';
+import _ from 'lodash';
+import styled from 'styled-components';
+import {
+ FormTextField,
+ FormSubmit,
+ FormArchiveFolderSelect,
+ FormRow,
+ FormLabel,
+ FormSelectField,
+ FormCheckboxField,
+} from './forms';
+import { Formik, Form, useFormikContext } from 'formik';
+
+const FormArgumentsWrapper = styled.div``;
+
+function FormArgument({ arg, namePrefix }) {
+ const name = `${namePrefix}${arg.name}`;
+ if (arg.type == 'text') {
+ return ;
+ }
+ if (arg.type == 'checkbox') {
+ return ;
+ }
+ if (arg.type == 'select') {
+ return (
+
+ {arg.options.map((opt) =>
+ _.isString(opt) ? :
+ )}
+
+ );
+ }
+ return null;
+}
+
+export default function FormArgumentList({ args, onChangeValues = undefined, namePrefix }) {
+ const { values } = useFormikContext();
+ React.useEffect(() => {
+ if (onChangeValues) onChangeValues(values);
+ }, [values]);
+ return (
+
+ {' '}
+ {args.map((arg) => (
+
+ ))}
+
+ );
+}