diff --git a/packages/api/src/shell/csvReader.js b/packages/api/src/shell/csvReader.js index d4016376a..deb987e36 100644 --- a/packages/api/src/shell/csvReader.js +++ b/packages/api/src/shell/csvReader.js @@ -43,6 +43,7 @@ async function csvReader({ fileName, encoding = 'utf-8', header = true, delimite // @ts-ignore delimiter, quoted, + skip_lines_with_error: true, to_line: limitRows ? limitRows + 1 : -1, }); const fileStream = fs.createReadStream(fileName, encoding); diff --git a/packages/web/src/fileformats/csv.ts b/packages/web/src/fileformats/csv.ts index 410a70d38..4bec4d46b 100644 --- a/packages/web/src/fileformats/csv.ts +++ b/packages/web/src/fileformats/csv.ts @@ -8,6 +8,20 @@ const csvFormat: FileFormatDefinition = { name: 'CSV', readerFunc: 'csvReader', writerFunc: 'csvWriter', + args: [ + { + type: 'select', + name: 'delimiter', + label: 'Delimiter', + options: [ + { name: 'Comma (,)', value: ',' }, + { name: 'Semicolon (;)', value: ';' }, + { name: 'Tab', value: '\t' }, + { name: 'Pipe (|)', value: '|' }, + ], + apiName: 'delimiter', + }, + ], }; export default csvFormat; diff --git a/packages/web/src/fileformats/types.ts b/packages/web/src/fileformats/types.ts index b44e65f19..0f21e710f 100644 --- a/packages/web/src/fileformats/types.ts +++ b/packages/web/src/fileformats/types.ts @@ -4,6 +4,7 @@ export interface FileFormatDefinition { name: string; readerFunc?: string; writerFunc?: string; + args?: any[]; addFilesToSourceList: ( file: { full: string; diff --git a/packages/web/src/freetable/MacroParameters.js b/packages/web/src/freetable/MacroParameters.js index a0fb251ce..5b2abdabd 100644 --- a/packages/web/src/freetable/MacroParameters.js +++ b/packages/web/src/freetable/MacroParameters.js @@ -1,54 +1,8 @@ import React from 'react'; import _ from 'lodash'; -import styled from 'styled-components'; -import { - FormTextField, - FormSubmit, - FormArchiveFolderSelect, - FormRow, - FormLabel, - FormSelectField, - FormCheckboxField, -} from '../utility/forms'; import { Formik, Form, useFormikContext } from 'formik'; +import FormArgumentList from '../utility/FormArgumentList'; -const MacroArgumentsWrapper = styled.div` -`; - -function MacroArgument({ arg, namePrefix }) { - const name = `${namePrefix}${arg.name}`; - if (arg.type == 'text') { - return ; - } - if (arg.type == 'checkbox') { - return ; - } - if (arg.type == 'select') { - return ( - - {arg.options.map((opt) => - _.isString(opt) ? : - )} - - ); - } - return null; -} - -function MacroArgumentList({ args, onChangeValues, namePrefix }) { - const { values } = useFormikContext(); - React.useEffect(() => { - if (onChangeValues) onChangeValues(values); - }, [values]); - return ( - - {' '} - {args.map((arg) => ( - - ))} - - ); -} export default function MacroParameters({ args, onChangeValues, macroValues, namePrefix }) { if (!args || args.length == 0) return null; @@ -59,7 +13,7 @@ export default function MacroParameters({ args, onChangeValues, macroValues, nam return ( {}}>
- +
); diff --git a/packages/web/src/impexp/ImportExportConfigurator.js b/packages/web/src/impexp/ImportExportConfigurator.js index f90172c4b..b416f3ca5 100644 --- a/packages/web/src/impexp/ImportExportConfigurator.js +++ b/packages/web/src/impexp/ImportExportConfigurator.js @@ -26,6 +26,7 @@ import { useUploadsProvider } from '../utility/UploadsProvider'; import { FontIcon } from '../icons'; import useTheme from '../theme/useTheme'; import { fileformats, findFileFormat, getFileFormatDirections } from '../fileformats'; +import FormArgumentList from '../utility/FormArgumentList'; const Container = styled.div` // max-height: 50vh; @@ -192,6 +193,7 @@ function SourceTargetConfig({ const storageType = values[storageTypeField]; const dbinfo = useDatabaseInfo({ conid: values[connectionIdField], database: values[databaseNameField] }); const archiveFiles = useArchiveFiles({ folder: values[archiveFolderField] }); + const format = findFileFormat(storageType); return ( {direction == 'source' && ( @@ -297,7 +299,14 @@ function SourceTargetConfig({ )} - {isFileStorage(storageType) && direction == 'source' && } + {!!format && direction == 'source' && } + + {format && format.args && ( + !arg.direction || arg.direction == direction)} + namePrefix={`${direction}_${format.storageType}_`} + /> + )} ); } @@ -379,6 +388,18 @@ export default function ImportExportConfigurator({ uploadedFile = undefined, onC handleChangePreviewSource(); }, [previewSource, supportsPreview]); + const oldValues = React.useRef({}); + React.useEffect(() => { + const changed = _.pickBy( + values, + (v, k) => k.startsWith(`source_${values.sourceStorageType}_`) && oldValues.current[k] != v + ); + if (!_.isEmpty(changed)) { + handleChangePreviewSource(); + } + oldValues.current = values; + }, [values]); + return ( diff --git a/packages/web/src/impexp/PreviewDataGrid.js b/packages/web/src/impexp/PreviewDataGrid.js index ca1849765..301348093 100644 --- a/packages/web/src/impexp/PreviewDataGrid.js +++ b/packages/web/src/impexp/PreviewDataGrid.js @@ -21,6 +21,7 @@ export default function PreviewDataGrid({ reader, ...other }) { setGrider(null); return; } + setErrorMessage(null); setIsLoading(true); const resp = await axios.post('runners/load-reader', reader); // @ts-ignore diff --git a/packages/web/src/impexp/createImpExpScript.js b/packages/web/src/impexp/createImpExpScript.js index 8afde4819..393a4493c 100644 --- a/packages/web/src/impexp/createImpExpScript.js +++ b/packages/web/src/impexp/createImpExpScript.js @@ -18,6 +18,14 @@ export function isFileStorage(storageType) { return !!findFileFormat(storageType); } +function extractApiParameters(values, direction, format) { + const pairs = (format.args || []) + .filter((arg) => arg.apiName) + .map((arg) => [arg.apiName, values[`${direction}_${format.storageType}_${arg.name}`]]) + .filter((x) => x[1]); + return _.fromPairs(pairs); +} + async function getConnection(storageType, conid, database) { if (storageType == 'database' || storageType == 'query') { const conn = await getConnectionInfo({ conid }); @@ -58,7 +66,13 @@ function getSourceExpr(sourceName, values, sourceConnection, sourceDriver) { const sourceFile = values[`sourceFile_${sourceName}`]; const format = findFileFormat(sourceStorageType); if (format && format.readerFunc) { - return [format.readerFunc, sourceFile]; + return [ + format.readerFunc, + { + ...sourceFile, + ...extractApiParameters(values, 'source', format), + }, + ]; } } if (sourceStorageType == 'jsldata') { @@ -103,9 +117,9 @@ function getTargetExpr(sourceName, values, targetConnection, targetDriver) { format.writerFunc, { fileName: getTargetName(sourceName, values), + ...extractApiParameters(values, 'target', format), }, ]; - } if (targetStorageType == 'database') { return [ diff --git a/packages/web/src/utility/FormArgumentList.js b/packages/web/src/utility/FormArgumentList.js new file mode 100644 index 000000000..fd9041763 --- /dev/null +++ b/packages/web/src/utility/FormArgumentList.js @@ -0,0 +1,50 @@ +import React from 'react'; +import _ from 'lodash'; +import styled from 'styled-components'; +import { + FormTextField, + FormSubmit, + FormArchiveFolderSelect, + FormRow, + FormLabel, + FormSelectField, + FormCheckboxField, +} from './forms'; +import { Formik, Form, useFormikContext } from 'formik'; + +const FormArgumentsWrapper = styled.div``; + +function FormArgument({ arg, namePrefix }) { + const name = `${namePrefix}${arg.name}`; + if (arg.type == 'text') { + return ; + } + if (arg.type == 'checkbox') { + return ; + } + if (arg.type == 'select') { + return ( + + {arg.options.map((opt) => + _.isString(opt) ? : + )} + + ); + } + return null; +} + +export default function FormArgumentList({ args, onChangeValues = undefined, namePrefix }) { + const { values } = useFormikContext(); + React.useEffect(() => { + if (onChangeValues) onChangeValues(values); + }, [values]); + return ( + + {' '} + {args.map((arg) => ( + + ))} + + ); +}