diff --git a/packages/api/src/controllers/files.js b/packages/api/src/controllers/files.js new file mode 100644 index 000000000..dbca12db5 --- /dev/null +++ b/packages/api/src/controllers/files.js @@ -0,0 +1,19 @@ +const exceljs = require('exceljs'); +const _ = require('lodash'); + +module.exports = { + openedReaders: {}, + + analyseExcel_meta: 'get', + async analyseExcel({ filePath }) { + const workbook = new exceljs.Workbook(); + await workbook.xlsx.readFile(filePath); + return { + tables: workbook.worksheets.map((sheet) => { + const header = sheet.getRow(1); + const columns = _.range(header.cellCount).map((index) => ({ columnName: header.getCell(index + 1).value })); + return { pureName: sheet.name, columns }; + }), + }; + }, +}; diff --git a/packages/api/src/main.js b/packages/api/src/main.js index 3423c0a50..d2bd44e4e 100644 --- a/packages/api/src/main.js +++ b/packages/api/src/main.js @@ -19,6 +19,7 @@ const sessions = require('./controllers/sessions'); const runners = require('./controllers/runners'); const jsldata = require('./controllers/jsldata'); const config = require('./controllers/config'); +const files = require('./controllers/files'); const { rundir } = require('./utility/directories'); @@ -53,6 +54,7 @@ function start(argument = null) { useController(app, '/runners', runners); useController(app, '/jsldata', jsldata); useController(app, '/config', config); + useController(app, '/files', files); if (process.env.PAGES_DIRECTORY) { app.use('/pages', express.static(process.env.PAGES_DIRECTORY)); diff --git a/packages/web/src/appobj/databaseAppObject.js b/packages/web/src/appobj/databaseAppObject.js index 1652810be..48c22f612 100644 --- a/packages/web/src/appobj/databaseAppObject.js +++ b/packages/web/src/appobj/databaseAppObject.js @@ -3,8 +3,9 @@ import _ from 'lodash'; import { DatabaseIcon } from '../icons'; import { DropDownMenuItem } from '../modals/DropDownMenu'; import { openNewTab } from '../utility/common'; +import ImportExportModal from '../modals/ImportExportModal'; -function Menu({ data, setOpenedTabs }) { +function Menu({ data, setOpenedTabs, showModal }) { const { connection, name } = data; const tooltip = `${connection.displayName || connection.server}\n${name}`; @@ -21,9 +22,24 @@ function Menu({ data, setOpenedTabs }) { }); }; + const handleImport = () => { + showModal((modalState) => ( + + )); + }; + return ( <> New query + Import ); } diff --git a/packages/web/src/impexp/ImportExportConfigurator.js b/packages/web/src/impexp/ImportExportConfigurator.js index 8e8e0dc55..e33201508 100644 --- a/packages/web/src/impexp/ImportExportConfigurator.js +++ b/packages/web/src/impexp/ImportExportConfigurator.js @@ -17,9 +17,17 @@ import { import { useConnectionList, useDatabaseList, useDatabaseInfo } from '../utility/metadataLoaders'; import TableControl, { TableColumn } from '../utility/TableControl'; import { TextField, SelectField } from '../utility/inputs'; -import { getActionOptions, getTargetName } from './createImpExpScript'; +import { getActionOptions, getTargetName, isFileStorage } from './createImpExpScript'; +import getElectron from '../utility/getElectron'; +import ErrorInfo from '../widgets/ErrorInfo'; +import getAsArray from '../utility/getAsArray'; +import axios from '../utility/axios'; +import LoadingInfo from '../widgets/LoadingInfo'; -const Container = styled.div``; +const Container = styled.div` + max-height: 50vh; + overflow-y: scroll; +`; const Wrapper = styled.div` display: flex; @@ -44,6 +52,19 @@ const Label = styled.div` color: #777; `; +const SourceNameWrapper = styled.div` + display: flex; + justify-content: space-between; +`; + +const TrashWrapper = styled.div` + &:hover { + background-color: #ccc; + } + cursor: pointer; + color: blue; +`; + function DatabaseSelector() { const connections = useConnectionList(); const connectionOptions = React.useMemo( @@ -70,6 +91,88 @@ function DatabaseSelector() { ); } +function getFileFilters(storageType) { + const res = []; + if (storageType == 'csv') res.push({ name: 'CSV files', extensions: ['csv'] }); + if (storageType == 'jsonl') res.push({ name: 'JSON lines', extensions: ['jsonl'] }); + if (storageType == 'excel') res.push({ name: 'MS Excel files', extensions: ['xlsx'] }); + res.push({ name: 'All Files', extensions: ['*'] }); + return res; +} + +async function addFilesToSourceList(files, values, setFieldValue) { + const newSources = []; + const storage = values.sourceStorageType; + for (const file of getAsArray(files)) { + if (isFileStorage(storage)) { + if (storage == 'excel') { + const resp = await axios.get(`files/analyse-excel?filePath=${encodeURIComponent(file.full)}`); + /** @type {import('@dbgate/types').DatabaseInfo} */ + const structure = resp.data; + for (const table of structure.tables) { + const sourceName = table.pureName; + newSources.push(sourceName); + setFieldValue(`sourceFile_${sourceName}`, { + fileName: file.full, + sheetName: table.pureName, + }); + } + } else { + const sourceName = file.name; + newSources.push(sourceName); + setFieldValue(`sourceFile_${sourceName}`, { + fileName: file.full, + }); + } + } + } + setFieldValue('sourceList', [...(values.sourceList || []).filter((x) => !newSources.includes(x)), ...newSources]); +} + +function ElectronFilesInput() { + const { values, setFieldValue } = useFormikContext(); + const electron = getElectron(); + const [isLoading, setIsLoading] = React.useState(false); + + const handleClick = async () => { + const files = electron.remote.dialog.showOpenDialogSync(electron.remote.getCurrentWindow(), { + properties: ['openFile', 'multiSelections'], + filters: getFileFilters(values.sourceStorageType), + }); + if (files) { + const path = window.require('path'); + try { + setIsLoading(true); + await addFilesToSourceList( + files.map((full) => ({ + full, + ...path.parse(full), + })), + values, + setFieldValue + ); + } finally { + setIsLoading(false); + } + } + }; + + return ( + <> + + {isLoading && } + + ); +} + +function FilesInput() { + const electron = getElectron(); + if (electron) { + return ; + } + return ; +} + function SourceTargetConfig({ direction, storageTypeField, @@ -80,16 +183,18 @@ function SourceTargetConfig({ }) { const types = [ { value: 'database', label: 'Database', directions: ['source', 'target'] }, - { value: 'csv', label: 'CSV file(s)', directions: ['target'] }, - { value: 'jsonl', label: 'JSON lines file(s)', directions: ['target'] }, + { value: 'csv', label: 'CSV file(s)', directions: ['source', 'target'] }, + { value: 'jsonl', label: 'JSON lines file(s)', directions: ['source', 'target'] }, + { value: 'excel', label: 'MS Excel file(s)', directions: ['source'] }, ]; const { values } = useFormikContext(); + const storageType = values[storageTypeField]; return ( {direction == 'source' && } {direction == 'target' && } x.directions.includes(direction))} name={storageTypeField} /> - {values[storageTypeField] == 'database' && ( + {storageType == 'database' && ( <> @@ -110,10 +215,30 @@ function SourceTargetConfig({ )} )} + {isFileStorage(storageType) && direction == 'source' && } ); } +function SourceName({ name }) { + const { values, setFieldValue } = useFormikContext(); + const handleDelete = () => { + setFieldValue( + 'sourceList', + values.sourceList.filter((x) => x != name) + ); + }; + + return ( + +
{name}
+ + + +
+ ); +} + export default function ImportExportConfigurator() { const { values, setFieldValue } = useFormikContext(); const targetDbinfo = useDatabaseInfo({ conid: values.targetConnectionId, database: values.targetDatabaseName }); @@ -139,7 +264,7 @@ export default function ImportExportConfigurator() { /> - row} /> + } />