workign excel import

packages/api/src/controllers/files.js

@@ -0,0 +1,19 @@
+const exceljs = require('exceljs');
+const _ = require('lodash');
+
+module.exports = {
+  openedReaders: {},
+
+  analyseExcel_meta: 'get',
+  async analyseExcel({ filePath }) {
+    const workbook = new exceljs.Workbook();
+    await workbook.xlsx.readFile(filePath);
+    return {
+      tables: workbook.worksheets.map((sheet) => {
+        const header = sheet.getRow(1);
+        const columns = _.range(header.cellCount).map((index) => ({ columnName: header.getCell(index + 1).value }));
+        return { pureName: sheet.name, columns };
+      }),
+    };
+  },
+};

packages/api/src/main.js

@@ -19,6 +19,7 @@ const sessions = require('./controllers/sessions');
 const runners = require('./controllers/runners');
 const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
+const files = require('./controllers/files');
 
 const { rundir } = require('./utility/directories');
 
@@ -53,6 +54,7 @@ function start(argument = null) {
   useController(app, '/runners', runners);
   useController(app, '/jsldata', jsldata);
   useController(app, '/config', config);
+  useController(app, '/files', files);
 
   if (process.env.PAGES_DIRECTORY) {
     app.use('/pages', express.static(process.env.PAGES_DIRECTORY));

packages/web/src/appobj/databaseAppObject.js

@@ -3,8 +3,9 @@ import _ from 'lodash';
 import { DatabaseIcon } from '../icons';
 import { DropDownMenuItem } from '../modals/DropDownMenu';
 import { openNewTab } from '../utility/common';
+import ImportExportModal from '../modals/ImportExportModal';
 
-function Menu({ data, setOpenedTabs }) {
+function Menu({ data, setOpenedTabs, showModal }) {
   const { connection, name } = data;
   const tooltip = `${connection.displayName || connection.server}\n${name}`;
 
@@ -21,9 +22,24 @@ function Menu({ data, setOpenedTabs }) {
     });
   };
 
+  const handleImport = () => {
+    showModal((modalState) => (
+      <ImportExportModal
+        modalState={modalState}
+        initialValues={{
+          sourceStorageType: 'csv',
+          targetStorageType: 'database',
+          targetConnectionId: data.connection._id,
+          targetDatabaseName: data.name,
+        }}
+      />
+    ));
+  };
+
   return (
     <>
       <DropDownMenuItem onClick={handleNewQuery}>New query</DropDownMenuItem>
+      <DropDownMenuItem onClick={handleImport}>Import</DropDownMenuItem>
     </>
   );
 }

packages/web/src/impexp/ImportExportConfigurator.js

@@ -17,9 +17,17 @@ import {
 import { useConnectionList, useDatabaseList, useDatabaseInfo } from '../utility/metadataLoaders';
 import TableControl, { TableColumn } from '../utility/TableControl';
 import { TextField, SelectField } from '../utility/inputs';
-import { getActionOptions, getTargetName } from './createImpExpScript';
+import { getActionOptions, getTargetName, isFileStorage } from './createImpExpScript';
+import getElectron from '../utility/getElectron';
+import ErrorInfo from '../widgets/ErrorInfo';
+import getAsArray from '../utility/getAsArray';
+import axios from '../utility/axios';
+import LoadingInfo from '../widgets/LoadingInfo';
 
-const Container = styled.div``;
+const Container = styled.div`
+  max-height: 50vh;
+  overflow-y: scroll;
+`;
 
 const Wrapper = styled.div`
   display: flex;
@@ -44,6 +52,19 @@ const Label = styled.div`
   color: #777;
 `;
 
+const SourceNameWrapper = styled.div`
+  display: flex;
+  justify-content: space-between;
+`;
+
+const TrashWrapper = styled.div`
+  &:hover {
+    background-color: #ccc;
+  }
+  cursor: pointer;
+  color: blue;
+`;
+
 function DatabaseSelector() {
   const connections = useConnectionList();
   const connectionOptions = React.useMemo(
@@ -70,6 +91,88 @@ function DatabaseSelector() {
   );
 }
 
+function getFileFilters(storageType) {
+  const res = [];
+  if (storageType == 'csv') res.push({ name: 'CSV files', extensions: ['csv'] });
+  if (storageType == 'jsonl') res.push({ name: 'JSON lines', extensions: ['jsonl'] });
+  if (storageType == 'excel') res.push({ name: 'MS Excel files', extensions: ['xlsx'] });
+  res.push({ name: 'All Files', extensions: ['*'] });
+  return res;
+}
+
+async function addFilesToSourceList(files, values, setFieldValue) {
+  const newSources = [];
+  const storage = values.sourceStorageType;
+  for (const file of getAsArray(files)) {
+    if (isFileStorage(storage)) {
+      if (storage == 'excel') {
+        const resp = await axios.get(`files/analyse-excel?filePath=${encodeURIComponent(file.full)}`);
+        /** @type {import('@dbgate/types').DatabaseInfo} */
+        const structure = resp.data;
+        for (const table of structure.tables) {
+          const sourceName = table.pureName;
+          newSources.push(sourceName);
+          setFieldValue(`sourceFile_${sourceName}`, {
+            fileName: file.full,
+            sheetName: table.pureName,
+          });
+        }
+      } else {
+        const sourceName = file.name;
+        newSources.push(sourceName);
+        setFieldValue(`sourceFile_${sourceName}`, {
+          fileName: file.full,
+        });
+      }
+    }
+  }
+  setFieldValue('sourceList', [...(values.sourceList || []).filter((x) => !newSources.includes(x)), ...newSources]);
+}
+
+function ElectronFilesInput() {
+  const { values, setFieldValue } = useFormikContext();
+  const electron = getElectron();
+  const [isLoading, setIsLoading] = React.useState(false);
+
+  const handleClick = async () => {
+    const files = electron.remote.dialog.showOpenDialogSync(electron.remote.getCurrentWindow(), {
+      properties: ['openFile', 'multiSelections'],
+      filters: getFileFilters(values.sourceStorageType),
+    });
+    if (files) {
+      const path = window.require('path');
+      try {
+        setIsLoading(true);
+        await addFilesToSourceList(
+          files.map((full) => ({
+            full,
+            ...path.parse(full),
+          })),
+          values,
+          setFieldValue
+        );
+      } finally {
+        setIsLoading(false);
+      }
+    }
+  };
+
+  return (
+    <>
+      <FormStyledButton type="button" value="Add file(s)" onClick={handleClick} />
+      {isLoading && <LoadingInfo message="Anaysing input files" />}
+    </>
+  );
+}
+
+function FilesInput() {
+  const electron = getElectron();
+  if (electron) {
+    return <ElectronFilesInput />;
+  }
+  return <ErrorInfo message="Import files is currently implemented only for electron client" />;
+}
+
 function SourceTargetConfig({
   direction,
   storageTypeField,
@@ -80,16 +183,18 @@ function SourceTargetConfig({
 }) {
   const types = [
     { value: 'database', label: 'Database', directions: ['source', 'target'] },
-    { value: 'csv', label: 'CSV file(s)', directions: ['target'] },
+    { value: 'csv', label: 'CSV file(s)', directions: ['source', 'target'] },
-    { value: 'jsonl', label: 'JSON lines file(s)', directions: ['target'] },
+    { value: 'jsonl', label: 'JSON lines file(s)', directions: ['source', 'target'] },
+    { value: 'excel', label: 'MS Excel file(s)', directions: ['source'] },
   ];
   const { values } = useFormikContext();
+  const storageType = values[storageTypeField];
   return (
     <Column>
       {direction == 'source' && <Label>Source configuration</Label>}
       {direction == 'target' && <Label>Target configuration</Label>}
       <FormReactSelect options={types.filter((x) => x.directions.includes(direction))} name={storageTypeField} />
-      {values[storageTypeField] == 'database' && (
+      {storageType == 'database' && (
         <>
           <Label>Server</Label>
           <FormConnectionSelect name={connectionIdField} />
@@ -110,10 +215,30 @@ function SourceTargetConfig({
           )}
         </>
       )}
+      {isFileStorage(storageType) && direction == 'source' && <FilesInput />}
     </Column>
   );
 }
 
+function SourceName({ name }) {
+  const { values, setFieldValue } = useFormikContext();
+  const handleDelete = () => {
+    setFieldValue(
+      'sourceList',
+      values.sourceList.filter((x) => x != name)
+    );
+  };
+
+  return (
+    <SourceNameWrapper>
+      <div>{name}</div>
+      <TrashWrapper onClick={handleDelete}>
+        <i className="fas fa-trash" />
+      </TrashWrapper>
+    </SourceNameWrapper>
+  );
+}
+
 export default function ImportExportConfigurator() {
   const { values, setFieldValue } = useFormikContext();
   const targetDbinfo = useDatabaseInfo({ conid: values.targetConnectionId, database: values.targetDatabaseName });
@@ -139,7 +264,7 @@ export default function ImportExportConfigurator() {
         />
       </Wrapper>
       <TableControl rows={sourceList || []}>
-        <TableColumn fieldName="source" header="Source" formatter={(row) => row} />
+        <TableColumn fieldName="source" header="Source" formatter={(row) => <SourceName name={row} />} />
         <TableColumn
           fieldName="action"
           header="Action"

packages/web/src/impexp/createImpExpScript.js

@@ -14,6 +14,10 @@ export function getTargetName(source, values) {
   return source;
 }
 
+export function isFileStorage(storageType) {
+  return storageType == 'csv' || storageType == 'jsonl' || storageType == 'excel';
+}
+
 async function getConnection(storageType, conid, database) {
   if (storageType == 'database') {
     const conn = await getConnectionInfo({ conid });
@@ -41,6 +45,18 @@ function getSourceExpr(sourceName, values, sourceConnection, sourceDriver) {
       },
     ];
   }
+  if (isFileStorage(values.sourceStorageType)) {
+    const sourceFile = values[`sourceFile_${sourceName}`];
+    if (values.sourceStorageType == 'excel') {
+      return ['excelSheetReader', sourceFile];
+    }
+    if (values.sourceStorageType == 'jsonl') {
+      return ['jsonLinesReader', sourceFile];
+    }
+    if (values.sourceStorageType == 'csv') {
+      return ['csvReader', sourceFile];
+    }
+  }
 }
 
 function getFlagsFroAction(action) {
@@ -106,7 +122,6 @@ export default async function createImpExpScript(values) {
     values.targetDatabaseName
   );
 
-  if (values.sourceStorageType == 'database') {
   const sourceList = getAsArray(values.sourceList);
   for (const sourceName of sourceList) {
     const sourceVar = script.allocVariable();
@@ -120,7 +135,6 @@ export default async function createImpExpScript(values) {
     script.copyStream(sourceVar, targetVar);
     script.put();
   }
-  }
   return script.s;
 }
 

packages/web/src/modals/ImportExportModal.js

@@ -28,7 +28,7 @@ export default function ImportExportModal({ modalState, initialValues }) {
         initialScript: code,
       },
     });
-    modalState.close();
+    // modalState.close();
   };
   return (
     <ModalBase modalState={modalState}>