preview in import dialog

packages/api/src/shell/archiveReader.js

@@ -2,9 +2,9 @@ const path = require('path');
 const { archivedir } = require('../utility/directories');
 const jsonLinesReader = require('./jsonLinesReader');
 
-function archiveReader({ folderName, fileName }) {
+function archiveReader({ folderName, fileName, ...other }) {
   const jsonlFile = path.join(archivedir(), folderName, `${fileName}.jsonl`);
-  const res = jsonLinesReader({ fileName: jsonlFile });
+  const res = jsonLinesReader({ fileName: jsonlFile, ...other });
   return res;
 }
 

packages/api/src/shell/csvReader.js

@@ -37,12 +37,13 @@ class CsvPrepareStream extends stream.Transform {
   }
 }
 
-async function csvReader({ fileName, encoding = 'utf-8', header = true, delimiter, quoted }) {
+async function csvReader({ fileName, encoding = 'utf-8', header = true, delimiter, quoted, limitRows = undefined }) {
   console.log(`Reading file ${fileName}`);
   const csvStream = csv.parse({
     // @ts-ignore
     delimiter,
     quoted,
+    to_line: limitRows ? limitRows + 1 : -1,
   });
   const fileStream = fs.createReadStream(fileName, encoding);
   const csvPrepare = new CsvPrepareStream({ header });

packages/api/src/shell/excelSheetReader.js

@@ -14,7 +14,7 @@ async function loadWorkbook(fileName) {
   return workbook;
 }
 
-async function excelSheetReader({ fileName, sheetName }) {
+async function excelSheetReader({ fileName, sheetName, limitRows = undefined }) {
   const workbook = await loadWorkbook(fileName);
   const sheet = workbook.getWorksheet(sheetName);
 
@@ -27,6 +27,7 @@ async function excelSheetReader({ fileName, sheetName }) {
   };
   pass.write(structure);
   for (let rowIndex = 2; rowIndex <= sheet.rowCount; rowIndex++) {
+    if (limitRows && rowIndex > limitRows + 1) break;
     const row = sheet.getRow(rowIndex);
     const rowData = _.fromPairs(structure.columns.map((col, index) => [col.columnName, row.getCell(index + 1).value]));
     if (_.isEmpty(_.omitBy(rowData, (v) => v == null || v.toString().trim().length == 0))) continue;

packages/api/src/shell/jslDataReader.js

@@ -1,9 +1,9 @@
 const getJslFileName = require('../utility/getJslFileName');
 const jsonLinesReader = require('./jsonLinesReader');
 
-function jslDataReader({ jslid }) {
+function jslDataReader({ jslid, ...other }) {
   const fileName = getJslFileName(jslid);
-  return jsonLinesReader({ fileName });
+  return jsonLinesReader({ fileName, ...other });
 }
 
 module.exports = jslDataReader;

packages/api/src/shell/jsonLinesReader.js

@@ -3,10 +3,12 @@ const stream = require('stream');
 const byline = require('byline');
 
 class ParseStream extends stream.Transform {
-  constructor({ header }) {
+  constructor({ header, limitRows }) {
     super({ objectMode: true });
     this.header = header;
     this.wasHeader = false;
+    this.limitRows = limitRows;
+    this.rowsWritten = 0;
   }
   _transform(chunk, encoding, done) {
     const obj = JSON.parse(chunk);
@@ -14,17 +16,19 @@ class ParseStream extends stream.Transform {
       if (!this.header) this.push({ columns: Object.keys(obj).map((columnName) => ({ columnName })) });
       this.wasHeader = true;
     }
+    if (!this.limitRows || this.rowsWritten < this.limitRows) {
       this.push(obj);
+    }
     done();
   }
 }
 
-async function jsonLinesReader({ fileName, encoding = 'utf-8', header = true }) {
+async function jsonLinesReader({ fileName, encoding = 'utf-8', header = true, limitRows = undefined }) {
   console.log(`Reading file ${fileName}`);
 
   const fileStream = fs.createReadStream(fileName, encoding);
   const liner = byline(fileStream);
-  const parser = new ParseStream({ header });
+  const parser = new ParseStream({ header, limitRows });
   liner.pipe(parser);
   return parser;
 }

packages/api/src/shell/tableReader.js

@@ -3,7 +3,7 @@ const driverConnect = require('../utility/driverConnect');
 
 const engines = require('@dbgate/engines');
 
-async function queryReader({ connection, pureName, schemaName }) {
+async function tableReader({ connection, pureName, schemaName }) {
   const driver = engines(connection);
   const pool = await driverConnect(driver, connection);
   console.log(`Connected.`);
@@ -25,4 +25,4 @@ async function queryReader({ connection, pureName, schemaName }) {
   return await driver.readQuery(pool, query);
 }
 
-module.exports = queryReader;
+module.exports = tableReader;

packages/web/src/impexp/ImportExportConfigurator.js

@@ -14,8 +14,8 @@ import {
 } from '../utility/forms';
 import { useArchiveFiles, useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders';
 import TableControl, { TableColumn } from '../utility/TableControl';
-import { TextField, SelectField } from '../utility/inputs';
+import { TextField, SelectField, CheckboxField } from '../utility/inputs';
-import { getActionOptions, getTargetName, isFileStorage } from './createImpExpScript';
+import { createPreviewReader, getActionOptions, getTargetName, isFileStorage } from './createImpExpScript';
 import getElectron from '../utility/getElectron';
 import ErrorInfo from '../widgets/ErrorInfo';
 import getAsArray from '../utility/getAsArray';
@@ -86,7 +86,7 @@ function getFileFilters(storageType) {
   return res;
 }
 
-async function addFilesToSourceList(files, values, setFieldValue, preferedStorageType) {
+async function addFilesToSourceList(files, values, setFieldValue, preferedStorageType, setPreviewSource) {
   const newSources = [];
   const storage = preferedStorageType || values.sourceStorageType;
   for (const file of getAsArray(files)) {
@@ -116,6 +116,9 @@ async function addFilesToSourceList(files, values, setFieldValue, preferedStorag
   if (preferedStorageType && preferedStorageType != values.sourceStorageType) {
     setFieldValue('sourceStorageType', preferedStorageType);
   }
+  if (setPreviewSource && newSources.length == 1) {
+    setPreviewSource(newSources[0]);
+  }
 }
 
 function ElectronFilesInput() {
@@ -308,7 +311,7 @@ function SourceName({ name }) {
   );
 }
 
-export default function ImportExportConfigurator({ uploadedFile = undefined }) {
+export default function ImportExportConfigurator({ uploadedFile = undefined, onChangePreview = undefined }) {
   const { values, setFieldValue } = useFormikContext();
   const targetDbinfo = useDatabaseInfo({ conid: values.targetConnectionId, database: values.targetDatabaseName });
   const sourceConnectionInfo = useConnectionInfo({ conid: values.sourceConnectionId });
@@ -316,6 +319,7 @@ export default function ImportExportConfigurator({ uploadedFile = undefined }) {
   const { sourceList } = values;
   const { setUploadListener } = useUploadsProvider();
   const theme = useTheme();
+  const [previewSource, setPreviewSource] = React.useState(null);
 
   const handleUpload = React.useCallback(
     (file) => {
@@ -328,7 +332,8 @@ export default function ImportExportConfigurator({ uploadedFile = undefined }) {
         ],
         values,
         setFieldValue,
-        !sourceList || sourceList.length == 0 ? file.storageType : null
+        !sourceList || sourceList.length == 0 ? file.storageType : null,
+        setPreviewSource
       );
       // setFieldValue('sourceList', [...(sourceList || []), file.originalName]);
     },
@@ -348,6 +353,21 @@ export default function ImportExportConfigurator({ uploadedFile = undefined }) {
     }
   }, []);
 
+  const supportsPreview = ['csv', 'jsonl', 'excel'].includes(values.sourceStorageType);
+
+  const handleChangePreviewSource = async () => {
+    if (previewSource && supportsPreview) {
+      const reader = await createPreviewReader(values, previewSource);
+      if (onChangePreview) onChangePreview(reader);
+    } else {
+      onChangePreview(null);
+    }
+  };
+
+  React.useEffect(() => {
+    handleChangePreviewSource();
+  }, [previewSource, supportsPreview]);
+
   return (
     <Container>
       <Wrapper>
@@ -396,6 +416,21 @@ export default function ImportExportConfigurator({ uploadedFile = undefined }) {
             />
           )}
         />
+        <TableColumn
+          fieldName="preview"
+          header="Preview"
+          formatter={(row) =>
+            supportsPreview ? (
+              <CheckboxField
+                checked={previewSource == row}
+                onChange={(e) => {
+                  if (e.target.checked) setPreviewSource(row);
+                  else setPreviewSource(null);
+                }}
+              />
+            ) : null
+          }
+        />
       </TableControl>
     </Container>
   );

packages/web/src/impexp/PreviewDataGrid.js

@@ -0,0 +1,59 @@
+import { createGridCache, createGridConfig, FreeTableGridDisplay } from '@dbgate/datalib';
+import React from 'react';
+import DataGridCore from '../datagrid/DataGridCore';
+import RowsArrayGrider from '../datagrid/RowsArrayGrider';
+import axios from '../utility/axios';
+import ErrorInfo from '../widgets/ErrorInfo';
+import LoadingInfo from '../widgets/LoadingInfo';
+
+export default function PreviewDataGrid({ reader, ...other }) {
+  const [isLoading, setIsLoading] = React.useState(false);
+  const [errorMessage, setErrorMessage] = React.useState(null);
+  const [model, setModel] = React.useState(null);
+  const [config, setConfig] = React.useState(createGridConfig());
+  const [cache, setCache] = React.useState(createGridCache());
+  const [grider, setGrider] = React.useState(null);
+
+  const handleLoadInitialData = async () => {
+    try {
+      if (!reader) {
+        setModel(null);
+        setGrider(null);
+        return;
+      }
+      setIsLoading(true);
+      const resp = await axios.post('runners/load-reader', reader);
+      // @ts-ignore
+      setModel(resp.data);
+      setGrider(new RowsArrayGrider(resp.data.rows));
+      setIsLoading(false);
+    } catch (err) {
+      setIsLoading(false);
+      const errorMessage = (err && err.response && err.response.data && err.response.data.error) || 'Loading failed';
+      setErrorMessage(errorMessage);
+      console.error(err.response);
+    }
+  };
+
+  React.useEffect(() => {
+    handleLoadInitialData();
+  }, [reader]);
+
+  const display = React.useMemo(() => new FreeTableGridDisplay(model, config, setConfig, cache, setCache), [
+    model,
+    config,
+    cache,
+    grider,
+  ]);
+
+  if (isLoading) {
+    return <LoadingInfo wrapper message="Loading data" />;
+  }
+  if (errorMessage) {
+    return <ErrorInfo message={errorMessage} />;
+  }
+
+  if (!grider) return null;
+
+  return <DataGridCore {...other} grider={grider} display={display} />;
+}

packages/web/src/impexp/createImpExpScript.js

@@ -211,3 +211,19 @@ export function getActionOptions(source, values, targetDbinfo) {
   }
   return res;
 }
+
+export async function createPreviewReader(values, sourceName) {
+  const [sourceConnection, sourceDriver] = await getConnection(
+    values.sourceStorageType,
+    values.sourceConnectionId,
+    values.sourceDatabaseName
+  );
+  const [functionName, props] = getSourceExpr(sourceName, values, sourceConnection, sourceDriver);
+  return {
+    functionName,
+    props: {
+      ...props,
+      limitRows: 100,
+    },
+  };
+}

packages/web/src/modals/ImportExportModal.js

@@ -1,4 +1,4 @@
-import React from 'react';
+import React, { useState } from 'react';
 import ModalBase from './ModalBase';
 import FormStyledButton from '../widgets/FormStyledButton';
 import { Formik, Form, useFormikContext } from 'formik';
@@ -16,6 +16,7 @@ import WidgetColumnBar, { WidgetColumnBarItem } from '../widgets/WidgetColumnBar
 import SocketMessagesView from '../query/SocketMessagesView';
 import RunnerOutputFiles from '../query/RunnerOuputFiles';
 import useTheme from '../theme/useTheme';
+import PreviewDataGrid from '../impexp/PreviewDataGrid';
 
 const headerHeight = '60px';
 const footerHeight = '60px';
@@ -106,6 +107,7 @@ export default function ImportExportModal({ modalState, initialValues, uploadedF
   const [runnerId, setRunnerId] = React.useState(null);
   const archive = useCurrentArchive();
   const theme = useTheme();
+  const [previewReader, setPreviewReader] = useState(0);
 
   const handleExecute = async (values) => {
     const script = await createImpExpScript(values);
@@ -134,13 +136,10 @@ export default function ImportExportModal({ modalState, initialValues, uploadedF
           <ModalHeader modalState={modalState}>Import/Export</ModalHeader>
           <Wrapper>
             <ContentWrapper theme={theme}>
-              <ImportExportConfigurator uploadedFile={uploadedFile} />
+              <ImportExportConfigurator uploadedFile={uploadedFile} onChangePreview={setPreviewReader} />
             </ContentWrapper>
             <WidgetColumnWrapper theme={theme}>
               <WidgetColumnBar>
-                {/* <WidgetColumnBarItem title="Preview" name="preview">
-                  Preview
-                </WidgetColumnBarItem> */}
                 <WidgetColumnBarItem title="Output files" name="output" height="20%">
                   <RunnerOutputFiles runnerId={runnerId} executeNumber={executeNumber} />
                 </WidgetColumnBarItem>
@@ -150,6 +149,11 @@ export default function ImportExportModal({ modalState, initialValues, uploadedF
                     executeNumber={executeNumber}
                   />
                 </WidgetColumnBarItem>
+                {previewReader && (
+                  <WidgetColumnBarItem title="Preview" name="preview">
+                    <PreviewDataGrid reader={previewReader} />
+                  </WidgetColumnBarItem>
+                )}
               </WidgetColumnBar>
             </WidgetColumnWrapper>
           </Wrapper>