better loading free table data

packages/api/src/controllers/runners.js

@@ -11,7 +11,18 @@ const scriptTemplate = (script) => `
 const dbgateApi = require(process.env.DBGATE_API || "@dbgate/api");
 require=null;
 async function run() {
-${script}
+const reader = ${script}
+}
+dbgateApi.runScript(run);
+`;
+
+const loaderScriptTemplate = (functionName, props, runid) => `
+const dbgateApi = require(process.env.DBGATE_API || "@dbgate/api");
+require=null;
+async function run() {
+const reader=await dbgateApi.${functionName}(${JSON.stringify(props)});
+const writer=await dbgateApi.collectorWriter({runid: '${runid}'});
+await dbgateApi.copyStream(reader, writer);
 }
 dbgateApi.runScript(run);
 `;
@@ -19,9 +30,10 @@ dbgateApi.runScript(run);
 module.exports = {
   /** @type {import('@dbgate/types').OpenedRunner[]} */
   opened: [],
+  requests: {},
 
   dispatchMessage(runid, message) {
-    // console.log('DISPATCHING', message);
+    if (message) console.log('...', message.message);
     if (_.isString(message)) {
       socket.emit(`runner-info-${runid}`, {
         message,
@@ -40,12 +52,24 @@ module.exports = {
 
   handle_ping() {},
 
-  start_meta: 'post',
-  async start({ script }) {
-    const runid = uuidv1();
+  handle_freeData(runid, { freeData }) {
+    const [resolve, reject] = this.requests[runid];
+    resolve(freeData);
+    delete this.requests[runid];
+  },
+
+  rejectRequest(runid, error) {
+    if (this.requests[runid]) {
+      const [resolve, reject] = this.requests[runid];
+      reject(error);
+      delete this.requests[runid];
+    }
+  },
+
+  startCore(runid, scriptText) {
     const directory = path.join(rundir(), runid);
     const scriptFile = path.join(uploadsdir(), runid + '.js');
-    fs.writeFileSync(`${scriptFile}`, scriptTemplate(script));
+    fs.writeFileSync(`${scriptFile}`, scriptText);
     fs.mkdirSync(directory);
     console.log(`RUNNING SCRIPT ${scriptFile}`);
     const subprocess = fork(scriptFile, ['--checkParent'], {
@@ -61,9 +85,13 @@ module.exports = {
     byline(subprocess.stdout).on('data', pipeDispatcher('info'));
     byline(subprocess.stderr).on('data', pipeDispatcher('error'));
     subprocess.on('exit', (code) => {
+      this.rejectRequest(runid, { message: 'No data retured, maybe input data source is too big' });
+      console.log('... EXIT process', code);
       socket.emit(`runner-done-${runid}`, code);
     });
     subprocess.on('error', (error) => {
+      this.rejectRequest(runid, { message: error && (error.message || error.toString()) });
+      console.error('... ERROR subprocess', error);
       this.dispatchMessage({
         severity: 'error',
         message: error.toString(),
@@ -81,6 +109,12 @@ module.exports = {
     return newOpened;
   },
 
+  start_meta: 'post',
+  async start({ script }) {
+    const runid = uuidv1();
+    return this.startCore(runid, scriptTemplate(script));
+  },
+
   cancel_meta: 'post',
   async cancel({ runid }) {
     const runner = this.opened.find((x) => x.runid == runid);
@@ -106,4 +140,14 @@ module.exports = {
     }
     return res;
   },
+
+  loadReader_meta: 'post',
+  async loadReader({ functionName, props }) {
+    const promise = new Promise((resolve, reject) => {
+      const runid = uuidv1();
+      this.requests[runid] = [resolve, reject];
+      this.startCore(runid, loaderScriptTemplate(functionName, props, runid));
+    });
+    return promise;
+  },
 };

packages/api/src/shell/collectorWriter.js

@@ -0,0 +1,33 @@
+const stream = require('stream');
+
+class CollectorWriterStream extends stream.Writable {
+  constructor(options) {
+    super(options);
+    this.rows = [];
+    this.structure = null;
+    this.runid = options.runid;
+  }
+  _write(chunk, enc, next) {
+    if (!this.structure) this.structure = chunk;
+    else this.rows.push(chunk);
+    next();
+  }
+
+  _final(callback) {
+    process.send({
+      msgtype: 'freeData',
+      runid: this.runid,
+      freeData: { rows: this.rows, structure: this.structure },
+    });
+    callback();
+  }
+}
+
+async function collectorWriter({ runid }) {
+  return new CollectorWriterStream({
+    objectMode: true,
+    runid,
+  });
+}
+
+module.exports = collectorWriter;

packages/api/src/shell/index.js

@@ -13,6 +13,7 @@ const jsonLinesReader = require('./jsonLinesReader');
 const jslDataReader = require('./jslDataReader');
 const archiveWriter = require('./archiveWriter');
 const archiveReader = require('./archiveReader');
+const collectorWriter = require('./collectorWriter');
 
 module.exports = {
   queryReader,
@@ -30,4 +31,5 @@ module.exports = {
   jslDataReader,
   archiveWriter,
   archiveReader,
+  collectorWriter,
 };

packages/web/src/appobj/archiveFileAppObject.js

@@ -29,14 +29,20 @@ function Menu({ data, setOpenedTabs }) {
     openArchive(setOpenedTabs, data.fileName, data.folderName);
   };
   const handleOpenWrite = async () => {
-    const resp = await axios.post('archive/load-free-table', { file: data.fileName, folder: data.folderName });
+    // const resp = await axios.post('archive/load-free-table', { file: data.fileName, folder: data.folderName });
 
     openNewTab(setOpenedTabs, {
       title: data.fileName,
       icon: 'freetable.svg',
       tabComponent: 'FreeTableTab',
       props: {
-        initialData: resp.data,
+        initialData: {
+          functionName: 'archiveReader',
+          props: {
+            fileName: data.fileName,
+            folderName: data.folderName,
+          },
+        },
         archiveFile: data.fileName,
         archiveFolder: data.folderName,
       },

packages/web/src/datagrid/DataGridCore.js

@@ -90,22 +90,6 @@ const RowCountLabel = styled.div`
   bottom: 20px;
 `;
 
-const LoadingInfoWrapper = styled.div`
-  position: absolute;
-  left: 0;
-  top: 0;
-  right: 0;
-  bottom: 0;
-  display: flex;
-  align-items: center;
-  justify-content: space-around;
-`;
-const LoadingInfoBox = styled.div`
-  background-color: #ccc;
-  padding: 10px;
-  border: 1px solid gray;
-`;
-
 /** @param props {import('./types').DataGridProps} */
 export default function DataGridCore(props) {
   const {
@@ -296,14 +280,7 @@ export default function DataGridCore(props) {
     return `Rows: ${allRowCount.toLocaleString()}`;
   }, [selectedCells, allRowCount, grider, visibleRealColumns]);
 
-  if (!columns || columns.length == 0)
-    return (
-      <LoadingInfoWrapper>
-        <LoadingInfoBox>
-          <LoadingInfo message="Waiting for structure" />
-        </LoadingInfoBox>
-      </LoadingInfoWrapper>
-    );
+  if (!columns || columns.length == 0) return <LoadingInfo wrapper message="Waiting for structure" />;
 
   if (errorMessage) {
     return <ErrorInfo message={errorMessage} />;
@@ -936,8 +913,8 @@ export default function DataGridCore(props) {
           )}
         </TableHead>
         <TableBody ref={tableBodyRef}>
-          {_.range(firstVisibleRowScrollIndex, firstVisibleRowScrollIndex + visibleRowCountUpperBound)
-            .map((rowIndex) => (
+          {_.range(firstVisibleRowScrollIndex, firstVisibleRowScrollIndex + visibleRowCountUpperBound).map(
+            (rowIndex) => (
               <DataGridRow
                 key={rowIndex}
                 grider={grider}
@@ -952,7 +929,8 @@ export default function DataGridCore(props) {
                 display={display}
                 focusedColumn={display.focusedColumn}
               />
-            ))}
+            )
+          )}
         </TableBody>
       </Table>
       <HorizontalScrollBar
@@ -972,23 +950,14 @@ export default function DataGridCore(props) {
         viewportRatio={visibleRowCountUpperBound / grider.rowCount}
       />
       {allRowCount && <RowCountLabel>{rowCountInfo}</RowCountLabel>}
-      {props.toolbarPortalRef && props.toolbarPortalRef.current &&
+      {props.toolbarPortalRef &&
+        props.toolbarPortalRef.current &&
         tabVisible &&
         ReactDOM.createPortal(
-          <DataGridToolbar
-            reload={() => display.reload()}
-            save={handleSave}
-            grider={grider}
-          />,
+          <DataGridToolbar reload={() => display.reload()} save={handleSave} grider={grider} />,
           props.toolbarPortalRef.current
         )}
-      {isLoading && (
-        <LoadingInfoWrapper>
-          <LoadingInfoBox>
-            <LoadingInfo message="Loading data" />
-          </LoadingInfoBox>
-        </LoadingInfoWrapper>
-      )}
+      {isLoading && <LoadingInfo wrapper message="Loading data" />}
     </GridContainer>
   );
 }

packages/web/src/tabs/FreeTableTab.js

@@ -9,14 +9,33 @@ import FreeTableGrid from '../freetable/FreeTableGrid';
 import SaveArchiveModal from '../modals/SaveArchiveModal';
 import useModalState from '../modals/useModalState';
 import axios from '../utility/axios';
+import LoadingInfo from '../widgets/LoadingInfo';
 import { changeTab } from '../utility/common';
+import ErrorInfo from '../widgets/ErrorInfo';
 
 export default function FreeDataTab({ archiveFolder, archiveFile, tabVisible, toolbarPortalRef, tabid, initialData }) {
   const [config, setConfig] = useGridConfig(tabid);
-  const [modelState, dispatchModel] = useUndoReducer(initialData || createFreeTableModel());
+  const [modelState, dispatchModel] = useUndoReducer(createFreeTableModel());
   const storageKey = `tabdata_freetable_${tabid}`;
   const saveSqlFileModalState = useModalState();
   const setOpenedTabs = useSetOpenedTabs();
+  const [isLoading, setIsLoading] = React.useState(false);
+  const [errorMessage, setErrorMessage] = React.useState(null);
+
+  const handleLoadInitialData = async () => {
+    setIsLoading(true);
+    try {
+      const resp = await axios.post('runners/load-reader', initialData);
+      // @ts-ignore
+      dispatchModel({ type: 'reset', value: resp.data });
+      setIsLoading(false);
+    } catch (err) {
+      setIsLoading(false);
+      const errorMessage = (err && err.response && err.response.data && err.response.data.error) || 'Loading failed';
+      setErrorMessage(errorMessage);
+      console.error(err.response);
+    }
+  };
 
   React.useEffect(() => {
     const existingData = localStorage.getItem(storageKey);
@@ -24,6 +43,8 @@ export default function FreeDataTab({ archiveFolder, archiveFile, tabVisible, to
       const value = JSON.parse(existingData);
       // @ts-ignore
       dispatchModel({ type: 'reset', value });
+    } else if (initialData) {
+      handleLoadInitialData();
     }
   }, []);
 
@@ -40,6 +61,13 @@ export default function FreeDataTab({ archiveFolder, archiveFile, tabVisible, to
     }));
   };
 
+  if (isLoading) {
+    return <LoadingInfo wrapper message="Loading data" />;
+  }
+  if (errorMessage) {
+    return <ErrorInfo message={errorMessage} />;
+  }
+
   return (
     <>
       <FreeTableGrid

packages/web/src/widgets/LoadingInfo.js

@@ -13,8 +13,24 @@ const Spinner = styled.div`
   margin: 10px;
 `;
 
-export default function LoadingInfo({ message }) {
-  return (
+const LoadingInfoWrapper = styled.div`
+  position: absolute;
+  left: 0;
+  top: 0;
+  right: 0;
+  bottom: 0;
+  display: flex;
+  align-items: center;
+  justify-content: space-around;
+`;
+const LoadingInfoBox = styled.div`
+  background-color: #ccc;
+  padding: 10px;
+  border: 1px solid gray;
+`;
+
+export default function LoadingInfo({ message, wrapper = false }) {
+  const core = (
     <Container>
       <Spinner>
         <i className="fas fa-spinner fa-spin" />
@@ -22,4 +38,13 @@ export default function LoadingInfo({ message }) {
       {message}
     </Container>
   );
+  if (wrapper) {
+    return (
+      <LoadingInfoWrapper>
+        <LoadingInfoBox>{core}</LoadingInfoBox>
+      </LoadingInfoWrapper>
+    );
+  } else {
+    return core;
+  }
 }