loading data from query fix

packages/api/src/controllers/jsldata.js

@@ -2,6 +2,7 @@ const path = require('path');
 const fs = require('fs');
 const lineReader = require('line-reader');
 const { jsldir } = require('../utility/directories');
+const socket = require('../utility/socket');
 
 module.exports = {
   openedReaders: {},
@@ -78,4 +79,15 @@ module.exports = {
     }
     return res;
   },
+
+  getStats_meta: 'get',
+  getStats({ jslid }) {
+    const file = path.join(jsldir(), `${jslid}.jsonl.stats`);
+    return JSON.parse(fs.readFileSync(file, 'utf-8'));
+  },
+
+  async notifyChangedStats(stats) {
+    await this.closeReader(stats.jslid);
+    socket.emit(`jsldata-stats-${stats.jslid}`, stats);
+  },
 };

packages/api/src/controllers/sessions.js

@@ -3,6 +3,7 @@ const uuidv1 = require('uuid/v1');
 const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
+const jsldata = require('./jsldata');
 
 module.exports = {
   /** @type {import('@dbgate/types').OpenedSession[]} */
@@ -53,6 +54,10 @@ module.exports = {
     socket.emit(`session-recordset-${sesid}`, { jslid });
   },
 
+  handle_stats(sesid, stats) {
+    jsldata.notifyChangedStats(stats);
+  },
+
   create_meta: 'post',
   async create({ conid, database }) {
     const sesid = uuidv1();

packages/api/src/proc/sessionProcess.js

@@ -2,6 +2,7 @@ const engines = require('@dbgate/engines');
 const uuidv1 = require('uuid/v1');
 const path = require('path');
 const fs = require('fs');
+const _ = require('lodash');
 
 const driverConnect = require('../utility/driverConnect');
 const { jsldir } = require('../utility/directories');
@@ -26,7 +27,27 @@ class StreamHandler {
   closeCurrentStream() {
     if (this.currentStream) {
       this.currentStream.end();
+      this.writeCurrentStats(true, true);
+
       this.currentStream = null;
+      this.jslid = null;
+      this.currentFile = null;
+      this.currentRowCount = null;
+      this.currentChangeIndex = null;
+    }
+  }
+
+  writeCurrentStats(isFinished = false, emitEvent = false) {
+    const stats = {
+      rowCount: this.currentRowCount,
+      changeIndex: this.currentChangeIndex,
+      isFinished,
+      jslid: this.jslid,
+    };
+    fs.writeFileSync(`${this.currentFile}.stats`, JSON.stringify(stats));
+    this.currentChangeIndex += 1;
+    if (emitEvent) {
+      process.send({ msgtype: 'stats', ...stats });
     }
   }
 
@@ -35,12 +56,23 @@ class StreamHandler {
     this.jslid = uuidv1();
     this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     this.currentStream = fs.createWriteStream(this.currentFile);
+    this.currentRowCount = 0;
+    this.currentChangeIndex = 0;
     fs.writeFileSync(`${this.currentFile}.info`, JSON.stringify(columns));
     process.send({ msgtype: 'recordset', jslid: this.jslid });
+    this.writeCurrentStats();
+
+    this.onRow = _.throttle((jslid) => {
+      if (jslid == this.jslid) {
+        this.writeCurrentStats(false, true);
+      }
+    }, 500);
   }
   row(row) {
     // console.log('ACCEPT ROW', row);
     this.currentStream.write(JSON.stringify(row) + '\n');
+    this.currentRowCount += 1;
+    this.onRow(this.jslid);
   }
   // error(error) {
   //   process.send({ msgtype: 'error', error });

packages/web/src/datagrid/DataGridCore.js

@@ -37,6 +37,7 @@ import ColumnHeaderControl from './ColumnHeaderControl';
 import InlineButton from '../widgets/InlineButton';
 import { showMenu } from '../modals/DropDownMenu';
 import DataGridContextMenu from './DataGridContextMenu';
+import useSocket from '../utility/SocketProvider';
 
 const GridContainer = styled.div`
   position: absolute;
@@ -143,6 +144,18 @@ function dataPageAvailable(props) {
 /** @param props {import('./types').DataGridProps} */
 async function loadRowCount(props) {
   const { display, conid, database, jslid } = props;
+
+  if (jslid) {
+    const response = await axios.request({
+      url: 'jsldata/get-stats',
+      method: 'get',
+      params: {
+        jslid,
+      },
+    });
+    return response.data.rowCount;
+  }
+
   const sql = display.getCountQuery();
 
   const response = await axios.request({
@@ -160,7 +173,7 @@ async function loadRowCount(props) {
 
 /** @param props {import('./types').DataGridProps} */
 export default function DataGridCore(props) {
-  const { conid, database, display, changeSetState, dispatchChangeSet, tabVisible } = props;
+  const { conid, database, display, changeSetState, dispatchChangeSet, tabVisible, jslid } = props;
   // console.log('RENDER GRID', display.baseTable.pureName);
   const columns = React.useMemo(() => display.getGridColumns(), [display]);
 
@@ -225,7 +238,6 @@ export default function DataGridCore(props) {
     setLoadProps((oldLoadProps) => ({
       ...oldLoadProps,
       isLoading: true,
-      allRowCount: null,
     }));
     const loadStart = new Date().getTime();
     loadedTimeRef.current = loadStart;
@@ -265,6 +277,7 @@ export default function DataGridCore(props) {
   // const { rows, columns } = data || {};
   const [firstVisibleRowScrollIndex, setFirstVisibleRowScrollIndex] = React.useState(0);
   const [firstVisibleColumnScrollIndex, setFirstVisibleColumnScrollIndex] = React.useState(0);
+  const socket = useSocket();
 
   const [headerRowRef, { height: rowHeight }] = useDimensions();
   const [tableBodyRef] = useDimensions();
@@ -357,6 +370,23 @@ export default function DataGridCore(props) {
     }
   }, [tabVisible, focusFieldRef.current]);
 
+  const handleJslDataStats = React.useCallback((stats) => {
+    setLoadProps((oldProps) => ({
+      ...oldProps,
+      allRowCount: stats.rowCount,
+      isLoadedAll: false,
+    }));
+  }, []);
+
+  React.useEffect(() => {
+    if (jslid && socket) {
+      socket.on(`jsldata-stats-${jslid}`, handleJslDataStats);
+      return () => {
+        socket.off(`jsldata-stats-${jslid}`, handleJslDataStats);
+      };
+    }
+  }, [jslid]);
+
   // const handleCloseInplaceEditor = React.useCallback(
   //   mode => {
   //     const [row, col] = currentCell || [];