diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index a938019d1..2aea7ea5a 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -2,6 +2,7 @@ const path = require('path'); const fs = require('fs'); const lineReader = require('line-reader'); const { jsldir } = require('../utility/directories'); +const socket = require('../utility/socket'); module.exports = { openedReaders: {}, @@ -78,4 +79,15 @@ module.exports = { } return res; }, + + getStats_meta: 'get', + getStats({ jslid }) { + const file = path.join(jsldir(), `${jslid}.jsonl.stats`); + return JSON.parse(fs.readFileSync(file, 'utf-8')); + }, + + async notifyChangedStats(stats) { + await this.closeReader(stats.jslid); + socket.emit(`jsldata-stats-${stats.jslid}`, stats); + }, }; diff --git a/packages/api/src/controllers/sessions.js b/packages/api/src/controllers/sessions.js index 2c9fac8c2..cc8e2a671 100644 --- a/packages/api/src/controllers/sessions.js +++ b/packages/api/src/controllers/sessions.js @@ -3,6 +3,7 @@ const uuidv1 = require('uuid/v1'); const connections = require('./connections'); const socket = require('../utility/socket'); const { fork } = require('child_process'); +const jsldata = require('./jsldata'); module.exports = { /** @type {import('@dbgate/types').OpenedSession[]} */ @@ -53,6 +54,10 @@ module.exports = { socket.emit(`session-recordset-${sesid}`, { jslid }); }, + handle_stats(sesid, stats) { + jsldata.notifyChangedStats(stats); + }, + create_meta: 'post', async create({ conid, database }) { const sesid = uuidv1(); diff --git a/packages/api/src/proc/sessionProcess.js b/packages/api/src/proc/sessionProcess.js index a6566ac57..67cc34a64 100644 --- a/packages/api/src/proc/sessionProcess.js +++ b/packages/api/src/proc/sessionProcess.js @@ -2,6 +2,7 @@ const engines = require('@dbgate/engines'); const uuidv1 = require('uuid/v1'); const path = require('path'); const fs = require('fs'); +const _ = require('lodash'); const driverConnect = require('../utility/driverConnect'); const { jsldir } = require('../utility/directories'); @@ -26,7 +27,27 @@ class StreamHandler { closeCurrentStream() { if (this.currentStream) { this.currentStream.end(); + this.writeCurrentStats(true, true); + this.currentStream = null; + this.jslid = null; + this.currentFile = null; + this.currentRowCount = null; + this.currentChangeIndex = null; + } + } + + writeCurrentStats(isFinished = false, emitEvent = false) { + const stats = { + rowCount: this.currentRowCount, + changeIndex: this.currentChangeIndex, + isFinished, + jslid: this.jslid, + }; + fs.writeFileSync(`${this.currentFile}.stats`, JSON.stringify(stats)); + this.currentChangeIndex += 1; + if (emitEvent) { + process.send({ msgtype: 'stats', ...stats }); } } @@ -35,12 +56,23 @@ class StreamHandler { this.jslid = uuidv1(); this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); this.currentStream = fs.createWriteStream(this.currentFile); + this.currentRowCount = 0; + this.currentChangeIndex = 0; fs.writeFileSync(`${this.currentFile}.info`, JSON.stringify(columns)); process.send({ msgtype: 'recordset', jslid: this.jslid }); + this.writeCurrentStats(); + + this.onRow = _.throttle((jslid) => { + if (jslid == this.jslid) { + this.writeCurrentStats(false, true); + } + }, 500); } row(row) { // console.log('ACCEPT ROW', row); this.currentStream.write(JSON.stringify(row) + '\n'); + this.currentRowCount += 1; + this.onRow(this.jslid); } // error(error) { // process.send({ msgtype: 'error', error }); diff --git a/packages/web/src/datagrid/DataGridCore.js b/packages/web/src/datagrid/DataGridCore.js index f72b5822f..6a5983b66 100644 --- a/packages/web/src/datagrid/DataGridCore.js +++ b/packages/web/src/datagrid/DataGridCore.js @@ -37,6 +37,7 @@ import ColumnHeaderControl from './ColumnHeaderControl'; import InlineButton from '../widgets/InlineButton'; import { showMenu } from '../modals/DropDownMenu'; import DataGridContextMenu from './DataGridContextMenu'; +import useSocket from '../utility/SocketProvider'; const GridContainer = styled.div` position: absolute; @@ -143,6 +144,18 @@ function dataPageAvailable(props) { /** @param props {import('./types').DataGridProps} */ async function loadRowCount(props) { const { display, conid, database, jslid } = props; + + if (jslid) { + const response = await axios.request({ + url: 'jsldata/get-stats', + method: 'get', + params: { + jslid, + }, + }); + return response.data.rowCount; + } + const sql = display.getCountQuery(); const response = await axios.request({ @@ -160,7 +173,7 @@ async function loadRowCount(props) { /** @param props {import('./types').DataGridProps} */ export default function DataGridCore(props) { - const { conid, database, display, changeSetState, dispatchChangeSet, tabVisible } = props; + const { conid, database, display, changeSetState, dispatchChangeSet, tabVisible, jslid } = props; // console.log('RENDER GRID', display.baseTable.pureName); const columns = React.useMemo(() => display.getGridColumns(), [display]); @@ -225,7 +238,6 @@ export default function DataGridCore(props) { setLoadProps((oldLoadProps) => ({ ...oldLoadProps, isLoading: true, - allRowCount: null, })); const loadStart = new Date().getTime(); loadedTimeRef.current = loadStart; @@ -265,6 +277,7 @@ export default function DataGridCore(props) { // const { rows, columns } = data || {}; const [firstVisibleRowScrollIndex, setFirstVisibleRowScrollIndex] = React.useState(0); const [firstVisibleColumnScrollIndex, setFirstVisibleColumnScrollIndex] = React.useState(0); + const socket = useSocket(); const [headerRowRef, { height: rowHeight }] = useDimensions(); const [tableBodyRef] = useDimensions(); @@ -357,6 +370,23 @@ export default function DataGridCore(props) { } }, [tabVisible, focusFieldRef.current]); + const handleJslDataStats = React.useCallback((stats) => { + setLoadProps((oldProps) => ({ + ...oldProps, + allRowCount: stats.rowCount, + isLoadedAll: false, + })); + }, []); + + React.useEffect(() => { + if (jslid && socket) { + socket.on(`jsldata-stats-${jslid}`, handleJslDataStats); + return () => { + socket.off(`jsldata-stats-${jslid}`, handleJslDataStats); + }; + } + }, [jslid]); + // const handleCloseInplaceEditor = React.useCallback( // mode => { // const [row, col] = currentCell || [];