diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js index 26dd76eb2..b5c093f1e 100644 --- a/packages/api/src/controllers/runners.js +++ b/packages/api/src/controllers/runners.js @@ -11,7 +11,18 @@ const scriptTemplate = (script) => ` const dbgateApi = require(process.env.DBGATE_API || "@dbgate/api"); require=null; async function run() { -${script} +const reader = ${script} +} +dbgateApi.runScript(run); +`; + +const loaderScriptTemplate = (functionName, props, runid) => ` +const dbgateApi = require(process.env.DBGATE_API || "@dbgate/api"); +require=null; +async function run() { +const reader=await dbgateApi.${functionName}(${JSON.stringify(props)}); +const writer=await dbgateApi.collectorWriter({runid: '${runid}'}); +await dbgateApi.copyStream(reader, writer); } dbgateApi.runScript(run); `; @@ -19,9 +30,10 @@ dbgateApi.runScript(run); module.exports = { /** @type {import('@dbgate/types').OpenedRunner[]} */ opened: [], + requests: {}, dispatchMessage(runid, message) { - // console.log('DISPATCHING', message); + if (message) console.log('...', message.message); if (_.isString(message)) { socket.emit(`runner-info-${runid}`, { message, @@ -40,12 +52,24 @@ module.exports = { handle_ping() {}, - start_meta: 'post', - async start({ script }) { - const runid = uuidv1(); + handle_freeData(runid, { freeData }) { + const [resolve, reject] = this.requests[runid]; + resolve(freeData); + delete this.requests[runid]; + }, + + rejectRequest(runid, error) { + if (this.requests[runid]) { + const [resolve, reject] = this.requests[runid]; + reject(error); + delete this.requests[runid]; + } + }, + + startCore(runid, scriptText) { const directory = path.join(rundir(), runid); const scriptFile = path.join(uploadsdir(), runid + '.js'); - fs.writeFileSync(`${scriptFile}`, scriptTemplate(script)); + fs.writeFileSync(`${scriptFile}`, scriptText); fs.mkdirSync(directory); console.log(`RUNNING SCRIPT ${scriptFile}`); const subprocess = fork(scriptFile, ['--checkParent'], { @@ -61,9 +85,13 @@ module.exports = { byline(subprocess.stdout).on('data', pipeDispatcher('info')); byline(subprocess.stderr).on('data', pipeDispatcher('error')); subprocess.on('exit', (code) => { + this.rejectRequest(runid, { message: 'No data retured, maybe input data source is too big' }); + console.log('... EXIT process', code); socket.emit(`runner-done-${runid}`, code); }); subprocess.on('error', (error) => { + this.rejectRequest(runid, { message: error && (error.message || error.toString()) }); + console.error('... ERROR subprocess', error); this.dispatchMessage({ severity: 'error', message: error.toString(), @@ -81,6 +109,12 @@ module.exports = { return newOpened; }, + start_meta: 'post', + async start({ script }) { + const runid = uuidv1(); + return this.startCore(runid, scriptTemplate(script)); + }, + cancel_meta: 'post', async cancel({ runid }) { const runner = this.opened.find((x) => x.runid == runid); @@ -106,4 +140,14 @@ module.exports = { } return res; }, + + loadReader_meta: 'post', + async loadReader({ functionName, props }) { + const promise = new Promise((resolve, reject) => { + const runid = uuidv1(); + this.requests[runid] = [resolve, reject]; + this.startCore(runid, loaderScriptTemplate(functionName, props, runid)); + }); + return promise; + }, }; diff --git a/packages/api/src/shell/collectorWriter.js b/packages/api/src/shell/collectorWriter.js new file mode 100644 index 000000000..c896e3d07 --- /dev/null +++ b/packages/api/src/shell/collectorWriter.js @@ -0,0 +1,33 @@ +const stream = require('stream'); + +class CollectorWriterStream extends stream.Writable { + constructor(options) { + super(options); + this.rows = []; + this.structure = null; + this.runid = options.runid; + } + _write(chunk, enc, next) { + if (!this.structure) this.structure = chunk; + else this.rows.push(chunk); + next(); + } + + _final(callback) { + process.send({ + msgtype: 'freeData', + runid: this.runid, + freeData: { rows: this.rows, structure: this.structure }, + }); + callback(); + } +} + +async function collectorWriter({ runid }) { + return new CollectorWriterStream({ + objectMode: true, + runid, + }); +} + +module.exports = collectorWriter; diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index 5b4bcb3e3..e0abbe3d6 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -13,6 +13,7 @@ const jsonLinesReader = require('./jsonLinesReader'); const jslDataReader = require('./jslDataReader'); const archiveWriter = require('./archiveWriter'); const archiveReader = require('./archiveReader'); +const collectorWriter = require('./collectorWriter'); module.exports = { queryReader, @@ -30,4 +31,5 @@ module.exports = { jslDataReader, archiveWriter, archiveReader, + collectorWriter, }; diff --git a/packages/web/src/appobj/archiveFileAppObject.js b/packages/web/src/appobj/archiveFileAppObject.js index d6b2e2fd7..4a6918343 100644 --- a/packages/web/src/appobj/archiveFileAppObject.js +++ b/packages/web/src/appobj/archiveFileAppObject.js @@ -29,14 +29,20 @@ function Menu({ data, setOpenedTabs }) { openArchive(setOpenedTabs, data.fileName, data.folderName); }; const handleOpenWrite = async () => { - const resp = await axios.post('archive/load-free-table', { file: data.fileName, folder: data.folderName }); + // const resp = await axios.post('archive/load-free-table', { file: data.fileName, folder: data.folderName }); openNewTab(setOpenedTabs, { title: data.fileName, icon: 'freetable.svg', tabComponent: 'FreeTableTab', props: { - initialData: resp.data, + initialData: { + functionName: 'archiveReader', + props: { + fileName: data.fileName, + folderName: data.folderName, + }, + }, archiveFile: data.fileName, archiveFolder: data.folderName, }, diff --git a/packages/web/src/datagrid/DataGridCore.js b/packages/web/src/datagrid/DataGridCore.js index 21d60c54e..6207b3aab 100644 --- a/packages/web/src/datagrid/DataGridCore.js +++ b/packages/web/src/datagrid/DataGridCore.js @@ -90,22 +90,6 @@ const RowCountLabel = styled.div` bottom: 20px; `; -const LoadingInfoWrapper = styled.div` - position: absolute; - left: 0; - top: 0; - right: 0; - bottom: 0; - display: flex; - align-items: center; - justify-content: space-around; -`; -const LoadingInfoBox = styled.div` - background-color: #ccc; - padding: 10px; - border: 1px solid gray; -`; - /** @param props {import('./types').DataGridProps} */ export default function DataGridCore(props) { const { @@ -296,14 +280,7 @@ export default function DataGridCore(props) { return `Rows: ${allRowCount.toLocaleString()}`; }, [selectedCells, allRowCount, grider, visibleRealColumns]); - if (!columns || columns.length == 0) - return ( - - - - - - ); + if (!columns || columns.length == 0) return ; if (errorMessage) { return ; @@ -936,8 +913,8 @@ export default function DataGridCore(props) { )} - {_.range(firstVisibleRowScrollIndex, firstVisibleRowScrollIndex + visibleRowCountUpperBound) - .map((rowIndex) => ( + {_.range(firstVisibleRowScrollIndex, firstVisibleRowScrollIndex + visibleRowCountUpperBound).map( + (rowIndex) => ( - ))} + ) + )} {allRowCount && {rowCountInfo}} - {props.toolbarPortalRef && props.toolbarPortalRef.current && + {props.toolbarPortalRef && + props.toolbarPortalRef.current && tabVisible && ReactDOM.createPortal( - display.reload()} - save={handleSave} - grider={grider} - />, + display.reload()} save={handleSave} grider={grider} />, props.toolbarPortalRef.current )} - {isLoading && ( - - - - - - )} + {isLoading && } ); } diff --git a/packages/web/src/tabs/FreeTableTab.js b/packages/web/src/tabs/FreeTableTab.js index a54bd35b1..a222a3a0b 100644 --- a/packages/web/src/tabs/FreeTableTab.js +++ b/packages/web/src/tabs/FreeTableTab.js @@ -9,14 +9,33 @@ import FreeTableGrid from '../freetable/FreeTableGrid'; import SaveArchiveModal from '../modals/SaveArchiveModal'; import useModalState from '../modals/useModalState'; import axios from '../utility/axios'; +import LoadingInfo from '../widgets/LoadingInfo'; import { changeTab } from '../utility/common'; +import ErrorInfo from '../widgets/ErrorInfo'; export default function FreeDataTab({ archiveFolder, archiveFile, tabVisible, toolbarPortalRef, tabid, initialData }) { const [config, setConfig] = useGridConfig(tabid); - const [modelState, dispatchModel] = useUndoReducer(initialData || createFreeTableModel()); + const [modelState, dispatchModel] = useUndoReducer(createFreeTableModel()); const storageKey = `tabdata_freetable_${tabid}`; const saveSqlFileModalState = useModalState(); const setOpenedTabs = useSetOpenedTabs(); + const [isLoading, setIsLoading] = React.useState(false); + const [errorMessage, setErrorMessage] = React.useState(null); + + const handleLoadInitialData = async () => { + setIsLoading(true); + try { + const resp = await axios.post('runners/load-reader', initialData); + // @ts-ignore + dispatchModel({ type: 'reset', value: resp.data }); + setIsLoading(false); + } catch (err) { + setIsLoading(false); + const errorMessage = (err && err.response && err.response.data && err.response.data.error) || 'Loading failed'; + setErrorMessage(errorMessage); + console.error(err.response); + } + }; React.useEffect(() => { const existingData = localStorage.getItem(storageKey); @@ -24,6 +43,8 @@ export default function FreeDataTab({ archiveFolder, archiveFile, tabVisible, to const value = JSON.parse(existingData); // @ts-ignore dispatchModel({ type: 'reset', value }); + } else if (initialData) { + handleLoadInitialData(); } }, []); @@ -40,6 +61,13 @@ export default function FreeDataTab({ archiveFolder, archiveFile, tabVisible, to })); }; + if (isLoading) { + return ; + } + if (errorMessage) { + return ; + } + return ( <> @@ -22,4 +38,13 @@ export default function LoadingInfo({ message }) { {message} ); + if (wrapper) { + return ( + + {core} + + ); + } else { + return core; + } }