From f68bdafd9fdad41304a798791d0e2236d0b791cc Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 11 Jun 2020 10:09:04 +0200 Subject: [PATCH] scripting engine --- packages/api/src/main.js | 2 +- packages/api/src/proc/sessionProcess.js | 11 +++++--- packages/api/src/shell/csvWriter.js | 31 +++++++++++++++++--- packages/api/src/shell/fakeObjectReader.js | 1 + packages/api/src/utility/goSplit.js | 18 ++++++++++++ packages/engines/mssql/index.js | 4 +++ packages/engines/mysql/index.js | 18 +++++++++++- packages/web/src/sqleditor/JslDataGrid.js | 5 ++-- packages/web/src/tabs/QueryTab.js | 11 +++++--- test/exportTable.js | 21 ++++++++++---- test/test.csv | 33 +++++++++++++++++----- 11 files changed, 127 insertions(+), 28 deletions(-) create mode 100644 packages/api/src/utility/goSplit.js diff --git a/packages/api/src/main.js b/packages/api/src/main.js index e15ef032a..3423c0a50 100644 --- a/packages/api/src/main.js +++ b/packages/api/src/main.js @@ -43,7 +43,7 @@ function start(argument = null) { } app.use(cors()); - app.use(bodyParser.json()); + app.use(bodyParser.json({ limit: '50mb' })); useController(app, '/connections', connections); useController(app, '/server-connections', serverConnections); diff --git a/packages/api/src/proc/sessionProcess.js b/packages/api/src/proc/sessionProcess.js index baaec3283..60bab4b4a 100644 --- a/packages/api/src/proc/sessionProcess.js +++ b/packages/api/src/proc/sessionProcess.js @@ -4,6 +4,7 @@ const path = require('path'); const fs = require('fs'); const _ = require('lodash'); const childProcessChecker = require('../utility/childProcessChecker'); +const goSplit = require('../utility/goSplit'); const driverConnect = require('../utility/driverConnect'); const { jsldir } = require('../utility/directories'); @@ -19,7 +20,7 @@ class TableWriter { this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); this.currentRowCount = 0; this.currentChangeIndex = 0; - fs.writeFileSync(this.currentFile, JSON.stringify(columns) + '\n'); + fs.writeFileSync(this.currentFile, JSON.stringify({ columns }) + '\n'); this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' }); this.writeCurrentStats(false, false); process.send({ msgtype: 'recordset', jslid: this.jslid }); @@ -140,9 +141,11 @@ async function handleExecuteQuery({ sql }) { await waitConnected(); const driver = engines(storedConnection); - const handler = new StreamHandler(); - const stream = await driver.stream(systemConnection, sql, handler); - handler.stream = stream; + for (const sqlItem of goSplit(sql)) { + const handler = new StreamHandler(); + const stream = await driver.stream(systemConnection, sqlItem, handler); + handler.stream = stream; + } } const messageHandlers = { diff --git a/packages/api/src/shell/csvWriter.js b/packages/api/src/shell/csvWriter.js index ec6c44054..a5297c651 100644 --- a/packages/api/src/shell/csvWriter.js +++ b/packages/api/src/shell/csvWriter.js @@ -1,13 +1,36 @@ const csv = require('csv'); const fs = require('fs'); +const stream = require('stream'); -async function csvWriter({ fileName, encoding = 'utf-8', ...options }) { +class CsvPrepareStream extends stream.Transform { + constructor({ header }) { + super({ objectMode: true }); + this.structure = null; + this.header = header; + } + _transform(chunk, encoding, done) { + if (this.structure) { + this.push(this.structure.columns.map((col) => chunk[col.columnName])); + done(); + } else { + this.structure = chunk; + if (this.header) { + this.push(chunk.columns.map((x) => x.columnName)); + } + done(); + } + } +} + +async function csvWriter({ fileName, encoding = 'utf-8', header = true, delimiter, quoted }) { console.log(`Writing file ${fileName}`); - const csvStream = csv.stringify(options); + const csvPrepare = new CsvPrepareStream({ header }); + const csvStream = csv.stringify({ delimiter, quoted }); const fileStream = fs.createWriteStream(fileName, encoding); + csvPrepare.pipe(csvStream); csvStream.pipe(fileStream); - csvStream['finisher'] = fileStream; - return csvStream; + csvPrepare['finisher'] = fileStream; + return csvPrepare; } module.exports = csvWriter; diff --git a/packages/api/src/shell/fakeObjectReader.js b/packages/api/src/shell/fakeObjectReader.js index 73532d5cf..a5e19ca44 100644 --- a/packages/api/src/shell/fakeObjectReader.js +++ b/packages/api/src/shell/fakeObjectReader.js @@ -5,6 +5,7 @@ async function fakeObjectReader({ delay = 0 } = {}) { objectMode: true, }); function doWrite() { + pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }] }); pass.write({ id: 1, country: 'Czechia' }); pass.write({ id: 2, country: 'Austria' }); pass.write({ country: 'Germany', id: 3 }); diff --git a/packages/api/src/utility/goSplit.js b/packages/api/src/utility/goSplit.js new file mode 100644 index 000000000..9555ca7ee --- /dev/null +++ b/packages/api/src/utility/goSplit.js @@ -0,0 +1,18 @@ +function goSplit(sql) { + if (!sql) return []; + const lines = sql.split('\n'); + const res = []; + let buffer = ''; + for (const line of lines) { + if (/^\s*go\s*$/i.test(line)) { + if (buffer.trim()) res.push(buffer); + buffer = ''; + } else { + buffer += line + '\n'; + } + } + if (buffer.trim()) res.push(buffer); + return res; +} + +module.exports = goSplit; diff --git a/packages/engines/mssql/index.js b/packages/engines/mssql/index.js index 62957871c..8a2d772c1 100644 --- a/packages/engines/mssql/index.js +++ b/packages/engines/mssql/index.js @@ -164,6 +164,10 @@ const driver = { }); request.stream = true; + request.on('recordset', (driverColumns) => { + const [columns, mapper] = extractColumns(driverColumns); + pass.write({ columns }); + }); request.on('row', (row) => pass.write(row)); request.on('error', (err) => { console.error(err); diff --git a/packages/engines/mysql/index.js b/packages/engines/mysql/index.js index b2058bb5c..39c0b4be3 100644 --- a/packages/engines/mysql/index.js +++ b/packages/engines/mysql/index.js @@ -86,7 +86,23 @@ const driver = { }, async readableStream(connection, sql) { const query = connection.query(sql); - return query.stream({ highWaterMark: 100 }); + const { stream } = connection._nativeModules; + + const pass = new stream.PassThrough({ + objectMode: true, + highWaterMark: 100, + }); + + query + .on('error', (err) => { + console.error(err); + pass.end(); + }) + .on('fields', (fields) => pass.write({ columns: extractColumns(fields) })) + .on('result', (row) => pass.write(row)) + .on('end', () => pass.end()); + + return pass; }, async getVersion(connection) { const { rows } = await this.query(connection, "show variables like 'version'"); diff --git a/packages/web/src/sqleditor/JslDataGrid.js b/packages/web/src/sqleditor/JslDataGrid.js index efab38edc..ad4dd3378 100644 --- a/packages/web/src/sqleditor/JslDataGrid.js +++ b/packages/web/src/sqleditor/JslDataGrid.js @@ -4,11 +4,12 @@ import { JslGridDisplay, createGridConfig, createGridCache } from '@dbgate/datal import useFetch from '../utility/useFetch'; export default function JslDataGrid({ jslid }) { - const columns = useFetch({ + const info = useFetch({ params: { jslid }, url: 'jsldata/get-info', - defaultValue: [], + defaultValue: {}, }); + const columns = (info && info.columns) || []; const [config, setConfig] = React.useState(createGridConfig()); const [cache, setCache] = React.useState(createGridCache()); const display = React.useMemo(() => new JslGridDisplay(jslid, columns, config, setConfig, cache, setCache), [ diff --git a/packages/web/src/tabs/QueryTab.js b/packages/web/src/tabs/QueryTab.js index c2a72d085..706e501a7 100644 --- a/packages/web/src/tabs/QueryTab.js +++ b/packages/web/src/tabs/QueryTab.js @@ -89,10 +89,13 @@ export default function QueryTab({ } }, [sqlFromTemplate]); - const saveToStorage = React.useCallback(() => localStorage.setItem(localStorageKey, queryTextRef.current), [ - localStorageKey, - queryTextRef, - ]); + const saveToStorage = React.useCallback(() => { + try { + localStorage.setItem(localStorageKey, queryTextRef.current); + } catch (err) { + console.error(err); + } + }, [localStorageKey, queryTextRef]); const saveToStorageDebounced = React.useMemo(() => _.debounce(saveToStorage, 5000), [saveToStorage]); React.useEffect(() => { diff --git a/test/exportTable.js b/test/exportTable.js index 35fffdba2..4fb9b4121 100644 --- a/test/exportTable.js +++ b/test/exportTable.js @@ -1,13 +1,24 @@ const dbgateApi = require('@dbgate/api'); async function run() { + // const queryReader = await dbgateApi.queryReader({ + // connection: { + // server: 'localhost', + // engine: 'mysql', + // user: 'root', + // password: 'test', + // port: '3307', + // database: 'Chinook', + // }, + // sql: 'SELECT * FROM Genre', + // }); + const queryReader = await dbgateApi.queryReader({ connection: { server: 'localhost', - engine: 'mysql', - user: 'root', - password: 'test', - port: '3307', + engine: 'mssql', + user: 'sa', + password: 'Pwd2020Db', database: 'Chinook', }, sql: 'SELECT * FROM Genre', @@ -17,7 +28,7 @@ async function run() { const csvWriter = await dbgateApi.csvWriter({ fileName: 'test.csv', - header: true, + // header: false, }); const consoleWriter = await dbgateApi.consoleObjectWriter(); diff --git a/test/test.csv b/test/test.csv index e86e72da5..a129fcf4c 100644 --- a/test/test.csv +++ b/test/test.csv @@ -1,7 +1,26 @@ -id,country -1,Czechia -2,Austria -3,Germany -4,Romania -5,Great Britain -6,"Bosna, Hecegovina" +GenreId,Name +1,Rock +2,Jazz +3,Metal +4,Alternative & Punk +5,Rock And Roll +6,Blues +7,Latin +8,Reggae +9,Pop +10,Soundtrack +11,Bossa Nova +12,Easy Listening +13,Heavy Metal +14,R&B/Soul +15,Electronica/Dance +16,World +17,Hip Hop/Rap +18,Science Fiction +19,TV Shows +20,Sci Fi & Fantasy +21,Drama +22,Comedy +23,Alternative +24,Classical +25,Opera