scripting engine

packages/api/src/main.js

@@ -43,7 +43,7 @@ function start(argument = null) {
   }
 
   app.use(cors());
-  app.use(bodyParser.json());
+  app.use(bodyParser.json({ limit: '50mb' }));
 
   useController(app, '/connections', connections);
   useController(app, '/server-connections', serverConnections);

packages/api/src/proc/sessionProcess.js

@@ -4,6 +4,7 @@ const path = require('path');
 const fs = require('fs');
 const _ = require('lodash');
 const childProcessChecker = require('../utility/childProcessChecker');
+const goSplit = require('../utility/goSplit');
 
 const driverConnect = require('../utility/driverConnect');
 const { jsldir } = require('../utility/directories');
@@ -19,7 +20,7 @@ class TableWriter {
     this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     this.currentRowCount = 0;
     this.currentChangeIndex = 0;
-    fs.writeFileSync(this.currentFile, JSON.stringify(columns) + '\n');
+    fs.writeFileSync(this.currentFile, JSON.stringify({ columns }) + '\n');
     this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
     this.writeCurrentStats(false, false);
     process.send({ msgtype: 'recordset', jslid: this.jslid });
@@ -140,9 +141,11 @@ async function handleExecuteQuery({ sql }) {
   await waitConnected();
   const driver = engines(storedConnection);
 
+  for (const sqlItem of goSplit(sql)) {
     const handler = new StreamHandler();
-  const stream = await driver.stream(systemConnection, sql, handler);
+    const stream = await driver.stream(systemConnection, sqlItem, handler);
     handler.stream = stream;
+  }
 }
 
 const messageHandlers = {

packages/api/src/shell/csvWriter.js

@@ -1,13 +1,36 @@
 const csv = require('csv');
 const fs = require('fs');
+const stream = require('stream');
 
-async function csvWriter({ fileName, encoding = 'utf-8', ...options }) {
+class CsvPrepareStream extends stream.Transform {
+  constructor({ header }) {
+    super({ objectMode: true });
+    this.structure = null;
+    this.header = header;
+  }
+  _transform(chunk, encoding, done) {
+    if (this.structure) {
+      this.push(this.structure.columns.map((col) => chunk[col.columnName]));
+      done();
+    } else {
+      this.structure = chunk;
+      if (this.header) {
+        this.push(chunk.columns.map((x) => x.columnName));
+      }
+      done();
+    }
+  }
+}
+
+async function csvWriter({ fileName, encoding = 'utf-8', header = true, delimiter, quoted }) {
   console.log(`Writing file ${fileName}`);
-  const csvStream = csv.stringify(options);
+  const csvPrepare = new CsvPrepareStream({ header });
+  const csvStream = csv.stringify({ delimiter, quoted });
   const fileStream = fs.createWriteStream(fileName, encoding);
+  csvPrepare.pipe(csvStream);
   csvStream.pipe(fileStream);
-  csvStream['finisher'] = fileStream;
-  return csvStream;
+  csvPrepare['finisher'] = fileStream;
+  return csvPrepare;
 }
 
 module.exports = csvWriter;

packages/api/src/shell/fakeObjectReader.js

@@ -5,6 +5,7 @@ async function fakeObjectReader({ delay = 0 } = {}) {
     objectMode: true,
   });
   function doWrite() {
+    pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }] });
     pass.write({ id: 1, country: 'Czechia' });
     pass.write({ id: 2, country: 'Austria' });
     pass.write({ country: 'Germany', id: 3 });

packages/api/src/utility/goSplit.js

@@ -0,0 +1,18 @@
+function goSplit(sql) {
+  if (!sql) return [];
+  const lines = sql.split('\n');
+  const res = [];
+  let buffer = '';
+  for (const line of lines) {
+    if (/^\s*go\s*$/i.test(line)) {
+      if (buffer.trim()) res.push(buffer);
+      buffer = '';
+    } else {
+      buffer += line + '\n';
+    }
+  }
+  if (buffer.trim()) res.push(buffer);
+  return res;
+}
+
+module.exports = goSplit;

packages/engines/mssql/index.js

@@ -164,6 +164,10 @@ const driver = {
     });
 
     request.stream = true;
+    request.on('recordset', (driverColumns) => {
+      const [columns, mapper] = extractColumns(driverColumns);
+      pass.write({ columns });
+    });
     request.on('row', (row) => pass.write(row));
     request.on('error', (err) => {
       console.error(err);

packages/engines/mysql/index.js

@@ -86,7 +86,23 @@ const driver = {
   },
   async readableStream(connection, sql) {
     const query = connection.query(sql);
-    return query.stream({ highWaterMark: 100 });
+    const { stream } = connection._nativeModules;
+
+    const pass = new stream.PassThrough({
+      objectMode: true,
+      highWaterMark: 100,
+    });
+
+    query
+      .on('error', (err) => {
+        console.error(err);
+        pass.end();
+      })
+      .on('fields', (fields) => pass.write({ columns: extractColumns(fields) }))
+      .on('result', (row) => pass.write(row))
+      .on('end', () => pass.end());
+
+    return pass;
   },
   async getVersion(connection) {
     const { rows } = await this.query(connection, "show variables like 'version'");

packages/web/src/sqleditor/JslDataGrid.js

@@ -4,11 +4,12 @@ import { JslGridDisplay, createGridConfig, createGridCache } from '@dbgate/datal
 import useFetch from '../utility/useFetch';
 
 export default function JslDataGrid({ jslid }) {
-  const columns = useFetch({
+  const info = useFetch({
     params: { jslid },
     url: 'jsldata/get-info',
-    defaultValue: [],
+    defaultValue: {},
   });
+  const columns = (info && info.columns) || [];
   const [config, setConfig] = React.useState(createGridConfig());
   const [cache, setCache] = React.useState(createGridCache());
   const display = React.useMemo(() => new JslGridDisplay(jslid, columns, config, setConfig, cache, setCache), [

packages/web/src/tabs/QueryTab.js

@@ -89,10 +89,13 @@ export default function QueryTab({
     }
   }, [sqlFromTemplate]);
 
-  const saveToStorage = React.useCallback(() => localStorage.setItem(localStorageKey, queryTextRef.current), [
-    localStorageKey,
-    queryTextRef,
-  ]);
+  const saveToStorage = React.useCallback(() => {
+    try {
+      localStorage.setItem(localStorageKey, queryTextRef.current);
+    } catch (err) {
+      console.error(err);
+    }
+  }, [localStorageKey, queryTextRef]);
   const saveToStorageDebounced = React.useMemo(() => _.debounce(saveToStorage, 5000), [saveToStorage]);
 
   React.useEffect(() => {

test/exportTable.js

@@ -1,13 +1,24 @@
 const dbgateApi = require('@dbgate/api');
 
 async function run() {
+  // const queryReader = await dbgateApi.queryReader({
+  //   connection: {
+  //     server: 'localhost',
+  //     engine: 'mysql',
+  //     user: 'root',
+  //     password: 'test',
+  //     port: '3307',
+  //     database: 'Chinook',
+  //   },
+  //   sql: 'SELECT * FROM Genre',
+  // });
+
   const queryReader = await dbgateApi.queryReader({
     connection: {
       server: 'localhost',
-      engine: 'mysql',
-      user: 'root',
-      password: 'test',
-      port: '3307',
+      engine: 'mssql',
+      user: 'sa',
+      password: 'Pwd2020Db',
       database: 'Chinook',
     },
     sql: 'SELECT * FROM Genre',
@@ -17,7 +28,7 @@ async function run() {
 
   const csvWriter = await dbgateApi.csvWriter({
     fileName: 'test.csv',
-    header: true,
+    // header: false,
   });
 
   const consoleWriter = await dbgateApi.consoleObjectWriter();

test/test.csv

@@ -1,7 +1,26 @@
-id,country
-1,Czechia
-2,Austria
-3,Germany
-4,Romania
-5,Great Britain
-6,"Bosna, Hecegovina"
+GenreId,Name
+1,Rock
+2,Jazz
+3,Metal
+4,Alternative & Punk
+5,Rock And Roll
+6,Blues
+7,Latin
+8,Reggae
+9,Pop
+10,Soundtrack
+11,Bossa Nova
+12,Easy Listening
+13,Heavy Metal
+14,R&B/Soul
+15,Electronica/Dance
+16,World
+17,Hip Hop/Rap
+18,Science Fiction
+19,TV Shows
+20,Sci Fi & Fantasy
+21,Drama
+22,Comedy
+23,Alternative
+24,Classical
+25,Opera