export CSV from mysql test

packages/api/package.json

@@ -1,5 +1,6 @@
 {
   "name": "@dbgate/api",
+  "main": "src/index.js",
   "version": "0.1.0",
   "private": true,
   "dependencies": {
@@ -10,6 +11,7 @@
     "bufferutil": "^4.0.1",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
+    "csv": "^5.3.2",
     "eslint": "^6.8.0",
     "express": "^4.17.1",
     "find-free-port": "^2.0.0",

packages/api/src/index.js

@@ -1,3 +1,4 @@
+const shell = require('./shell');
 // require('socket.io-client');
 
 // "socket.io-client": "^2.3.0",
@@ -11,8 +12,10 @@ if (argument && argument.endsWith('Process')) {
 
   const module = proc[argument];
   module.start();
-} else {
+} else if (!module['parent']) {
   const main = require('./main');
-  
+
   main.start(argument);
 }
+
+module.exports = shell;

packages/api/src/shell/copyStream.js

@@ -0,0 +1,10 @@
+function copyStream(input, output) {
+  return new Promise((resolve, reject) => {
+    const finisher = output['finisher'] || output;
+    finisher.on('finish', resolve);
+    finisher.on('error', reject);
+    input.pipe(output);
+  });
+}
+
+module.exports = copyStream;

packages/api/src/shell/csvWriter.js

@@ -0,0 +1,13 @@
+const csv = require('csv');
+const fs = require('fs');
+
+async function csvWriter({ fileName, encoding = 'utf-8', ...options }) {
+  console.log(`Writing file ${fileName}`);
+  const csvStream = csv.stringify(options);
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  csvStream.pipe(fileStream);
+  csvStream['finisher'] = fileStream;
+  return csvStream;
+}
+
+module.exports = csvWriter;

packages/api/src/shell/fakeObjectReader.js

@@ -0,0 +1,24 @@
+const stream = require('stream');
+
+async function fakeObjectReader({ delay = 0 } = {}) {
+  const pass = new stream.PassThrough({
+    objectMode: true,
+  });
+  function doWrite() {
+    pass.write({ id: 1, country: 'Czechia' });
+    pass.write({ id: 2, country: 'Austria' });
+    pass.write({ id: 3, country: 'Germany' });
+    pass.write({ id: 4, country: 'Romania' });
+    pass.end();
+  }
+
+  if (delay) {
+    setTimeout(doWrite, delay);
+  } else {
+    doWrite();
+  }
+
+  return pass;
+}
+
+module.exports = fakeObjectReader;

packages/api/src/shell/index.js

@@ -0,0 +1,13 @@
+const queryReader = require('./queryReader');
+const csvWriter = require('./csvWriter');
+const runScript = require('./runScript');
+const copyStream = require('./copyStream');
+const fakeObjectReader = require('./fakeObjectReader');
+
+module.exports = {
+  queryReader,
+  csvWriter,
+  runScript,
+  copyStream,
+  fakeObjectReader,
+};

packages/api/src/shell/queryReader.js

@@ -0,0 +1,14 @@
+const driverConnect = require('../utility/driverConnect');
+
+const engines = require('@dbgate/engines');
+
+async function queryReader({ connection, sql }) {
+  console.log(`Reading query ${sql}`);
+
+  const driver = engines(connection);
+  const pool = await driverConnect(driver, connection);
+  console.log(`Connected.`);
+  return driver.readableStream(pool, sql);
+}
+
+module.exports = queryReader;

packages/api/src/shell/runScript.js

@@ -0,0 +1,11 @@
+async function runScript(func) {
+  try {
+    await func();
+    process.exit(0);
+  } catch (err) {
+    console.log(err);
+    process.exit(1);
+  }
+}
+
+module.exports = runScript;

packages/engines/mysql/index.js

@@ -84,6 +84,10 @@ const driver = {
 
     return query;
   },
+  readableStream(connection, sql) {
+    const query = connection.query(sql);
+    return query.stream({ highWaterMark: 100 });
+  },
   async getVersion(connection) {
     const { rows } = await this.query(connection, "show variables like 'version'");
     const version = rows[0].Value;

packages/types/engines.d.ts

@@ -1,3 +1,4 @@
+import stream from 'stream'
 import { QueryResult } from './query';
 import { SqlDialect } from './dialect';
 import { SqlDumper } from './dumper';
@@ -16,6 +17,7 @@ export interface EngineDriver {
   connect(nativeModules, { server, port, user, password, database }): any;
   query(pool: any, sql: string): Promise<QueryResult>;
   stream(pool: any, sql: string, options: StreamOptions);
+  readableStream(pool: any, sql: string): stream.Readable;
   getVersion(pool: any): Promise<{ version: string }>;
   listDatabases(
     pool: any

test/exportTable.js

@@ -0,0 +1,25 @@
+const dbgateApi = require('@dbgate/api');
+
+async function run() {
+  const queryReader = await dbgateApi.queryReader({
+    connection: {
+      server: 'localhost',
+      engine: 'mysql',
+      user: 'root',
+      password: 'test',
+      port: '3307',
+      database: 'Chinook',
+    },
+    sql: 'SELECT * FROM Genre',
+  });
+
+  // const queryReader = await dbgateApi.fakeObjectReader({ delay: 1000 });
+
+  const csvWriter = await dbgateApi.csvWriter({
+    fileName: 'test.csv',
+  });
+
+  await dbgateApi.copyStream(queryReader, csvWriter);
+}
+
+dbgateApi.runScript(run);

test/test.csv

@@ -0,0 +1,25 @@
+1,Rock
+2,Jazz
+3,Metal
+4,Alternative & Punk
+5,Rock And Roll
+6,Blues
+7,Latin
+8,Reggae
+9,Pop
+10,Soundtrack
+11,Bossa Nova
+12,Easy Listening
+13,Heavy Metal
+14,R&B/Soul
+15,Electronica/Dance
+16,World
+17,Hip Hop/Rap
+18,Science Fiction
+19,TV Shows
+20,Sci Fi & Fantasy
+21,Drama
+22,Comedy
+23,Alternative
+24,Classical
+25,Opera

yarn.lock

@@ -3715,6 +3715,31 @@ csstype@^2.2.0:
   resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.6.8.tgz#0fb6fc2417ffd2816a418c9336da74d7f07db431"
   integrity sha512-msVS9qTuMT5zwAGCVm4mxfrZ18BNc6Csd0oJAtiFMZ1FAx1CCvy2+5MDmYoix63LM/6NDbNtodCiGYGmFgO0dA==
 
+csv-generate@^3.2.4:
+  version "3.2.4"
+  resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-3.2.4.tgz#440dab9177339ee0676c9e5c16f50e2b3463c019"
+  integrity sha512-qNM9eqlxd53TWJeGtY1IQPj90b563Zx49eZs8e0uMyEvPgvNVmX1uZDtdzAcflB3PniuH9creAzcFOdyJ9YGvA==
+
+csv-parse@^4.8.8:
+  version "4.10.1"
+  resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.10.1.tgz#1e26ba63d29c75e94d0eba6e9de9a8aaf89d72a6"
+  integrity sha512-gdDJVchi0oSLIcYXz1H/VSgLE6htHDqJyFsRU/vTkQgmVOZ3S0IR2LXnNbWUYG7VD76dYVwdfBLyx8AX9+An8A==
+
+csv-stringify@^5.3.6:
+  version "5.5.0"
+  resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-5.5.0.tgz#0bdeaaf60d6e15b89c752a0eceb4b4c2c8af5a8a"
+  integrity sha512-G05575DSO/9vFzQxZN+Srh30cNyHk0SM0ePyiTChMD5WVt7GMTVPBQf4rtgMF6mqhNCJUPw4pN8LDe8MF9EYOA==
+
+csv@^5.3.2:
+  version "5.3.2"
+  resolved "https://registry.yarnpkg.com/csv/-/csv-5.3.2.tgz#50b344e25dfbb8c62684a1bcec18c22468b2161e"
+  integrity sha512-odDyucr9OgJTdGM2wrMbJXbOkJx3nnUX3Pt8SFOwlAMOpsUQlz1dywvLMXJWX/4Ib0rjfOsaawuuwfI5ucqBGQ==
+  dependencies:
+    csv-generate "^3.2.4"
+    csv-parse "^4.8.8"
+    csv-stringify "^5.3.6"
+    stream-transform "^2.0.1"
+
 cyclist@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"
@@ -7474,6 +7499,11 @@ mixin-object@^2.0.1:
     for-in "^0.1.3"
     is-extendable "^0.1.1"
 
+mixme@^0.3.1:
+  version "0.3.5"
+  resolved "https://registry.yarnpkg.com/mixme/-/mixme-0.3.5.tgz#304652cdaf24a3df0487205e61ac6162c6906ddd"
+  integrity sha512-SyV9uPETRig5ZmYev0ANfiGeB+g6N2EnqqEfBbCGmmJ6MgZ3E4qv5aPbnHVdZ60KAHHXV+T3sXopdrnIXQdmjQ==
+
 mkdirp@0.5.1, mkdirp@0.x, mkdirp@^0.5.1, mkdirp@~0.5.1:
   version "0.5.1"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903"
@@ -10711,6 +10741,13 @@ stream-shift@^1.0.0:
   resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d"
   integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==
 
+stream-transform@^2.0.1:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-2.0.2.tgz#3cb7a14c802eb39bc40caaab0535e584f3a65caf"
+  integrity sha512-J+D5jWPF/1oX+r9ZaZvEXFbu7znjxSkbNAHJ9L44bt/tCVuOEWZlDqU9qJk7N2xBU1S+K2DPpSKeR/MucmCA1Q==
+  dependencies:
+    mixme "^0.3.1"
+
 strict-uri-encode@^1.0.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"