diff --git a/packages/api/package.json b/packages/api/package.json index ee229037a..bfc749891 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,5 +1,6 @@ { "name": "@dbgate/api", + "main": "src/index.js", "version": "0.1.0", "private": true, "dependencies": { @@ -10,6 +11,7 @@ "bufferutil": "^4.0.1", "cors": "^2.8.5", "cross-env": "^6.0.3", + "csv": "^5.3.2", "eslint": "^6.8.0", "express": "^4.17.1", "find-free-port": "^2.0.0", diff --git a/packages/api/src/index.js b/packages/api/src/index.js index a0e1c7543..366b28250 100644 --- a/packages/api/src/index.js +++ b/packages/api/src/index.js @@ -1,3 +1,4 @@ +const shell = require('./shell'); // require('socket.io-client'); // "socket.io-client": "^2.3.0", @@ -11,8 +12,10 @@ if (argument && argument.endsWith('Process')) { const module = proc[argument]; module.start(); -} else { +} else if (!module['parent']) { const main = require('./main'); - + main.start(argument); } + +module.exports = shell; \ No newline at end of file diff --git a/packages/api/src/shell/copyStream.js b/packages/api/src/shell/copyStream.js new file mode 100644 index 000000000..dab470ee6 --- /dev/null +++ b/packages/api/src/shell/copyStream.js @@ -0,0 +1,10 @@ +function copyStream(input, output) { + return new Promise((resolve, reject) => { + const finisher = output['finisher'] || output; + finisher.on('finish', resolve); + finisher.on('error', reject); + input.pipe(output); + }); +} + +module.exports = copyStream; diff --git a/packages/api/src/shell/csvWriter.js b/packages/api/src/shell/csvWriter.js new file mode 100644 index 000000000..ec6c44054 --- /dev/null +++ b/packages/api/src/shell/csvWriter.js @@ -0,0 +1,13 @@ +const csv = require('csv'); +const fs = require('fs'); + +async function csvWriter({ fileName, encoding = 'utf-8', ...options }) { + console.log(`Writing file ${fileName}`); + const csvStream = csv.stringify(options); + const fileStream = fs.createWriteStream(fileName, encoding); + csvStream.pipe(fileStream); + csvStream['finisher'] = fileStream; + return csvStream; +} + +module.exports = csvWriter; diff --git a/packages/api/src/shell/fakeObjectReader.js b/packages/api/src/shell/fakeObjectReader.js new file mode 100644 index 000000000..8b7fa4f0d --- /dev/null +++ b/packages/api/src/shell/fakeObjectReader.js @@ -0,0 +1,24 @@ +const stream = require('stream'); + +async function fakeObjectReader({ delay = 0 } = {}) { + const pass = new stream.PassThrough({ + objectMode: true, + }); + function doWrite() { + pass.write({ id: 1, country: 'Czechia' }); + pass.write({ id: 2, country: 'Austria' }); + pass.write({ id: 3, country: 'Germany' }); + pass.write({ id: 4, country: 'Romania' }); + pass.end(); + } + + if (delay) { + setTimeout(doWrite, delay); + } else { + doWrite(); + } + + return pass; +} + +module.exports = fakeObjectReader; diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js new file mode 100644 index 000000000..2cb673e27 --- /dev/null +++ b/packages/api/src/shell/index.js @@ -0,0 +1,13 @@ +const queryReader = require('./queryReader'); +const csvWriter = require('./csvWriter'); +const runScript = require('./runScript'); +const copyStream = require('./copyStream'); +const fakeObjectReader = require('./fakeObjectReader'); + +module.exports = { + queryReader, + csvWriter, + runScript, + copyStream, + fakeObjectReader, +}; diff --git a/packages/api/src/shell/queryReader.js b/packages/api/src/shell/queryReader.js new file mode 100644 index 000000000..8231fbae8 --- /dev/null +++ b/packages/api/src/shell/queryReader.js @@ -0,0 +1,14 @@ +const driverConnect = require('../utility/driverConnect'); + +const engines = require('@dbgate/engines'); + +async function queryReader({ connection, sql }) { + console.log(`Reading query ${sql}`); + + const driver = engines(connection); + const pool = await driverConnect(driver, connection); + console.log(`Connected.`); + return driver.readableStream(pool, sql); +} + +module.exports = queryReader; diff --git a/packages/api/src/shell/runScript.js b/packages/api/src/shell/runScript.js new file mode 100644 index 000000000..49803e8bb --- /dev/null +++ b/packages/api/src/shell/runScript.js @@ -0,0 +1,11 @@ +async function runScript(func) { + try { + await func(); + process.exit(0); + } catch (err) { + console.log(err); + process.exit(1); + } +} + +module.exports = runScript; diff --git a/packages/engines/mysql/index.js b/packages/engines/mysql/index.js index c32baf6c7..2f1b18920 100644 --- a/packages/engines/mysql/index.js +++ b/packages/engines/mysql/index.js @@ -84,6 +84,10 @@ const driver = { return query; }, + readableStream(connection, sql) { + const query = connection.query(sql); + return query.stream({ highWaterMark: 100 }); + }, async getVersion(connection) { const { rows } = await this.query(connection, "show variables like 'version'"); const version = rows[0].Value; diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index fc0d9f646..3108c327b 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -1,3 +1,4 @@ +import stream from 'stream' import { QueryResult } from './query'; import { SqlDialect } from './dialect'; import { SqlDumper } from './dumper'; @@ -16,6 +17,7 @@ export interface EngineDriver { connect(nativeModules, { server, port, user, password, database }): any; query(pool: any, sql: string): Promise; stream(pool: any, sql: string, options: StreamOptions); + readableStream(pool: any, sql: string): stream.Readable; getVersion(pool: any): Promise<{ version: string }>; listDatabases( pool: any diff --git a/test/exportTable.js b/test/exportTable.js new file mode 100644 index 000000000..7801ce6df --- /dev/null +++ b/test/exportTable.js @@ -0,0 +1,25 @@ +const dbgateApi = require('@dbgate/api'); + +async function run() { + const queryReader = await dbgateApi.queryReader({ + connection: { + server: 'localhost', + engine: 'mysql', + user: 'root', + password: 'test', + port: '3307', + database: 'Chinook', + }, + sql: 'SELECT * FROM Genre', + }); + + // const queryReader = await dbgateApi.fakeObjectReader({ delay: 1000 }); + + const csvWriter = await dbgateApi.csvWriter({ + fileName: 'test.csv', + }); + + await dbgateApi.copyStream(queryReader, csvWriter); +} + +dbgateApi.runScript(run); diff --git a/test/test.csv b/test/test.csv new file mode 100644 index 000000000..0547517bb --- /dev/null +++ b/test/test.csv @@ -0,0 +1,25 @@ +1,Rock +2,Jazz +3,Metal +4,Alternative & Punk +5,Rock And Roll +6,Blues +7,Latin +8,Reggae +9,Pop +10,Soundtrack +11,Bossa Nova +12,Easy Listening +13,Heavy Metal +14,R&B/Soul +15,Electronica/Dance +16,World +17,Hip Hop/Rap +18,Science Fiction +19,TV Shows +20,Sci Fi & Fantasy +21,Drama +22,Comedy +23,Alternative +24,Classical +25,Opera diff --git a/yarn.lock b/yarn.lock index 1ac152c0b..a64f2c136 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3715,6 +3715,31 @@ csstype@^2.2.0: resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.6.8.tgz#0fb6fc2417ffd2816a418c9336da74d7f07db431" integrity sha512-msVS9qTuMT5zwAGCVm4mxfrZ18BNc6Csd0oJAtiFMZ1FAx1CCvy2+5MDmYoix63LM/6NDbNtodCiGYGmFgO0dA== +csv-generate@^3.2.4: + version "3.2.4" + resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-3.2.4.tgz#440dab9177339ee0676c9e5c16f50e2b3463c019" + integrity sha512-qNM9eqlxd53TWJeGtY1IQPj90b563Zx49eZs8e0uMyEvPgvNVmX1uZDtdzAcflB3PniuH9creAzcFOdyJ9YGvA== + +csv-parse@^4.8.8: + version "4.10.1" + resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.10.1.tgz#1e26ba63d29c75e94d0eba6e9de9a8aaf89d72a6" + integrity sha512-gdDJVchi0oSLIcYXz1H/VSgLE6htHDqJyFsRU/vTkQgmVOZ3S0IR2LXnNbWUYG7VD76dYVwdfBLyx8AX9+An8A== + +csv-stringify@^5.3.6: + version "5.5.0" + resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-5.5.0.tgz#0bdeaaf60d6e15b89c752a0eceb4b4c2c8af5a8a" + integrity sha512-G05575DSO/9vFzQxZN+Srh30cNyHk0SM0ePyiTChMD5WVt7GMTVPBQf4rtgMF6mqhNCJUPw4pN8LDe8MF9EYOA== + +csv@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/csv/-/csv-5.3.2.tgz#50b344e25dfbb8c62684a1bcec18c22468b2161e" + integrity sha512-odDyucr9OgJTdGM2wrMbJXbOkJx3nnUX3Pt8SFOwlAMOpsUQlz1dywvLMXJWX/4Ib0rjfOsaawuuwfI5ucqBGQ== + dependencies: + csv-generate "^3.2.4" + csv-parse "^4.8.8" + csv-stringify "^5.3.6" + stream-transform "^2.0.1" + cyclist@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" @@ -7474,6 +7499,11 @@ mixin-object@^2.0.1: for-in "^0.1.3" is-extendable "^0.1.1" +mixme@^0.3.1: + version "0.3.5" + resolved "https://registry.yarnpkg.com/mixme/-/mixme-0.3.5.tgz#304652cdaf24a3df0487205e61ac6162c6906ddd" + integrity sha512-SyV9uPETRig5ZmYev0ANfiGeB+g6N2EnqqEfBbCGmmJ6MgZ3E4qv5aPbnHVdZ60KAHHXV+T3sXopdrnIXQdmjQ== + mkdirp@0.5.1, mkdirp@0.x, mkdirp@^0.5.1, mkdirp@~0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" @@ -10711,6 +10741,13 @@ stream-shift@^1.0.0: resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== +stream-transform@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-2.0.2.tgz#3cb7a14c802eb39bc40caaab0535e584f3a65caf" + integrity sha512-J+D5jWPF/1oX+r9ZaZvEXFbu7znjxSkbNAHJ9L44bt/tCVuOEWZlDqU9qJk7N2xBU1S+K2DPpSKeR/MucmCA1Q== + dependencies: + mixme "^0.3.1" + strict-uri-encode@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"