diff --git a/.github/workflows/build-npm.yaml b/.github/workflows/build-npm.yaml index d78684ba3..30d8a68a2 100644 --- a/.github/workflows/build-npm.yaml +++ b/.github/workflows/build-npm.yaml @@ -79,11 +79,6 @@ jobs: run: | npm publish - - name: Publish query-splitter - working-directory: packages/query-splitter - run: | - npm publish - - name: Publish web working-directory: packages/web run: | diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 8e14f7aec..9bea98ca2 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -31,11 +31,6 @@ jobs: run: | cd packages/filterparser yarn test:ci - - name: Query spliiter tests - if: always() - run: | - cd packages/query-splitter - yarn test:ci - uses: tanmen/jest-reporter@v1 if: always() with: @@ -48,12 +43,6 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} result-file: packages/filterparser/result.json action-name: Filter parser test results - - uses: tanmen/jest-reporter@v1 - if: always() - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - result-file: packages/query-splitter/result.json - action-name: Query splitter test results services: postgres: diff --git a/package.json b/package.json index 4d0b5bc8a..5dbdfa21a 100644 --- a/package.json +++ b/package.json @@ -17,13 +17,11 @@ "start:tools": "yarn workspace dbgate-tools start", "start:datalib": "yarn workspace dbgate-datalib start", "start:filterparser": "yarn workspace dbgate-filterparser start", - "start:querysplitter": "yarn workspace dbgate-query-splitter start", "build:sqltree": "yarn workspace dbgate-sqltree build", "build:datalib": "yarn workspace dbgate-datalib build", "build:filterparser": "yarn workspace dbgate-filterparser build", - "build:querysplitter": "yarn workspace dbgate-query-splitter build", "build:tools": "yarn workspace dbgate-tools build", - "build:lib": "yarn build:querysplitter && yarn build:sqltree && yarn build:tools && yarn build:filterparser && yarn build:datalib", + "build:lib": "yarn yarn build:sqltree && yarn build:tools && yarn build:filterparser && yarn build:datalib", "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build", "build:api": "yarn workspace dbgate-api build", "build:web:docker": "yarn workspace dbgate-web build", @@ -43,7 +41,7 @@ "install:sqlite:docker": "cd docker && yarn init --yes && yarn add better-sqlite3 && cd ..", "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build && yarn install:sqlite:docker", "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"", - "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn start:querysplitter\" \"yarn build:plugins:frontend:watch\"", + "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn build:plugins:frontend:watch\"", "ts:api": "yarn workspace dbgate-api ts", "ts:web": "yarn workspace dbgate-web ts", "ts": "yarn ts:api && yarn ts:web", diff --git a/packages/api/package.json b/packages/api/package.json index 6c8238eed..3bb7cceb3 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -25,7 +25,7 @@ "compare-versions": "^3.6.0", "cors": "^2.8.5", "cross-env": "^6.0.3", - "dbgate-query-splitter": "^4.1.1", + "dbgate-query-splitter": "^4.8.3", "dbgate-sqltree": "^4.1.1", "dbgate-tools": "^4.1.1", "diff": "^5.0.0", diff --git a/packages/query-splitter/.gitignore b/packages/query-splitter/.gitignore deleted file mode 100644 index 7951405f8..000000000 --- a/packages/query-splitter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -lib \ No newline at end of file diff --git a/packages/query-splitter/README.md b/packages/query-splitter/README.md deleted file mode 100644 index a74c22134..000000000 --- a/packages/query-splitter/README.md +++ /dev/null @@ -1,53 +0,0 @@ -[![NPM version](https://img.shields.io/npm/v/dbgate-query-splitter.svg)](https://www.npmjs.com/package/dbgate-query-splitter) - -# dbgate-query-splitter - -Splits long SQL query into into particular statements. Designed to have zero dependencies and to be fast. Also supports nodejs-streams. - -Supports following SQL dialects: - -- MySQL -- PostgreSQL -- SQLite -- Microsoft SQL Server - -## Usage - -```js -import { splitQuery, mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitterOptions } from 'dbgate-query-splitter'; - -const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`;', mysqlSplitterOptions); - -// output is ['SELECT * FROM `table1`', 'SELECT * FROM `table2`'] -``` - -## Streaming support in nodejs -Function splitQueryStream accepts input stream and query options. Result is object stream, each object for one splitted query. -Tokens must not be divided into more input chunks. This can be accomplished eg. when input stream emits one chunk per line (eg. using byline module) - -```js -const { mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitterOptions } = require('dbgate-query-splitter'); -const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream'); -const fs = require('fs'); -const byline = require('byline'); - -const fileStream = fs.createReadStream('INPUT_FILE_NAME', 'utf-8'); -const lineStream = byline(fileStream); -const splittedStream = splitQueryStream(lineStream, mysqlSplitterOptions); - -``` - -## Contributing - -Please run tests before pushing any changes. - -```sh -yarn test -``` - -## Supported syntax - -- Comments -- Dollar strings (PostgreSQL) -- GO separators (MS SQL) -- Custom delimiter, setby DELIMITER keyword (MySQL) diff --git a/packages/query-splitter/jest.config.js b/packages/query-splitter/jest.config.js deleted file mode 100644 index 790050941..000000000 --- a/packages/query-splitter/jest.config.js +++ /dev/null @@ -1,5 +0,0 @@ -module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - moduleFileExtensions: ['js'], -}; diff --git a/packages/query-splitter/package.json b/packages/query-splitter/package.json deleted file mode 100644 index 7d2cce9c2..000000000 --- a/packages/query-splitter/package.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version": "4.1.1", - "name": "dbgate-query-splitter", - "main": "lib/index.js", - "typings": "lib/index.d.ts", - "description": "SQL Query splitter for verious database engines", - "homepage": "https://github.com/dbgate/dbgate/tree/master/packages/query-splitter", - "repository": { - "type": "git", - "url": "https://github.com/dbgate/dbgate" - }, - "author": "Jan Prochazka", - "license": "MIT", - "keywords": [ - "SQL", - "query", - "split", - "parse" - ], - "scripts": { - "build": "tsc", - "start": "tsc --watch", - "test": "jest", - "test:ci": "jest --json --outputFile=result.json --testLocationInResults" - }, - "files": [ - "lib" - ], - "devDependencies": { - "dbgate-types": "^4.1.1", - "@types/jest": "^25.1.4", - "@types/node": "^13.7.0", - "jest": "^24.9.0", - "ts-jest": "^25.2.1", - "typescript": "^4.4.3" - } -} \ No newline at end of file diff --git a/packages/query-splitter/src/index.ts b/packages/query-splitter/src/index.ts deleted file mode 100644 index bce756683..000000000 --- a/packages/query-splitter/src/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { splitQuery } from './splitQuery'; -export * from './options'; diff --git a/packages/query-splitter/src/options.ts b/packages/query-splitter/src/options.ts deleted file mode 100644 index 45c568d13..000000000 --- a/packages/query-splitter/src/options.ts +++ /dev/null @@ -1,93 +0,0 @@ -export interface SplitterOptions { - stringsBegins: string[]; - stringsEnds: { [begin: string]: string }; - stringEscapes: { [begin: string]: string }; - - allowSemicolon: boolean; - allowCustomDelimiter: boolean; - allowGoDelimiter: boolean; - allowDollarDollarString: boolean; - noSplit: boolean; - doubleDashComments: boolean; - multilineComments: boolean; - javaScriptComments: boolean; - - returnRichInfo: boolean; - splitByLines: boolean; -} - -export const defaultSplitterOptions: SplitterOptions = { - stringsBegins: ["'"], - stringsEnds: { "'": "'" }, - stringEscapes: { "'": "'" }, - - allowSemicolon: true, - allowCustomDelimiter: false, - allowGoDelimiter: false, - allowDollarDollarString: false, - noSplit: false, - - doubleDashComments: true, - multilineComments: true, - javaScriptComments: false, - - returnRichInfo: false, - splitByLines: false, -}; - -export const mysqlSplitterOptions: SplitterOptions = { - ...defaultSplitterOptions, - - allowCustomDelimiter: true, - stringsBegins: ["'", '`'], - stringsEnds: { "'": "'", '`': '`' }, - stringEscapes: { "'": '\\', '`': '`' }, -}; - -export const mssqlSplitterOptions: SplitterOptions = { - ...defaultSplitterOptions, - allowSemicolon: false, - allowGoDelimiter: true, - - stringsBegins: ["'", '['], - stringsEnds: { "'": "'", '[': ']' }, - stringEscapes: { "'": "'" }, -}; - -export const postgreSplitterOptions: SplitterOptions = { - ...defaultSplitterOptions, - - allowDollarDollarString: true, - - stringsBegins: ["'", '"'], - stringsEnds: { "'": "'", '"': '"' }, - stringEscapes: { "'": "'", '"': '"' }, -}; - -export const sqliteSplitterOptions: SplitterOptions = { - ...defaultSplitterOptions, - - stringsBegins: ["'", '"'], - stringsEnds: { "'": "'", '"': '"' }, - stringEscapes: { "'": "'", '"': '"' }, -}; - -export const mongoSplitterOptions: SplitterOptions = { - ...defaultSplitterOptions, - - stringsBegins: ["'", '"'], - stringsEnds: { "'": "'", '"': '"' }, - stringEscapes: { "'": '\\', '"': '\\' }, -}; - -export const noSplitSplitterOptions: SplitterOptions = { - ...defaultSplitterOptions, - - noSplit: true, -}; - -export const redisSplitterOptions: SplitterOptions = { - ...defaultSplitterOptions, - - splitByLines: true, -}; diff --git a/packages/query-splitter/src/splitQuery.ts b/packages/query-splitter/src/splitQuery.ts deleted file mode 100644 index 71dafb5ef..000000000 --- a/packages/query-splitter/src/splitQuery.ts +++ /dev/null @@ -1,431 +0,0 @@ -import { SplitterOptions, defaultSplitterOptions } from './options'; - -const SEMICOLON = ';'; - -export interface SplitStreamContext { - options: SplitterOptions; - currentDelimiter: string; - pushOutput: (item: SplitResultItem) => void; - commandPart: string; - - line: number; - column: number; - streamPosition: number; - - commandStartPosition: number; - commandStartLine: number; - commandStartColumn: number; -} - -export interface SplitLineContext extends SplitStreamContext { - source: string; - position: number; - // output: string[]; - end: number; - wasDataOnLine: boolean; - currentCommandStart: number; - - // unread: string; - // currentStatement: string; - // semicolonKeyTokenRegex: RegExp; -} - -export interface SplitPositionDefinition { - position: number; - line: number; - column: number; -} - -export interface SplitResultItemRich { - text: string; - start: SplitPositionDefinition; - end: SplitPositionDefinition; - trimStart?: SplitPositionDefinition; - trimEnd?: SplitPositionDefinition; -} - -export type SplitResultItem = string | SplitResultItemRich; - -function movePosition(context: SplitLineContext, count: number) { - if (context.options.returnRichInfo) { - let { source, position, line, column, streamPosition } = context; - while (count > 0) { - if (source[position] == '\n') { - line += 1; - column = 0; - } else { - column += 1; - } - position += 1; - streamPosition += 1; - count -= 1; - } - context.position = position; - context.streamPosition = streamPosition; - context.line = line; - context.column = column; - } else { - context.position += count; - } -} - -function isStringEnd(s: string, pos: number, endch: string, escapech: string) { - if (!escapech) { - return s[pos] == endch; - } - if (endch == escapech) { - return s[pos] == endch && s[pos + 1] != endch; - } else { - return s[pos] == endch && s[pos - 1] != escapech; - } -} - -interface Token { - type: 'string' | 'delimiter' | 'whitespace' | 'eoln' | 'data' | 'set_delimiter' | 'comment' | 'go_delimiter'; - length: number; - value?: string; -} - -const WHITESPACE_TOKEN: Token = { - type: 'whitespace', - length: 1, -}; -const EOLN_TOKEN: Token = { - type: 'eoln', - length: 1, -}; -const DATA_TOKEN: Token = { - type: 'data', - length: 1, -}; - -function scanDollarQuotedString(context: SplitLineContext): Token { - if (!context.options.allowDollarDollarString) return null; - - let pos = context.position; - const s = context.source; - - const match = /^(\$[a-zA-Z0-9_]*\$)/.exec(s.slice(pos)); - if (!match) return null; - const label = match[1]; - pos += label.length; - - while (pos < context.end) { - if (s.slice(pos).startsWith(label)) { - return { - type: 'string', - length: pos + label.length - context.position, - }; - } - pos++; - } - - return null; -} - -function scanToken(context: SplitLineContext): Token { - let pos = context.position; - const s = context.source; - const ch = s[pos]; - - if (context.options.stringsBegins.includes(ch)) { - pos++; - const endch = context.options.stringsEnds[ch]; - const escapech = context.options.stringEscapes[ch]; - while (pos < context.end && !isStringEnd(s, pos, endch, escapech)) { - if (endch == escapech && s[pos] == endch && s[pos + 1] == endch) { - pos += 2; - } else { - pos++; - } - } - return { - type: 'string', - length: pos - context.position + 1, - }; - } - - if (context.currentDelimiter && s.slice(pos).startsWith(context.currentDelimiter)) { - return { - type: 'delimiter', - length: context.currentDelimiter.length, - }; - } - - if (ch == ' ' || ch == '\t' || ch == '\r') { - return WHITESPACE_TOKEN; - } - - if (ch == '\n') { - return EOLN_TOKEN; - } - - if (context.options.doubleDashComments && ch == '-' && s[pos + 1] == '-') { - while (pos < context.end && s[pos] != '\n') pos++; - return { - type: 'comment', - length: pos - context.position, - }; - } - - if (context.options.multilineComments && ch == '/' && s[pos + 1] == '*') { - pos += 2; - while (pos < context.end) { - if (s[pos] == '*' && s[pos + 1] == '/') break; - pos++; - } - return { - type: 'comment', - length: pos - context.position + 2, - }; - } - - if (context.options.allowCustomDelimiter && !context.wasDataOnLine) { - const m = s.slice(pos).match(/^DELIMITER[ \t]+([^\n]+)/i); - if (m) { - return { - type: 'set_delimiter', - value: m[1].trim(), - length: m[0].length, - }; - } - } - - if (context.options.allowGoDelimiter && !context.wasDataOnLine) { - const m = s.slice(pos).match(/^GO[\t\r ]*(\n|$)/i); - if (m) { - return { - type: 'go_delimiter', - length: m[0].length - 1, - }; - } - } - - const dollarString = scanDollarQuotedString(context); - if (dollarString) return dollarString; - - return DATA_TOKEN; -} - -function pushQuery(context: SplitLineContext) { - const sql = (context.commandPart || '') + context.source.slice(context.currentCommandStart, context.position); - const trimmed = sql.trim(); - if (trimmed) { - if (context.options.returnRichInfo) { - context.pushOutput( - countTrimmedPositions(sql, { - text: trimmed, - - start: { - position: context.commandStartPosition, - line: context.commandStartLine, - column: context.commandStartColumn, - }, - - end: { - position: context.streamPosition, - line: context.line, - column: context.column, - }, - }) - ); - } else { - context.pushOutput(trimmed); - } - } -} - -function countTrimmedPositions(full: string, positions: SplitResultItemRich): SplitResultItemRich { - const startIndex = full.indexOf(positions.text); - - const trimStart = { ...positions.start }; - for (let i = 0; i < startIndex; i += 1) { - if (full[i] == '\n') { - trimStart.position += 1; - trimStart.line += 1; - trimStart.column = 0; - } else { - trimStart.position += 1; - trimStart.column += 1; - } - } - - return { - ...positions, - trimStart, - trimEnd: positions.end, - }; -} - -function markStartCommand(context: SplitLineContext) { - if (context.options.returnRichInfo) { - context.commandStartPosition = context.streamPosition; - context.commandStartLine = context.line; - context.commandStartColumn = context.column; - } -} - -function splitByLines(context: SplitLineContext) { - while (context.position < context.end) { - if (context.source[context.position] == '\n') { - pushQuery(context); - context.commandPart = ''; - movePosition(context, 1); - context.currentCommandStart = context.position; - markStartCommand(context); - } else { - movePosition(context, 1); - } - } - - if (context.end > context.currentCommandStart) { - context.commandPart += context.source.slice(context.currentCommandStart, context.position); - } -} - -export function splitQueryLine(context: SplitLineContext) { - if (context.options.splitByLines) { - splitByLines(context); - return; - } - - while (context.position < context.end) { - const token = scanToken(context); - if (!token) { - // nothing special, move forward - movePosition(context, 1); - continue; - } - switch (token.type) { - case 'string': - movePosition(context, token.length); - context.wasDataOnLine = true; - break; - case 'comment': - movePosition(context, token.length); - context.wasDataOnLine = true; - break; - case 'eoln': - movePosition(context, token.length); - context.wasDataOnLine = false; - break; - case 'data': - movePosition(context, token.length); - context.wasDataOnLine = true; - break; - case 'whitespace': - movePosition(context, token.length); - break; - case 'set_delimiter': - pushQuery(context); - context.commandPart = ''; - context.currentDelimiter = token.value; - movePosition(context, token.length); - context.currentCommandStart = context.position; - markStartCommand(context); - break; - case 'go_delimiter': - pushQuery(context); - context.commandPart = ''; - movePosition(context, token.length); - context.currentCommandStart = context.position; - markStartCommand(context); - break; - case 'delimiter': - pushQuery(context); - context.commandPart = ''; - movePosition(context, token.length); - context.currentCommandStart = context.position; - markStartCommand(context); - break; - } - } - - if (context.end > context.currentCommandStart) { - context.commandPart += context.source.slice(context.currentCommandStart, context.position); - } -} - -export function getInitialDelimiter(options: SplitterOptions) { - return options?.allowSemicolon === false ? null : SEMICOLON; -} - -export function finishSplitStream(context: SplitStreamContext) { - const trimmed = context.commandPart.trim(); - if (trimmed) { - if (context.options.returnRichInfo) { - context.pushOutput( - countTrimmedPositions(context.commandPart, { - text: trimmed, - - start: { - position: context.commandStartPosition, - line: context.commandStartLine, - column: context.commandStartColumn, - }, - - end: { - position: context.streamPosition, - line: context.line, - column: context.column, - }, - }) - ); - } else { - context.pushOutput(trimmed); - } - } -} - -export function splitQuery(sql: string, options: SplitterOptions = null): SplitResultItem[] { - const usedOptions = { - ...defaultSplitterOptions, - ...options, - }; - - if (usedOptions.noSplit) { - if (usedOptions.returnRichInfo) { - const lines = sql.split('\n'); - return [ - { - text: sql, - start: { - position: 0, - line: 0, - column: 0, - }, - end: { - position: sql.length, - line: lines.length, - column: lines[lines.length - 1]?.length || 0, - }, - }, - ]; - } - return [sql]; - } - - const output = []; - const context: SplitLineContext = { - source: sql, - end: sql.length, - currentDelimiter: getInitialDelimiter(options), - position: 0, - column: 0, - line: 0, - currentCommandStart: 0, - commandStartLine: 0, - commandStartColumn: 0, - commandStartPosition: 0, - streamPosition: 0, - pushOutput: cmd => output.push(cmd), - wasDataOnLine: false, - options: usedOptions, - commandPart: '', - }; - - splitQueryLine(context); - finishSplitStream(context); - - return output; -} diff --git a/packages/query-splitter/src/splitQueryStream.ts b/packages/query-splitter/src/splitQueryStream.ts deleted file mode 100644 index 4f3dae226..000000000 --- a/packages/query-splitter/src/splitQueryStream.ts +++ /dev/null @@ -1,52 +0,0 @@ -import stream from 'stream'; -import { - SplitStreamContext, - getInitialDelimiter, - SplitLineContext, - splitQueryLine, - finishSplitStream, -} from './splitQuery'; -import { SplitterOptions } from './options'; - -export class SplitQueryStream extends stream.Transform { - context: SplitStreamContext; - - constructor(options: SplitterOptions) { - super({ objectMode: true }); - this.context = { - commandPart: '', - commandStartLine: 0, - commandStartColumn: 0, - commandStartPosition: 0, - streamPosition: 0, - line: 0, - column: 0, - options, - currentDelimiter: getInitialDelimiter(options), - pushOutput: cmd => this.push(cmd), - }; - } - _transform(chunk, encoding, done) { - const lineContext: SplitLineContext = { - ...this.context, - position: 0, - currentCommandStart: 0, - wasDataOnLine: false, - source: chunk, - end: chunk.length, - }; - splitQueryLine(lineContext); - this.context.commandPart = lineContext.commandPart; - done(); - } - _flush(done) { - finishSplitStream(this.context); - done(); - } -} - -export function splitQueryStream(sourceStream, options: SplitterOptions) { - const splitter = new SplitQueryStream(options); - sourceStream.pipe(splitter); - return splitter; -} diff --git a/packages/query-splitter/src/splitter.test.ts b/packages/query-splitter/src/splitter.test.ts deleted file mode 100644 index 6669cfe9c..000000000 --- a/packages/query-splitter/src/splitter.test.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { - mysqlSplitterOptions, - mssqlSplitterOptions, - postgreSplitterOptions, - mongoSplitterOptions, - noSplitSplitterOptions, - redisSplitterOptions, -} from './options'; -import { splitQuery } from './splitQuery'; - -test('simple query', () => { - const output = splitQuery('select * from A'); - expect(output).toEqual(['select * from A']); -}); - -test('correct split 2 queries', () => { - const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`;', mysqlSplitterOptions); - expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']); -}); - -test('correct split 2 queries - no end semicolon', () => { - const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`', mysqlSplitterOptions); - expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']); -}); - -test('delete empty query', () => { - const output = splitQuery(';;;\n;;SELECT * FROM `table1`;;;;;SELECT * FROM `table2`;;; ;;;', mysqlSplitterOptions); - expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']); -}); - -test('should handle double backtick', () => { - const input = ['CREATE TABLE `a``b` (`c"d` INT)', 'CREATE TABLE `a````b` (`c"d` INT)']; - const output = splitQuery(input.join(';\n') + ';', mysqlSplitterOptions); - expect(output).toEqual(input); -}); - -test('semicolon inside string', () => { - const input = ['CREATE TABLE a', "INSERT INTO a (x) VALUES ('1;2;3;4')"]; - const output = splitQuery(input.join(';\n') + ';', mysqlSplitterOptions); - expect(output).toEqual(input); -}); - -test('semicolon inside identyifier - mssql', () => { - const input = ['CREATE TABLE [a;1]', "INSERT INTO [a;1] (x) VALUES ('1')"]; - const output = splitQuery(input.join(';\n') + ';', { - ...mssqlSplitterOptions, - allowSemicolon: true, - }); - expect(output).toEqual(input); -}); - -test('delimiter test', () => { - const input = 'SELECT 1;\n DELIMITER $$\n SELECT 2; SELECT 3; \n DELIMITER ;'; - const output = splitQuery(input, mysqlSplitterOptions); - expect(output).toEqual(['SELECT 1', 'SELECT 2; SELECT 3;']); -}); - -test('one line comment test', () => { - const input = 'SELECT 1 -- comment1;comment2\n;SELECT 2'; - const output = splitQuery(input, mysqlSplitterOptions); - expect(output).toEqual(['SELECT 1 -- comment1;comment2', 'SELECT 2']); -}); - -test('multi line comment test', () => { - const input = 'SELECT 1 /* comment1;comment2\ncomment3*/;SELECT 2'; - const output = splitQuery(input, mysqlSplitterOptions); - expect(output).toEqual(['SELECT 1 /* comment1;comment2\ncomment3*/', 'SELECT 2']); -}); - -test('dollar string', () => { - const input = 'CREATE PROC $$ SELECT 1; SELECT 2; $$ ; SELECT 3'; - const output = splitQuery(input, postgreSplitterOptions); - expect(output).toEqual(['CREATE PROC $$ SELECT 1; SELECT 2; $$', 'SELECT 3']); -}); - -test('go delimiter', () => { - const input = 'SELECT 1\ngo\nSELECT 2'; - const output = splitQuery(input, mssqlSplitterOptions); - expect(output).toEqual(['SELECT 1', 'SELECT 2']); -}); - -test('no split', () => { - const input = 'SELECT 1;SELECT 2'; - const output = splitQuery(input, noSplitSplitterOptions); - expect(output).toEqual(['SELECT 1;SELECT 2']); -}); - -test('split mongo', () => { - const input = 'db.collection.insert({x:1});db.collection.insert({y:2})'; - const output = splitQuery(input, mongoSplitterOptions); - expect(output).toEqual(['db.collection.insert({x:1})', 'db.collection.insert({y:2})']); -}); - -test('redis split by newline', () => { - const output = splitQuery('SET x 1\nSET y 2', redisSplitterOptions); - expect(output).toEqual(['SET x 1', 'SET y 2']); -}); - -test('redis split by newline 2', () => { - const output = splitQuery('SET x 1\n\nSET y 2\n', redisSplitterOptions); - expect(output).toEqual(['SET x 1', 'SET y 2']); -}); - -test('count lines', () => { - const output = splitQuery('SELECT * FROM `table1`;\nSELECT * FROM `table2`;', { - ...mysqlSplitterOptions, - returnRichInfo: true, - }); - expect(output).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - text: 'SELECT * FROM `table1`', - - trimStart: expect.objectContaining({ - position: 0, - line: 0, - column: 0, - }), - - end: expect.objectContaining({ - position: 22, - line: 0, - column: 22, - }), - }), - expect.objectContaining({ - text: 'SELECT * FROM `table2`', - - trimStart: expect.objectContaining({ - position: 24, - line: 1, - column: 0, - }), - - end: expect.objectContaining({ - position: 46, - line: 1, - column: 22, - }), - }), - ]) - ); -}); - -test('count lines with flush', () => { - const output = splitQuery('SELECT * FROM `table1`;\nSELECT * FROM `table2`', { - ...mysqlSplitterOptions, - returnRichInfo: true, - }); - expect(output).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - text: 'SELECT * FROM `table1`', - - trimStart: expect.objectContaining({ - position: 0, - line: 0, - column: 0, - }), - - end: expect.objectContaining({ - position: 22, - line: 0, - column: 22, - }), - }), - expect.objectContaining({ - text: 'SELECT * FROM `table2`', - - trimStart: expect.objectContaining({ - position: 24, - line: 1, - column: 0, - }), - - end: expect.objectContaining({ - position: 46, - line: 1, - column: 22, - }), - }), - ]) - ); -}); diff --git a/packages/query-splitter/src/splitterStream.test.ts b/packages/query-splitter/src/splitterStream.test.ts deleted file mode 100644 index 1c33d48dd..000000000 --- a/packages/query-splitter/src/splitterStream.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitterOptions, noSplitSplitterOptions } from './options'; -import stream from 'stream'; -import { splitQueryStream } from './splitQueryStream'; - -function createInputStream(...lines) { - const pass = new stream.PassThrough({ - objectMode: true, - }); - lines.forEach(line => pass.write(line)); - pass.end(); - return pass; -} - -function streamToArray(streamSource) { - return new Promise((resolve, reject) => { - const res = []; - streamSource.on('data', x => res.push(x)); - streamSource.on('end', () => resolve(res)); - }); -} - -test('stream: simple query', async () => { - const output = await streamToArray(splitQueryStream(createInputStream('select * from A'), mysqlSplitterOptions)); - expect(output).toEqual(['select * from A']); -}); - -test('stream: query on 2 lines', async () => { - const output = await streamToArray(splitQueryStream(createInputStream('select * ', 'from A'), mysqlSplitterOptions)); - expect(output).toEqual(['select * from A']); -}); - -test('stream: query on 2 lines', async () => { - const output = await streamToArray( - splitQueryStream( - createInputStream('SELECT * ', 'FROM `table1`;', 'SELECT *', ' FROM `table2`'), - mysqlSplitterOptions - ) - ); - expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']); -}); diff --git a/packages/query-splitter/tsconfig.json b/packages/query-splitter/tsconfig.json deleted file mode 100644 index b2671e70a..000000000 --- a/packages/query-splitter/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2015", - "module": "commonjs", - "declaration": true, - "skipLibCheck": true, - "outDir": "lib", - "preserveWatchOutput": true, - "esModuleInterop": true - }, - "include": [ - "src/**/*" - ] -} diff --git a/packages/tools/package.json b/packages/tools/package.json index 56c759c04..09380a33b 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -32,7 +32,7 @@ }, "dependencies": { "lodash": "^4.17.21", - "dbgate-query-splitter": "^4.1.1", + "dbgate-query-splitter": "^4.8.3", "dbgate-sqltree": "^4.1.1", "uuid": "^3.4.0" } diff --git a/packages/web/package.json b/packages/web/package.json index f51ebaede..d0c99337a 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -24,7 +24,7 @@ "chartjs-adapter-moment": "^1.0.0", "cross-env": "^7.0.3", "dbgate-datalib": "^4.1.1", - "dbgate-query-splitter": "^4.1.1", + "dbgate-query-splitter": "^4.8.3", "dbgate-sqltree": "^4.1.1", "dbgate-tools": "^4.1.1", "dbgate-types": "^4.1.1", diff --git a/plugins/dbgate-plugin-mongo/package.json b/plugins/dbgate-plugin-mongo/package.json index 6c808c0ac..352d5c88f 100644 --- a/plugins/dbgate-plugin-mongo/package.json +++ b/plugins/dbgate-plugin-mongo/package.json @@ -32,7 +32,7 @@ }, "devDependencies": { "dbgate-plugin-tools": "^1.0.7", - "dbgate-query-splitter": "^4.1.1", + "dbgate-query-splitter": "^4.8.3", "webpack": "^4.42.0", "webpack-cli": "^3.3.11", "dbgate-tools": "^4.1.1", diff --git a/plugins/dbgate-plugin-mssql/package.json b/plugins/dbgate-plugin-mssql/package.json index 18410cd07..582e52717 100644 --- a/plugins/dbgate-plugin-mssql/package.json +++ b/plugins/dbgate-plugin-mssql/package.json @@ -32,7 +32,7 @@ }, "devDependencies": { "dbgate-plugin-tools": "^1.0.7", - "dbgate-query-splitter": "^4.1.1", + "dbgate-query-splitter": "^4.8.3", "webpack": "^4.42.0", "webpack-cli": "^3.3.11", "dbgate-tools": "^4.1.1", diff --git a/plugins/dbgate-plugin-mysql/package.json b/plugins/dbgate-plugin-mysql/package.json index b1f658354..565e0bfe4 100644 --- a/plugins/dbgate-plugin-mysql/package.json +++ b/plugins/dbgate-plugin-mysql/package.json @@ -32,7 +32,7 @@ }, "devDependencies": { "dbgate-plugin-tools": "^1.0.7", - "dbgate-query-splitter": "^4.1.1", + "dbgate-query-splitter": "^4.8.3", "webpack": "^4.42.0", "webpack-cli": "^3.3.11", "dbgate-tools": "^4.1.1", diff --git a/plugins/dbgate-plugin-postgres/package.json b/plugins/dbgate-plugin-postgres/package.json index 7e14bf5d5..9431e7781 100644 --- a/plugins/dbgate-plugin-postgres/package.json +++ b/plugins/dbgate-plugin-postgres/package.json @@ -31,7 +31,7 @@ }, "devDependencies": { "dbgate-plugin-tools": "^1.0.7", - "dbgate-query-splitter": "^4.1.1", + "dbgate-query-splitter": "^4.8.3", "dbgate-tools": "^4.1.1", "lodash": "^4.17.21", "pg": "^8.7.1", diff --git a/plugins/dbgate-plugin-redis/package.json b/plugins/dbgate-plugin-redis/package.json index 5b83146cf..88f7f8e76 100644 --- a/plugins/dbgate-plugin-redis/package.json +++ b/plugins/dbgate-plugin-redis/package.json @@ -30,7 +30,7 @@ }, "devDependencies": { "dbgate-plugin-tools": "^1.0.7", - "dbgate-query-splitter": "^4.1.1", + "dbgate-query-splitter": "^4.8.3", "dbgate-tools": "^4.1.1", "lodash": "^4.17.21", "webpack": "^4.42.0", diff --git a/plugins/dbgate-plugin-sqlite/package.json b/plugins/dbgate-plugin-sqlite/package.json index d64918f8f..79422584a 100644 --- a/plugins/dbgate-plugin-sqlite/package.json +++ b/plugins/dbgate-plugin-sqlite/package.json @@ -32,7 +32,7 @@ "devDependencies": { "dbgate-tools": "^4.1.1", "dbgate-plugin-tools": "^1.0.4", - "dbgate-query-splitter": "^4.1.1", + "dbgate-query-splitter": "^4.8.3", "byline": "^5.0.0", "webpack": "^4.42.0", "webpack-cli": "^3.3.11" diff --git a/setCurrentVersion.js b/setCurrentVersion.js index b65acdd4f..0c4c733ad 100644 --- a/setCurrentVersion.js +++ b/setCurrentVersion.js @@ -4,7 +4,9 @@ const path = require('path'); function changeDependencies(deps, version) { if (!deps) return; for (const key of Object.keys(deps)) { - if (key.startsWith('dbgate-') && key != 'dbgate-plugin-tools') deps[key] = `^${version}`; + if (key.startsWith('dbgate-') && key != 'dbgate-plugin-tools' && key != 'dbgate-query-splitter') { + deps[key] = `^${version}`; + } } } @@ -40,7 +42,6 @@ changePackageFile('packages/datalib', json.version); changePackageFile('packages/dbgate', json.version); changePackageFile('packages/serve', json.version); changePackageFile('packages/filterparser', json.version); -changePackageFile('packages/query-splitter', json.version); changePackageFile('plugins/dbgate-plugin-csv', json.version); changePackageFile('plugins/dbgate-plugin-xml', json.version);