dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-17 23:45:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

query splitter extracted into separate repository

Browse Source
This commit is contained in:
Jan Prochazka
2022-04-07 08:17:38 +02:00
parent 30e52723dd
commit 85e449953f
24 changed files with 14 additions and 943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/build-npm.yaml vendored
View File

@@ -79,11 +79,6 @@ jobs:
run: |
npm publish
- name: Publish query-splitter
working-directory: packages/query-splitter
run: |
npm publish
- name: Publish web
working-directory: packages/web
run: |

11
.github/workflows/run-tests.yaml vendored
View File

@@ -31,11 +31,6 @@ jobs:
run: |
cd packages/filterparser
yarn test:ci
- name: Query spliiter tests
if: always()
run: |
cd packages/query-splitter
yarn test:ci
- uses: tanmen/jest-reporter@v1
if: always()
with:
@@ -48,12 +43,6 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
result-file: packages/filterparser/result.json
action-name: Filter parser test results
- uses: tanmen/jest-reporter@v1
if: always()
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
result-file: packages/query-splitter/result.json
action-name: Query splitter test results
services:
postgres:

6
package.json
View File

@@ -17,13 +17,11 @@
"start:tools": "yarn workspace dbgate-tools start",
"start:datalib": "yarn workspace dbgate-datalib start",
"start:filterparser": "yarn workspace dbgate-filterparser start",
"start:querysplitter": "yarn workspace dbgate-query-splitter start",
"build:sqltree": "yarn workspace dbgate-sqltree build",
"build:datalib": "yarn workspace dbgate-datalib build",
"build:filterparser": "yarn workspace dbgate-filterparser build",
"build:querysplitter": "yarn workspace dbgate-query-splitter build",
"build:tools": "yarn workspace dbgate-tools build",
"build:lib": "yarn build:querysplitter && yarn build:sqltree && yarn build:tools && yarn build:filterparser && yarn build:datalib",
"build:lib": "yarn yarn build:sqltree && yarn build:tools && yarn build:filterparser && yarn build:datalib",
"build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
"build:api": "yarn workspace dbgate-api build",
"build:web:docker": "yarn workspace dbgate-web build",
@@ -43,7 +41,7 @@
"install:sqlite:docker": "cd docker && yarn init --yes && yarn add better-sqlite3 && cd ..",
"prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build && yarn install:sqlite:docker",
"start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
"lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn start:querysplitter\" \"yarn build:plugins:frontend:watch\"",
"lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn build:plugins:frontend:watch\"",
"ts:api": "yarn workspace dbgate-api ts",
"ts:web": "yarn workspace dbgate-web ts",
"ts": "yarn ts:api && yarn ts:web",

2
packages/api/package.json
View File

@@ -25,7 +25,7 @@
"compare-versions": "^3.6.0",
"cors": "^2.8.5",
"cross-env": "^6.0.3",
"dbgate-query-splitter": "^4.1.1",
"dbgate-query-splitter": "^4.8.3",
"dbgate-sqltree": "^4.1.1",
"dbgate-tools": "^4.1.1",
"diff": "^5.0.0",

1
packages/query-splitter/.gitignore vendored
View File

@@ -1 +0,0 @@
lib

53
packages/query-splitter/README.md
View File

@@ -1,53 +0,0 @@
[![NPM version](https://img.shields.io/npm/v/dbgate-query-splitter.svg)](https://www.npmjs.com/package/dbgate-query-splitter)
# dbgate-query-splitter
Splits long SQL query into into particular statements. Designed to have zero dependencies and to be fast. Also supports nodejs-streams.
Supports following SQL dialects:
- MySQL
- PostgreSQL
- SQLite
- Microsoft SQL Server
## Usage
```js
import { splitQuery, mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitterOptions } from 'dbgate-query-splitter';
const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`;', mysqlSplitterOptions);
// output is ['SELECT * FROM `table1`', 'SELECT * FROM `table2`']
```
## Streaming support in nodejs
Function splitQueryStream accepts input stream and query options. Result is object stream, each object for one splitted query.
Tokens must not be divided into more input chunks. This can be accomplished eg. when input stream emits one chunk per line (eg. using byline module)
```js
const { mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitterOptions } = require('dbgate-query-splitter');
const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream');
const fs = require('fs');
const byline = require('byline');
const fileStream = fs.createReadStream('INPUT_FILE_NAME', 'utf-8');
const lineStream = byline(fileStream);
const splittedStream = splitQueryStream(lineStream, mysqlSplitterOptions);
```
## Contributing
Please run tests before pushing any changes.
```sh
yarn test
```
## Supported syntax
- Comments
- Dollar strings (PostgreSQL)
- GO separators (MS SQL)
- Custom delimiter, setby DELIMITER keyword (MySQL)

5
packages/query-splitter/jest.config.js
View File

@@ -1,5 +0,0 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
moduleFileExtensions: ['js'],
};

37
packages/query-splitter/package.json
View File

@@ -1,37 +0,0 @@
{
"version": "4.1.1",
"name": "dbgate-query-splitter",
"main": "lib/index.js",
"typings": "lib/index.d.ts",
"description": "SQL Query splitter for verious database engines",
"homepage": "https://github.com/dbgate/dbgate/tree/master/packages/query-splitter",
"repository": {
"type": "git",
"url": "https://github.com/dbgate/dbgate"
},
"author": "Jan Prochazka",
"license": "MIT",
"keywords": [
"SQL",
"query",
"split",
"parse"
],
"scripts": {
"build": "tsc",
"start": "tsc --watch",
"test": "jest",
"test:ci": "jest --json --outputFile=result.json --testLocationInResults"
},
"files": [
"lib"
],
"devDependencies": {
"dbgate-types": "^4.1.1",
"@types/jest": "^25.1.4",
"@types/node": "^13.7.0",
"jest": "^24.9.0",
"ts-jest": "^25.2.1",
"typescript": "^4.4.3"
}
}

2
packages/query-splitter/src/index.ts
View File

@@ -1,2 +0,0 @@
export { splitQuery } from './splitQuery';
export * from './options';

93
packages/query-splitter/src/options.ts
View File

@@ -1,93 +0,0 @@
export interface SplitterOptions {
stringsBegins: string[];
stringsEnds: { [begin: string]: string };
stringEscapes: { [begin: string]: string };
allowSemicolon: boolean;
allowCustomDelimiter: boolean;
allowGoDelimiter: boolean;
allowDollarDollarString: boolean;
noSplit: boolean;
doubleDashComments: boolean;
multilineComments: boolean;
javaScriptComments: boolean;
returnRichInfo: boolean;
splitByLines: boolean;
}
export const defaultSplitterOptions: SplitterOptions = {
stringsBegins: ["'"],
stringsEnds: { "'": "'" },
stringEscapes: { "'": "'" },
allowSemicolon: true,
allowCustomDelimiter: false,
allowGoDelimiter: false,
allowDollarDollarString: false,
noSplit: false,
doubleDashComments: true,
multilineComments: true,
javaScriptComments: false,
returnRichInfo: false,
splitByLines: false,
};
export const mysqlSplitterOptions: SplitterOptions = {
...defaultSplitterOptions,
allowCustomDelimiter: true,
stringsBegins: ["'", '`'],
stringsEnds: { "'": "'", '`': '`' },
stringEscapes: { "'": '\\', '`': '`' },
};
export const mssqlSplitterOptions: SplitterOptions = {
...defaultSplitterOptions,
allowSemicolon: false,
allowGoDelimiter: true,
stringsBegins: ["'", '['],
stringsEnds: { "'": "'", '[': ']' },
stringEscapes: { "'": "'" },
};
export const postgreSplitterOptions: SplitterOptions = {
...defaultSplitterOptions,
allowDollarDollarString: true,
stringsBegins: ["'", '"'],
stringsEnds: { "'": "'", '"': '"' },
stringEscapes: { "'": "'", '"': '"' },
};
export const sqliteSplitterOptions: SplitterOptions = {
...defaultSplitterOptions,
stringsBegins: ["'", '"'],
stringsEnds: { "'": "'", '"': '"' },
stringEscapes: { "'": "'", '"': '"' },
};
export const mongoSplitterOptions: SplitterOptions = {
...defaultSplitterOptions,
stringsBegins: ["'", '"'],
stringsEnds: { "'": "'", '"': '"' },
stringEscapes: { "'": '\\', '"': '\\' },
};
export const noSplitSplitterOptions: SplitterOptions = {
...defaultSplitterOptions,
noSplit: true,
};
export const redisSplitterOptions: SplitterOptions = {
...defaultSplitterOptions,
splitByLines: true,
};

431
packages/query-splitter/src/splitQuery.ts
View File

@@ -1,431 +0,0 @@
import { SplitterOptions, defaultSplitterOptions } from './options';
const SEMICOLON = ';';
export interface SplitStreamContext {
options: SplitterOptions;
currentDelimiter: string;
pushOutput: (item: SplitResultItem) => void;
commandPart: string;
line: number;
column: number;
streamPosition: number;
commandStartPosition: number;
commandStartLine: number;
commandStartColumn: number;
}
export interface SplitLineContext extends SplitStreamContext {
source: string;
position: number;
// output: string[];
end: number;
wasDataOnLine: boolean;
currentCommandStart: number;
// unread: string;
// currentStatement: string;
// semicolonKeyTokenRegex: RegExp;
}
export interface SplitPositionDefinition {
position: number;
line: number;
column: number;
}
export interface SplitResultItemRich {
text: string;
start: SplitPositionDefinition;
end: SplitPositionDefinition;
trimStart?: SplitPositionDefinition;
trimEnd?: SplitPositionDefinition;
}
export type SplitResultItem = string | SplitResultItemRich;
function movePosition(context: SplitLineContext, count: number) {
if (context.options.returnRichInfo) {
let { source, position, line, column, streamPosition } = context;
while (count > 0) {
if (source[position] == '\n') {
line += 1;
column = 0;
} else {
column += 1;
}
position += 1;
streamPosition += 1;
count -= 1;
}
context.position = position;
context.streamPosition = streamPosition;
context.line = line;
context.column = column;
} else {
context.position += count;
}
}
function isStringEnd(s: string, pos: number, endch: string, escapech: string) {
if (!escapech) {
return s[pos] == endch;
}
if (endch == escapech) {
return s[pos] == endch && s[pos + 1] != endch;
} else {
return s[pos] == endch && s[pos - 1] != escapech;
}
}
interface Token {
type: 'string' | 'delimiter' | 'whitespace' | 'eoln' | 'data' | 'set_delimiter' | 'comment' | 'go_delimiter';
length: number;
value?: string;
}
const WHITESPACE_TOKEN: Token = {
type: 'whitespace',
length: 1,
};
const EOLN_TOKEN: Token = {
type: 'eoln',
length: 1,
};
const DATA_TOKEN: Token = {
type: 'data',
length: 1,
};
function scanDollarQuotedString(context: SplitLineContext): Token {
if (!context.options.allowDollarDollarString) return null;
let pos = context.position;
const s = context.source;
const match = /^(\$[a-zA-Z0-9_]*\$)/.exec(s.slice(pos));
if (!match) return null;
const label = match[1];
pos += label.length;
while (pos < context.end) {
if (s.slice(pos).startsWith(label)) {
return {
type: 'string',
length: pos + label.length - context.position,
};
}
pos++;
}
return null;
}
function scanToken(context: SplitLineContext): Token {
let pos = context.position;
const s = context.source;
const ch = s[pos];
if (context.options.stringsBegins.includes(ch)) {
pos++;
const endch = context.options.stringsEnds[ch];
const escapech = context.options.stringEscapes[ch];
while (pos < context.end && !isStringEnd(s, pos, endch, escapech)) {
if (endch == escapech && s[pos] == endch && s[pos + 1] == endch) {
pos += 2;
} else {
pos++;
}
}
return {
type: 'string',
length: pos - context.position + 1,
};
}
if (context.currentDelimiter && s.slice(pos).startsWith(context.currentDelimiter)) {
return {
type: 'delimiter',
length: context.currentDelimiter.length,
};
}
if (ch == ' ' || ch == '\t' || ch == '\r') {
return WHITESPACE_TOKEN;
}
if (ch == '\n') {
return EOLN_TOKEN;
}
if (context.options.doubleDashComments && ch == '-' && s[pos + 1] == '-') {
while (pos < context.end && s[pos] != '\n') pos++;
return {
type: 'comment',
length: pos - context.position,
};
}
if (context.options.multilineComments && ch == '/' && s[pos + 1] == '*') {
pos += 2;
while (pos < context.end) {
if (s[pos] == '*' && s[pos + 1] == '/') break;
pos++;
}
return {
type: 'comment',
length: pos - context.position + 2,
};
}
if (context.options.allowCustomDelimiter && !context.wasDataOnLine) {
const m = s.slice(pos).match(/^DELIMITER[ \t]+([^\n]+)/i);
if (m) {
return {
type: 'set_delimiter',
value: m[1].trim(),
length: m[0].length,
};
}
}
if (context.options.allowGoDelimiter && !context.wasDataOnLine) {
const m = s.slice(pos).match(/^GO[\t\r ]*(\n|$)/i);
if (m) {
return {
type: 'go_delimiter',
length: m[0].length - 1,
};
}
}
const dollarString = scanDollarQuotedString(context);
if (dollarString) return dollarString;
return DATA_TOKEN;
}
function pushQuery(context: SplitLineContext) {
const sql = (context.commandPart || '') + context.source.slice(context.currentCommandStart, context.position);
const trimmed = sql.trim();
if (trimmed) {
if (context.options.returnRichInfo) {
context.pushOutput(
countTrimmedPositions(sql, {
text: trimmed,
start: {
position: context.commandStartPosition,
line: context.commandStartLine,
column: context.commandStartColumn,
},
end: {
position: context.streamPosition,
line: context.line,
column: context.column,
},
})
);
} else {
context.pushOutput(trimmed);
}
}
}
function countTrimmedPositions(full: string, positions: SplitResultItemRich): SplitResultItemRich {
const startIndex = full.indexOf(positions.text);
const trimStart = { ...positions.start };
for (let i = 0; i < startIndex; i += 1) {
if (full[i] == '\n') {
trimStart.position += 1;
trimStart.line += 1;
trimStart.column = 0;
} else {
trimStart.position += 1;
trimStart.column += 1;
}
}
return {
...positions,
trimStart,
trimEnd: positions.end,
};
}
function markStartCommand(context: SplitLineContext) {
if (context.options.returnRichInfo) {
context.commandStartPosition = context.streamPosition;
context.commandStartLine = context.line;
context.commandStartColumn = context.column;
}
}
function splitByLines(context: SplitLineContext) {
while (context.position < context.end) {
if (context.source[context.position] == '\n') {
pushQuery(context);
context.commandPart = '';
movePosition(context, 1);
context.currentCommandStart = context.position;
markStartCommand(context);
} else {
movePosition(context, 1);
}
}
if (context.end > context.currentCommandStart) {
context.commandPart += context.source.slice(context.currentCommandStart, context.position);
}
}
export function splitQueryLine(context: SplitLineContext) {
if (context.options.splitByLines) {
splitByLines(context);
return;
}
while (context.position < context.end) {
const token = scanToken(context);
if (!token) {
// nothing special, move forward
movePosition(context, 1);
continue;
}
switch (token.type) {
case 'string':
movePosition(context, token.length);
context.wasDataOnLine = true;
break;
case 'comment':
movePosition(context, token.length);
context.wasDataOnLine = true;
break;
case 'eoln':
movePosition(context, token.length);
context.wasDataOnLine = false;
break;
case 'data':
movePosition(context, token.length);
context.wasDataOnLine = true;
break;
case 'whitespace':
movePosition(context, token.length);
break;
case 'set_delimiter':
pushQuery(context);
context.commandPart = '';
context.currentDelimiter = token.value;
movePosition(context, token.length);
context.currentCommandStart = context.position;
markStartCommand(context);
break;
case 'go_delimiter':
pushQuery(context);
context.commandPart = '';
movePosition(context, token.length);
context.currentCommandStart = context.position;
markStartCommand(context);
break;
case 'delimiter':
pushQuery(context);
context.commandPart = '';
movePosition(context, token.length);
context.currentCommandStart = context.position;
markStartCommand(context);
break;
}
}
if (context.end > context.currentCommandStart) {
context.commandPart += context.source.slice(context.currentCommandStart, context.position);
}
}
export function getInitialDelimiter(options: SplitterOptions) {
return options?.allowSemicolon === false ? null : SEMICOLON;
}
export function finishSplitStream(context: SplitStreamContext) {
const trimmed = context.commandPart.trim();
if (trimmed) {
if (context.options.returnRichInfo) {
context.pushOutput(
countTrimmedPositions(context.commandPart, {
text: trimmed,
start: {
position: context.commandStartPosition,
line: context.commandStartLine,
column: context.commandStartColumn,
},
end: {
position: context.streamPosition,
line: context.line,
column: context.column,
},
})
);
} else {
context.pushOutput(trimmed);
}
}
}
export function splitQuery(sql: string, options: SplitterOptions = null): SplitResultItem[] {
const usedOptions = {
...defaultSplitterOptions,
...options,
};
if (usedOptions.noSplit) {
if (usedOptions.returnRichInfo) {
const lines = sql.split('\n');
return [
{
text: sql,
start: {
position: 0,
line: 0,
column: 0,
},
end: {
position: sql.length,
line: lines.length,
column: lines[lines.length - 1]?.length || 0,
},
},
];
}
return [sql];
}
const output = [];
const context: SplitLineContext = {
source: sql,
end: sql.length,
currentDelimiter: getInitialDelimiter(options),
position: 0,
column: 0,
line: 0,
currentCommandStart: 0,
commandStartLine: 0,
commandStartColumn: 0,
commandStartPosition: 0,
streamPosition: 0,
pushOutput: cmd => output.push(cmd),
wasDataOnLine: false,
options: usedOptions,
commandPart: '',
};
splitQueryLine(context);
finishSplitStream(context);
return output;
}

52
packages/query-splitter/src/splitQueryStream.ts
View File

@@ -1,52 +0,0 @@
import stream from 'stream';
import {
SplitStreamContext,
getInitialDelimiter,
SplitLineContext,
splitQueryLine,
finishSplitStream,
} from './splitQuery';
import { SplitterOptions } from './options';
export class SplitQueryStream extends stream.Transform {
context: SplitStreamContext;
constructor(options: SplitterOptions) {
super({ objectMode: true });
this.context = {
commandPart: '',
commandStartLine: 0,
commandStartColumn: 0,
commandStartPosition: 0,
streamPosition: 0,
line: 0,
column: 0,
options,
currentDelimiter: getInitialDelimiter(options),
pushOutput: cmd => this.push(cmd),
};
}
_transform(chunk, encoding, done) {
const lineContext: SplitLineContext = {
...this.context,
position: 0,
currentCommandStart: 0,
wasDataOnLine: false,
source: chunk,
end: chunk.length,
};
splitQueryLine(lineContext);
this.context.commandPart = lineContext.commandPart;
done();
}
_flush(done) {
finishSplitStream(this.context);
done();
}
}
export function splitQueryStream(sourceStream, options: SplitterOptions) {
const splitter = new SplitQueryStream(options);
sourceStream.pipe(splitter);
return splitter;
}

184
packages/query-splitter/src/splitter.test.ts
View File

@@ -1,184 +0,0 @@
import {
mysqlSplitterOptions,
mssqlSplitterOptions,
postgreSplitterOptions,
mongoSplitterOptions,
noSplitSplitterOptions,
redisSplitterOptions,
} from './options';
import { splitQuery } from './splitQuery';
test('simple query', () => {
const output = splitQuery('select * from A');
expect(output).toEqual(['select * from A']);
});
test('correct split 2 queries', () => {
const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`;', mysqlSplitterOptions);
expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
});
test('correct split 2 queries - no end semicolon', () => {
const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`', mysqlSplitterOptions);
expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
});
test('delete empty query', () => {
const output = splitQuery(';;;\n;;SELECT * FROM `table1`;;;;;SELECT * FROM `table2`;;; ;;;', mysqlSplitterOptions);
expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
});
test('should handle double backtick', () => {
const input = ['CREATE TABLE `a``b` (`c"d` INT)', 'CREATE TABLE `a````b` (`c"d` INT)'];
const output = splitQuery(input.join(';\n') + ';', mysqlSplitterOptions);
expect(output).toEqual(input);
});
test('semicolon inside string', () => {
const input = ['CREATE TABLE a', "INSERT INTO a (x) VALUES ('1;2;3;4')"];
const output = splitQuery(input.join(';\n') + ';', mysqlSplitterOptions);
expect(output).toEqual(input);
});
test('semicolon inside identyifier - mssql', () => {
const input = ['CREATE TABLE [a;1]', "INSERT INTO [a;1] (x) VALUES ('1')"];
const output = splitQuery(input.join(';\n') + ';', {
...mssqlSplitterOptions,
allowSemicolon: true,
});
expect(output).toEqual(input);
});
test('delimiter test', () => {
const input = 'SELECT 1;\n DELIMITER $$\n SELECT 2; SELECT 3; \n DELIMITER ;';
const output = splitQuery(input, mysqlSplitterOptions);
expect(output).toEqual(['SELECT 1', 'SELECT 2; SELECT 3;']);
});
test('one line comment test', () => {
const input = 'SELECT 1 -- comment1;comment2\n;SELECT 2';
const output = splitQuery(input, mysqlSplitterOptions);
expect(output).toEqual(['SELECT 1 -- comment1;comment2', 'SELECT 2']);
});
test('multi line comment test', () => {
const input = 'SELECT 1 /* comment1;comment2\ncomment3*/;SELECT 2';
const output = splitQuery(input, mysqlSplitterOptions);
expect(output).toEqual(['SELECT 1 /* comment1;comment2\ncomment3*/', 'SELECT 2']);
});
test('dollar string', () => {
const input = 'CREATE PROC $$ SELECT 1; SELECT 2; $$ ; SELECT 3';
const output = splitQuery(input, postgreSplitterOptions);
expect(output).toEqual(['CREATE PROC $$ SELECT 1; SELECT 2; $$', 'SELECT 3']);
});
test('go delimiter', () => {
const input = 'SELECT 1\ngo\nSELECT 2';
const output = splitQuery(input, mssqlSplitterOptions);
expect(output).toEqual(['SELECT 1', 'SELECT 2']);
});
test('no split', () => {
const input = 'SELECT 1;SELECT 2';
const output = splitQuery(input, noSplitSplitterOptions);
expect(output).toEqual(['SELECT 1;SELECT 2']);
});
test('split mongo', () => {
const input = 'db.collection.insert({x:1});db.collection.insert({y:2})';
const output = splitQuery(input, mongoSplitterOptions);
expect(output).toEqual(['db.collection.insert({x:1})', 'db.collection.insert({y:2})']);
});
test('redis split by newline', () => {
const output = splitQuery('SET x 1\nSET y 2', redisSplitterOptions);
expect(output).toEqual(['SET x 1', 'SET y 2']);
});
test('redis split by newline 2', () => {
const output = splitQuery('SET x 1\n\nSET y 2\n', redisSplitterOptions);
expect(output).toEqual(['SET x 1', 'SET y 2']);
});
test('count lines', () => {
const output = splitQuery('SELECT * FROM `table1`;\nSELECT * FROM `table2`;', {
...mysqlSplitterOptions,
returnRichInfo: true,
});
expect(output).toEqual(
expect.arrayContaining([
expect.objectContaining({
text: 'SELECT * FROM `table1`',
trimStart: expect.objectContaining({
position: 0,
line: 0,
column: 0,
}),
end: expect.objectContaining({
position: 22,
line: 0,
column: 22,
}),
}),
expect.objectContaining({
text: 'SELECT * FROM `table2`',
trimStart: expect.objectContaining({
position: 24,
line: 1,
column: 0,
}),
end: expect.objectContaining({
position: 46,
line: 1,
column: 22,
}),
}),
])
);
});
test('count lines with flush', () => {
const output = splitQuery('SELECT * FROM `table1`;\nSELECT * FROM `table2`', {
...mysqlSplitterOptions,
returnRichInfo: true,
});
expect(output).toEqual(
expect.arrayContaining([
expect.objectContaining({
text: 'SELECT * FROM `table1`',
trimStart: expect.objectContaining({
position: 0,
line: 0,
column: 0,
}),
end: expect.objectContaining({
position: 22,
line: 0,
column: 22,
}),
}),
expect.objectContaining({
text: 'SELECT * FROM `table2`',
trimStart: expect.objectContaining({
position: 24,
line: 1,
column: 0,
}),
end: expect.objectContaining({
position: 46,
line: 1,
column: 22,
}),
}),
])
);
});

40
packages/query-splitter/src/splitterStream.test.ts
View File

@@ -1,40 +0,0 @@
import { mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitterOptions, noSplitSplitterOptions } from './options';
import stream from 'stream';
import { splitQueryStream } from './splitQueryStream';
function createInputStream(...lines) {
const pass = new stream.PassThrough({
objectMode: true,
});
lines.forEach(line => pass.write(line));
pass.end();
return pass;
}
function streamToArray(streamSource) {
return new Promise((resolve, reject) => {
const res = [];
streamSource.on('data', x => res.push(x));
streamSource.on('end', () => resolve(res));
});
}
test('stream: simple query', async () => {
const output = await streamToArray(splitQueryStream(createInputStream('select * from A'), mysqlSplitterOptions));
expect(output).toEqual(['select * from A']);
});
test('stream: query on 2 lines', async () => {
const output = await streamToArray(splitQueryStream(createInputStream('select * ', 'from A'), mysqlSplitterOptions));
expect(output).toEqual(['select * from A']);
});
test('stream: query on 2 lines', async () => {
const output = await streamToArray(
splitQueryStream(
createInputStream('SELECT * ', 'FROM `table1`;', 'SELECT *', ' FROM `table2`'),
mysqlSplitterOptions
)
);
expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
});

14
packages/query-splitter/tsconfig.json
View File

@@ -1,14 +0,0 @@
{
"compilerOptions": {
"target": "ES2015",
"module": "commonjs",
"declaration": true,
"skipLibCheck": true,
"outDir": "lib",
"preserveWatchOutput": true,
"esModuleInterop": true
},
"include": [
"src/**/*"
]
}

2
packages/tools/package.json
View File

@@ -32,7 +32,7 @@
},
"dependencies": {
"lodash": "^4.17.21",
"dbgate-query-splitter": "^4.1.1",
"dbgate-query-splitter": "^4.8.3",
"dbgate-sqltree": "^4.1.1",
"uuid": "^3.4.0"
}

2
packages/web/package.json
View File

@@ -24,7 +24,7 @@
"chartjs-adapter-moment": "^1.0.0",
"cross-env": "^7.0.3",
"dbgate-datalib": "^4.1.1",
"dbgate-query-splitter": "^4.1.1",
"dbgate-query-splitter": "^4.8.3",
"dbgate-sqltree": "^4.1.1",
"dbgate-tools": "^4.1.1",
"dbgate-types": "^4.1.1",

2
plugins/dbgate-plugin-mongo/package.json
View File

@@ -32,7 +32,7 @@
},
"devDependencies": {
"dbgate-plugin-tools": "^1.0.7",
"dbgate-query-splitter": "^4.1.1",
"dbgate-query-splitter": "^4.8.3",
"webpack": "^4.42.0",
"webpack-cli": "^3.3.11",
"dbgate-tools": "^4.1.1",

2
plugins/dbgate-plugin-mssql/package.json
View File

@@ -32,7 +32,7 @@
},
"devDependencies": {
"dbgate-plugin-tools": "^1.0.7",
"dbgate-query-splitter": "^4.1.1",
"dbgate-query-splitter": "^4.8.3",
"webpack": "^4.42.0",
"webpack-cli": "^3.3.11",
"dbgate-tools": "^4.1.1",

2
plugins/dbgate-plugin-mysql/package.json
View File

@@ -32,7 +32,7 @@
},
"devDependencies": {
"dbgate-plugin-tools": "^1.0.7",
"dbgate-query-splitter": "^4.1.1",
"dbgate-query-splitter": "^4.8.3",
"webpack": "^4.42.0",
"webpack-cli": "^3.3.11",
"dbgate-tools": "^4.1.1",

2
plugins/dbgate-plugin-postgres/package.json
View File

@@ -31,7 +31,7 @@
},
"devDependencies": {
"dbgate-plugin-tools": "^1.0.7",
"dbgate-query-splitter": "^4.1.1",
"dbgate-query-splitter": "^4.8.3",
"dbgate-tools": "^4.1.1",
"lodash": "^4.17.21",
"pg": "^8.7.1",

2
plugins/dbgate-plugin-redis/package.json
View File

@@ -30,7 +30,7 @@
},
"devDependencies": {
"dbgate-plugin-tools": "^1.0.7",
"dbgate-query-splitter": "^4.1.1",
"dbgate-query-splitter": "^4.8.3",
"dbgate-tools": "^4.1.1",
"lodash": "^4.17.21",
"webpack": "^4.42.0",

2
plugins/dbgate-plugin-sqlite/package.json
View File

@@ -32,7 +32,7 @@
"devDependencies": {
"dbgate-tools": "^4.1.1",
"dbgate-plugin-tools": "^1.0.4",
"dbgate-query-splitter": "^4.1.1",
"dbgate-query-splitter": "^4.8.3",
"byline": "^5.0.0",
"webpack": "^4.42.0",
"webpack-cli": "^3.3.11"

5
setCurrentVersion.js
View File

@@ -4,7 +4,9 @@ const path = require('path');
function changeDependencies(deps, version) {
if (!deps) return;
for (const key of Object.keys(deps)) {
if (key.startsWith('dbgate-') && key != 'dbgate-plugin-tools') deps[key] = `^${version}`;
if (key.startsWith('dbgate-') && key != 'dbgate-plugin-tools' && key != 'dbgate-query-splitter') {
deps[key] = `^${version}`;
}
}
}
@@ -40,7 +42,6 @@ changePackageFile('packages/datalib', json.version);
changePackageFile('packages/dbgate', json.version);
changePackageFile('packages/serve', json.version);
changePackageFile('packages/filterparser', json.version);
changePackageFile('packages/query-splitter', json.version);
changePackageFile('plugins/dbgate-plugin-csv', json.version);
changePackageFile('plugins/dbgate-plugin-xml', json.version);