diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 7fa500e96..bb0ddf940 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -5,7 +5,7 @@ on: - master jobs: - test: + test-runner: runs-on: ubuntu-latest container: node:10.18-jessie @@ -30,6 +30,11 @@ jobs: run: | cd packages/filterparser yarn test:ci + - name: Query spliiter tests + if: always() + run: | + cd packages/querysplitter + yarn test:ci - uses: tanmen/jest-reporter@v1 if: always() with: @@ -41,7 +46,13 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} result-file: packages/filterparser/result.json - action-name: Filter parser tests + action-name: Filter parser test results + - uses: tanmen/jest-reporter@v1 + if: always() + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + result-file: packages/querysplitter/result.json + action-name: Query splitter test results services: postgres: diff --git a/package.json b/package.json index e4d73cf73..86b46ce9c 100644 --- a/package.json +++ b/package.json @@ -16,11 +16,13 @@ "start:tools": "yarn workspace dbgate-tools start", "start:datalib": "yarn workspace dbgate-datalib start", "start:filterparser": "yarn workspace dbgate-filterparser start", + "start:querysplitter": "yarn workspace dbgate-querysplitter start", "build:sqltree": "yarn workspace dbgate-sqltree build", "build:datalib": "yarn workspace dbgate-datalib build", "build:filterparser": "yarn workspace dbgate-filterparser build", + "build:querysplitter": "yarn workspace dbgate-querysplitter build", "build:tools": "yarn workspace dbgate-tools build", - "build:lib": "yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib", + "build:lib": "yarn build:querysplitter && yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib", "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build", "build:app:mac": "yarn plugins:copydist && cd app && yarn install && yarn build:mac", "build:api": "yarn workspace dbgate-api build", @@ -40,7 +42,7 @@ "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2", "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build", "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"", - "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn build:plugins:frontend:watch\"", + "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn start:querysplitter\" \"yarn build:plugins:frontend:watch\"", "ts:api": "yarn workspace dbgate-api ts", "ts:web": "yarn workspace dbgate-web ts", "ts": "yarn ts:api && yarn ts:web", diff --git a/packages/querysplitter/.gitignore b/packages/querysplitter/.gitignore new file mode 100644 index 000000000..7951405f8 --- /dev/null +++ b/packages/querysplitter/.gitignore @@ -0,0 +1 @@ +lib \ No newline at end of file diff --git a/packages/querysplitter/jest.config.js b/packages/querysplitter/jest.config.js new file mode 100644 index 000000000..790050941 --- /dev/null +++ b/packages/querysplitter/jest.config.js @@ -0,0 +1,5 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + moduleFileExtensions: ['js'], +}; diff --git a/packages/querysplitter/package.json b/packages/querysplitter/package.json new file mode 100644 index 000000000..c3d8a6a55 --- /dev/null +++ b/packages/querysplitter/package.json @@ -0,0 +1,26 @@ +{ + "version": "4.1.1", + "name": "dbgate-querysplitter", + "main": "lib/index.js", + "typings": "lib/index.d.ts", + "scripts": { + "build": "tsc", + "start": "tsc --watch", + "test": "jest", + "test:ci": "jest --json --outputFile=result.json --testLocationInResults" + }, + "files": [ + "lib" + ], + "devDependencies": { + "dbgate-types": "^4.1.1", + "@types/jest": "^25.1.4", + "@types/node": "^13.7.0", + "jest": "^24.9.0", + "ts-jest": "^25.2.1", + "typescript": "^3.7.5" + }, + "dependencies": { + "lodash": "^4.17.21" + } +} \ No newline at end of file diff --git a/packages/querysplitter/src/index.ts b/packages/querysplitter/src/index.ts new file mode 100644 index 000000000..3d42416b3 --- /dev/null +++ b/packages/querysplitter/src/index.ts @@ -0,0 +1 @@ +export * from './splitQuery'; diff --git a/packages/querysplitter/src/splitQuery.ts b/packages/querysplitter/src/splitQuery.ts new file mode 100644 index 000000000..b557b8074 --- /dev/null +++ b/packages/querysplitter/src/splitQuery.ts @@ -0,0 +1,217 @@ +const SINGLE_QUOTE = "'"; +const DOUBLE_QUOTE = '"'; +const BACKTICK = '`'; +const DOUBLE_DASH_COMMENT_START = '--'; +const HASH_COMMENT_START = '#'; +const C_STYLE_COMMENT_START = '/*'; +const SEMICOLON = ';'; +const LINE_FEED = '\n'; +const DELIMITER_KEYWORD = 'DELIMITER'; + +export interface SplitQueryOptions {} + +interface SplitExecutionContext { + options: SplitQueryOptions; + unread: string; + currentDelimiter: string; + currentStatement: string; + output: string[]; +} + +interface FindExpResult { + expIndex: number; + exp: string | null; + nextIndex: number; +} + +const regexEscapeSetRegex = /[-/\\^$*+?.()|[\]{}]/g; +const singleQuoteStringEndRegex = /(? = { + [SINGLE_QUOTE]: singleQuoteStringEndRegex, + [DOUBLE_QUOTE]: doubleQuoteStringEndRegex, + [BACKTICK]: backtickQuoteEndRegex, +}; + +function escapeRegex(value: string): string { + return value.replace(regexEscapeSetRegex, '\\$&'); +} + +function buildKeyTokenRegex(delimiter: string): RegExp { + return new RegExp( + '(?:' + + [ + escapeRegex(delimiter), + SINGLE_QUOTE, + DOUBLE_QUOTE, + BACKTICK, + doubleDashCommentStartRegex.source, + HASH_COMMENT_START, + cStyleCommentStartRegex.source, + delimiterStartRegex.source, + ].join('|') + + ')', + 'i' + ); +} + +function findExp(content: string, regex: RegExp): FindExpResult { + const match = content.match(regex); + let result: FindExpResult; + if (match?.index !== undefined) { + result = { + expIndex: match.index, + exp: match[0], + nextIndex: match.index + match[0].length, + }; + } else { + result = { + expIndex: -1, + exp: null, + nextIndex: content.length, + }; + } + return result; +} + +function findKeyToken(content: string, currentDelimiter: string): FindExpResult { + let regex; + if (currentDelimiter === SEMICOLON) { + regex = semicolonKeyTokenRegex; + } else { + regex = buildKeyTokenRegex(currentDelimiter); + } + return findExp(content, regex); +} + +function findEndQuote(content: string, quote: string): FindExpResult { + if (!(quote in quoteEndRegexDict)) { + throw new TypeError(`Incorrect quote ${quote} supplied`); + } + return findExp(content, quoteEndRegexDict[quote]); +} + +function read(context: SplitExecutionContext, readToIndex: number, nextUnreadIndex?: number): void { + const readContent = context.unread.slice(0, readToIndex); + context.currentStatement += readContent; + if (nextUnreadIndex !== undefined && nextUnreadIndex > 0) { + context.unread = context.unread.slice(nextUnreadIndex); + } else { + context.unread = context.unread.slice(readToIndex); + } +} + +function readTillNewLine(context: SplitExecutionContext): void { + const findResult = findExp(context.unread, newLineRegex); + read(context, findResult.expIndex, findResult.expIndex); +} + +function discard(context: SplitExecutionContext, nextUnreadIndex: number): void { + if (nextUnreadIndex > 0) { + context.unread = context.unread.slice(nextUnreadIndex); + } +} + +function discardTillNewLine(context: SplitExecutionContext): void { + const findResult = findExp(context.unread, newLineRegex); + discard(context, findResult.expIndex); +} + +function publishStatement(context: SplitExecutionContext): void { + const trimmed = context.currentStatement.trim(); + if (trimmed) { + context.output.push(trimmed); + } + context.currentStatement = ''; +} + +function handleKeyTokenFindResult(context: SplitExecutionContext, findResult: FindExpResult): void { + switch (findResult.exp?.trim()) { + case context.currentDelimiter: + read(context, findResult.expIndex, findResult.nextIndex); + publishStatement(context); + break; + case SINGLE_QUOTE: + case DOUBLE_QUOTE: + case BACKTICK: { + read(context, findResult.nextIndex); + const findQuoteResult = findEndQuote(context.unread, findResult.exp); + read(context, findQuoteResult.nextIndex, undefined); + break; + } + case DOUBLE_DASH_COMMENT_START: { + read(context, findResult.nextIndex); + readTillNewLine(context); + break; + } + case HASH_COMMENT_START: { + read(context, findResult.nextIndex); + readTillNewLine(context); + break; + } + case C_STYLE_COMMENT_START: { + read(context, findResult.nextIndex); + const findCommentResult = findExp(context.unread, cStyleCommentEndRegex); + read(context, findCommentResult.nextIndex); + break; + } + case DELIMITER_KEYWORD: { + read(context, findResult.expIndex, findResult.nextIndex); + // MySQL client will return `DELIMITER cannot contain a backslash character` if backslash is used + // Shall we reject backslash as well? + const matched = context.unread.match(delimiterTokenRegex); + if (matched?.index !== undefined) { + context.currentDelimiter = matched[0].trim(); + discard(context, matched[0].length); + } + discardTillNewLine(context); + break; + } + case undefined: + case null: + read(context, findResult.nextIndex); + publishStatement(context); + break; + default: + // This should never happen + throw new Error(`Unknown token '${findResult.exp ?? '(null)'}'`); + } +} + +export function splitQuery(sql: string, options: SplitQueryOptions = {}): string[] { + const context: SplitExecutionContext = { + unread: sql, + currentDelimiter: SEMICOLON, + currentStatement: '', + output: [], + options, + }; + let findResult: FindExpResult = { + expIndex: -1, + exp: null, + nextIndex: 0, + }; + let lastUnreadLength; + do { + // console.log('context.unread', context.unread); + lastUnreadLength = context.unread.length; + findResult = findKeyToken(context.unread, context.currentDelimiter); + handleKeyTokenFindResult(context, findResult); + // Prevent infinite loop by returning incorrect result + if (lastUnreadLength === context.unread.length) { + read(context, context.unread.length); + } + } while (context.unread !== ''); + publishStatement(context); + // console.log('RESULT', context.output); + return context.output; +} diff --git a/packages/querysplitter/src/splitter.test.ts b/packages/querysplitter/src/splitter.test.ts new file mode 100644 index 000000000..a27df60e5 --- /dev/null +++ b/packages/querysplitter/src/splitter.test.ts @@ -0,0 +1,33 @@ +import { splitQuery } from './splitQuery'; + +test('simple query', () => { + const output = splitQuery('select * from A'); + expect(output).toEqual(['select * from A']); +}); + +test('correct split 2 queries', () => { + const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`;'); + expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']); +}); + +test('correct split 2 queries - no end semicolon', () => { + const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`'); + expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']); +}); + +test('delete empty query', () => { + const output = splitQuery(';;;\n;;SELECT * FROM `table1`;;;;;SELECT * FROM `table2`;;; ;;;'); + expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']); +}); + +test('should handle double backtick', () => { + const input = ['CREATE TABLE `a``b` (`c"d` INT)', 'CREATE TABLE `a````b` (`c"d` INT)']; + const output = splitQuery(input.join(';\n') + ';'); + expect(output).toEqual(input); +}); + +test('semicolon inside string', () => { + const input = ['CREATE TABLE [a1]', "INSERT INTO [a1] (x) VALUES ('1;2;3;4')"]; + const output = splitQuery(input.join(';\n') + ';'); + expect(output).toEqual(input); +}); diff --git a/packages/querysplitter/tsconfig.json b/packages/querysplitter/tsconfig.json new file mode 100644 index 000000000..b2671e70a --- /dev/null +++ b/packages/querysplitter/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2015", + "module": "commonjs", + "declaration": true, + "skipLibCheck": true, + "outDir": "lib", + "preserveWatchOutput": true, + "esModuleInterop": true + }, + "include": [ + "src/**/*" + ] +}