query splitter initial version

.github/workflows/run-tests.yaml

@@ -5,7 +5,7 @@ on:
       - master
 
 jobs:
-  test:
+  test-runner:
     runs-on: ubuntu-latest
     container: node:10.18-jessie
 
@@ -30,6 +30,11 @@ jobs:
         run: |
           cd packages/filterparser
           yarn test:ci
+      - name: Query spliiter tests
+        if: always()
+        run: |
+          cd packages/querysplitter
+          yarn test:ci
       - uses: tanmen/jest-reporter@v1
         if: always()
         with:
@@ -41,7 +46,13 @@ jobs:
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           result-file: packages/filterparser/result.json
-          action-name: Filter parser tests
+          action-name: Filter parser test results
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/querysplitter/result.json
+          action-name: Query splitter test results
 
     services:
       postgres:

package.json

@@ -16,11 +16,13 @@
     "start:tools": "yarn workspace dbgate-tools start",
     "start:datalib": "yarn workspace dbgate-datalib start",
     "start:filterparser": "yarn workspace dbgate-filterparser start",
+    "start:querysplitter": "yarn workspace dbgate-querysplitter start",
     "build:sqltree": "yarn workspace dbgate-sqltree build",
     "build:datalib": "yarn workspace dbgate-datalib build",
     "build:filterparser": "yarn workspace dbgate-filterparser build",
+    "build:querysplitter": "yarn workspace dbgate-querysplitter build",
     "build:tools": "yarn workspace dbgate-tools build",
-    "build:lib": "yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
+    "build:lib": "yarn build:querysplitter && yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
     "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
     "build:app:mac": "yarn plugins:copydist && cd app && yarn install && yarn build:mac",
     "build:api": "yarn workspace dbgate-api build",
@@ -40,7 +42,7 @@
     "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
     "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build",
     "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
-    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn build:plugins:frontend:watch\"",
+    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn start:querysplitter\" \"yarn build:plugins:frontend:watch\"",
     "ts:api": "yarn workspace dbgate-api ts",
     "ts:web": "yarn workspace dbgate-web ts",
     "ts": "yarn ts:api && yarn ts:web",

packages/querysplitter/.gitignore

@@ -0,0 +1 @@
+lib

packages/querysplitter/jest.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  moduleFileExtensions: ['js'],
+};

packages/querysplitter/package.json

@@ -0,0 +1,26 @@
+{
+  "version": "4.1.1",
+  "name": "dbgate-querysplitter",
+  "main": "lib/index.js",
+  "typings": "lib/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "start": "tsc --watch",
+    "test": "jest",
+    "test:ci": "jest --json --outputFile=result.json --testLocationInResults"
+  },
+  "files": [
+    "lib"
+  ],
+  "devDependencies": {
+    "dbgate-types": "^4.1.1",
+    "@types/jest": "^25.1.4",
+    "@types/node": "^13.7.0",
+    "jest": "^24.9.0",
+    "ts-jest": "^25.2.1",
+    "typescript": "^3.7.5"
+  },
+  "dependencies": {
+    "lodash": "^4.17.21"
+  }
+}

packages/querysplitter/src/index.ts

@@ -0,0 +1 @@
+export * from './splitQuery';

packages/querysplitter/src/splitQuery.ts

@@ -0,0 +1,217 @@
+const SINGLE_QUOTE = "'";
+const DOUBLE_QUOTE = '"';
+const BACKTICK = '`';
+const DOUBLE_DASH_COMMENT_START = '--';
+const HASH_COMMENT_START = '#';
+const C_STYLE_COMMENT_START = '/*';
+const SEMICOLON = ';';
+const LINE_FEED = '\n';
+const DELIMITER_KEYWORD = 'DELIMITER';
+
+export interface SplitQueryOptions {}
+
+interface SplitExecutionContext {
+  options: SplitQueryOptions;
+  unread: string;
+  currentDelimiter: string;
+  currentStatement: string;
+  output: string[];
+}
+
+interface FindExpResult {
+  expIndex: number;
+  exp: string | null;
+  nextIndex: number;
+}
+
+const regexEscapeSetRegex = /[-/\\^$*+?.()|[\]{}]/g;
+const singleQuoteStringEndRegex = /(?<!\\)'/;
+const doubleQuoteStringEndRegex = /(?<!\\)"/;
+const backtickQuoteEndRegex = /(?<!`)`(?!`)/;
+const doubleDashCommentStartRegex = /--[ \f\n\r\t\v]/;
+const cStyleCommentStartRegex = /\/\*/;
+const cStyleCommentEndRegex = /(?<!\/)\*\//;
+const newLineRegex = /(?:[\r\n]+|$)/;
+const delimiterStartRegex = /(?:^|[\n\r]+)[ \f\t\v]*DELIMITER[ \t]+/i;
+// Best effort only, unable to find a syntax specification on delimiter
+const delimiterTokenRegex = /^(?:'(.+)'|"(.+)"|`(.+)`|([^\s]+))/;
+const semicolonKeyTokenRegex = buildKeyTokenRegex(SEMICOLON);
+const quoteEndRegexDict: Record<string, RegExp> = {
+  [SINGLE_QUOTE]: singleQuoteStringEndRegex,
+  [DOUBLE_QUOTE]: doubleQuoteStringEndRegex,
+  [BACKTICK]: backtickQuoteEndRegex,
+};
+
+function escapeRegex(value: string): string {
+  return value.replace(regexEscapeSetRegex, '\\$&');
+}
+
+function buildKeyTokenRegex(delimiter: string): RegExp {
+  return new RegExp(
+    '(?:' +
+      [
+        escapeRegex(delimiter),
+        SINGLE_QUOTE,
+        DOUBLE_QUOTE,
+        BACKTICK,
+        doubleDashCommentStartRegex.source,
+        HASH_COMMENT_START,
+        cStyleCommentStartRegex.source,
+        delimiterStartRegex.source,
+      ].join('|') +
+      ')',
+    'i'
+  );
+}
+
+function findExp(content: string, regex: RegExp): FindExpResult {
+  const match = content.match(regex);
+  let result: FindExpResult;
+  if (match?.index !== undefined) {
+    result = {
+      expIndex: match.index,
+      exp: match[0],
+      nextIndex: match.index + match[0].length,
+    };
+  } else {
+    result = {
+      expIndex: -1,
+      exp: null,
+      nextIndex: content.length,
+    };
+  }
+  return result;
+}
+
+function findKeyToken(content: string, currentDelimiter: string): FindExpResult {
+  let regex;
+  if (currentDelimiter === SEMICOLON) {
+    regex = semicolonKeyTokenRegex;
+  } else {
+    regex = buildKeyTokenRegex(currentDelimiter);
+  }
+  return findExp(content, regex);
+}
+
+function findEndQuote(content: string, quote: string): FindExpResult {
+  if (!(quote in quoteEndRegexDict)) {
+    throw new TypeError(`Incorrect quote ${quote} supplied`);
+  }
+  return findExp(content, quoteEndRegexDict[quote]);
+}
+
+function read(context: SplitExecutionContext, readToIndex: number, nextUnreadIndex?: number): void {
+  const readContent = context.unread.slice(0, readToIndex);
+  context.currentStatement += readContent;
+  if (nextUnreadIndex !== undefined && nextUnreadIndex > 0) {
+    context.unread = context.unread.slice(nextUnreadIndex);
+  } else {
+    context.unread = context.unread.slice(readToIndex);
+  }
+}
+
+function readTillNewLine(context: SplitExecutionContext): void {
+  const findResult = findExp(context.unread, newLineRegex);
+  read(context, findResult.expIndex, findResult.expIndex);
+}
+
+function discard(context: SplitExecutionContext, nextUnreadIndex: number): void {
+  if (nextUnreadIndex > 0) {
+    context.unread = context.unread.slice(nextUnreadIndex);
+  }
+}
+
+function discardTillNewLine(context: SplitExecutionContext): void {
+  const findResult = findExp(context.unread, newLineRegex);
+  discard(context, findResult.expIndex);
+}
+
+function publishStatement(context: SplitExecutionContext): void {
+  const trimmed = context.currentStatement.trim();
+  if (trimmed) {
+    context.output.push(trimmed);
+  }
+  context.currentStatement = '';
+}
+
+function handleKeyTokenFindResult(context: SplitExecutionContext, findResult: FindExpResult): void {
+  switch (findResult.exp?.trim()) {
+    case context.currentDelimiter:
+      read(context, findResult.expIndex, findResult.nextIndex);
+      publishStatement(context);
+      break;
+    case SINGLE_QUOTE:
+    case DOUBLE_QUOTE:
+    case BACKTICK: {
+      read(context, findResult.nextIndex);
+      const findQuoteResult = findEndQuote(context.unread, findResult.exp);
+      read(context, findQuoteResult.nextIndex, undefined);
+      break;
+    }
+    case DOUBLE_DASH_COMMENT_START: {
+      read(context, findResult.nextIndex);
+      readTillNewLine(context);
+      break;
+    }
+    case HASH_COMMENT_START: {
+      read(context, findResult.nextIndex);
+      readTillNewLine(context);
+      break;
+    }
+    case C_STYLE_COMMENT_START: {
+      read(context, findResult.nextIndex);
+      const findCommentResult = findExp(context.unread, cStyleCommentEndRegex);
+      read(context, findCommentResult.nextIndex);
+      break;
+    }
+    case DELIMITER_KEYWORD: {
+      read(context, findResult.expIndex, findResult.nextIndex);
+      // MySQL client will return `DELIMITER cannot contain a backslash character` if backslash is used
+      // Shall we reject backslash as well?
+      const matched = context.unread.match(delimiterTokenRegex);
+      if (matched?.index !== undefined) {
+        context.currentDelimiter = matched[0].trim();
+        discard(context, matched[0].length);
+      }
+      discardTillNewLine(context);
+      break;
+    }
+    case undefined:
+    case null:
+      read(context, findResult.nextIndex);
+      publishStatement(context);
+      break;
+    default:
+      // This should never happen
+      throw new Error(`Unknown token '${findResult.exp ?? '(null)'}'`);
+  }
+}
+
+export function splitQuery(sql: string, options: SplitQueryOptions = {}): string[] {
+  const context: SplitExecutionContext = {
+    unread: sql,
+    currentDelimiter: SEMICOLON,
+    currentStatement: '',
+    output: [],
+    options,
+  };
+  let findResult: FindExpResult = {
+    expIndex: -1,
+    exp: null,
+    nextIndex: 0,
+  };
+  let lastUnreadLength;
+  do {
+    // console.log('context.unread', context.unread);
+    lastUnreadLength = context.unread.length;
+    findResult = findKeyToken(context.unread, context.currentDelimiter);
+    handleKeyTokenFindResult(context, findResult);
+    // Prevent infinite loop by returning incorrect result
+    if (lastUnreadLength === context.unread.length) {
+      read(context, context.unread.length);
+    }
+  } while (context.unread !== '');
+  publishStatement(context);
+  // console.log('RESULT', context.output);
+  return context.output;
+}

packages/querysplitter/src/splitter.test.ts

@@ -0,0 +1,33 @@
+import { splitQuery } from './splitQuery';
+
+test('simple query', () => {
+  const output = splitQuery('select * from A');
+  expect(output).toEqual(['select * from A']);
+});
+
+test('correct split 2 queries', () => {
+  const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`;');
+  expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
+});
+
+test('correct split 2 queries - no end semicolon', () => {
+  const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`');
+  expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
+});
+
+test('delete empty query', () => {
+  const output = splitQuery(';;;\n;;SELECT * FROM `table1`;;;;;SELECT * FROM `table2`;;; ;;;');
+  expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
+});
+
+test('should handle double backtick', () => {
+  const input = ['CREATE TABLE `a``b` (`c"d` INT)', 'CREATE TABLE `a````b` (`c"d` INT)'];
+  const output = splitQuery(input.join(';\n') + ';');
+  expect(output).toEqual(input);
+});
+
+test('semicolon inside string', () => {
+  const input = ['CREATE TABLE [a1]', "INSERT INTO [a1] (x) VALUES ('1;2;3;4')"];
+  const output = splitQuery(input.join(';\n') + ';');
+  expect(output).toEqual(input);
+});

packages/querysplitter/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "ES2015",
+    "module": "commonjs",
+    "declaration": true,
+    "skipLibCheck": true,
+    "outDir": "lib",
+    "preserveWatchOutput": true,
+    "esModuleInterop": true
+  },
+  "include": [
+    "src/**/*"
+  ]
+}