Enhance binary data handling by converting hex strings to base64 in filter parser and updating MongoDB driver for BinData support

packages/filterparser/src/parseFilter.ts

@@ -2,7 +2,7 @@ import P from 'parsimmon';
 import moment from 'moment';
 import { Condition } from 'dbgate-sqltree';
 import { interpretEscapes, token, word, whitespace } from './common';
-import { hexStringToArray, parseNumberSafe } from 'dbgate-tools';
+import { hexToBase64, parseNumberSafe } from 'dbgate-tools';
 import { FilterBehaviour, TransformType } from 'dbgate-types';
 
 const binaryCondition =
@@ -385,10 +385,7 @@ const createParser = (filterBehaviour: FilterBehaviour) => {
 
     hexstring: () =>
       token(P.regexp(/0x(([0-9a-fA-F][0-9a-fA-F])+)/, 1))
-        .map(x => ({
-          type: 'Buffer',
-          data: hexStringToArray(x),
-        }))
+        .map(x => ({ $binary: { base64: hexToBase64(x) } }))
         .desc('hex string'),
 
     noQuotedString: () => P.regexp(/[^\s^,^'^"]+/).desc('string unquoted'),

packages/tools/src/filterBehaviours.ts

@@ -47,6 +47,7 @@ export const mongoFilterBehaviour: FilterBehaviour = {
   allowStringToken: true,
   allowNumberDualTesting: true,
   allowObjectIdTesting: true,
+  allowHexString: true,
 };
 
 export const evalFilterBehaviour: FilterBehaviour = {

plugins/dbgate-plugin-mongo/src/backend/driver.js

@@ -51,6 +51,10 @@ function findArrayResult(resValue) {
   return null;
 }
 
+function BinData(_subType, base64) {
+  return Buffer.from(base64, 'base64');
+}
+
 async function getScriptableDb(dbhan) {
   const db = dbhan.getDatabase();
   db.getCollection = (name) => db.collection(name);
@@ -156,9 +160,9 @@ const driver = {
     //   return printable;
     // }
     let func;
-    func = eval(`(db,ObjectId) => ${sql}`);
+    func = eval(`(db,ObjectId,BinData) => ${sql}`);
     const db = await getScriptableDb(dbhan);
-    const res = func(db, ObjectId.createFromHexString);
+    const res = func(db, ObjectId.createFromHexString, BinData);
     if (isPromise(res)) await res;
   },
   async operation(dbhan, operation, options) {
@@ -285,7 +289,7 @@ const driver = {
     } else {
       let func;
       try {
-        func = eval(`(db,ObjectId) => ${sql}`);
+        func = eval(`(db,ObjectId,BinData) => ${sql}`);
       } catch (err) {
         options.info({
           message: 'Error compiling expression: ' + err.message,
@@ -299,7 +303,7 @@ const driver = {
 
       let exprValue;
       try {
-        exprValue = func(db, ObjectId.createFromHexString);
+        exprValue = func(db, ObjectId.createFromHexString, BinData);
       } catch (err) {
         options.info({
           message: 'Error evaluating expression: ' + err.message,
@@ -411,9 +415,9 @@ const driver = {
     //   highWaterMark: 100,
     // });
 
-    func = eval(`(db,ObjectId) => ${sql}`);
+    func = eval(`(db,ObjectId,BinData) => ${sql}`);
     const db = await getScriptableDb(dbhan);
-    exprValue = func(db, ObjectId.createFromHexString);
+    exprValue = func(db, ObjectId.createFromHexString, BinData);
 
     const pass = new stream.PassThrough({
       objectMode: true,

plugins/dbgate-plugin-mongo/src/frontend/driver.js

@@ -15,7 +15,10 @@ function mongoReplacer(key, value) {
 function jsonStringifyWithObjectId(obj) {
   return JSON.stringify(obj, mongoReplacer, 2)
     .replace(/\{\s*\"\$oid\"\s*\:\s*\"([0-9a-f]+)\"\s*\}/g, (m, id) => `ObjectId("${id}")`)
-    .replace(/\{\s*\"\$bigint\"\s*\:\s*\"([0-9]+)\"\s*\}/g, (m, num) => `${num}n`);
+    .replace(/\{\s*\"\$bigint\"\s*\:\s*\"([0-9]+)\"\s*\}/g, (m, num) => `${num}n`)
+    .replace(/\{\s*"\$binary"\s*:\s*\{\s*"base64"\s*:\s*"([^"]+)"(?:\s*,\s*"subType"\s*:\s*"([0-9a-fA-F]{2})")?\s*\}\s*\}/g, (m, base64, subType) => {
+      return `BinData(${parseInt(subType || "00", 16)}, "${base64}")`;
+    });
 }
 
 /** @type {import('dbgate-types').SqlDialect} */
@@ -129,7 +132,7 @@ const driver = {
 
   getCollectionExportQueryScript(collection, condition, sort) {
     return `db.getCollection('${collection}')
-  .find(${JSON.stringify(convertToMongoCondition(condition) || {})})
+  .find(${jsonStringifyWithObjectId(convertToMongoCondition(condition) || {})})
   .sort(${JSON.stringify(convertToMongoSort(sort) || {})})`;
   },
   getCollectionExportQueryJson(collection, condition, sort) {
@@ -148,6 +151,7 @@ const driver = {
     parseJsonObject: true,
     parseObjectIdAsDollar: true,
     parseDateAsDollar: true,
+    parseHexAsBuffer: true,
 
     explicitDataType: true,
     supportNumberType: true,