From aa7fb7431265a83610c0e21dd31f14902755d01d Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Wed, 29 Oct 2025 12:22:13 +0100 Subject: [PATCH 01/16] PostgreSQL export to SQL and XML bytea contents #1228 --- packages/api/src/utility/handleQueryStream.js | 8 +++++--- packages/tools/src/SqlDumper.ts | 1 + packages/tools/src/stringTools.ts | 15 +++++++++++++++ .../src/backend/drivers.js | 17 ++++++++++++++--- plugins/dbgate-plugin-xml/src/backend/writer.js | 4 ++++ 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/api/src/utility/handleQueryStream.js b/packages/api/src/utility/handleQueryStream.js index 617991ea3..7f9e0cd2c 100644 --- a/packages/api/src/utility/handleQueryStream.js +++ b/packages/api/src/utility/handleQueryStream.js @@ -16,7 +16,7 @@ class QueryStreamTableWriter { this.sesid = sesid; } - initializeFromQuery(structure, resultIndex, chartDefinition, autoDetectCharts = false) { + initializeFromQuery(structure, resultIndex, chartDefinition, autoDetectCharts = false, options = {}) { this.jslid = crypto.randomUUID(); this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); fs.writeFileSync( @@ -24,6 +24,7 @@ class QueryStreamTableWriter { JSON.stringify({ ...structure, __isStreamHeader: true, + ...options }) + '\n' ); this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' }); @@ -166,7 +167,7 @@ class StreamHandler { } } - recordset(columns) { + recordset(columns, options) { if (this.rowsLimitOverflow) { return; } @@ -176,7 +177,8 @@ class StreamHandler { Array.isArray(columns) ? { columns } : columns, this.queryStreamInfoHolder.resultIndex, this.frontMatter?.[`chart-${this.queryStreamInfoHolder.resultIndex + 1}`], - this.autoDetectCharts + this.autoDetectCharts, + options ); this.queryStreamInfoHolder.resultIndex += 1; this.rowCounter = 0; diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index ac347857b..686377925 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -78,6 +78,7 @@ export class SqlDumper implements AlterProcessor { else if (_isNumber(value)) this.putRaw(value.toString()); else if (_isDate(value)) this.putStringValue(new Date(value).toISOString()); else if (value?.type == 'Buffer' && _isArray(value?.data)) this.putByteArrayValue(value?.data); + else if (value?.$binary) this.putByteArrayValue(Buffer.from(value?.$binary.base64, 'base64')); else if (value?.$bigint) this.putRaw(value?.$bigint); else if (_isPlainObject(value) || _isArray(value)) this.putStringValue(JSON.stringify(value)); else this.put('^null'); diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index 1abd2a50b..65b227d17 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -43,6 +43,14 @@ export function hexStringToArray(inputString) { return res; } +export function base64ToHex(base64String) { + const binaryString = atob(base64String); + const hexString = Array.from(binaryString, c => + c.charCodeAt(0).toString(16).padStart(2, '0') + ).join(''); + return '0x' + hexString.toUpperCase(); +}; + export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) { if (!_isString(value)) return value; @@ -230,6 +238,13 @@ export function stringifyCellValue( if (value === true) return { value: 'true', gridStyle: 'valueCellStyle' }; if (value === false) return { value: 'false', gridStyle: 'valueCellStyle' }; + if (value?.$binary?.base64) { + return { + value: base64ToHex(value.$binary.base64), + gridStyle: 'valueCellStyle', + }; + } + if (editorTypes?.parseHexAsBuffer) { if (value?.type == 'Buffer' && _isArray(value.data)) { return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' }; diff --git a/plugins/dbgate-plugin-postgres/src/backend/drivers.js b/plugins/dbgate-plugin-postgres/src/backend/drivers.js index bf75c1518..8dcc6039d 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/drivers.js +++ b/plugins/dbgate-plugin-postgres/src/backend/drivers.js @@ -47,6 +47,9 @@ function transformRow(row, columnsToTransform) { if (dataTypeName == 'geography') { row[columnName] = extractGeographyDate(row[columnName]); } + else if (dataTypeName == 'bytea' && row[columnName]) { + row[columnName] = { $binary: { base64: Buffer.from(row[columnName]).toString('base64') } }; + } } return row; @@ -142,7 +145,7 @@ const drivers = driverBases.map(driverBase => ({ conid, }; - const datatypes = await this.query(dbhan, `SELECT oid, typname FROM pg_type WHERE typname in ('geography')`); + const datatypes = await this.query(dbhan, `SELECT oid, typname FROM pg_type WHERE typname in ('geography', 'bytea')`); const typeIdToName = _.fromPairs(datatypes.rows.map(cur => [cur.oid, cur.typname])); dbhan['typeIdToName'] = typeIdToName; @@ -164,7 +167,14 @@ const drivers = driverBases.map(driverBase => ({ } const res = await dbhan.client.query({ text: sql, rowMode: 'array' }); const columns = extractPostgresColumns(res, dbhan); - return { rows: (res.rows || []).map(row => zipDataRow(row, columns)), columns }; + + const transormableTypeNames = Object.values(dbhan.typeIdToName ?? {}); + const columnsToTransform = columns.filter(x => transormableTypeNames.includes(x.dataTypeName)); + + const zippedRows = (res.rows || []).map(row => zipDataRow(row, columns)); + const transformedRows = zippedRows.map(row => transformRow(row, columnsToTransform)); + + return { rows: transformedRows, columns }; }, stream(dbhan, sql, options) { const handleNotice = notice => { @@ -191,7 +201,7 @@ const drivers = driverBases.map(driverBase => ({ if (!wasHeader) { columns = extractPostgresColumns(query._result, dbhan); if (columns && columns.length > 0) { - options.recordset(columns); + options.recordset(columns, { engine: driverBase.engine }); } wasHeader = true; } @@ -310,6 +320,7 @@ const drivers = driverBases.map(driverBase => ({ columns = extractPostgresColumns(query._result, dbhan); pass.write({ __isStreamHeader: true, + engine: driverBase.engine, ...(structure || { columns }), }); wasHeader = true; diff --git a/plugins/dbgate-plugin-xml/src/backend/writer.js b/plugins/dbgate-plugin-xml/src/backend/writer.js index 11314b706..f79236806 100644 --- a/plugins/dbgate-plugin-xml/src/backend/writer.js +++ b/plugins/dbgate-plugin-xml/src/backend/writer.js @@ -45,6 +45,10 @@ class StringifyStream extends stream.Transform { elementValue(element, value) { this.startElement(element); + if (value?.$binary?.base64) { + const buffer = Buffer.from(value.$binary.base64, 'base64'); + value = '0x' +buffer.toString('hex').toUpperCase(); + } this.push(escapeXml(`${value}`)); this.endElement(element); } From 417334d1407a9dfe11c3c4f3a214d024e8319021 Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Wed, 29 Oct 2025 15:08:57 +0100 Subject: [PATCH 02/16] Add base64 handling for binary data in filter and grid components --- packages/filterparser/src/filterTool.ts | 6 ++++-- packages/web/src/celldata/PictureCellView.svelte | 3 +++ packages/web/src/datagrid/DataGridCore.svelte | 5 ++++- packages/web/src/modals/ValueLookupModal.svelte | 3 ++- packages/web/src/perspectives/PerspectiveCell.svelte | 3 +++ 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/filterparser/src/filterTool.ts b/packages/filterparser/src/filterTool.ts index 864e16021..efab31696 100644 --- a/packages/filterparser/src/filterTool.ts +++ b/packages/filterparser/src/filterTool.ts @@ -1,4 +1,4 @@ -import { arrayToHexString, evalFilterBehaviour, isTypeDateTime } from 'dbgate-tools'; +import { arrayToHexString, base64ToHex, evalFilterBehaviour, isTypeDateTime } from 'dbgate-tools'; import { format, toDate } from 'date-fns'; import _isString from 'lodash/isString'; import _cloneDeepWith from 'lodash/cloneDeepWith'; @@ -24,7 +24,9 @@ export function getFilterValueExpression(value, dataType?) { if (value.type == 'Buffer' && Array.isArray(value.data)) { return '0x' + arrayToHexString(value.data); } - + if (value?.$binary?.base64) { + return base64ToHex(value.$binary.base64); + } return `="${value}"`; } diff --git a/packages/web/src/celldata/PictureCellView.svelte b/packages/web/src/celldata/PictureCellView.svelte index 1d6eff082..fe5b4efbc 100644 --- a/packages/web/src/celldata/PictureCellView.svelte +++ b/packages/web/src/celldata/PictureCellView.svelte @@ -10,6 +10,9 @@ if (value?.type == 'Buffer' && _.isArray(value?.data)) { return 'data:image/png;base64, ' + btoa(String.fromCharCode.apply(null, value?.data)); } + if (value?.$binary?.base64) { + return 'data:image/png;base64, ' + value.$binary.base64; + } return null; } catch (err) { console.log('Error showing picture', err); diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index df5d6da95..2df3efae1 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -361,6 +361,7 @@ detectSqlFilterBehaviour, stringifyCellValue, shouldOpenMultilineDialog, + base64ToHex, } from 'dbgate-tools'; import { getContext, onDestroy } from 'svelte'; import _, { map } from 'lodash'; @@ -758,7 +759,7 @@ export function saveCellToFileEnabled() { const value = getSelectedExportableCell(); - return _.isString(value) || (value?.type == 'Buffer' && _.isArray(value?.data)); + return _.isString(value) || (value?.type == 'Buffer' && _.isArray(value?.data)) || (value?.$binary?.base64); } export async function saveCellToFile() { @@ -771,6 +772,8 @@ fs.promises.writeFile(file, value); } else if (value?.type == 'Buffer' && _.isArray(value?.data)) { fs.promises.writeFile(file, window['Buffer'].from(value.data)); + } else if (value?.$binary?.base64) { + fs.promises.writeFile(file, window['Buffer'].from(value.$binary.base64, 'base64')); } } } diff --git a/packages/web/src/modals/ValueLookupModal.svelte b/packages/web/src/modals/ValueLookupModal.svelte index 56c2e9653..a4c24ac8b 100644 --- a/packages/web/src/modals/ValueLookupModal.svelte +++ b/packages/web/src/modals/ValueLookupModal.svelte @@ -15,6 +15,7 @@ import _ from 'lodash'; import { apiCall } from '../utility/api'; import ErrorInfo from '../elements/ErrorInfo.svelte'; + import { base64ToHex } from 'dbgate-tools'; export let onConfirm; export let conid; @@ -112,7 +113,7 @@ { fieldName: 'value', header: 'Value', - formatter: row => (row.value == null ? '(NULL)' : row.value), + formatter: row => (row.value == null ? '(NULL)' : row.value?.$binary?.base64 ? base64ToHex(row.value.$binary.base64) : row.value), }, ]} > diff --git a/packages/web/src/perspectives/PerspectiveCell.svelte b/packages/web/src/perspectives/PerspectiveCell.svelte index f72f3968d..8da6db2e7 100644 --- a/packages/web/src/perspectives/PerspectiveCell.svelte +++ b/packages/web/src/perspectives/PerspectiveCell.svelte @@ -15,6 +15,9 @@ if (force && value?.type == 'Buffer' && _.isArray(value.data)) { return String.fromCharCode.apply(String, value.data); } + else if (force && value?.$binary?.base64) { + return atob(value.$binary.base64); + } return stringifyCellValue(value, 'gridCellIntent').value; } From 9c1819467add4b405f55053ddac89b738c05d10d Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Mon, 3 Nov 2025 09:17:11 +0100 Subject: [PATCH 03/16] test binary data type --- integration-tests/__tests__/query.spec.js | 31 +++++++++++++++++++++++ integration-tests/engines.js | 1 + packages/types/test-engines.d.ts | 2 ++ 3 files changed, 34 insertions(+) diff --git a/integration-tests/__tests__/query.spec.js b/integration-tests/__tests__/query.spec.js index b8afeadd4..933558f44 100644 --- a/integration-tests/__tests__/query.spec.js +++ b/integration-tests/__tests__/query.spec.js @@ -223,4 +223,35 @@ describe('Query', () => { expect(row[keys[0]] == 1).toBeTruthy(); }) ); + + test.each(engines.filter(x => x.binaryDataType).map(engine => [engine.label, engine]))( + 'Binary', + testWrapper(async (conn, driver, engine) => { + await runCommandOnDriver(conn, driver, dmp => + // bytea + dmp.createTable({ + pureName: 't1', + columns: [ + { columnName: 'id', dataType: 'int', notNull: true, autoIncrement: true }, + { columnName: 'val', dataType: engine.binaryDataType }, + ], + primaryKey: { + columns: [{ columnName: 'id' }], + }, + }) + ); + + const structure = await driver.analyseFull(conn); + const table = structure.tables.find(x => x.pureName == 't1'); + + await runCommandOnDriver(conn, driver, dmp => dmp.put("INSERT INTO t1 (val) VALUES (%v)", {$binary: {base64: 'iVBORw0KWgo='}})); + + const res2 = await runQueryOnDriver(conn, driver, dmp => dmp.put('SELECT val FROM t1')); + + const row = res2.rows[0]; + const keys = Object.keys(row); + expect(keys.length).toEqual(1); + expect(row[keys[0]]).toEqual({$binary: {base64: 'iVBORw0KWgo='}}); + }) + ); }); diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 6c16040a9..77b78790e 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -216,6 +216,7 @@ const postgreSqlEngine = { supportSchemas: true, supportRenameSqlObject: true, defaultSchemaName: 'public', + binaryDataType: 'bytea', dumpFile: 'data/chinook-postgre.sql', dumpChecks: [ { diff --git a/packages/types/test-engines.d.ts b/packages/types/test-engines.d.ts index 9f4e2e3e9..74e58de59 100644 --- a/packages/types/test-engines.d.ts +++ b/packages/types/test-engines.d.ts @@ -96,4 +96,6 @@ export type TestEngineInfo = { }>; objects?: Array; + + binaryDataType?: string; }; From d6ae3d4f1646b2beca2d5d9a828c4976647c6c4e Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Mon, 3 Nov 2025 13:31:43 +0100 Subject: [PATCH 04/16] Refactor binary data handling in SQL dumper and update test for binary insertion --- integration-tests/__tests__/query.spec.js | 7 ++----- packages/tools/src/SqlDumper.ts | 4 +++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/integration-tests/__tests__/query.spec.js b/integration-tests/__tests__/query.spec.js index 933558f44..317b6d241 100644 --- a/integration-tests/__tests__/query.spec.js +++ b/integration-tests/__tests__/query.spec.js @@ -228,7 +228,6 @@ describe('Query', () => { 'Binary', testWrapper(async (conn, driver, engine) => { await runCommandOnDriver(conn, driver, dmp => - // bytea dmp.createTable({ pureName: 't1', columns: [ @@ -240,13 +239,11 @@ describe('Query', () => { }, }) ); - const structure = await driver.analyseFull(conn); const table = structure.tables.find(x => x.pureName == 't1'); - await runCommandOnDriver(conn, driver, dmp => dmp.put("INSERT INTO t1 (val) VALUES (%v)", {$binary: {base64: 'iVBORw0KWgo='}})); - - const res2 = await runQueryOnDriver(conn, driver, dmp => dmp.put('SELECT val FROM t1')); + await runCommandOnDriver(conn, driver, dmp => dmp.put("INSERT INTO ~t1 (~val) VALUES (%v)", {$binary: {base64: 'iVBORw0KWgo='}})); + const res2 = await runQueryOnDriver(conn, driver, dmp => dmp.put('SELECT ~val FROM ~t1')); const row = res2.rows[0]; const keys = Object.keys(row); diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index 686377925..b11ab0077 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -78,7 +78,9 @@ export class SqlDumper implements AlterProcessor { else if (_isNumber(value)) this.putRaw(value.toString()); else if (_isDate(value)) this.putStringValue(new Date(value).toISOString()); else if (value?.type == 'Buffer' && _isArray(value?.data)) this.putByteArrayValue(value?.data); - else if (value?.$binary) this.putByteArrayValue(Buffer.from(value?.$binary.base64, 'base64')); + else if (value?.$binary?.base64) { + this.putByteArrayValue(Array.from(Buffer.from(value.$binary.base64, 'base64'))); + } else if (value?.$bigint) this.putRaw(value?.$bigint); else if (_isPlainObject(value) || _isArray(value)) this.putStringValue(JSON.stringify(value)); else this.put('^null'); From 0f2af6eb37191cffc2b701bbacedaf74f4e2d6b8 Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Wed, 5 Nov 2025 14:09:58 +0100 Subject: [PATCH 05/16] Add binary data handling in query tests with enhanced stream handler --- integration-tests/__tests__/query.spec.js | 59 ++++++++++++++++++++--- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/integration-tests/__tests__/query.spec.js b/integration-tests/__tests__/query.spec.js index 317b6d241..64fc09aae 100644 --- a/integration-tests/__tests__/query.spec.js +++ b/integration-tests/__tests__/query.spec.js @@ -49,6 +49,32 @@ class StreamHandler { } } +class BinaryTestStreamHandler { + constructor(resolve, reject, expectedValue) { + this.resolve = resolve; + this.reject = reject; + this.expectedValue = expectedValue; + this.rowsReceived = []; + } + row(row) { + try { + this.rowsReceived.push(row); + if (this.expectedValue) { + expect(row).toEqual(this.expectedValue); + } + } catch (error) { + this.reject(error); + return; + } + } + recordset(columns) {} + done(result) { + this.resolve(this.rowsReceived); + } + info(msg) {} +} + + function executeStreamItem(driver, conn, sql) { return new Promise(resolve => { const handler = new StreamHandler(resolve); @@ -226,8 +252,8 @@ describe('Query', () => { test.each(engines.filter(x => x.binaryDataType).map(engine => [engine.label, engine]))( 'Binary', - testWrapper(async (conn, driver, engine) => { - await runCommandOnDriver(conn, driver, dmp => + testWrapper(async (dbhan, driver, engine) => { + await runCommandOnDriver(dbhan, driver, dmp => dmp.createTable({ pureName: 't1', columns: [ @@ -239,16 +265,35 @@ describe('Query', () => { }, }) ); - const structure = await driver.analyseFull(conn); + const structure = await driver.analyseFull(dbhan); const table = structure.tables.find(x => x.pureName == 't1'); - await runCommandOnDriver(conn, driver, dmp => dmp.put("INSERT INTO ~t1 (~val) VALUES (%v)", {$binary: {base64: 'iVBORw0KWgo='}})); - const res2 = await runQueryOnDriver(conn, driver, dmp => dmp.put('SELECT ~val FROM ~t1')); - - const row = res2.rows[0]; + const dmp = driver.createDumper(); + dmp.putCmd("INSERT INTO ~t1 (~val) VALUES (%v)", { + $binary: { base64: 'iVBORw0KWgo=' }, + }); + await driver.query(dbhan, dmp.s, {discardResult: true}); + + const dmp2 = driver.createDumper(); + dmp2.put('SELECT ~val FROM ~t1'); + const res = await driver.query(dbhan, dmp2.s); + + const row = res.rows[0]; const keys = Object.keys(row); expect(keys.length).toEqual(1); expect(row[keys[0]]).toEqual({$binary: {base64: 'iVBORw0KWgo='}}); + + const res2 = await driver.readQuery(dbhan, dmp2.s); + const rows = await Array.fromAsync(res2); + const rowsVal = rows.filter(r => r.val != null); + + expect(rowsVal.length).toEqual(1); + expect(rowsVal[0].val).toEqual({$binary: {base64: 'iVBORw0KWgo='}}); + + const res3 = await new Promise((resolve, reject) => { + const handler = new BinaryTestStreamHandler(resolve, reject, {val: {$binary: {base64: 'iVBORw0KWgo='}}}); + driver.stream(dbhan, dmp2.s, handler); + }); }) ); }); From 37d54811e0dae0ef3e1059b329f8e92a92486c87 Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Wed, 5 Nov 2025 16:07:14 +0100 Subject: [PATCH 06/16] Enhance binary data handling in transformRow to serialize Buffer as base64 --- .../src/backend/drivers.js | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/plugins/dbgate-plugin-mysql/src/backend/drivers.js b/plugins/dbgate-plugin-mysql/src/backend/drivers.js index f684e0b84..3df30c1d4 100644 --- a/plugins/dbgate-plugin-mysql/src/backend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/backend/drivers.js @@ -21,6 +21,15 @@ function extractColumns(fields) { return null; } +function transformRow(row, columns) { + columns.forEach((col) => { + if (Buffer.isBuffer(row[col.columnName])) { + row[col.columnName] = { $binary: { base64: Buffer.from(row[col.columnName]).toString('base64') } }; + } + }); + return row; +} + function zipDataRow(rowArray, columns) { return _.zipObject( columns.map(x => x.columnName), @@ -96,8 +105,8 @@ const drivers = driverBases.map(driverBase => ({ return new Promise((resolve, reject) => { dbhan.client.query(sql, function (error, results, fields) { if (error) reject(error); - const columns = extractColumns(fields); - resolve({ rows: results && columns && results.map && results.map(row => zipDataRow(row, columns)), columns }); + const columns = extractColumns(fields); + resolve({ rows: results && columns && results.map && results.map(row => transformRow(zipDataRow(row, columns), columns)), columns }); }); }); }, @@ -127,16 +136,17 @@ const drivers = driverBases.map(driverBase => ({ time: new Date(), severity: 'info', }); + options.recordset(columns, { engine: driverBase.engine }); } else { if (columns) { - options.row(zipDataRow(row, columns)); + options.row(transformRow(zipDataRow(row, columns), columns)); } } }; const handleFields = fields => { columns = extractColumns(fields); - if (columns) options.recordset(columns); + if (columns) options.recordset(columns, { engine: driverBase.engine }); }; const handleError = error => { @@ -170,10 +180,11 @@ const drivers = driverBases.map(driverBase => ({ columns = extractColumns(fields); pass.write({ __isStreamHeader: true, + engine: driverBase.engine, ...(structure || { columns }), }); }) - .on('result', row => pass.write(zipDataRow(row, columns))) + .on('result', row => pass.write(transformRow(zipDataRow(row, columns), columns))) .on('end', () => pass.end()); return pass; From dca9ea24d7acb5916992cf154393ea7cdc9b169b Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Thu, 6 Nov 2025 12:30:48 +0100 Subject: [PATCH 07/16] Implement base64 encoding for binary data in SQL dumper and modify row handling in MySQL and MSSQL drivers --- packages/tools/src/SqlDumper.ts | 7 +++- packages/tools/src/stringTools.ts | 19 ++++++--- .../src/backend/tediousDriver.js | 39 ++++++++++++++----- .../src/backend/drivers.js | 8 ++-- 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index b11ab0077..9302fa368 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -79,7 +79,12 @@ export class SqlDumper implements AlterProcessor { else if (_isDate(value)) this.putStringValue(new Date(value).toISOString()); else if (value?.type == 'Buffer' && _isArray(value?.data)) this.putByteArrayValue(value?.data); else if (value?.$binary?.base64) { - this.putByteArrayValue(Array.from(Buffer.from(value.$binary.base64, 'base64'))); + const binary = atob(value.$binary.base64); + const bytes = new Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + this.putByteArrayValue(bytes); } else if (value?.$bigint) this.putRaw(value?.$bigint); else if (_isPlainObject(value) || _isArray(value)) this.putStringValue(JSON.stringify(value)); diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index 65b227d17..e9470ac9f 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -51,6 +51,11 @@ export function base64ToHex(base64String) { return '0x' + hexString.toUpperCase(); }; +export function hexToBase64(hexString) { + const binaryString = hexString.match(/.{1,2}/g).map(byte => String.fromCharCode(parseInt(byte, 16))).join(''); + return btoa(binaryString); +} + export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) { if (!_isString(value)) return value; @@ -62,9 +67,10 @@ export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) { const mHex = value.match(/^0x([0-9a-fA-F][0-9a-fA-F])+$/); if (mHex) { return { - type: 'Buffer', - data: hexStringToArray(value.substring(2)), - }; + $binary: { + base64: hexToBase64(value.substring(2)) + } + } } } @@ -246,10 +252,11 @@ export function stringifyCellValue( } if (editorTypes?.parseHexAsBuffer) { - if (value?.type == 'Buffer' && _isArray(value.data)) { - return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' }; - } + // if (value?.type == 'Buffer' && _isArray(value.data)) { + // return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' }; + // } } + if (editorTypes?.parseObjectIdAsDollar) { if (value?.$oid) { switch (intent) { diff --git a/plugins/dbgate-plugin-mssql/src/backend/tediousDriver.js b/plugins/dbgate-plugin-mssql/src/backend/tediousDriver.js index ac4fd9e45..112a61a9e 100644 --- a/plugins/dbgate-plugin-mssql/src/backend/tediousDriver.js +++ b/plugins/dbgate-plugin-mssql/src/backend/tediousDriver.js @@ -24,6 +24,15 @@ function extractTediousColumns(columns, addDriverNativeColumn = false) { return res; } +function modifyRow(row, columns) { + columns.forEach((col) => { + if (Buffer.isBuffer(row[col.columnName])) { + row[col.columnName] = { $binary: { base64: Buffer.from(row[col.columnName]).toString('base64') } }; + } + }); + return row; +} + async function getDefaultAzureSqlToken() { const credential = new ManagedIdentityCredential(); const tokenResponse = await credential.getToken('https://database.windows.net/.default'); @@ -121,9 +130,12 @@ async function tediousQueryCore(dbhan, sql, options) { }); request.on('row', function (columns) { result.rows.push( - _.zipObject( - result.columns.map(x => x.columnName), - columns.map(x => x.value) + modifyRow( + _.zipObject( + result.columns.map(x => x.columnName), + columns.map(x => x.value) + ), + result.columns ) ); }); @@ -148,13 +160,17 @@ async function tediousReadQuery(dbhan, sql, structure) { currentColumns = extractTediousColumns(columns); pass.write({ __isStreamHeader: true, + engine: 'mssql@dbgate-plugin-mssql', ...(structure || { columns: currentColumns }), }); }); request.on('row', function (columns) { - const row = _.zipObject( - currentColumns.map(x => x.columnName), - columns.map(x => x.value) + const row = modifyRow( + _.zipObject( + currentColumns.map(x => x.columnName), + columns.map(x => x.value) + ), + currentColumns ); pass.write(row); }); @@ -204,12 +220,15 @@ async function tediousStream(dbhan, sql, options) { }); request.on('columnMetadata', function (columns) { currentColumns = extractTediousColumns(columns); - options.recordset(currentColumns); + options.recordset(currentColumns, { engine: 'mssql@dbgate-plugin-mssql' }); }); request.on('row', function (columns) { - const row = _.zipObject( - currentColumns.map(x => x.columnName), - columns.map(x => x.value) + const row = modifyRow( + _.zipObject( + currentColumns.map(x => x.columnName), + columns.map(x => x.value) + ), + currentColumns ); options.row(row); }); diff --git a/plugins/dbgate-plugin-mysql/src/backend/drivers.js b/plugins/dbgate-plugin-mysql/src/backend/drivers.js index 3df30c1d4..019934db2 100644 --- a/plugins/dbgate-plugin-mysql/src/backend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/backend/drivers.js @@ -21,7 +21,7 @@ function extractColumns(fields) { return null; } -function transformRow(row, columns) { +function modifyRow(row, columns) { columns.forEach((col) => { if (Buffer.isBuffer(row[col.columnName])) { row[col.columnName] = { $binary: { base64: Buffer.from(row[col.columnName]).toString('base64') } }; @@ -106,7 +106,7 @@ const drivers = driverBases.map(driverBase => ({ dbhan.client.query(sql, function (error, results, fields) { if (error) reject(error); const columns = extractColumns(fields); - resolve({ rows: results && columns && results.map && results.map(row => transformRow(zipDataRow(row, columns), columns)), columns }); + resolve({ rows: results && columns && results.map && results.map(row => modifyRow(zipDataRow(row, columns), columns)), columns }); }); }); }, @@ -139,7 +139,7 @@ const drivers = driverBases.map(driverBase => ({ options.recordset(columns, { engine: driverBase.engine }); } else { if (columns) { - options.row(transformRow(zipDataRow(row, columns), columns)); + options.row(modifyRow(zipDataRow(row, columns), columns)); } } }; @@ -184,7 +184,7 @@ const drivers = driverBases.map(driverBase => ({ ...(structure || { columns }), }); }) - .on('result', row => pass.write(transformRow(zipDataRow(row, columns), columns))) + .on('result', row => pass.write(modifyRow(zipDataRow(row, columns), columns))) .on('end', () => pass.end()); return pass; From 98f2b5dd08100da315abffd91a5306fc42359354 Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Thu, 6 Nov 2025 13:08:50 +0100 Subject: [PATCH 08/16] Enhance binary data handling in Oracle driver and adjust dumper for byte array values --- .../src/backend/driver.js | 24 +++++++++++++------ .../src/frontend/Dumper.js | 6 ++--- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/plugins/dbgate-plugin-oracle/src/backend/driver.js b/plugins/dbgate-plugin-oracle/src/backend/driver.js index 0fb226a0d..889175bff 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/driver.js +++ b/plugins/dbgate-plugin-oracle/src/backend/driver.js @@ -37,6 +37,15 @@ function zipDataRow(rowArray, columns) { return obj; } +function modifyRow(row, columns) { + columns.forEach(col => { + if (Buffer.isBuffer(row[col.columnName])) { + row[col.columnName] = { $binary: { base64: row[col.columnName].toString('base64') } }; + } + }); + return row; +} + let oracleClientInitialized = false; /** @type {import('dbgate-types').EngineDriver} */ @@ -106,7 +115,7 @@ const driver = { const res = await dbhan.client.execute(sql); try { const columns = extractOracleColumns(res.metaData); - return { rows: (res.rows || []).map(row => zipDataRow(row, columns)), columns }; + return { rows: (res.rows || []).map(row => modifyRow(zipDataRow(row, columns), columns)), columns }; } catch (err) { return { rows: [], @@ -134,7 +143,7 @@ const driver = { if (!wasHeader) { columns = extractOracleColumns(row); if (columns && columns.length > 0) { - options.recordset(columns); + options.recordset(columns, { engine: driverBase.engine }); } wasHeader = true; } @@ -147,11 +156,11 @@ const driver = { if (!wasHeader) { columns = extractOracleColumns(row); if (columns && columns.length > 0) { - options.recordset(columns); + options.recordset(columns, { engine: driverBase.engine }); } wasHeader = true; } - options.row(zipDataRow(row, columns)); + options.row(modifyRow(zipDataRow(row, columns), columns)); }); query.on('end', () => { @@ -214,9 +223,9 @@ const driver = { if (rows && metaData) { const columns = extractOracleColumns(metaData); - options.recordset(columns); + options.recordset(columns, { engine: driverBase.engine }); for (const row of rows) { - options.row(zipDataRow(row, columns)); + options.row(modifyRow(zipDataRow(row, columns), columns)); } } else if (rowsAffected) { options.info({ @@ -302,6 +311,7 @@ const driver = { if (columns && columns.length > 0) { pass.write({ __isStreamHeader: true, + engine: driverBase.engine, ...(structure || { columns }), }); } @@ -310,7 +320,7 @@ const driver = { }); query.on('data', row => { - pass.write(zipDataRow(row, columns)); + pass.write(modifyRow(zipDataRow(row, columns), columns)); }); query.on('end', () => { diff --git a/plugins/dbgate-plugin-oracle/src/frontend/Dumper.js b/plugins/dbgate-plugin-oracle/src/frontend/Dumper.js index 99d592d4f..84d2ba84b 100644 --- a/plugins/dbgate-plugin-oracle/src/frontend/Dumper.js +++ b/plugins/dbgate-plugin-oracle/src/frontend/Dumper.js @@ -136,9 +136,9 @@ class Dumper extends SqlDumper { // else super.putValue(value); // } - // putByteArrayValue(value) { - // this.putRaw(`e'\\\\x${arrayToHexString(value)}'`); - // } + putByteArrayValue(value) { + this.putRaw(`HEXTORAW('${arrayToHexString(value)}')`); + } putValue(value, dataType) { if (dataType?.toLowerCase() == 'timestamp') { From dfeb910ac90b3b4a53cb701f2c28cf5138c1936f Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Thu, 6 Nov 2025 14:14:24 +0100 Subject: [PATCH 09/16] Enhance binary data handling by integrating modifyRow function in SQLite driver and helpers --- .../src/backend/driver.libsql.js | 10 ++++++---- .../src/backend/driver.sqlite.js | 10 ++++++---- .../dbgate-plugin-sqlite/src/backend/helpers.js | 17 ++++++++++++++--- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/plugins/dbgate-plugin-sqlite/src/backend/driver.libsql.js b/plugins/dbgate-plugin-sqlite/src/backend/driver.libsql.js index 8b8720093..993711a9f 100644 --- a/plugins/dbgate-plugin-sqlite/src/backend/driver.libsql.js +++ b/plugins/dbgate-plugin-sqlite/src/backend/driver.libsql.js @@ -4,7 +4,7 @@ const stream = require('stream'); const driverBases = require('../frontend/drivers'); const Analyser = require('./Analyser'); const { splitQuery, sqliteSplitterOptions } = require('dbgate-query-splitter'); -const { runStreamItem, waitForDrain } = require('./helpers'); +const { runStreamItem, waitForDrain, modifyRow } = require('./helpers'); const { getLogger, createBulkInsertStreamBase, extractErrorLogData } = global.DBGATE_PACKAGES['dbgate-tools']; const logger = getLogger('sqliteDriver'); @@ -51,7 +51,7 @@ const libsqlDriver = { const columns = stmtColumns.length > 0 ? stmtColumns : extractColumns(rows[0]); return { - rows, + rows: rows.map((row) => modifyRow(row, columns)), columns: columns.map((col) => ({ columnName: col.name, dataType: col.type, @@ -66,7 +66,7 @@ const libsqlDriver = { console.log('#stream', sql); const inTransaction = dbhan.client.transaction(() => { for (const sqlItem of sqlSplitted) { - runStreamItem(dbhan, sqlItem, options, rowCounter); + runStreamItem(dbhan, sqlItem, options, rowCounter, driverBases[1].engine); } if (rowCounter.date) { @@ -114,9 +114,10 @@ const libsqlDriver = { async readQueryTask(stmt, pass) { // let sent = 0; + const columns = stmt.columns(); for (const row of stmt.iterate()) { // sent++; - if (!pass.write(row)) { + if (!pass.write(modifyRow(row, columns))) { // console.log('WAIT DRAIN', sent); await waitForDrain(pass); } @@ -134,6 +135,7 @@ const libsqlDriver = { pass.write({ __isStreamHeader: true, + engine: driverBases[1].engine, ...(structure || { columns: columns.map((col) => ({ columnName: col.name, diff --git a/plugins/dbgate-plugin-sqlite/src/backend/driver.sqlite.js b/plugins/dbgate-plugin-sqlite/src/backend/driver.sqlite.js index 440c1df61..64eeb6b0a 100644 --- a/plugins/dbgate-plugin-sqlite/src/backend/driver.sqlite.js +++ b/plugins/dbgate-plugin-sqlite/src/backend/driver.sqlite.js @@ -5,7 +5,7 @@ const Analyser = require('./Analyser'); const driverBases = require('../frontend/drivers'); const { splitQuery, sqliteSplitterOptions } = require('dbgate-query-splitter'); const { getLogger, createBulkInsertStreamBase, extractErrorLogData } = global.DBGATE_PACKAGES['dbgate-tools']; -const { runStreamItem, waitForDrain } = require('./helpers'); +const { runStreamItem, waitForDrain, modifyRow } = require('./helpers'); const logger = getLogger('sqliteDriver'); @@ -40,7 +40,7 @@ const driver = { const columns = stmt.columns(); const rows = stmt.all(); return { - rows, + rows: rows.map((row) => modifyRow(row, columns)), columns: columns.map((col) => ({ columnName: col.name, dataType: col.type, @@ -61,7 +61,7 @@ const driver = { const inTransaction = dbhan.client.transaction(() => { for (const sqlItem of sqlSplitted) { - runStreamItem(dbhan, sqlItem, options, rowCounter); + runStreamItem(dbhan, sqlItem, options, rowCounter, driverBases[0].engine); } if (rowCounter.date) { @@ -102,9 +102,10 @@ const driver = { async readQueryTask(stmt, pass) { // let sent = 0; + const columns = stmt.columns(); for (const row of stmt.iterate()) { // sent++; - if (!pass.write(row)) { + if (!pass.write(modifyRow(row, columns))) { // console.log('WAIT DRAIN', sent); await waitForDrain(pass); } @@ -122,6 +123,7 @@ const driver = { pass.write({ __isStreamHeader: true, + engine: driverBases[0].engine, ...(structure || { columns: columns.map((col) => ({ columnName: col.name, diff --git a/plugins/dbgate-plugin-sqlite/src/backend/helpers.js b/plugins/dbgate-plugin-sqlite/src/backend/helpers.js index 67cc3164f..6f24d06cc 100644 --- a/plugins/dbgate-plugin-sqlite/src/backend/helpers.js +++ b/plugins/dbgate-plugin-sqlite/src/backend/helpers.js @@ -1,6 +1,6 @@ // @ts-check -function runStreamItem(dbhan, sql, options, rowCounter) { +function runStreamItem(dbhan, sql, options, rowCounter, engine) { const stmt = dbhan.client.prepare(sql); console.log(stmt); console.log(stmt.reader); @@ -12,11 +12,12 @@ function runStreamItem(dbhan, sql, options, rowCounter) { columns.map((col) => ({ columnName: col.name, dataType: col.type, - })) + })), + { engine } ); for (const row of stmt.iterate()) { - options.row(row); + options.row(modifyRow(row, columns)); } } else { const info = stmt.run(); @@ -43,7 +44,17 @@ async function waitForDrain(stream) { }); } +function modifyRow(row, columns) { + columns.forEach((col) => { + if (row[col.name] instanceof Uint8Array) { + row[col.name] = { $binary: { base64: row[col.name].toString('base64') } }; + } + }); + return row; +} + module.exports = { runStreamItem, waitForDrain, + modifyRow, }; From 94b35e3d5f22ba13fa0dfcd26831913cbbbeda2f Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Tue, 11 Nov 2025 14:39:43 +0100 Subject: [PATCH 10/16] Enhance binary data handling by converting hex strings to base64 in filter parser and updating MongoDB driver for BinData support --- packages/filterparser/src/parseFilter.ts | 7 ++----- packages/tools/src/filterBehaviours.ts | 1 + .../dbgate-plugin-mongo/src/backend/driver.js | 16 ++++++++++------ .../dbgate-plugin-mongo/src/frontend/driver.js | 8 ++++++-- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/packages/filterparser/src/parseFilter.ts b/packages/filterparser/src/parseFilter.ts index 8b24c6cc9..60fb58071 100644 --- a/packages/filterparser/src/parseFilter.ts +++ b/packages/filterparser/src/parseFilter.ts @@ -2,7 +2,7 @@ import P from 'parsimmon'; import moment from 'moment'; import { Condition } from 'dbgate-sqltree'; import { interpretEscapes, token, word, whitespace } from './common'; -import { hexStringToArray, parseNumberSafe } from 'dbgate-tools'; +import { hexToBase64, parseNumberSafe } from 'dbgate-tools'; import { FilterBehaviour, TransformType } from 'dbgate-types'; const binaryCondition = @@ -385,10 +385,7 @@ const createParser = (filterBehaviour: FilterBehaviour) => { hexstring: () => token(P.regexp(/0x(([0-9a-fA-F][0-9a-fA-F])+)/, 1)) - .map(x => ({ - type: 'Buffer', - data: hexStringToArray(x), - })) + .map(x => ({ $binary: { base64: hexToBase64(x) } })) .desc('hex string'), noQuotedString: () => P.regexp(/[^\s^,^'^"]+/).desc('string unquoted'), diff --git a/packages/tools/src/filterBehaviours.ts b/packages/tools/src/filterBehaviours.ts index 61ebdf5a6..0eeb2f073 100644 --- a/packages/tools/src/filterBehaviours.ts +++ b/packages/tools/src/filterBehaviours.ts @@ -47,6 +47,7 @@ export const mongoFilterBehaviour: FilterBehaviour = { allowStringToken: true, allowNumberDualTesting: true, allowObjectIdTesting: true, + allowHexString: true, }; export const evalFilterBehaviour: FilterBehaviour = { diff --git a/plugins/dbgate-plugin-mongo/src/backend/driver.js b/plugins/dbgate-plugin-mongo/src/backend/driver.js index 109e294f2..cdbac08ba 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/backend/driver.js @@ -51,6 +51,10 @@ function findArrayResult(resValue) { return null; } +function BinData(_subType, base64) { + return Buffer.from(base64, 'base64'); +} + async function getScriptableDb(dbhan) { const db = dbhan.getDatabase(); db.getCollection = (name) => db.collection(name); @@ -156,9 +160,9 @@ const driver = { // return printable; // } let func; - func = eval(`(db,ObjectId) => ${sql}`); + func = eval(`(db,ObjectId,BinData) => ${sql}`); const db = await getScriptableDb(dbhan); - const res = func(db, ObjectId.createFromHexString); + const res = func(db, ObjectId.createFromHexString, BinData); if (isPromise(res)) await res; }, async operation(dbhan, operation, options) { @@ -285,7 +289,7 @@ const driver = { } else { let func; try { - func = eval(`(db,ObjectId) => ${sql}`); + func = eval(`(db,ObjectId,BinData) => ${sql}`); } catch (err) { options.info({ message: 'Error compiling expression: ' + err.message, @@ -299,7 +303,7 @@ const driver = { let exprValue; try { - exprValue = func(db, ObjectId.createFromHexString); + exprValue = func(db, ObjectId.createFromHexString, BinData); } catch (err) { options.info({ message: 'Error evaluating expression: ' + err.message, @@ -411,9 +415,9 @@ const driver = { // highWaterMark: 100, // }); - func = eval(`(db,ObjectId) => ${sql}`); + func = eval(`(db,ObjectId,BinData) => ${sql}`); const db = await getScriptableDb(dbhan); - exprValue = func(db, ObjectId.createFromHexString); + exprValue = func(db, ObjectId.createFromHexString, BinData); const pass = new stream.PassThrough({ objectMode: true, diff --git a/plugins/dbgate-plugin-mongo/src/frontend/driver.js b/plugins/dbgate-plugin-mongo/src/frontend/driver.js index e61458888..a8163ec89 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/driver.js @@ -15,7 +15,10 @@ function mongoReplacer(key, value) { function jsonStringifyWithObjectId(obj) { return JSON.stringify(obj, mongoReplacer, 2) .replace(/\{\s*\"\$oid\"\s*\:\s*\"([0-9a-f]+)\"\s*\}/g, (m, id) => `ObjectId("${id}")`) - .replace(/\{\s*\"\$bigint\"\s*\:\s*\"([0-9]+)\"\s*\}/g, (m, num) => `${num}n`); + .replace(/\{\s*\"\$bigint\"\s*\:\s*\"([0-9]+)\"\s*\}/g, (m, num) => `${num}n`) + .replace(/\{\s*"\$binary"\s*:\s*\{\s*"base64"\s*:\s*"([^"]+)"(?:\s*,\s*"subType"\s*:\s*"([0-9a-fA-F]{2})")?\s*\}\s*\}/g, (m, base64, subType) => { + return `BinData(${parseInt(subType || "00", 16)}, "${base64}")`; + }); } /** @type {import('dbgate-types').SqlDialect} */ @@ -129,7 +132,7 @@ const driver = { getCollectionExportQueryScript(collection, condition, sort) { return `db.getCollection('${collection}') - .find(${JSON.stringify(convertToMongoCondition(condition) || {})}) + .find(${jsonStringifyWithObjectId(convertToMongoCondition(condition) || {})}) .sort(${JSON.stringify(convertToMongoSort(sort) || {})})`; }, getCollectionExportQueryJson(collection, condition, sort) { @@ -148,6 +151,7 @@ const driver = { parseJsonObject: true, parseObjectIdAsDollar: true, parseDateAsDollar: true, + parseHexAsBuffer: true, explicitDataType: true, supportNumberType: true, From a14c08f122ec2c6369b80e8c2d5cbcf78acb9c61 Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Wed, 12 Nov 2025 17:03:35 +0100 Subject: [PATCH 11/16] Handle binary data in load cell from file by converting Buffer to base64 --- packages/web/src/datagrid/DataGridCore.svelte | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index 2df3efae1..117bdba96 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -799,8 +799,9 @@ isText ? data : { - type: 'Buffer', - data: [...data], + $binary: { + base64: data.toString('base64'), + }, } ); } From e1e4eb5d6f9ab34874f7f16fa8dfee4167e9ae92 Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Thu, 13 Nov 2025 09:35:39 +0100 Subject: [PATCH 12/16] Added binary data types for engines --- integration-tests/engines.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 77b78790e..99b1308ea 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -44,6 +44,7 @@ const mysqlEngine = { supportRenameSqlObject: false, dbSnapshotBySeconds: true, dumpFile: 'data/chinook-mysql.sql', + binaryDataType: 'blob', dumpChecks: [ { sql: 'select count(*) as res from genre', @@ -447,6 +448,7 @@ const sqlServerEngine = { supportTableComments: true, supportColumnComments: true, // skipSeparateSchemas: true, + binaryDataType: 'varbinary(100)', triggers: [ { testName: 'triggers before each row', @@ -507,6 +509,7 @@ const sqliteEngine = { }, }, ], + binaryDataType: 'blob', }; const libsqlFileEngine = { @@ -620,6 +623,7 @@ const oracleEngine = { }, }, ], + binaryDataType: 'blob', }; /** @type {import('dbgate-types').TestEngineInfo} */ @@ -753,18 +757,18 @@ const enginesOnCi = [ const enginesOnLocal = [ // all engines, which would be run on local test // cassandraEngine, - // mysqlEngine, + //mysqlEngine, // mariaDbEngine, - // postgreSqlEngine, - // sqlServerEngine, - // sqliteEngine, + //postgreSqlEngine, + //sqlServerEngine, + sqliteEngine, // cockroachDbEngine, // clickhouseEngine, // libsqlFileEngine, // libsqlWsEngine, - // oracleEngine, + //oracleEngine, // duckdbEngine, - firebirdEngine, + //firebirdEngine, ]; /** @type {import('dbgate-types').TestEngineInfo[] & Record} */ From 83b6c939f783cf28000f7fb545c050c69e220379 Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Thu, 13 Nov 2025 10:10:33 +0100 Subject: [PATCH 13/16] Test binary - added engine label --- integration-tests/__tests__/query.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/__tests__/query.spec.js b/integration-tests/__tests__/query.spec.js index 64fc09aae..b35f10c91 100644 --- a/integration-tests/__tests__/query.spec.js +++ b/integration-tests/__tests__/query.spec.js @@ -251,7 +251,7 @@ describe('Query', () => { ); test.each(engines.filter(x => x.binaryDataType).map(engine => [engine.label, engine]))( - 'Binary', + 'Binary - %s', testWrapper(async (dbhan, driver, engine) => { await runCommandOnDriver(dbhan, driver, dmp => dmp.createTable({ From c867d39d8d728b7169c8a0945b8a5a8fb4960b87 Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Thu, 13 Nov 2025 10:25:39 +0100 Subject: [PATCH 14/16] Remove recordset call in MySQL driver --- plugins/dbgate-plugin-mysql/src/backend/drivers.js | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/dbgate-plugin-mysql/src/backend/drivers.js b/plugins/dbgate-plugin-mysql/src/backend/drivers.js index 019934db2..f995f97d2 100644 --- a/plugins/dbgate-plugin-mysql/src/backend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/backend/drivers.js @@ -136,7 +136,6 @@ const drivers = driverBases.map(driverBase => ({ time: new Date(), severity: 'info', }); - options.recordset(columns, { engine: driverBase.engine }); } else { if (columns) { options.row(modifyRow(zipDataRow(row, columns), columns)); From 7d112a208f7960baf11bbed09f393f72fba56585 Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Thu, 13 Nov 2025 13:02:29 +0100 Subject: [PATCH 15/16] Enhance binary data handling in modifyRow function to support ArrayBuffer conversion to base64 --- plugins/dbgate-plugin-sqlite/src/backend/helpers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/dbgate-plugin-sqlite/src/backend/helpers.js b/plugins/dbgate-plugin-sqlite/src/backend/helpers.js index 6f24d06cc..9a3bf0bb6 100644 --- a/plugins/dbgate-plugin-sqlite/src/backend/helpers.js +++ b/plugins/dbgate-plugin-sqlite/src/backend/helpers.js @@ -46,8 +46,8 @@ async function waitForDrain(stream) { function modifyRow(row, columns) { columns.forEach((col) => { - if (row[col.name] instanceof Uint8Array) { - row[col.name] = { $binary: { base64: row[col.name].toString('base64') } }; + if (row[col.name] instanceof Uint8Array || row[col.name] instanceof ArrayBuffer) { + row[col.name] = { $binary: { base64: Buffer.from(row[col.name]).toString('base64') } }; } }); return row; From 0f4f154637b362b4d4dd16bed78d41db9979a7cf Mon Sep 17 00:00:00 2001 From: Stela Augustinova Date: Thu, 13 Nov 2025 13:37:33 +0100 Subject: [PATCH 16/16] Add support for base64 image source from binary object --- packages/tools/src/stringTools.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index e9470ac9f..f44017b69 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -504,6 +504,9 @@ export function getAsImageSrc(obj) { if (obj?.type == 'Buffer' && _isArray(obj?.data)) { return `data:image/png;base64, ${arrayBufferToBase64(obj?.data)}`; } + if (obj?.$binary?.base64) { + return `data:image/png;base64, ${obj.$binary.base64}`; + } if (_isString(obj) && (obj.startsWith('http://') || obj.startsWith('https://'))) { return obj;