PostgreSQL export to SQL and XML bytea contents #1228

packages/api/src/utility/handleQueryStream.js

@@ -16,7 +16,7 @@ class QueryStreamTableWriter {
     this.sesid = sesid;
   }
 
-  initializeFromQuery(structure, resultIndex, chartDefinition, autoDetectCharts = false) {
+  initializeFromQuery(structure, resultIndex, chartDefinition, autoDetectCharts = false, options = {}) {
     this.jslid = crypto.randomUUID();
     this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     fs.writeFileSync(
@@ -24,6 +24,7 @@ class QueryStreamTableWriter {
       JSON.stringify({
         ...structure,
         __isStreamHeader: true,
+        ...options
       }) + '\n'
     );
     this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
@@ -166,7 +167,7 @@ class StreamHandler {
     }
   }
 
-  recordset(columns) {
+  recordset(columns, options) {
     if (this.rowsLimitOverflow) {
       return;
     }
@@ -176,7 +177,8 @@ class StreamHandler {
       Array.isArray(columns) ? { columns } : columns,
       this.queryStreamInfoHolder.resultIndex,
       this.frontMatter?.[`chart-${this.queryStreamInfoHolder.resultIndex + 1}`],
-      this.autoDetectCharts
+      this.autoDetectCharts,
+      options
     );
     this.queryStreamInfoHolder.resultIndex += 1;
     this.rowCounter = 0;

packages/tools/src/SqlDumper.ts

@@ -78,6 +78,7 @@ export class SqlDumper implements AlterProcessor {
     else if (_isNumber(value)) this.putRaw(value.toString());
     else if (_isDate(value)) this.putStringValue(new Date(value).toISOString());
     else if (value?.type == 'Buffer' && _isArray(value?.data)) this.putByteArrayValue(value?.data);
+    else if (value?.$binary) this.putByteArrayValue(Buffer.from(value?.$binary.base64, 'base64'));
     else if (value?.$bigint) this.putRaw(value?.$bigint);
     else if (_isPlainObject(value) || _isArray(value)) this.putStringValue(JSON.stringify(value));
     else this.put('^null');

packages/tools/src/stringTools.ts

@@ -43,6 +43,14 @@ export function hexStringToArray(inputString) {
   return res;
 }
 
+export function base64ToHex(base64String) {
+  const binaryString = atob(base64String);
+  const hexString = Array.from(binaryString, c =>
+    c.charCodeAt(0).toString(16).padStart(2, '0')
+  ).join('');
+  return '0x' + hexString.toUpperCase();
+};
+
 export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) {
   if (!_isString(value)) return value;
 
@@ -230,6 +238,13 @@ export function stringifyCellValue(
   if (value === true) return { value: 'true', gridStyle: 'valueCellStyle' };
   if (value === false) return { value: 'false', gridStyle: 'valueCellStyle' };
 
+  if (value?.$binary?.base64) {
+    return {
+      value: base64ToHex(value.$binary.base64),
+      gridStyle: 'valueCellStyle',
+    };
+  }
+
   if (editorTypes?.parseHexAsBuffer) {
     if (value?.type == 'Buffer' && _isArray(value.data)) {
       return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' };

plugins/dbgate-plugin-postgres/src/backend/drivers.js

@@ -47,6 +47,9 @@ function transformRow(row, columnsToTransform) {
     if (dataTypeName == 'geography') {
       row[columnName] = extractGeographyDate(row[columnName]);
     }
+    else if (dataTypeName == 'bytea' && row[columnName]) {
+      row[columnName] = { $binary: { base64: Buffer.from(row[columnName]).toString('base64') } };
+    }
   }
 
   return row;
@@ -142,7 +145,7 @@ const drivers = driverBases.map(driverBase => ({
       conid,
     };
 
-    const datatypes = await this.query(dbhan, `SELECT oid, typname FROM pg_type WHERE typname in ('geography')`);
+    const datatypes = await this.query(dbhan, `SELECT oid, typname FROM pg_type WHERE typname in ('geography', 'bytea')`);
     const typeIdToName = _.fromPairs(datatypes.rows.map(cur => [cur.oid, cur.typname]));
     dbhan['typeIdToName'] = typeIdToName;
 
@@ -164,7 +167,14 @@ const drivers = driverBases.map(driverBase => ({
     }
     const res = await dbhan.client.query({ text: sql, rowMode: 'array' });
     const columns = extractPostgresColumns(res, dbhan);
-    return { rows: (res.rows || []).map(row => zipDataRow(row, columns)), columns };
+
+    const transormableTypeNames = Object.values(dbhan.typeIdToName ?? {});
+    const columnsToTransform = columns.filter(x => transormableTypeNames.includes(x.dataTypeName));
+
+    const zippedRows = (res.rows || []).map(row => zipDataRow(row, columns));
+    const transformedRows = zippedRows.map(row => transformRow(row, columnsToTransform));
+
+    return { rows: transformedRows, columns };
   },
   stream(dbhan, sql, options) {
     const handleNotice = notice => {
@@ -191,7 +201,7 @@ const drivers = driverBases.map(driverBase => ({
       if (!wasHeader) {
         columns = extractPostgresColumns(query._result, dbhan);
         if (columns && columns.length > 0) {
-          options.recordset(columns);
+          options.recordset(columns, { engine: driverBase.engine });
         }
         wasHeader = true;
       }
@@ -310,6 +320,7 @@ const drivers = driverBases.map(driverBase => ({
         columns = extractPostgresColumns(query._result, dbhan);
         pass.write({
           __isStreamHeader: true,
+          engine: driverBase.engine,
           ...(structure || { columns }),
         });
         wasHeader = true;

plugins/dbgate-plugin-xml/src/backend/writer.js

@@ -45,6 +45,10 @@ class StringifyStream extends stream.Transform {
 
   elementValue(element, value) {
     this.startElement(element);
+    if (value?.$binary?.base64) {
+      const buffer = Buffer.from(value.$binary.base64, 'base64');
+      value = '0x' +buffer.toString('hex').toUpperCase();
+    }
     this.push(escapeXml(`${value}`));
     this.endElement(element);
   }