Merge branch 'master' into feature/postgresql-export-bytea

plugins/dbgate-plugin-cassandra/package.json

@@ -5,6 +5,10 @@
   "license": "GPL-3.0",
   "author": "Jan Prochazka",
   "description": "cassandra connector for DbGate",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate.git"
+  },
   "keywords": [
     "dbgate",
     "cassandra",

plugins/dbgate-plugin-clickhouse/package.json

@@ -14,6 +14,10 @@
     "dist",
     "icon.svg"
   ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate.git"
+  },
   "scripts": {
     "build:frontend": "webpack --config webpack-frontend.config",
     "build:frontend:watch": "webpack  --watch --config webpack-frontend.config",

plugins/dbgate-plugin-csv/src/backend/writer.js

@@ -6,6 +6,54 @@ const { getLogger } = global.DBGATE_PACKAGES['dbgate-tools'];
 
 const logger = getLogger('csvWriter');
 
+class RecodeTransform extends stream.Transform {
+  constructor(toEncoding = 'utf8') {
+    super({ readableObjectMode: false, writableObjectMode: false });
+    this.to = String(toEncoding).toLowerCase();
+    this.decoder = new (global.TextDecoder || require('util').TextDecoder)('utf-8', { fatal: false });
+  }
+
+  _encodeString(str) {
+    if (this.to === 'utf8' || this.to === 'utf-8') {
+      return Buffer.from(str, 'utf8');
+    }
+    if (this.to === 'utf16le' || this.to === 'ucs2' || this.to === 'utf-16le') {
+      return Buffer.from(str, 'utf16le');
+    }
+    if (this.to === 'utf16be' || this.to === 'utf-16be') {
+      const le = Buffer.from(str, 'utf16le');
+      for (let i = 0; i + 1 < le.length; i += 2) {
+        const a = le[i];
+        le[i] = le[i + 1];
+        le[i + 1] = a;
+      }
+      return le;
+    }
+    throw new Error(`Unsupported target encoding: ${this.to}`);
+  }
+
+  _transform(chunk, enc, cb) {
+    try {
+      if (!Buffer.isBuffer(chunk)) chunk = Buffer.from(chunk, enc);
+      const part = this.decoder.decode(chunk, { stream: true });
+      if (part.length) this.push(this._encodeString(part));
+      cb();
+    } catch (e) {
+      cb(e);
+    }
+  }
+
+  _flush(cb) {
+    try {
+      const rest = this.decoder.decode();
+      if (rest.length) this.push(this._encodeString(rest));
+      cb();
+    } catch (e) {
+      cb(e);
+    }
+  }
+}
+
 class CsvPrepareStream extends stream.Transform {
   constructor({ header }) {
     super({ objectMode: true });
@@ -26,18 +74,59 @@ class CsvPrepareStream extends stream.Transform {
   }
 }
 
-async function writer({ fileName, encoding = 'utf-8', header = true, delimiter, quoted }) {
+async function writer({
+  fileName,
+  encoding = 'utf-8',
+  header = true,
+  delimiter,
+  quoted,
+  writeBom,
+  writeSepHeader,
+  recordDelimiter,
+}) {
   logger.info(`DBGM-00133 Writing file ${fileName}`);
   const csvPrepare = new CsvPrepareStream({ header });
-  const csvStream = csv.stringify({ delimiter, quoted });
+  const csvStream = csv.stringify({ delimiter, quoted, record_delimiter: recordDelimiter || undefined });
   const fileStream = fs.createWriteStream(fileName, encoding);
-  // csvPrepare.pipe(csvStream);
-  // csvStream.pipe(fileStream);
-  // csvPrepare['finisher'] = fileStream;
+  if (writeBom) {
+    switch (encoding.toLowerCase()) {
+      case 'utf-8':
+      case 'utf8':
+        fileStream.write(Buffer.from([0xef, 0xbb, 0xbf]));
+        break;
+      case 'utf-16':
+      case 'utf16':
+        fileStream.write(Buffer.from([0xff, 0xfe]));
+        break;
+      case 'utf-16le':
+      case 'utf16le':
+        fileStream.write(Buffer.from([0xff, 0xfe]));
+        break;
+      case 'utf-16be':
+      case 'utf16be':
+        fileStream.write(Buffer.from([0xfe, 0xff]));
+        break;
+      case 'utf-32le':
+      case 'utf32le':
+        fileStream.write(Buffer.from([0xff, 0xfe, 0x00, 0x00]));
+        break;
+      case 'utf-32be':
+      case 'utf32be':
+        fileStream.write(Buffer.from([0x00, 0x00, 0xfe, 0xff]));
+        break;
+      default:
+        break;
+    }
+  }
+  if (writeSepHeader) {
+    fileStream.write(`sep=${delimiter}${recordDelimiter || '\n'}`);
+  }
+
   csvPrepare.requireFixedStructure = true;
 
-  return [csvPrepare, csvStream, fileStream];
-  // return csvPrepare;
+  return encoding.toLowerCase() === 'utf8' || encoding.toLowerCase() === 'utf-8'
+    ? [csvPrepare, csvStream, fileStream]
+    : [csvPrepare, csvStream, new RecodeTransform(encoding), fileStream];
 }
 
 module.exports = writer;

plugins/dbgate-plugin-csv/src/frontend/index.js

@@ -39,6 +39,24 @@ const fileFormat = {
       apiName: 'header',
       default: true,
     },
+    {
+      type: 'checkbox',
+      name: 'writeBom',
+      label: 'Write BOM (Byte Order Mark)',
+      apiName: 'writeBom',
+      direction: 'target',
+    },
+    {
+      type: 'select',
+      name: 'recordDelimiter',
+      label: 'Record Delimiter',
+      options: [
+        { name: 'CR', value: '\r' },
+        { name: 'CRLF', value: '\r\n' },
+      ],
+      apiName: 'recordDelimiter',
+      direction: 'target',
+    },
   ],
 };
 
@@ -68,5 +86,31 @@ export default {
         },
       }),
     },
+    {
+      label: 'CSV file for MS Excel',
+      extension: 'csv',
+      createWriter: (fileName) => ({
+        functionName: 'writer@dbgate-plugin-csv',
+        props: {
+          fileName,
+          delimiter: ';',
+          recordDelimiter: '\r\n',
+          encoding: 'utf16le',
+          writeSepHeader: true,
+          writeBom: true,
+        },
+      }),
+    },
+    {
+      label: 'TSV file (tab separated)',
+      extension: 'tsv',
+      createWriter: (fileName) => ({
+        functionName: 'writer@dbgate-plugin-csv',
+        props: {
+          fileName,
+          delimiter: '\t',
+        },
+      }),
+    },
   ],
 };

plugins/dbgate-plugin-mssql/src/backend/tediousDriver.js

@@ -78,7 +78,11 @@ async function tediousConnect(storedConnection) {
     const [host, instance] = (server || '').split('\\');
     const connectionOptions = {
       instanceName: instance,
-      encrypt: !!ssl || authType == 'msentra' || authType == 'azureManagedIdentity',
+      encrypt:
+        !!ssl ||
+        authType == 'msentra' ||
+        authType == 'azureManagedIdentity' ||
+        server?.endsWith('.database.windows.net'),
       cryptoCredentialsDetails: ssl ? _.pick(ssl, ['ca', 'cert', 'key']) : undefined,
       trustServerCertificate: ssl ? (!ssl.ca && !ssl.cert && !ssl.key ? true : ssl.rejectUnauthorized) : undefined,
       enableArithAbort: true,
@@ -181,6 +185,7 @@ async function tediousReadQuery(dbhan, sql, structure) {
 
 async function tediousStream(dbhan, sql, options) {
   let currentColumns = [];
+  let skipAffectedMessage = false;
 
   const handleInfo = info => {
     const { message, lineNumber, procName } = info;
@@ -202,7 +207,11 @@ async function tediousStream(dbhan, sql, options) {
       severity: 'error',
     });
   };
+  const handleDatabaseChange = database => {
+    options.changedCurrentDatabase(database);
+  };
 
+  dbhan.client.on('databaseChange', handleDatabaseChange);
   dbhan.client.on('infoMessage', handleInfo);
   dbhan.client.on('errorMessage', handleError);
   const request = new tedious.Request(sql, (err, rowCount) => {
@@ -212,11 +221,14 @@ async function tediousStream(dbhan, sql, options) {
     dbhan.client.off('infoMessage', handleInfo);
     dbhan.client.off('errorMessage', handleError);
 
-    options.info({
-      message: `${rowCount} rows affected`,
-      time: new Date(),
-      severity: 'info',
-    });
+    if (!skipAffectedMessage) {
+      options.info({
+        message: `${rowCount} rows affected`,
+        time: new Date(),
+        severity: 'info',
+        rowsAffected: rowCount,
+      });
+    }
   });
   request.on('columnMetadata', function (columns) {
     currentColumns = extractTediousColumns(columns);
@@ -231,6 +243,7 @@ async function tediousStream(dbhan, sql, options) {
       currentColumns
     );
     options.row(row);
+    skipAffectedMessage = true;
   });
   dbhan.client.execSqlBatch(request);
 }

plugins/dbgate-plugin-mysql/src/backend/drivers.js

@@ -3,6 +3,7 @@ const stream = require('stream');
 const driverBases = require('../frontend/drivers');
 const Analyser = require('./Analyser');
 const mysql2 = require('mysql2');
+const fs = require('fs');
 const { getLogger, createBulkInsertStreamBase, makeUniqueColumnNames, extractErrorLogData } =
   global.DBGATE_PACKAGES['dbgate-tools'];
 
@@ -14,6 +15,7 @@ function extractColumns(fields) {
   if (fields) {
     const res = fields.map(col => ({
       columnName: col.name,
+      pureName: col.orgTable,
     }));
     makeUniqueColumnNames(res);
     return res;
@@ -62,6 +64,7 @@ const drivers = driverBases.map(driverBase => ({
       supportBigNumbers: true,
       bigNumberStrings: true,
       dateStrings: true,
+      infileStreamFactory: path => fs.createReadStream(path),
       // TODO: test following options
       // multipleStatements: true,
     };
@@ -135,7 +138,11 @@ const drivers = driverBases.map(driverBase => ({
           message: `${row.affectedRows} rows affected`,
           time: new Date(),
           severity: 'info',
+          rowsAffected: row.affectedRows,
         });
+        if (row.stateChanges?.schema) {
+          options.changedCurrentDatabase(row.stateChanges.schema);
+        }
       } else {
         if (columns) {
           options.row(modifyRow(zipDataRow(row, columns), columns));

plugins/dbgate-plugin-oracle/package.json

@@ -7,7 +7,7 @@
   "homepage": "https://dbgate.org",
   "repository": {
     "type": "git",
-    "url": "https://github.com/rinie/dbgate-plugin-oracle"
+    "url": "https://github.com/dbgate/dbgate.git"
   },
   "author": "Rinie Kervel",
   "keywords": [

plugins/dbgate-plugin-oracle/src/backend/driver.js

@@ -232,6 +232,7 @@ const driver = {
               message: `${rowsAffected} rows affected`,
               time: new Date(),
               severity: 'info',
+              rowsAffected,
             });
           }
         }

plugins/dbgate-plugin-postgres/src/backend/drivers.js

@@ -13,6 +13,7 @@ const {
   makeUniqueColumnNames,
   extractDbNameFromComposite,
   extractErrorLogData,
+  getConflictingColumnNames,
 } = global.DBGATE_PACKAGES['dbgate-tools'];
 
 let authProxy;
@@ -63,7 +64,23 @@ function extractPostgresColumns(result, dbhan) {
     columnName: fld.name,
     dataTypeId: fld.dataTypeID,
     dataTypeName: typeIdToName[fld.dataTypeID],
+    tableId: fld.tableID,
   }));
+
+  // const conflictingNames = getConflictingColumnNames(res);
+  // if (conflictingNames.size > 0) {
+  //   const requiredTableIds = res.filter(x => conflictingNames.has(x.columnName)).map(x => x.tableId);
+  //   const tableIdResult = await dbhan.client.query(
+  //     `SELECT DISTINCT c.oid AS table_id, c.relname AS table_name
+  //      FROM pg_class c
+  //      WHERE c.oid IN (${requiredTableIds.join(',')})`
+  //   );
+  //   const tableIdToTableName = _.fromPairs(tableIdResult.rows.map(cur => [cur.table_id, cur.table_name]));
+  //   for (const col of res) {
+  //     col.pureName = tableIdToTableName[col.tableId];
+  //   }
+  // }
+
   makeUniqueColumnNames(res);
   return res;
 }
@@ -225,6 +242,7 @@ const drivers = driverBases.map(driverBase => ({
           message: `${rowCount} rows affected`,
           time: new Date(),
           severity: 'info',
+          rowsAffected: rowCount,
         });
       }
 

plugins/dbgate-plugin-sqlite/src/backend/driver.libsql.js

@@ -74,6 +74,7 @@ const libsqlDriver = {
           message: `${rowCounter.count} rows affected`,
           time: new Date(),
           severity: 'info',
+          rowsAffected: rowCounter.count,
         });
       }
     });

plugins/dbgate-plugin-sqlite/src/backend/driver.sqlite.js

@@ -69,6 +69,7 @@ const driver = {
           message: `${rowCounter.count} rows affected`,
           time: new Date(),
           severity: 'info',
+          rowsAffected: rowCounter.count,
         });
       }
     });

plugins/dbgate-plugin-sqlite/src/backend/helpers.js

@@ -28,6 +28,7 @@ function runStreamItem(dbhan, sql, options, rowCounter, engine) {
         message: `${rowCounter.count} rows affected`,
         time: new Date(),
         severity: 'info',
+        rowsAffected: rowCounter.count,
       });
       rowCounter.count = 0;
       rowCounter.date = null;