better error reporting

packages/api/src/controllers/archive.js

@@ -6,7 +6,7 @@ const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('..
 const socket = require('../utility/socket');
 const loadFilesRecursive = require('../utility/loadFilesRecursive');
 const getJslFileName = require('../utility/getJslFileName');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const dbgateApi = require('../shell');
 const jsldata = require('./jsldata');
 const platformInfo = require('../utility/platformInfo');
@@ -74,7 +74,7 @@ module.exports = {
         ...fileType('.matview.sql', 'matview.sql'),
       ];
     } catch (err) {
-      logger.error({ err }, 'Error reading archive files');
+      logger.error(extractErrorLogData(err), 'Error reading archive files');
       return [];
     }
   },

packages/api/src/controllers/auth.js

@@ -1,7 +1,7 @@
 const axios = require('axios');
 const jwt = require('jsonwebtoken');
 const getExpressPath = require('../utility/getExpressPath');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const AD = require('activedirectory2').promiseWrapper;
 const crypto = require('crypto');
 const { getTokenSecret, getTokenLifetime } = require('../auth/authCommon');
@@ -71,7 +71,7 @@ function authMiddleware(req, res, next) {
       return next();
     }
 
-    logger.error({ err }, 'Sending invalid token error');
+    logger.error(extractErrorLogData(err), 'Sending invalid token error');
 
     return unauthorizedResponse(req, res, 'invalid token');
   }

packages/api/src/controllers/connections.js

@@ -12,7 +12,7 @@ const { pickSafeConnectionInfo } = require('../utility/crypting');
 const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
 
 const processArgs = require('../utility/processArgs');
-const { safeJsonParse, getLogger } = require('dbgate-tools');
+const { safeJsonParse, getLogger, extractErrorLogData } = require('dbgate-tools');
 const platformInfo = require('../utility/platformInfo');
 const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission');
 const pipeForkLogs = require('../utility/pipeForkLogs');
@@ -430,7 +430,7 @@ module.exports = {
       socket.emit('got-volatile-token', { strmid, savedConId: conid, volatileConId: volatile._id });
       return { success: true };
     } catch (err) {
-      logger.error({ err }, 'Error getting DB token');
+      logger.error(extractErrorLogData(err), 'Error getting DB token');
       return { error: err.message };
     }
   },
@@ -446,7 +446,7 @@ module.exports = {
       const resp = await authProvider.login(null, null, { conid: volatile._id });
       return resp;
     } catch (err) {
-      logger.error({ err }, 'Error getting DB token');
+      logger.error(extractErrorLogData(err), 'Error getting DB token');
       return { error: err.message };
     }
   },

packages/api/src/controllers/databaseConnections.js

@@ -12,6 +12,7 @@ const {
   extendDatabaseInfo,
   modelCompareDbDiffOptions,
   getLogger,
+  extractErrorLogData,
 } = require('dbgate-tools');
 const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -146,7 +147,7 @@ module.exports = {
       try {
         conn.subprocess.send({ msgid, ...message });
       } catch (err) {
-        logger.error({ err }, 'Error sending request do process');
+        logger.error(extractErrorLogData(err), 'Error sending request do process');
         this.close(conn.conid, conn.database);
       }
     });
@@ -318,7 +319,7 @@ module.exports = {
       try {
         existing.subprocess.send({ msgtype: 'ping' });
       } catch (err) {
-        logger.error({ err }, 'Error pinging DB connection');
+        logger.error(extractErrorLogData(err), 'Error pinging DB connection');
         this.close(conid, database);
 
         return {
@@ -362,7 +363,7 @@ module.exports = {
         try {
           existing.subprocess.kill();
         } catch (err) {
-          logger.error({ err }, 'Error killing subprocess');
+          logger.error(extractErrorLogData(err), 'Error killing subprocess');
         }
       }
       this.opened = this.opened.filter(x => x.conid != conid || x.database != database);

packages/api/src/controllers/serverConnections.js

@@ -11,7 +11,7 @@ const processArgs = require('../utility/processArgs');
 const { testConnectionPermission } = require('../utility/hasPermission');
 const { MissingCredentialsError } = require('../utility/exceptions');
 const pipeForkLogs = require('../utility/pipeForkLogs');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 
 const logger = getLogger('serverConnection');
 
@@ -112,7 +112,7 @@ module.exports = {
         try {
           existing.subprocess.kill();
         } catch (err) {
-          logger.error({ err }, 'Error killing subprocess');
+          logger.error(extractErrorLogData(err), 'Error killing subprocess');
         }
       }
       this.opened = this.opened.filter(x => x.conid != conid);
@@ -167,7 +167,7 @@ module.exports = {
         try {
           opened.subprocess.send({ msgtype: 'ping' });
         } catch (err) {
-          logger.error({ err }, 'Error pinging server connection');
+          logger.error(extractErrorLogData(err), 'Error pinging server connection');
           this.close(conid);
         }
       })
@@ -217,7 +217,7 @@ module.exports = {
       try {
         conn.subprocess.send({ msgid, ...message });
       } catch (err) {
-        logger.error({ err }, 'Error sending request');
+        logger.error(extractErrorLogData(err), 'Error sending request');
         this.close(conn.conid);
       }
     });

packages/api/src/controllers/sessions.js

@@ -8,7 +8,7 @@ const path = require('path');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
 const { appdir } = require('../utility/directories');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const pipeForkLogs = require('../utility/pipeForkLogs');
 const config = require('./config');
 
@@ -222,7 +222,7 @@ module.exports = {
     try {
       session.subprocess.send({ msgtype: 'ping' });
     } catch (err) {
-      logger.error({ err }, 'Error pinging session');
+      logger.error(extractErrorLogData(err), 'Error pinging session');
 
       return {
         status: 'error',

packages/api/src/controllers/uploads.js

@@ -1,7 +1,7 @@
 const crypto = require('crypto');
 const path = require('path');
 const { uploadsdir, getLogsFilePath } = require('../utility/directories');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const logger = getLogger('uploads');
 const axios = require('axios');
 const os = require('os');
@@ -110,7 +110,7 @@ module.exports = {
 
       return response.data;
     } catch (err) {
-      logger.error({ err }, 'Error uploading gist');
+      logger.error(extractErrorLogData(err), 'Error uploading gist');
 
       return {
         apiErrorMessage: err.message,

packages/api/src/proc/databaseConnectionProcess.js

@@ -7,6 +7,8 @@ const {
   getLogger,
   isCompositeDbName,
   dbNameLogCategory,
+  extractErrorMessage,
+  extractErrorLogData,
 } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
@@ -34,25 +36,6 @@ function getStatusCounter() {
   return statusCounter;
 }
 
-function extractErrorMessage(err, defaultMessage) {
-  if (!err) {
-    return defaultMessage;
-  }
-  if (err.errors) {
-    try {
-      return err.errors.map(x => x.message).join('\n');
-    } catch (e2) {}
-  }
-  if (err.message) {
-    return err.message;
-  }
-  const s = `${err}`;
-  if (s && (!s.endsWith('Error') || s.includes(' '))) {
-    return s;
-  }
-  return defaultMessage;
-}
-
 async function checkedAsyncCall(promise) {
   try {
     const res = await promise;
@@ -261,7 +244,7 @@ async function handleDriverDataCore(msgid, callMethod, { logName }) {
     const result = await callMethod(driver);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
-    logger.error(err, `Error when handling message ${logName}`);
+    logger.error(extractErrorLogData(err, { logName }), `Error when handling message ${logName}`);
     process.send({ msgtype: 'response', msgid, errorMessage: extractErrorMessage(err, 'Error executing DB data') });
   }
 }
@@ -436,7 +419,7 @@ function start() {
     try {
       await handleMessage(message);
     } catch (err) {
-      logger.error({ err }, 'Error in DB connection');
+      logger.error(extractErrorLogData(err), 'Error in DB connection');
       process.send({ msgtype: 'error', error: extractErrorMessage(err, 'Error processing message') });
     }
   });

packages/api/src/proc/sshForwardProcess.js

@@ -3,7 +3,7 @@ const platformInfo = require('../utility/platformInfo');
 const childProcessChecker = require('../utility/childProcessChecker');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SSHConnection } = require('../utility/SSHConnection');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 
 const logger = getLogger('sshProcess');
 
@@ -40,7 +40,7 @@ async function handleStart({ connection, tunnelConfig }) {
       tunnelConfig,
     });
   } catch (err) {
-    logger.error({ err }, 'Error creating SSH tunnel connection:');
+    logger.error(extractErrorLogData(err), 'Error creating SSH tunnel connection:');
 
     process.send({
       msgtype: 'error',

packages/api/src/shell/runScript.js

@@ -1,4 +1,4 @@
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const processArgs = require('../utility/processArgs');
 const logger = getLogger();
@@ -11,7 +11,7 @@ async function runScript(func) {
     await func();
     process.exit(0);
   } catch (err) {
-    logger.error({ err }, `Error running script: ${err.message || err}`);
+    logger.error(extractErrorLogData(err), `Error running script`);
     process.exit(1);
   }
 }

packages/api/src/utility/DatastoreProxy.js

@@ -3,7 +3,7 @@ const { fork } = require('child_process');
 const { handleProcessCommunication } = require('./processComm');
 const processArgs = require('../utility/processArgs');
 const pipeForkLogs = require('./pipeForkLogs');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const logger = getLogger('DatastoreProxy');
 
 class DatastoreProxy {
@@ -73,7 +73,7 @@ class DatastoreProxy {
       try {
         this.subprocess.send({ msgtype: 'read', msgid, offset, limit });
       } catch (err) {
-        logger.error({ err }, 'Error getting rows');
+        logger.error(extractErrorLogData(err), 'Error getting rows');
         this.subprocess = null;
       }
     });
@@ -87,7 +87,7 @@ class DatastoreProxy {
       try {
         this.subprocess.send({ msgtype: 'notify', msgid });
       } catch (err) {
-        logger.error({ err }, 'Error notifying subprocess');
+        logger.error(extractErrorLogData(err), 'Error notifying subprocess');
         this.subprocess = null;
       }
     });

packages/api/src/utility/childProcessChecker.js

@@ -1,4 +1,4 @@
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 
 const logger = getLogger('childProcessChecked');
 
@@ -12,7 +12,7 @@ function childProcessChecker() {
       // This will come once parent dies.
       // One way can be to check for error code ERR_IPC_CHANNEL_CLOSED
       //     and call process.exit()
-      logger.error({ err }, 'parent died');
+      logger.error(extractErrorLogData(err), 'parent died');
       process.exit(1);
     }
   }, 1000);

packages/api/src/utility/sshTunnel.js

@@ -5,7 +5,7 @@ const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
 const { fork } = require('child_process');
 const processArgs = require('../utility/processArgs');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const pipeForkLogs = require('./pipeForkLogs');
 const logger = getLogger('sshTunnel');
 
@@ -40,7 +40,7 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
       tunnelConfig,
     });
   } catch (err) {
-    logger.error({ err }, 'Error connecting SSH');
+    logger.error(extractErrorLogData(err), 'Error connecting SSH');
   }
   return new Promise((resolve, reject) => {
     subprocess.on('message', resp => {

packages/api/src/utility/sshTunnelProxy.js

@@ -1,5 +1,5 @@
 const crypto = require('crypto');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const { getSshTunnel } = require('./sshTunnel');
 const logger = getLogger('sshTunnelProxy');
 
@@ -10,7 +10,7 @@ async function handleGetSshTunnelRequest({ msgid, connection }, subprocess) {
   try {
     subprocess.send({ msgtype: 'getsshtunnel-response', msgid, response });
   } catch (err) {
-    logger.error({ err }, 'Error sending to SSH tunnel');
+    logger.error(extractErrorLogData(err), 'Error sending to SSH tunnel');
   }
 }
 

packages/api/src/utility/useController.js

@@ -2,7 +2,7 @@ const _ = require('lodash');
 const express = require('express');
 const getExpressPath = require('./getExpressPath');
 const { MissingCredentialsError } = require('./exceptions');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 
 const logger = getLogger('useController');
 /**
@@ -16,7 +16,7 @@ module.exports = function useController(app, electron, route, controller) {
     try {
       controller._init();
     } catch (err) {
-      logger.error({ err }, `Error initializing controller, exiting application`);
+      logger.error(extractErrorLogData(err), `Error initializing controller, exiting application`);
       process.exit(1);
     }
   }
@@ -78,7 +78,7 @@ module.exports = function useController(app, electron, route, controller) {
           const data = await controller[key]({ ...req.body, ...req.query }, req);
           res.json(data);
         } catch (err) {
-          logger.error({ err }, `Error when processing route ${route}/${key}`);
+          logger.error(extractErrorLogData(err), `Error when processing route ${route}/${key}`);
           if (err instanceof MissingCredentialsError) {
             res.json({
               missingCredentials: true,

packages/datalib/src/DataDuplicator.ts

@@ -1,4 +1,10 @@
-import { createAsyncWriteStream, getLogger, runCommandOnDriver, runQueryOnDriver } from 'dbgate-tools';
+import {
+  createAsyncWriteStream,
+  extractErrorLogData,
+  getLogger,
+  runCommandOnDriver,
+  runQueryOnDriver,
+} from 'dbgate-tools';
 import { DatabaseInfo, EngineDriver, ForeignKeyInfo, TableInfo } from 'dbgate-types';
 import _pick from 'lodash/pick';
 import _omit from 'lodash/omit';
@@ -255,7 +261,7 @@ export class DataDuplicator {
         );
       }
     } catch (err) {
-      logger.error({ err }, `Failed duplicator job, rollbacking. ${err.message}`);
+      logger.error(extractErrorLogData(err), `Failed duplicator job, rollbacking. ${err.message}`);
       await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction());
       return;
     }

packages/dbmodel/bin/dbmodel.js

@@ -13,6 +13,7 @@ global.IS_DB_MODEL = true;
 const program = require('commander');
 const dbgateApi = require('dbgate-api');
 const { createLogger } = require('pinomin');
+const { extractErrorLogData } = require('dbgate-tools');
 
 const logger = createLogger('dbmodel');
 
@@ -22,7 +23,7 @@ async function runAndExit(promise) {
     logger.info('Success');
     process.exit();
   } catch (err) {
-    logger.error({ err }, 'Processing failed');
+    logger.error(extractErrorLogData(err), 'Processing failed');
     process.exit(1);
   }
 }

packages/tools/src/DatabaseAnalyser.ts

@@ -6,6 +6,7 @@ import _compact from 'lodash/compact';
 import { getLogger } from './getLogger';
 import { type Logger } from 'pinomin';
 import { dbNameLogCategory, isCompositeDbName, splitCompositeDbName } from './schemaInfoTools';
+import { extractErrorLogData } from './stringTools';
 
 const logger = getLogger('dbAnalyser');
 
@@ -90,7 +91,9 @@ export class DatabaseAnalyser {
   }
 
   async incrementalAnalysis(structure) {
-    logger.info(`Performing incremental analysis, DB=${dbNameLogCategory(this.dbhan.database)}, engine=${this.driver.engine}`);
+    logger.info(
+      `Performing incremental analysis, DB=${dbNameLogCategory(this.dbhan.database)}, engine=${this.driver.engine}`
+    );
     this.structure = structure;
 
     const modifications = await this.getModifications();
@@ -336,7 +339,7 @@ export class DatabaseAnalyser {
       this.logger.debug({ rows: res.rows.length, template }, `Loaded analyser query`);
       return res;
     } catch (err) {
-      logger.error({ err, message: err.message, template }, 'Error running analyser query');
+      logger.error(extractErrorLogData(err, { template }), 'Error running analyser query');
       return {
         rows: [],
       };

packages/tools/src/SqlGenerator.ts

@@ -12,6 +12,7 @@ import _uniqBy from 'lodash/uniqBy';
 import { getLogger } from './getLogger';
 import { SqlDumper } from './SqlDumper';
 import { extendDatabaseInfo } from './structureTools';
+import { extractErrorLogData } from './stringTools';
 
 const logger = getLogger('sqlGenerator');
 
@@ -85,7 +86,7 @@ export class SqlGenerator {
   }
 
   private handleException = error => {
-    logger.error({ error }, 'Unhandled error');
+    logger.error(extractErrorLogData(error), 'Unhandled error');
     this.isUnhandledException = true;
   };
 

packages/tools/src/stringTools.ts

@@ -19,7 +19,8 @@ export type EditorDataType =
 const dateTimeStorageRegex =
   /^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|()|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$/;
 
-const dateTimeParseRegex = /^(\d{4})-(\d{2})-(\d{2})[Tt ](\d{2}):(\d{2}):(\d{2})(\.[0-9]+)?(([Zz])|()|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$/;
+const dateTimeParseRegex =
+  /^(\d{4})-(\d{2})-(\d{2})[Tt ](\d{2}):(\d{2}):(\d{2})(\.[0-9]+)?(([Zz])|()|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$/;
 
 export function arrayToHexString(byteArray) {
   return byteArray.reduce((output, elem) => output + ('0' + elem.toString(16)).slice(-2), '').toUpperCase();
@@ -463,3 +464,31 @@ export function getConvertValueMenu(value, onSetValue, editorTypes?: DataEditorT
     },
   ];
 }
+
+export function extractErrorMessage(err, defaultMessage = 'Unknown error') {
+  if (!err) {
+    return defaultMessage;
+  }
+  if (_isArray(err.errors)) {
+    try {
+      return err.errors.map(x => x.message).join('\n');
+    } catch (e2) {}
+  }
+  if (err.message) {
+    return err.message;
+  }
+  const s = `${err}`;
+  if (s && (!s.endsWith('Error') || s.includes(' '))) {
+    return s;
+  }
+  return defaultMessage;
+}
+
+export function extractErrorLogData(err, additionalFields = {}) {
+  if (!err) return null;
+  return {
+    errorMessage: extractErrorMessage(err),
+    errorObject: err,
+    ...additionalFields,
+  };
+}

plugins/dbgate-plugin-mysql/src/backend/drivers.js

@@ -3,7 +3,8 @@ const stream = require('stream');
 const driverBases = require('../frontend/drivers');
 const Analyser = require('./Analyser');
 const mysql2 = require('mysql2');
-const { getLogger, createBulkInsertStreamBase, makeUniqueColumnNames } = global.DBGATE_PACKAGES['dbgate-tools'];
+const { getLogger, createBulkInsertStreamBase, makeUniqueColumnNames, extractErrorLogData } =
+  global.DBGATE_PACKAGES['dbgate-tools'];
 const { MySqlDumper } = require('antares-mysql-dumper');
 
 const logger = getLogger('mysqlDriver');
@@ -128,7 +129,7 @@ const drivers = driverBases.map(driverBase => ({
     };
 
     const handleError = error => {
-      logger.error({ error }, 'Stream error');
+      logger.error(extractErrorLogData(error), 'Stream error');
       const { message } = error;
       options.info({
         message,

plugins/dbgate-plugin-postgres/src/backend/drivers.js

@@ -5,8 +5,13 @@ const driverBases = require('../frontend/drivers');
 const Analyser = require('./Analyser');
 const pg = require('pg');
 const pgCopyStreams = require('pg-copy-streams');
-const { getLogger, createBulkInsertStreamBase, makeUniqueColumnNames, extractDbNameFromComposite } =
-  global.DBGATE_PACKAGES['dbgate-tools'];
+const {
+  getLogger,
+  createBulkInsertStreamBase,
+  makeUniqueColumnNames,
+  extractDbNameFromComposite,
+  extractErrorLogData,
+} = global.DBGATE_PACKAGES['dbgate-tools'];
 
 const logger = getLogger('postreDriver');
 
@@ -155,7 +160,7 @@ const drivers = driverBases.map(driverBase => ({
     });
 
     query.on('error', error => {
-      logger.error({ error }, 'Stream error');
+      logger.error(extractErrorLogData(error), 'Stream error');
       const { message, position, procName } = error;
       let line = null;
       if (position) {

plugins/dbgate-plugin-sqlite/src/backend/driver.js

@@ -3,7 +3,7 @@ const stream = require('stream');
 const driverBase = require('../frontend/driver');
 const Analyser = require('./Analyser');
 const { splitQuery, sqliteSplitterOptions } = require('dbgate-query-splitter');
-const { getLogger, createBulkInsertStreamBase, makeUniqueColumnNames } = global.DBGATE_PACKAGES['dbgate-tools'];
+const { getLogger, createBulkInsertStreamBase, extractErrorLogData } = global.DBGATE_PACKAGES['dbgate-tools'];
 
 const logger = getLogger('sqliteDriver');
 
@@ -116,7 +116,7 @@ const driver = {
     try {
       inTransaction();
     } catch (error) {
-      logger.error({ error }, 'Stream error');
+      logger.error(extractErrorLogData(error), 'Stream error');
       const { message, procName } = error;
       options.info({
         message,