SYNC: Merge branch 'feature/dblogs'

packages/api/src/controllers/files.js

@@ -1,7 +1,7 @@
 const fs = require('fs-extra');
 const path = require('path');
 const crypto = require('crypto');
-const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir } = require('../utility/directories');
+const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir, jsldir } = require('../utility/directories');
 const getChartExport = require('../utility/getChartExport');
 const { hasPermission } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
@@ -13,7 +13,7 @@ const dbgateApi = require('../shell');
 const { getLogger } = require('dbgate-tools');
 const platformInfo = require('../utility/platformInfo');
 const { checkSecureFilePathsWithoutDirectory, checkSecureDirectories } = require('../utility/security');
-const { AppLogDatastore, getRecentAppLogRecords } = require('../utility/AppLogDatastore');
+const { copyAppLogsIntoFile, getRecentAppLogRecords } = require('../utility/appLogStore');
 const logger = getLogger('files');
 
 function serialize(format, data) {
@@ -29,9 +29,6 @@ function deserialize(format, text) {
 }
 
 module.exports = {
-  currentLogReader: null,
-  currentLogParamsKey: null,
-
   list_meta: true,
   async list({ folder }, req) {
     if (!hasPermission(`files/${folder}/read`, req)) return [];
@@ -316,19 +313,14 @@ module.exports = {
     return true;
   },
 
-  getAppLog_meta: true,
-  async getAppLog({ offset = 0, limit = 100, dateFrom = 0, dateTo = new Date().getTime(), filters = {} }) {
-    const paramsKey = `${dateFrom}-${dateTo}`;
-    if (paramsKey != this.currentLogParamsKey) {
-      if (this.currentLogReader) {
-        this.currentLogReader._closeReader();
-        this.currentLogReader = null;
-      }
-      this.currentLogReader = new AppLogDatastore({ timeFrom: dateFrom, timeTo: dateTo });
-      this.currentLogParamsKey = paramsKey;
-    }
-
-    return this.currentLogReader.getRows(offset, limit, filters);
+  fillAppLogs_meta: true,
+  async fillAppLogs({ dateFrom = 0, dateTo = new Date().getTime() }) {
+    const jslid = crypto.randomUUID();
+    const outputFile = path.join(jsldir(), `${jslid}.jsonl`);
+    await copyAppLogsIntoFile(dateFrom, dateTo, outputFile);
+    return {
+      jslid,
+    };
   },
 
   getRecentAppLog_meta: true,

packages/api/src/index.js

@@ -33,7 +33,7 @@ if (processArgs.processDisplayName) {
 // }
 
 function configureLogger() {
-  const { initializeRecentLogProvider, pushToRecentLogs } = require('./utility/AppLogDatastore');
+  const { initializeRecentLogProvider, pushToRecentLogs } = require('./utility/appLogStore');
   initializeRecentLogProvider();
 
   const logsFilePath = path.join(logsdir(), `${moment().format('YYYY-MM-DD-HH-mm')}-${process.pid}.ndjson`);

packages/api/src/proc/databaseConnectionProcess.js

@@ -6,7 +6,6 @@ const {
   extractIntSettingsValue,
   getLogger,
   isCompositeDbName,
-  dbNameLogCategory,
   extractErrorMessage,
   extractErrorLogData,
   ScriptWriterEval,
@@ -45,6 +44,14 @@ function getStatusCounter() {
   return statusCounter;
 }
 
+function getLogInfo() {
+  return {
+    database: dbhan ? dbhan.database : undefined,
+    conid: dbhan ? dbhan.conid : undefined,
+    engine: storedConnection ? storedConnection.engine : undefined,
+  };
+}
+
 async function checkedAsyncCall(promise) {
   try {
     const res = await promise;
@@ -131,10 +138,10 @@ async function readVersion() {
   const driver = requireEngineDriver(storedConnection);
   try {
     const version = await driver.getVersion(dbhan);
-    logger.debug(`DBGM-00037 Got server version: ${version.version}`);
+    logger.debug(getLogInfo(), `DBGM-00037 Got server version: ${version.version}`);
     serverVersion = version;
   } catch (err) {
-    logger.error(extractErrorLogData(err), 'DBGM-00149 Error getting DB server version');
+    logger.error(extractErrorLogData(err, getLogInfo()), 'DBGM-00149 Error getting DB server version');
     serverVersion = { version: 'Unknown' };
   }
   process.send({ msgtype: 'version', version: serverVersion });
@@ -148,9 +155,8 @@ async function handleConnect({ connection, structure, globalSettings }) {
   const driver = requireEngineDriver(storedConnection);
   dbhan = await checkedAsyncCall(connectUtility(driver, storedConnection, 'app'));
   logger.debug(
-    `DBGM-00038 Connected to database, driver: ${storedConnection.engine}, separate schemas: ${
-      storedConnection.useSeparateSchemas ? 'YES' : 'NO'
-    }, 'DB: ${dbNameLogCategory(dbhan.database)}`
+    getLogInfo(),
+    `DBGM-00038 Connected to database, separate schemas: ${storedConnection.useSeparateSchemas ? 'YES' : 'NO'}`
   );
   dbhan.feedback = feedback => setStatus({ feedback });
   await checkedAsyncCall(readVersion());
@@ -257,13 +263,16 @@ async function handleDriverDataCore(msgid, callMethod, { logName }) {
     const result = await callMethod(driver);
     process.send({ msgtype: 'response', msgid, result: serializeJsTypesForJsonStringify(result) });
   } catch (err) {
-    logger.error(extractErrorLogData(err, { logName }), `DBGM-00150 Error when handling message ${logName}`);
+    logger.error(
+      extractErrorLogData(err, { logName, ...getLogInfo() }),
+      `DBGM-00150 Error when handling message ${logName}`
+    );
     process.send({ msgtype: 'response', msgid, errorMessage: extractErrorMessage(err, 'Error executing DB data') });
   }
 }
 
 async function handleSchemaList({ msgid }) {
-  logger.debug('DBGM-00039 Loading schema list');
+  logger.debug(getLogInfo(), 'DBGM-00039 Loading schema list');
   return handleDriverDataCore(msgid, driver => driver.listSchemas(dbhan), { logName: 'listSchemas' });
 }
 
@@ -351,7 +360,7 @@ async function handleSqlPreview({ msgid, objects, options }) {
     process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
     if (generator.isUnhandledException) {
       setTimeout(async () => {
-        logger.error('DBGM-00151 Exiting because of unhandled exception');
+        logger.error(getLogInfo(), 'DBGM-00151 Exiting because of unhandled exception');
         await driver.close(dbhan);
         process.exit(0);
       }, 500);
@@ -485,7 +494,7 @@ function start() {
   setInterval(async () => {
     const time = new Date().getTime();
     if (time - lastPing > 40 * 1000) {
-      logger.info('DBGM-00040 Database connection not alive, exiting');
+      logger.info(getLogInfo(), 'DBGM-00040 Database connection not alive, exiting');
       const driver = requireEngineDriver(storedConnection);
       await driver.close(dbhan);
       process.exit(0);
@@ -497,7 +506,7 @@ function start() {
     try {
       await handleMessage(message);
     } catch (err) {
-      logger.error(extractErrorLogData(err), 'DBGM-00041 Error in DB connection');
+      logger.error(extractErrorLogData(err, getLogInfo()), 'DBGM-00041 Error in DB connection');
       process.send({
         msgtype: 'error',
         error: extractErrorMessage(err, 'DBGM-00042 Error processing message'),

packages/api/src/utility/JsonLinesDatastore.js

@@ -12,10 +12,9 @@ const { jsldir } = require('./directories');
 const LineReader = require('./LineReader');
 
 class JsonLinesDatastore {
-  constructor(file, formatterFunction, resolveNextFile = null) {
+  constructor(file, formatterFunction) {
     this.file = file;
     this.formatterFunction = formatterFunction;
-    this.resolveNextFile = resolveNextFile;
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
@@ -23,12 +22,10 @@ class JsonLinesDatastore {
     this.notifyChangedCallback = null;
     this.currentFilter = null;
     this.currentSort = null;
-    this.currentFileName = null;
     if (formatterFunction) {
       const requirePluginFunction = require('./requirePluginFunction');
       this.rowFormatter = requirePluginFunction(formatterFunction);
     }
-
     this.sortedFiles = {};
   }
 
@@ -72,7 +69,6 @@ class JsonLinesDatastore {
     // this.firstRowToBeReturned = null;
     this.currentFilter = null;
     this.currentSort = null;
-    this.currentFileName = null;
     await reader.close();
   }
 
@@ -106,18 +102,8 @@ class JsonLinesDatastore {
     //   return res;
     // }
     for (;;) {
-      let line = await this.reader.readLine();
-      while (!line) {
-        if (!this.currentSort && this.resolveNextFile) {
-          const nextFile = await this.resolveNextFile(this.currentFileName);
-          if (nextFile) {
-            await this.reader.close();
-            this.reader = await this._openReader(nextFile);
-            this.currentFileName = nextFile;
-            line = await this.reader.readLine();
-            continue;
-          }
-        }
+      const line = await this.reader.readLine();
+      if (!line) {
         // EOF
         return null;
       }
@@ -189,7 +175,6 @@ class JsonLinesDatastore {
     }
     if (!this.reader) {
       const reader = await this._openReader(sort ? this.sortedFiles[stableStringify(sort)] : this.file);
-      this.currentFileName = this.file;
       this.reader = reader;
       this.currentFilter = filter;
       this.currentSort = sort;

packages/api/src/utility/appLogStore.js

@@ -17,75 +17,6 @@ async function getLogFiles(timeFrom, timeTo) {
   return logFiles.sort().map(x => path.join(dir, x));
 }
 
-class AppLogDatastore {
-  constructor({ timeFrom, timeTo }) {
-    this.timeFrom = timeFrom;
-    this.timeTo = timeTo;
-  }
-
-  async resolveNextFile(file) {
-    const files = await getLogFiles(this.timeFrom, this.timeTo);
-    const index = files.indexOf(file);
-    if (index < 0 || index >= files.length - 1) return null;
-    return files[index + 1];
-  }
-
-  async getRows(offset = 0, limit = 100, filters = {}) {
-    if (!this.linesReader) {
-      const files = await getLogFiles(this.timeFrom, this.timeTo);
-      this.linesReader = new JsonLinesDatastore(files[0], null, file => this.resolveNextFile(file));
-    }
-
-    const conditions = [
-      {
-        conditionType: 'binary',
-        operator: '>=',
-        left: { exprType: 'column', columnName: 'time' },
-        right: { exprType: 'value', value: this.timeFrom },
-      },
-      {
-        conditionType: 'binary',
-        operator: '<=',
-        left: { exprType: 'column', columnName: 'time' },
-        right: { exprType: 'value', value: this.timeTo },
-      },
-    ];
-    for (const [key, values] of Object.entries(filters)) {
-      if (values.length == 1 && values[0] == null) {
-        // @ts-ignore
-        conditions.push({
-          conditionType: 'isNull',
-          expr: { exprType: 'column', columnName: key },
-        });
-        continue;
-      }
-      // @ts-ignore
-      conditions.push({
-        conditionType: 'in',
-        expr: { exprType: 'column', columnName: key },
-        values,
-      });
-    }
-
-    return this.linesReader.getRows(
-      offset,
-      limit,
-      {
-        conditionType: 'and',
-        conditions,
-      },
-      null
-    );
-  }
-
-  _closeReader() {
-    if (this.linesReader) {
-      this.linesReader._closeReader();
-      this.linesReader = null;
-    }
-  }
-}
-
 const RECENT_LOG_LIMIT = 1000;
 
 let recentLogs = null;
@@ -97,6 +28,27 @@ function adjustRecentLogs() {
   }
 }
 
+async function copyAppLogsIntoFile(timeFrom, timeTo, fileName) {
+  const writeStream = fs.createWriteStream(fileName);
+
+  for (const file of await getLogFiles(timeFrom, timeTo)) {
+    const readStream = fs.createReadStream(file);
+    const reader = new LineReader(readStream);
+    do {
+      const line = await reader.readLine();
+      if (line == null) break;
+      try {
+        const logEntry = JSON.parse(line);
+        if (logEntry.time >= timeFrom && logEntry.time <= timeTo) {
+          writeStream.write(JSON.stringify(logEntry) + '\n');
+        }
+      } catch (e) {
+        continue;
+      }
+    } while (true);
+  }
+}
+
 async function initializeRecentLogProvider() {
   const logs = [];
   for (const file of await getLogFiles(startOfDay(new Date()), new Date())) {
@@ -141,8 +93,8 @@ function getRecentAppLogRecords() {
 }
 
 module.exports = {
-  AppLogDatastore,
   initializeRecentLogProvider,
   getRecentAppLogRecords,
   pushToRecentLogs,
+  copyAppLogsIntoFile,
 };

packages/api/src/utility/connectUtility.js

@@ -132,7 +132,7 @@ async function connectUtility(driver, storedConnection, connectionMode, addition
 
   connection.ssl = await extractConnectionSslParams(connection);
 
-  const conn = await driver.connect({ ...connection, ...additionalOptions });
+  const conn = await driver.connect({ conid: connectionLoaded?._id, ...connection, ...additionalOptions });
   return conn;
 }