diff --git a/app/package.json b/app/package.json index 3b3e266c9..4ed844dce 100644 --- a/app/package.json +++ b/app/package.json @@ -6,7 +6,9 @@ "description": "Opensource database administration tool", "dependencies": { "electron-log": "^4.4.1", + "electron-unhandled": "^4.0.1", "electron-updater": "^4.6.1", + "electron-util": "^0.17.2", "lodash.clonedeepwith": "^4.5.0", "patch-package": "^6.4.7" }, @@ -113,7 +115,7 @@ }, "optionalDependencies": { "better-sqlite3": "7.6.2", - "oracledb": "^5.5.0", - "msnodesqlv8": "^2.6.0" + "msnodesqlv8": "^2.6.0", + "oracledb": "^5.5.0" } } diff --git a/app/src/electron.js b/app/src/electron.js index b03f05378..b5e702a44 100644 --- a/app/src/electron.js +++ b/app/src/electron.js @@ -1,6 +1,8 @@ const electron = require('electron'); const os = require('os'); const fs = require('fs'); +const unhandled = require('electron-unhandled'); +const { openNewGitHubIssue, debugInfo } = require('electron-util'); const { Menu, ipcMain } = require('electron'); const { autoUpdater } = require('electron-updater'); const log = require('electron-log'); @@ -22,9 +24,25 @@ const configRootPath = path.join(app.getPath('userData'), 'config-root.json'); let initialConfig = {}; let apiLoaded = false; let mainModule; +let getLogger; +let loadLogsContent; const isMac = () => os.platform() == 'darwin'; +unhandled({ + showDialog: true, + reportButton: error => { + openNewGitHubIssue({ + user: 'dbgate', + repo: 'dbgate', + body: `PLEASE DELETE SENSITIVE INFO BEFORE POSTING ISSUE!!!\n\n\`\`\`\n${ + error.stack + }\n\`\`\`\n\n---\n\n${debugInfo()}\n\n\`\`\`\n${loadLogsContent ? loadLogsContent(50) : ''}\n\`\`\``, + }); + }, + logger: error => (getLogger ? getLogger().fatal(error) : console.error(error)), +}); + try { initialConfig = JSON.parse(fs.readFileSync(configRootPath, { encoding: 'utf-8' })); } catch (err) { @@ -333,9 +351,12 @@ function createWindow() { // path.join(__dirname, process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js') // ) // ); + api.configureLogger(); const main = api.getMainModule(); main.useAllControllers(null, electron); mainModule = main; + getLogger = api.getLogger; + loadLogsContent = api.loadLogsContent; apiLoaded = true; } mainModule.setElectronSender(mainWindow.webContents); diff --git a/app/src/mainMenuDefinition.js b/app/src/mainMenuDefinition.js index 04d846c61..7efa484ec 100644 --- a/app/src/mainMenuDefinition.js +++ b/app/src/mainMenuDefinition.js @@ -86,6 +86,9 @@ module.exports = ({ editMenu }) => [ { command: 'sql.generator', hideDisabled: true }, { command: 'file.import', hideDisabled: true }, { command: 'new.modelCompare', hideDisabled: true }, + { divider: true }, + { command: 'folder.showLogs', hideDisabled: true }, + { command: 'folder.showData', hideDisabled: true }, ], }, { diff --git a/app/yarn.lock b/app/yarn.lock index 2d262e719..0d2d1871c 100644 --- a/app/yarn.lock +++ b/app/yarn.lock @@ -547,6 +547,11 @@ ci-info@^3.2.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.3.2.tgz#6d2967ffa407466481c6c90b6e16b3098f080128" integrity sha512-xmDt/QIAdeZ9+nfdPsaBCpMvHNLFiLdjj59qjqn+6iPe6YmHGQ35sBnQ8uslRBXFmXkiZQOJRjvQeoGppoTjjg== +clean-stack@^2.1.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" + integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== + cli-boxes@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.1.tgz#ddd5035d25094fce220e9cab40a45840a440318f" @@ -887,6 +892,16 @@ electron-builder@23.1.0: update-notifier "^5.1.0" yargs "^17.0.1" +electron-is-dev@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/electron-is-dev/-/electron-is-dev-1.2.0.tgz#2e5cea0a1b3ccf1c86f577cee77363ef55deb05e" + integrity sha512-R1oD5gMBPS7PVU8gJwH6CtT0e6VSoD0+SzSnYpNm+dBkcijgA+K7VAMHDfnRq/lkKPZArpzplTW6jfiMYosdzw== + +electron-is-dev@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/electron-is-dev/-/electron-is-dev-2.0.0.tgz#833487a069b8dad21425c67a19847d9064ab19bd" + integrity sha512-3X99K852Yoqu9AcW50qz3ibYBWY79/pBhlMCab8ToEWS48R0T9tyxRiQhwylE7zQdXrMnx2JKqUJyMPmt5FBqA== + electron-log@^4.4.1: version "4.4.8" resolved "https://registry.yarnpkg.com/electron-log/-/electron-log-4.4.8.tgz#fcb9f714dbcaefb6ac7984c4683912c74730248a" @@ -925,6 +940,17 @@ electron-publish@23.0.9: lazy-val "^1.0.5" mime "^2.5.2" +electron-unhandled@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/electron-unhandled/-/electron-unhandled-4.0.1.tgz#82bbcb5ee5423f611f6122e006bf4842dfd7c908" + integrity sha512-6BsLnBg+i96eUnbaIFZyYdyfNX3f80/Nlfqy34YEMxXT9JP3ddNsNnUeiOF8ezN4+et4t4D37gjghKTP0V3jyw== + dependencies: + clean-stack "^2.1.0" + electron-is-dev "^2.0.0" + ensure-error "^2.0.0" + lodash.debounce "^4.0.8" + serialize-error "^8.1.0" + electron-updater@^4.6.1: version "4.6.5" resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-4.6.5.tgz#e9a75458bbfd6bb41a58a829839e150ad2eb2d3d" @@ -939,6 +965,14 @@ electron-updater@^4.6.1: lodash.isequal "^4.5.0" semver "^7.3.5" +electron-util@^0.17.2: + version "0.17.2" + resolved "https://registry.yarnpkg.com/electron-util/-/electron-util-0.17.2.tgz#6b0fe798ae0154585e7e0e96a707bfeae592be05" + integrity sha512-4Kg/aZxJ2BZklgyfH86px/D2GyROPyIcnAZar+7KiNmKI2I5l09pwQTP7V95zM3FVhgDQwV9iuJta5dyEvuWAw== + dependencies: + electron-is-dev "^1.1.0" + new-github-issue-url "^0.2.1" + electron@17.4.10: version "17.4.10" resolved "https://registry.yarnpkg.com/electron/-/electron-17.4.10.tgz#904247a9eed0b09825f4d9f221f2442c7679f9e6" @@ -965,6 +999,11 @@ end-of-stream@^1.1.0, end-of-stream@^1.4.1: dependencies: once "^1.4.0" +ensure-error@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/ensure-error/-/ensure-error-2.1.0.tgz#f11fbe383c0cf4a54850ac77acceb7bc06e0f99d" + integrity sha512-+BMSJHw9gxiJAAp2ZR1E0TNcL09dD3lOvkl7WVm4+Y6xnes/pMetP/TzCHiDduh8ihNDjbGfuYxl7l4PA1xZ8A== + env-paths@^2.2.0: version "2.2.1" resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.1.tgz#420399d416ce1fbe9bc0a07c62fa68d67fd0f8f2" @@ -1597,6 +1636,11 @@ lodash.clonedeepwith@^4.5.0: resolved "https://registry.yarnpkg.com/lodash.clonedeepwith/-/lodash.clonedeepwith-4.5.0.tgz#6ee30573a03a1a60d670a62ef33c10cf1afdbdd4" integrity sha512-QRBRSxhbtsX1nc0baxSkkK5WlVTTm/s48DSukcGcWZwIyI8Zz+lB+kFiELJXtzfH4Aj6kMWQ1VWW4U5uUDgZMA== +lodash.debounce@^4.0.8: + version "4.0.8" + resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" + integrity sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow== + lodash.escaperegexp@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz#64762c48618082518ac3df4ccf5d5886dae20347" @@ -1765,6 +1809,11 @@ napi-build-utils@^1.0.1: resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806" integrity sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg== +new-github-issue-url@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/new-github-issue-url/-/new-github-issue-url-0.2.1.tgz#e17be1f665a92de465926603e44b9f8685630c1d" + integrity sha512-md4cGoxuT4T4d/HDOXbrUHkTKrp/vp+m3aOA7XXVYwNsUNMK49g3SQicTSeV5GIz/5QVGAeYRAOlyp9OvlgsYA== + nice-try@^1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" @@ -2208,6 +2257,13 @@ serialize-error@^7.0.1: dependencies: type-fest "^0.13.1" +serialize-error@^8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/serialize-error/-/serialize-error-8.1.0.tgz#3a069970c712f78634942ddd50fbbc0eaebe2f67" + integrity sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ== + dependencies: + type-fest "^0.20.2" + shebang-command@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" diff --git a/package.json b/package.json index 42265b9d8..0eaf3cf3b 100644 --- a/package.json +++ b/package.json @@ -8,16 +8,17 @@ "integration-tests" ], "scripts": { - "start:api": "yarn workspace dbgate-api start", - "start:app": "cd app && yarn start", + "start:api": "yarn workspace dbgate-api start | pino-pretty", + "start:api:json": "yarn workspace dbgate-api start", + "start:app": "cd app && yarn start | pino-pretty", "start:api:debug": "cross-env DEBUG=* yarn workspace dbgate-api start", "start:app:debug": "cd app && cross-env DEBUG=* yarn start", "start:api:debug:ssh": "cross-env DEBUG=ssh yarn workspace dbgate-api start", "start:app:debug:ssh": "cd app && cross-env DEBUG=ssh yarn start", - "start:api:portal": "yarn workspace dbgate-api start:portal", - "start:api:singledb": "yarn workspace dbgate-api start:singledb", - "start:api:auth": "yarn workspace dbgate-api start:auth", - "start:api:dblogin": "yarn workspace dbgate-api start:dblogin", + "start:api:portal": "yarn workspace dbgate-api start:portal | pino-pretty", + "start:api:singledb": "yarn workspace dbgate-api start:singledb | pino-pretty", + "start:api:auth": "yarn workspace dbgate-api start:auth | pino-pretty", + "start:api:dblogin": "yarn workspace dbgate-api start:dblogin | pino-pretty", "start:web": "yarn workspace dbgate-web dev", "start:sqltree": "yarn workspace dbgate-sqltree start", "start:tools": "yarn workspace dbgate-tools start", @@ -57,7 +58,8 @@ }, "dependencies": { "concurrently": "^5.1.0", - "patch-package": "^6.2.1" + "patch-package": "^6.2.1", + "pino-pretty": "^9.1.1" }, "devDependencies": { "copyfiles": "^2.2.0", diff --git a/packages/api/package.json b/packages/api/package.json index 64caf1d12..1de46fe97 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -49,6 +49,8 @@ "ncp": "^2.0.0", "node-cron": "^2.0.3", "on-finished": "^2.4.1", + "pino": "^8.8.0", + "pino-multi-stream": "^6.0.0", "portfinder": "^1.0.28", "simple-encryptor": "^4.0.0", "ssh2": "^1.11.0", @@ -79,7 +81,7 @@ }, "optionalDependencies": { "better-sqlite3": "7.6.2", - "oracledb": "^5.5.0", - "msnodesqlv8": "^2.6.0" + "msnodesqlv8": "^2.6.0", + "oracledb": "^5.5.0" } } diff --git a/packages/api/src/controllers/archive.js b/packages/api/src/controllers/archive.js index 50bb96fa6..fbde95256 100644 --- a/packages/api/src/controllers/archive.js +++ b/packages/api/src/controllers/archive.js @@ -6,6 +6,9 @@ const socket = require('../utility/socket'); const { saveFreeTableData } = require('../utility/freeTableStorage'); const loadFilesRecursive = require('../utility/loadFilesRecursive'); const getJslFileName = require('../utility/getJslFileName'); +const { getLogger } = require('dbgate-tools'); + +const logger = getLogger('archive'); module.exports = { folders_meta: true, @@ -68,7 +71,7 @@ module.exports = { ...fileType('.matview.sql', 'matview.sql'), ]; } catch (err) { - console.log('Error reading archive files', err.message); + logger.error({ err }, 'Error reading archive files'); return []; } }, diff --git a/packages/api/src/controllers/auth.js b/packages/api/src/controllers/auth.js index 167029b4f..74c5ad450 100644 --- a/packages/api/src/controllers/auth.js +++ b/packages/api/src/controllers/auth.js @@ -3,8 +3,11 @@ const jwt = require('jsonwebtoken'); const getExpressPath = require('../utility/getExpressPath'); const uuidv1 = require('uuid/v1'); const { getLogins } = require('../utility/hasPermission'); +const { getLogger } = require('dbgate-tools'); const AD = require('activedirectory2').promiseWrapper; +const logger = getLogger('auth'); + const tokenSecret = uuidv1(); function shouldAuthorizeApi() { @@ -51,7 +54,7 @@ function authMiddleware(req, res, next) { return next(); } - console.log('Sending invalid token error', err.message); + logger.error({ err }, 'Sending invalid token error'); return unauthorizedResponse(req, res, 'invalid token'); } @@ -74,7 +77,7 @@ module.exports = { const payload = jwt.decode(access_token); - console.log('User payload returned from OAUTH:', payload); + logger.info({ payload }, 'User payload returned from OAUTH'); const login = process.env.OAUTH_LOGIN_FIELD && payload && payload[process.env.OAUTH_LOGIN_FIELD] @@ -122,7 +125,7 @@ module.exports = { accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }), }; } catch (err) { - console.log('Failed active directory authentization', err.message); + logger.error({ err }, 'Failed active directory authentization'); return { error: err.message, }; diff --git a/packages/api/src/controllers/config.js b/packages/api/src/controllers/config.js index 5f20c02e8..c19c31f83 100644 --- a/packages/api/src/controllers/config.js +++ b/packages/api/src/controllers/config.js @@ -2,7 +2,7 @@ const fs = require('fs-extra'); const os = require('os'); const path = require('path'); const axios = require('axios'); -const { datadir } = require('../utility/directories'); +const { datadir, getLogsFilePath } = require('../utility/directories'); const { hasPermission, getLogins } = require('../utility/hasPermission'); const socket = require('../utility/socket'); const _ = require('lodash'); @@ -48,6 +48,8 @@ module.exports = { oauthScope: process.env.OAUTH_SCOPE, oauthLogout: process.env.OAUTH_LOGOUT, isLoginForm: !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH), + logsFilePath: getLogsFilePath(), + connectionsFilePath: path.join(datadir(), 'connections.jsonl'), ...currentVersion, }; }, diff --git a/packages/api/src/controllers/connections.js b/packages/api/src/controllers/connections.js index 58504e87d..4fc612012 100644 --- a/packages/api/src/controllers/connections.js +++ b/packages/api/src/controllers/connections.js @@ -12,9 +12,12 @@ const { pickSafeConnectionInfo } = require('../utility/crypting'); const JsonLinesDatabase = require('../utility/JsonLinesDatabase'); const processArgs = require('../utility/processArgs'); -const { safeJsonParse } = require('dbgate-tools'); +const { safeJsonParse, getLogger } = require('dbgate-tools'); const platformInfo = require('../utility/platformInfo'); const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission'); +const pipeForkLogs = require('../utility/pipeForkLogs'); + +const logger = getLogger('connections'); let volatileConnections = {}; @@ -86,13 +89,13 @@ function getPortalCollections() { sslKeyFile: process.env[`SSL_KEY_FILE_${id}`], sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`], })); - console.log('Using connections from ENV variables:'); - console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2)); + + logger.info({ connections: connections.map(pickSafeConnectionInfo) }, 'Using connections from ENV variables'); const noengine = connections.filter(x => !x.engine); if (noengine.length > 0) { - console.log( - 'Warning: Invalid CONNECTIONS configutation, missing ENGINE for connection ID:', - noengine.map(x => x._id) + logger.warn( + { connections: noengine.map(x => x._id) }, + 'Invalid CONNECTIONS configutation, missing ENGINE for connection ID' ); } return connections; @@ -203,13 +206,20 @@ module.exports = { test_meta: true, test(connection) { - const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [ - '--is-forked-api', - '--start-process', - 'connectProcess', - ...processArgs.getPassArgs(), - // ...process.argv.slice(3), - ]); + const subprocess = fork( + global['API_PACKAGE'] || process.argv[1], + [ + '--is-forked-api', + '--start-process', + 'connectProcess', + ...processArgs.getPassArgs(), + // ...process.argv.slice(3), + ], + { + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + } + ); + pipeForkLogs(subprocess); subprocess.send(connection); return new Promise(resolve => { subprocess.on('message', resp => { diff --git a/packages/api/src/controllers/databaseConnections.js b/packages/api/src/controllers/databaseConnections.js index 3601b4734..c9e0caa91 100644 --- a/packages/api/src/controllers/databaseConnections.js +++ b/packages/api/src/controllers/databaseConnections.js @@ -12,6 +12,7 @@ const { matchPairedObjects, extendDatabaseInfo, modelCompareDbDiffOptions, + getLogger, } = require('dbgate-tools'); const { html, parse } = require('diff2html'); const { handleProcessCommunication } = require('../utility/processComm'); @@ -28,6 +29,9 @@ const diff2htmlPage = require('../utility/diff2htmlPage'); const processArgs = require('../utility/processArgs'); const { testConnectionPermission } = require('../utility/hasPermission'); const { MissingCredentialsError } = require('../utility/exceptions'); +const pipeForkLogs = require('../utility/pipeForkLogs'); + +const logger = getLogger('databaseConnections'); module.exports = { /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */ @@ -60,7 +64,7 @@ module.exports = { handle_error(conid, database, props) { const { error } = props; - console.log(`Error in database connection ${conid}, database ${database}: ${error}`); + logger.error(`Error in database connection ${conid}, database ${database}: ${error}`); }, handle_response(conid, database, { msgid, ...response }) { const [resolve, reject] = this.requests[msgid]; @@ -85,13 +89,20 @@ module.exports = { if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') { throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode }); } - const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [ - '--is-forked-api', - '--start-process', - 'databaseConnectionProcess', - ...processArgs.getPassArgs(), - // ...process.argv.slice(3), - ]); + const subprocess = fork( + global['API_PACKAGE'] || process.argv[1], + [ + '--is-forked-api', + '--start-process', + 'databaseConnectionProcess', + ...processArgs.getPassArgs(), + // ...process.argv.slice(3), + ], + { + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + } + ); + pipeForkLogs(subprocess); const lastClosed = this.closed[`${conid}/${database}`]; const newOpened = { conid, @@ -137,7 +148,7 @@ module.exports = { queryData_meta: true, async queryData({ conid, database, sql }, req) { testConnectionPermission(conid, req); - console.log(`Processing query, conid=${conid}, database=${database}, sql=${sql}`); + logger.info({ conid, database, sql }, 'Processing query'); const opened = await this.ensureOpened(conid, database); // if (opened && opened.status && opened.status.name == 'error') { // return opened.status; @@ -157,7 +168,7 @@ module.exports = { runScript_meta: true, async runScript({ conid, database, sql }, req) { testConnectionPermission(conid, req); - console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`); + logger.info({ conid, database, sql }, 'Processing script'); const opened = await this.ensureOpened(conid, database); const res = await this.sendRequest(opened, { msgtype: 'runScript', sql }); return res; diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js index 06f72b5f4..393c64f84 100644 --- a/packages/api/src/controllers/runners.js +++ b/packages/api/src/controllers/runners.js @@ -6,10 +6,17 @@ const byline = require('byline'); const socket = require('../utility/socket'); const { fork } = require('child_process'); const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories'); -const { extractShellApiPlugins, extractShellApiFunctionName, jsonScriptToJavascript } = require('dbgate-tools'); +const { + extractShellApiPlugins, + extractShellApiFunctionName, + jsonScriptToJavascript, + getLogger, + safeJsonParse, +} = require('dbgate-tools'); const { handleProcessCommunication } = require('../utility/processComm'); const processArgs = require('../utility/processArgs'); const platformInfo = require('../utility/platformInfo'); +const logger = getLogger('runners'); function extractPlugins(script) { const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g; @@ -29,13 +36,14 @@ const requirePluginsTemplate = (plugins, isExport) => const scriptTemplate = (script, isExport) => ` const dbgateApi = require(${isExport ? `'dbgate-api'` : 'process.env.DBGATE_API'}); +const logger = dbgateApi.getLogger('script'); dbgateApi.initializeApiEnvironment(); ${requirePluginsTemplate(extractPlugins(script), isExport)} require=null; async function run() { ${script} await dbgateApi.finalizer.run(); -console.log('Finished job script'); +logger.info('Finished job script'); } dbgateApi.runScript(run); `; @@ -59,19 +67,17 @@ module.exports = { requests: {}, dispatchMessage(runid, message) { - if (message) console.log('...', message.message); - if (_.isString(message)) { - socket.emit(`runner-info-${runid}`, { - message, - time: new Date(), - severity: 'info', - }); - } - if (_.isPlainObject(message)) { + if (message) { + const json = safeJsonParse(message.message); + + if (json) logger.info(json); + else logger.info(message.message); + socket.emit(`runner-info-${runid}`, { time: new Date(), severity: 'info', ...message, + message: json ? json.msg : message.message, }); } }, @@ -98,13 +104,15 @@ module.exports = { fs.writeFileSync(`${scriptFile}`, scriptText); fs.mkdirSync(directory); const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList); - console.log(`RUNNING SCRIPT ${scriptFile}`); + logger.info({ scriptFile }, 'Running script'); // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], { const subprocess = fork( scriptFile, [ '--checkParent', // ...process.argv.slice(3) '--is-forked-api', + '--process-display-name', + 'script', ...processArgs.getPassArgs(), ], { @@ -124,7 +132,7 @@ module.exports = { byline(subprocess.stderr).on('data', pipeDispatcher('error')); subprocess.on('exit', code => { this.rejectRequest(runid, { message: 'No data retured, maybe input data source is too big' }); - console.log('... EXIT process', code); + logger.info({ code, pid: subprocess.pid }, 'Exited process'); socket.emit(`runner-done-${runid}`, code); }); subprocess.on('error', error => { diff --git a/packages/api/src/controllers/scheduler.js b/packages/api/src/controllers/scheduler.js index b3ff47098..d6a7ffd3c 100644 --- a/packages/api/src/controllers/scheduler.js +++ b/packages/api/src/controllers/scheduler.js @@ -4,6 +4,9 @@ const path = require('path'); const cron = require('node-cron'); const runners = require('./runners'); const { hasPermission } = require('../utility/hasPermission'); +const { getLogger } = require('dbgate-tools'); + +const logger = getLogger('scheduler'); const scheduleRegex = /\s*\/\/\s*@schedule\s+([^\n]+)\n/; @@ -21,7 +24,7 @@ module.exports = { if (!match) return; const pattern = match[1]; if (!cron.validate(pattern)) return; - console.log(`Schedule script ${file} with pattern ${pattern}`); + logger.info(`Schedule script ${file} with pattern ${pattern}`); const task = cron.schedule(pattern, () => runners.start({ script: text })); this.tasks.push(task); }, diff --git a/packages/api/src/controllers/serverConnections.js b/packages/api/src/controllers/serverConnections.js index 5417a2ce4..dd1c9104d 100644 --- a/packages/api/src/controllers/serverConnections.js +++ b/packages/api/src/controllers/serverConnections.js @@ -10,6 +10,7 @@ const config = require('./config'); const processArgs = require('../utility/processArgs'); const { testConnectionPermission } = require('../utility/hasPermission'); const { MissingCredentialsError } = require('../utility/exceptions'); +const pipeForkLogs = require('../utility/pipeForkLogs'); module.exports = { opened: [], @@ -53,13 +54,20 @@ module.exports = { if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') { throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode }); } - const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [ - '--is-forked-api', - '--start-process', - 'serverConnectionProcess', - ...processArgs.getPassArgs(), - // ...process.argv.slice(3), - ]); + const subprocess = fork( + global['API_PACKAGE'] || process.argv[1], + [ + '--is-forked-api', + '--start-process', + 'serverConnectionProcess', + ...processArgs.getPassArgs(), + // ...process.argv.slice(3), + ], + { + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + } + ); + pipeForkLogs(subprocess); const newOpened = { conid, subprocess, diff --git a/packages/api/src/controllers/sessions.js b/packages/api/src/controllers/sessions.js index a010dba4e..b4db80ada 100644 --- a/packages/api/src/controllers/sessions.js +++ b/packages/api/src/controllers/sessions.js @@ -8,6 +8,10 @@ const path = require('path'); const { handleProcessCommunication } = require('../utility/processComm'); const processArgs = require('../utility/processArgs'); const { appdir } = require('../utility/directories'); +const { getLogger } = require('dbgate-tools'); +const pipeForkLogs = require('../utility/pipeForkLogs'); + +const logger = getLogger('sessions'); module.exports = { /** @type {import('dbgate-types').OpenedSession[]} */ @@ -82,13 +86,20 @@ module.exports = { async create({ conid, database }) { const sesid = uuidv1(); const connection = await connections.getCore({ conid }); - const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [ - '--is-forked-api', - '--start-process', - 'sessionProcess', - ...processArgs.getPassArgs(), - // ...process.argv.slice(3), - ]); + const subprocess = fork( + global['API_PACKAGE'] || process.argv[1], + [ + '--is-forked-api', + '--start-process', + 'sessionProcess', + ...processArgs.getPassArgs(), + // ...process.argv.slice(3), + ], + { + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + } + ); + pipeForkLogs(subprocess); const newOpened = { conid, database, @@ -120,7 +131,7 @@ module.exports = { throw new Error('Invalid session'); } - console.log(`Processing query, sesid=${sesid}, sql=${sql}`); + logger.info({ sesid, sql }, 'Processing query'); this.dispatchMessage(sesid, 'Query execution started'); session.subprocess.send({ msgtype: 'executeQuery', sql }); @@ -158,7 +169,7 @@ module.exports = { throw new Error('Invalid session'); } - console.log(`Starting profiler, sesid=${sesid}`); + logger.info({ sesid }, 'Starting profiler'); session.loadingReader_jslid = jslid; session.subprocess.send({ msgtype: 'startProfiler', jslid }); diff --git a/packages/api/src/controllers/uploads.js b/packages/api/src/controllers/uploads.js index 1d2cafe64..e4f68d555 100644 --- a/packages/api/src/controllers/uploads.js +++ b/packages/api/src/controllers/uploads.js @@ -1,6 +1,8 @@ const path = require('path'); const { uploadsdir } = require('../utility/directories'); const uuidv1 = require('uuid/v1'); +const { getLogger } = require('dbgate-tools'); +const logger = getLogger('uploads'); module.exports = { upload_meta: { @@ -15,7 +17,7 @@ module.exports = { } const uploadName = uuidv1(); const filePath = path.join(uploadsdir(), uploadName); - console.log(`Uploading file ${data.name}, size=${data.size}`); + logger.info(`Uploading file ${data.name}, size=${data.size}`); data.mv(filePath, () => { res.json({ diff --git a/packages/api/src/index.js b/packages/api/src/index.js index e557a8c60..65fc0f9f3 100644 --- a/packages/api/src/index.js +++ b/packages/api/src/index.js @@ -1,5 +1,57 @@ -const shell = require('./shell'); +const { setLogger, getLogger, setLoggerName } = require('dbgate-tools'); const processArgs = require('./utility/processArgs'); +const pino = require('pino'); +const pinoms = require('pino-multi-stream'); +const fs = require('fs'); +const moment = require('moment'); +const path = require('path'); +const { logsdir, setLogsFilePath, getLogsFilePath } = require('./utility/directories'); + +if (processArgs.startProcess) { + setLoggerName(processArgs.startProcess.replace(/Process$/, '')); +} +if (processArgs.processDisplayName) { + setLoggerName(processArgs.processDisplayName); +} + +function loadLogsContent(maxLines) { + const text = fs.readFileSync(getLogsFilePath(), { encoding: 'utf8' }); + if (maxLines) { + const lines = text + .split('\n') + .map(x => x.trim()) + .filter(x => x); + return lines.slice(-maxLines).join('\n'); + } + return text; +} + +function configureLogger() { + const logsFilePath = path.join(logsdir(), `${moment().format('YYYY-MM-DD-HH-mm')}-${process.pid}.ndjson`); + setLogsFilePath(logsFilePath); + setLoggerName('main'); + + let logger = pinoms({ + streams: [ + { + stream: process.stdout, + level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info', + }, + { + stream: fs.createWriteStream(logsFilePath), + level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info', + }, + ], + }); + + setLogger(logger); +} + +if (processArgs.listenApi) { + configureLogger(); +} + +const shell = require('./shell'); const dbgateTools = require('dbgate-tools'); global['DBGATE_TOOLS'] = dbgateTools; @@ -8,7 +60,7 @@ if (processArgs.startProcess) { const proc = require('./proc'); const module = proc[processArgs.startProcess]; module.start(); -} +} if (processArgs.listenApi) { const main = require('./main'); @@ -17,5 +69,8 @@ if (processArgs.listenApi) { module.exports = { ...shell, + getLogger, + configureLogger, + loadLogsContent, getMainModule: () => require('./main'), }; diff --git a/packages/api/src/main.js b/packages/api/src/main.js index 5fdf9e625..d9d81c61f 100644 --- a/packages/api/src/main.js +++ b/packages/api/src/main.js @@ -33,6 +33,9 @@ const platformInfo = require('./utility/platformInfo'); const getExpressPath = require('./utility/getExpressPath'); const { getLogins } = require('./utility/hasPermission'); const _ = require('lodash'); +const { getLogger } = require('dbgate-tools'); + +const logger = getLogger('main'); function start() { // console.log('process.argv', process.argv); @@ -60,8 +63,8 @@ function start() { } else if (platformInfo.isNpmDist) { app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public'))); } else if (process.env.DEVWEB) { - console.log('__dirname', __dirname); - console.log(path.join(__dirname, '../../web/public/build')); + // console.log('__dirname', __dirname); + // console.log(path.join(__dirname, '../../web/public/build')); app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public'))); } else { app.get(getExpressPath('/'), (req, res) => { @@ -109,7 +112,7 @@ function start() { if (platformInfo.isDocker) { const port = process.env.PORT || 3000; - console.log('DbGate API listening on port (docker build)', port); + logger.info(`DbGate API listening on port ${port} (docker build)`); server.listen(port); } else if (platformInfo.isNpmDist) { getPort({ @@ -119,27 +122,27 @@ function start() { ), }).then(port => { server.listen(port, () => { - console.log(`DbGate API listening on port ${port} (NPM build)`); + logger.info(`DbGate API listening on port ${port} (NPM build)`); }); }); } else if (process.env.DEVWEB) { const port = process.env.PORT || 3000; - console.log('DbGate API & web listening on port (dev web build)', port); + logger.info(`DbGate API & web listening on port ${port} (dev web build)`); server.listen(port); } else { const port = process.env.PORT || 3000; - console.log('DbGate API listening on port (dev API build)', port); + logger.info(`DbGate API listening on port ${port} (dev API build)`); server.listen(port); } function shutdown() { - console.log('\nShutting down DbGate API server'); + logger.info('\nShutting down DbGate API server'); server.close(() => { - console.log('Server shut down, terminating'); + logger.info('Server shut down, terminating'); process.exit(0); }); setTimeout(() => { - console.log('Server close timeout, terminating'); + logger.info('Server close timeout, terminating'); process.exit(0); }, 1000); } diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index 96de90996..cc94ebe8f 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -1,7 +1,7 @@ const stableStringify = require('json-stable-stringify'); const { splitQuery } = require('dbgate-query-splitter'); const childProcessChecker = require('../utility/childProcessChecker'); -const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools'); +const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools'); const requireEngineDriver = require('../utility/requireEngineDriver'); const connectUtility = require('../utility/connectUtility'); const { handleProcessCommunication } = require('../utility/processComm'); @@ -9,6 +9,8 @@ const { SqlGenerator } = require('dbgate-tools'); const generateDeploySql = require('../shell/generateDeploySql'); const { dumpSqlSelect } = require('dbgate-sqltree'); +const logger = getLogger('dbconnProcess'); + let systemConnection; let storedConnection; let afterConnectCallbacks = []; @@ -269,7 +271,7 @@ async function handleSqlPreview({ msgid, objects, options }) { process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated }); if (generator.isUnhandledException) { setTimeout(() => { - console.log('Exiting because of unhandled exception'); + getLogger.info('Exiting because of unhandled exception'); process.exit(0); }, 500); } @@ -336,7 +338,7 @@ function start() { setInterval(() => { const time = new Date().getTime(); if (time - lastPing > 40 * 1000) { - console.log('Database connection not alive, exiting'); + logger.info('Database connection not alive, exiting'); process.exit(0); } }, 10 * 1000); @@ -345,9 +347,9 @@ function start() { if (handleProcessCommunication(message)) return; try { await handleMessage(message); - } catch (e) { - console.error('Error in DB connection', e); - process.send({ msgtype: 'error', error: e.message }); + } catch (err) { + logger.error({ err }, 'Error in DB connection'); + process.send({ msgtype: 'error', error: err.message }); } }); } diff --git a/packages/api/src/proc/serverConnectionProcess.js b/packages/api/src/proc/serverConnectionProcess.js index c720b8889..98f88a5e5 100644 --- a/packages/api/src/proc/serverConnectionProcess.js +++ b/packages/api/src/proc/serverConnectionProcess.js @@ -1,9 +1,10 @@ const stableStringify = require('json-stable-stringify'); -const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools'); +const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools'); const childProcessChecker = require('../utility/childProcessChecker'); const requireEngineDriver = require('../utility/requireEngineDriver'); const connectUtility = require('../utility/connectUtility'); const { handleProcessCommunication } = require('../utility/processComm'); +const logger = getLogger('srvconnProcess'); let systemConnection; let storedConnection; @@ -101,7 +102,7 @@ async function handleDatabaseOp(op, { name }) { } else { const dmp = driver.createDumper(); dmp[op](name); - console.log(`RUNNING SCRIPT: ${dmp.s}`); + logger.info({ sql: dmp.s }, 'Running script'); await driver.query(systemConnection, dmp.s); } await handleRefresh(); @@ -146,7 +147,7 @@ function start() { setInterval(() => { const time = new Date().getTime(); if (time - lastPing > 40 * 1000) { - console.log('Server connection not alive, exiting'); + logger.info('Server connection not alive, exiting'); process.exit(0); } }, 10 * 1000); diff --git a/packages/api/src/proc/sessionProcess.js b/packages/api/src/proc/sessionProcess.js index 4772e4404..e8e65809a 100644 --- a/packages/api/src/proc/sessionProcess.js +++ b/packages/api/src/proc/sessionProcess.js @@ -10,6 +10,9 @@ const requireEngineDriver = require('../utility/requireEngineDriver'); const { decryptConnection } = require('../utility/crypting'); const connectUtility = require('../utility/connectUtility'); const { handleProcessCommunication } = require('../utility/processComm'); +const { getLogger } = require('dbgate-tools'); + +const logger = getLogger('sessionProcess'); let systemConnection; let storedConnection; @@ -325,7 +328,7 @@ function start() { setInterval(() => { const time = new Date().getTime(); if (time - lastPing > 25 * 1000) { - console.log('Session not alive, exiting'); + logger.info('Session not alive, exiting'); process.exit(0); } }, 10 * 1000); diff --git a/packages/api/src/proc/sshForwardProcess.js b/packages/api/src/proc/sshForwardProcess.js index 75a5a85a1..8f1461a9d 100644 --- a/packages/api/src/proc/sshForwardProcess.js +++ b/packages/api/src/proc/sshForwardProcess.js @@ -3,6 +3,9 @@ const platformInfo = require('../utility/platformInfo'); const childProcessChecker = require('../utility/childProcessChecker'); const { handleProcessCommunication } = require('../utility/processComm'); const { SSHConnection } = require('../utility/SSHConnection'); +const { getLogger } = require('dbgate-tools'); + +const logger = getLogger('sshProcess'); async function getSshConnection(connection) { const sshConfig = { @@ -35,8 +38,8 @@ async function handleStart({ connection, tunnelConfig }) { tunnelConfig, }); } catch (err) { - console.log('Error creating SSH tunnel connection:', err.message); - + logger.error({ err }, 'Error creating SSH tunnel connection:'); + process.send({ msgtype: 'error', connection, diff --git a/packages/api/src/shell/archiveWriter.js b/packages/api/src/shell/archiveWriter.js index 7f7911c7a..c8eb17516 100644 --- a/packages/api/src/shell/archiveWriter.js +++ b/packages/api/src/shell/archiveWriter.js @@ -3,11 +3,14 @@ const fs = require('fs'); const { archivedir, resolveArchiveFolder } = require('../utility/directories'); // const socket = require('../utility/socket'); const jsonLinesWriter = require('./jsonLinesWriter'); +const { getLogger } = require('dbgate-tools'); + +const logger = getLogger(); function archiveWriter({ folderName, fileName }) { const dir = resolveArchiveFolder(folderName); if (!fs.existsSync(dir)) { - console.log(`Creating directory ${dir}`); + logger.info(`Creating directory ${dir}`); fs.mkdirSync(dir); } const jsonlFile = path.join(dir, `${fileName}.jsonl`); diff --git a/packages/api/src/shell/dumpDatabase.js b/packages/api/src/shell/dumpDatabase.js index f777ba5e1..0e2599151 100644 --- a/packages/api/src/shell/dumpDatabase.js +++ b/packages/api/src/shell/dumpDatabase.js @@ -1,5 +1,8 @@ const requireEngineDriver = require('../utility/requireEngineDriver'); const connectUtility = require('../utility/connectUtility'); +const { getLogger } = require('dbgate-tools'); + +const logger = getLogger('dumpDb'); function doDump(dumper) { return new Promise((resolve, reject) => { @@ -21,11 +24,11 @@ async function dumpDatabase({ databaseName, schemaName, }) { - console.log(`Dumping database`); + logger.info(`Dumping database`); if (!driver) driver = requireEngineDriver(connection); const pool = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true })); - console.log(`Connected.`); + logger.info(`Connected.`); const dumper = await driver.createBackupDumper(pool, { outputFile, diff --git a/packages/api/src/shell/executeQuery.js b/packages/api/src/shell/executeQuery.js index 4c277986e..a13361bda 100644 --- a/packages/api/src/shell/executeQuery.js +++ b/packages/api/src/shell/executeQuery.js @@ -1,12 +1,15 @@ const requireEngineDriver = require('../utility/requireEngineDriver'); const connectUtility = require('../utility/connectUtility'); +const { getLogger } = require('dbgate-tools'); + +const logger = getLogger('execQuery'); async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) { - console.log(`Execute query ${sql}`); + logger.info({ sql }, `Execute query`); if (!driver) driver = requireEngineDriver(connection); const pool = systemConnection || (await connectUtility(driver, connection, 'script')); - console.log(`Connected.`); + logger.info(`Connected.`); await driver.script(pool, sql); } diff --git a/packages/api/src/shell/importDatabase.js b/packages/api/src/shell/importDatabase.js index 6cb52961d..b59f9d4b9 100644 --- a/packages/api/src/shell/importDatabase.js +++ b/packages/api/src/shell/importDatabase.js @@ -4,6 +4,9 @@ const connectUtility = require('../utility/connectUtility'); const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream'); const download = require('./download'); const stream = require('stream'); +const { getLogger } = require('dbgate-tools'); + +const logger = getLogger('importDb'); class ImportStream extends stream.Transform { constructor(pool, driver) { @@ -38,11 +41,11 @@ function awaitStreamEnd(stream) { } async function importDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, inputFile }) { - console.log(`Importing database`); + logger.info(`Importing database`); if (!driver) driver = requireEngineDriver(connection); const pool = systemConnection || (await connectUtility(driver, connection, 'write')); - console.log(`Connected.`); + logger.info(`Connected.`); const downloadedFile = await download(inputFile); diff --git a/packages/api/src/shell/jsonArrayWriter.js b/packages/api/src/shell/jsonArrayWriter.js index 39e1831be..ed18b86f6 100644 --- a/packages/api/src/shell/jsonArrayWriter.js +++ b/packages/api/src/shell/jsonArrayWriter.js @@ -1,6 +1,9 @@ +const { getLogger } = require('dbgate-tools'); const fs = require('fs'); const stream = require('stream'); +const logger = getLogger('jsonArrayWriter'); + class StringifyStream extends stream.Transform { constructor() { super({ objectMode: true }); @@ -38,7 +41,7 @@ class StringifyStream extends stream.Transform { } async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) { - console.log(`Writing file ${fileName}`); + logger.info(`Writing file ${fileName}`); const stringify = new StringifyStream(); const fileStream = fs.createWriteStream(fileName, encoding); stringify.pipe(fileStream); diff --git a/packages/api/src/shell/jsonLinesReader.js b/packages/api/src/shell/jsonLinesReader.js index 113a9a788..399002ca1 100644 --- a/packages/api/src/shell/jsonLinesReader.js +++ b/packages/api/src/shell/jsonLinesReader.js @@ -1,6 +1,8 @@ const fs = require('fs'); const stream = require('stream'); const byline = require('byline'); +const { getLogger } = require('dbgate-tools'); +const logger = getLogger('jsonLinesReader'); class ParseStream extends stream.Transform { constructor({ limitRows }) { @@ -31,7 +33,7 @@ class ParseStream extends stream.Transform { } async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) { - console.log(`Reading file ${fileName}`); + logger.info(`Reading file ${fileName}`); const fileStream = fs.createReadStream(fileName, encoding); const liner = byline(fileStream); diff --git a/packages/api/src/shell/jsonLinesWriter.js b/packages/api/src/shell/jsonLinesWriter.js index 495c89560..a559e3174 100644 --- a/packages/api/src/shell/jsonLinesWriter.js +++ b/packages/api/src/shell/jsonLinesWriter.js @@ -1,5 +1,7 @@ +const { getLogger } = require('dbgate-tools'); const fs = require('fs'); const stream = require('stream'); +const logger = getLogger('jsonLinesWriter'); class StringifyStream extends stream.Transform { constructor({ header }) { @@ -21,7 +23,7 @@ class StringifyStream extends stream.Transform { } async function jsonLinesWriter({ fileName, encoding = 'utf-8', header = true }) { - console.log(`Writing file ${fileName}`); + logger.info(`Writing file ${fileName}`); const stringify = new StringifyStream({ header }); const fileStream = fs.createWriteStream(fileName, encoding); stringify.pipe(fileStream); diff --git a/packages/api/src/shell/queryReader.js b/packages/api/src/shell/queryReader.js index 1eea7147f..ec1634104 100644 --- a/packages/api/src/shell/queryReader.js +++ b/packages/api/src/shell/queryReader.js @@ -1,5 +1,7 @@ const requireEngineDriver = require('../utility/requireEngineDriver'); const connectUtility = require('../utility/connectUtility'); +const { getLogger } = require('dbgate-tools'); +const logger = getLogger('queryReader'); async function queryReader({ connection, @@ -14,12 +16,12 @@ async function queryReader({ // if (!sql && !json) { // throw new Error('One of sql or json must be set'); // } - console.log(`Reading query ${query || sql}`); + logger.info({ sql: query || sql }, `Reading query`); // else console.log(`Reading query ${JSON.stringify(json)}`); const driver = requireEngineDriver(connection); const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script'); - console.log(`Connected.`); + logger.info(`Connected.`); const reader = queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql); return reader; diff --git a/packages/api/src/shell/requirePlugin.js b/packages/api/src/shell/requirePlugin.js index 1c63bd1de..7e1c46769 100644 --- a/packages/api/src/shell/requirePlugin.js +++ b/packages/api/src/shell/requirePlugin.js @@ -3,6 +3,8 @@ const fs = require('fs'); const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories'); const nativeModules = require('../nativeModules'); const platformInfo = require('../utility/platformInfo'); +const { getLogger } = require('dbgate-tools'); +const logger = getLogger('requirePlugin'); const loadedPlugins = {}; @@ -17,7 +19,7 @@ function requirePlugin(packageName, requiredPlugin = null) { if (requiredPlugin == null) { let module; const modulePath = getPluginBackendPath(packageName); - console.log(`Loading module ${packageName} from ${modulePath}`); + logger.info(`Loading module ${packageName} from ${modulePath}`); try { // @ts-ignore module = __non_webpack_require__(modulePath); diff --git a/packages/api/src/shell/runScript.js b/packages/api/src/shell/runScript.js index 6b5fb2a54..331e4e1c8 100644 --- a/packages/api/src/shell/runScript.js +++ b/packages/api/src/shell/runScript.js @@ -1,5 +1,7 @@ +const { getLogger } = require('dbgate-tools'); const childProcessChecker = require('../utility/childProcessChecker'); const processArgs = require('../utility/processArgs'); +const logger = getLogger(); async function runScript(func) { if (processArgs.checkParent) { @@ -9,7 +11,7 @@ async function runScript(func) { await func(); process.exit(0); } catch (err) { - console.log(err); + logger.error('Error running script', err); process.exit(1); } } diff --git a/packages/api/src/shell/sqlDataWriter.js b/packages/api/src/shell/sqlDataWriter.js index 710f6dfbf..3dd6bad26 100644 --- a/packages/api/src/shell/sqlDataWriter.js +++ b/packages/api/src/shell/sqlDataWriter.js @@ -1,8 +1,9 @@ const fs = require('fs'); const stream = require('stream'); const path = require('path'); -const { driverBase } = require('dbgate-tools'); +const { driverBase, getLogger } = require('dbgate-tools'); const requireEngineDriver = require('../utility/requireEngineDriver'); +const logger = getLogger('sqlDataWriter'); class SqlizeStream extends stream.Transform { constructor({ fileName, dataName }) { @@ -40,7 +41,7 @@ class SqlizeStream extends stream.Transform { } async function sqlDataWriter({ fileName, dataName, driver, encoding = 'utf-8' }) { - console.log(`Writing file ${fileName}`); + logger.info(`Writing file ${fileName}`); const stringify = new SqlizeStream({ fileName, dataName }); const fileStream = fs.createWriteStream(fileName, encoding); stringify.pipe(fileStream); diff --git a/packages/api/src/shell/tableReader.js b/packages/api/src/shell/tableReader.js index 954f82055..84661f2ca 100644 --- a/packages/api/src/shell/tableReader.js +++ b/packages/api/src/shell/tableReader.js @@ -1,17 +1,18 @@ -const { quoteFullName, fullNameToString } = require('dbgate-tools'); +const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools'); const requireEngineDriver = require('../utility/requireEngineDriver'); const connectUtility = require('../utility/connectUtility'); +const logger = getLogger('tableReader'); async function tableReader({ connection, pureName, schemaName }) { const driver = requireEngineDriver(connection); const pool = await connectUtility(driver, connection, 'read'); - console.log(`Connected.`); + logger.info(`Connected.`); const fullName = { pureName, schemaName }; if (driver.databaseEngineTypes.includes('document')) { // @ts-ignore - console.log(`Reading collection ${fullNameToString(fullName)}`); + logger.info(`Reading collection ${fullNameToString(fullName)}`); // @ts-ignore return await driver.readQuery(pool, JSON.stringify(fullName)); } @@ -20,14 +21,14 @@ async function tableReader({ connection, pureName, schemaName }) { const query = `select * from ${quoteFullName(driver.dialect, fullName)}`; if (table) { // @ts-ignore - console.log(`Reading table ${fullNameToString(table)}`); + logger.info(`Reading table ${fullNameToString(table)}`); // @ts-ignore return await driver.readQuery(pool, query, table); } const view = await driver.analyseSingleObject(pool, fullName, 'views'); if (view) { // @ts-ignore - console.log(`Reading view ${fullNameToString(view)}`); + logger.info(`Reading view ${fullNameToString(view)}`); // @ts-ignore return await driver.readQuery(pool, query, view); } diff --git a/packages/api/src/shell/tableWriter.js b/packages/api/src/shell/tableWriter.js index e38fb9624..e03b55ad6 100644 --- a/packages/api/src/shell/tableWriter.js +++ b/packages/api/src/shell/tableWriter.js @@ -1,16 +1,17 @@ -const { fullNameToString } = require('dbgate-tools'); +const { fullNameToString, getLogger } = require('dbgate-tools'); const requireEngineDriver = require('../utility/requireEngineDriver'); const connectUtility = require('../utility/connectUtility'); +const logger = getLogger('tableWriter'); async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) { - console.log(`Writing table ${fullNameToString({ schemaName, pureName })}`); + logger.info(`Writing table ${fullNameToString({ schemaName, pureName })}`); if (!driver) { driver = requireEngineDriver(connection); } const pool = systemConnection || (await connectUtility(driver, connection, 'write')); - console.log(`Connected.`); + logger.info(`Connected.`); return await driver.writeTable(pool, { schemaName, pureName }, options); } diff --git a/packages/api/src/utility/DatastoreProxy.js b/packages/api/src/utility/DatastoreProxy.js index 0da243170..7bd8889d1 100644 --- a/packages/api/src/utility/DatastoreProxy.js +++ b/packages/api/src/utility/DatastoreProxy.js @@ -2,6 +2,7 @@ const { fork } = require('child_process'); const uuidv1 = require('uuid/v1'); const { handleProcessCommunication } = require('./processComm'); const processArgs = require('../utility/processArgs'); +const pipeForkLogs = require('./pipeForkLogs'); class DatastoreProxy { constructor(file) { @@ -30,13 +31,20 @@ class DatastoreProxy { async ensureSubprocess() { if (!this.subprocess) { - this.subprocess = fork(global['API_PACKAGE'] || process.argv[1], [ - '--is-forked-api', - '--start-process', - 'jslDatastoreProcess', - ...processArgs.getPassArgs(), - // ...process.argv.slice(3), - ]); + this.subprocess = fork( + global['API_PACKAGE'] || process.argv[1], + [ + '--is-forked-api', + '--start-process', + 'jslDatastoreProcess', + ...processArgs.getPassArgs(), + // ...process.argv.slice(3), + ], + { + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + } + ); + pipeForkLogs(this.subprocess); this.subprocess.on('message', message => { // @ts-ignore diff --git a/packages/api/src/utility/childProcessChecker.js b/packages/api/src/utility/childProcessChecker.js index f1dad6ab1..c206d407c 100644 --- a/packages/api/src/utility/childProcessChecker.js +++ b/packages/api/src/utility/childProcessChecker.js @@ -1,14 +1,18 @@ +const { getLogger } = require('dbgate-tools'); + +const logger = getLogger('childProcessChecked'); + let counter = 0; function childProcessChecker() { setInterval(() => { try { process.send({ msgtype: 'ping', counter: counter++ }); - } catch (ex) { + } catch (err) { // This will come once parent dies. // One way can be to check for error code ERR_IPC_CHANNEL_CLOSED // and call process.exit() - console.log('parent died', ex.toString()); + logger.error({ err }, 'parent died'); process.exit(1); } }, 1000); diff --git a/packages/api/src/utility/cleanDirectory.js b/packages/api/src/utility/cleanDirectory.js index e1e659912..df55fa2f6 100644 --- a/packages/api/src/utility/cleanDirectory.js +++ b/packages/api/src/utility/cleanDirectory.js @@ -2,7 +2,7 @@ const fs = require('fs-extra'); const path = require('path'); const ageSeconds = 3600; -async function cleanDirectory(directory) { +async function cleanDirectory(directory, age = undefined) { const files = await fs.readdir(directory); const now = new Date().getTime(); @@ -10,7 +10,7 @@ async function cleanDirectory(directory) { const full = path.join(directory, file); const stat = await fs.stat(full); const mtime = stat.mtime.getTime(); - const expirationTime = mtime + ageSeconds * 1000; + const expirationTime = mtime + (age || ageSeconds) * 1000; if (now > expirationTime) { if (stat.isDirectory()) { await fs.rmdir(full, { recursive: true }); diff --git a/packages/api/src/utility/directories.js b/packages/api/src/utility/directories.js index c7e4f4695..841ccb846 100644 --- a/packages/api/src/utility/directories.js +++ b/packages/api/src/utility/directories.js @@ -1,20 +1,24 @@ const os = require('os'); const path = require('path'); const fs = require('fs'); +const _ = require('lodash'); const cleanDirectory = require('./cleanDirectory'); const platformInfo = require('./platformInfo'); const processArgs = require('./processArgs'); const consoleObjectWriter = require('../shell/consoleObjectWriter'); +const { getLogger } = require('dbgate-tools'); + +let logsFilePath; const createDirectories = {}; const ensureDirectory = (dir, clean) => { if (!createDirectories[dir]) { if (clean && fs.existsSync(dir) && !platformInfo.isForkedApi) { - console.log(`Cleaning directory ${dir}`); - cleanDirectory(dir); + getLogger('directories').info(`Cleaning directory ${dir}`); + cleanDirectory(dir, _.isNumber(clean) ? clean : null); } if (!fs.existsSync(dir)) { - console.log(`Creating directory ${dir}`); + getLogger('directories').info(`Creating directory ${dir}`); fs.mkdirSync(dir); } createDirectories[dir] = true; @@ -38,7 +42,7 @@ function datadir() { return dir; } -const dirFunc = (dirname, clean = false) => () => { +const dirFunc = (dirname, clean) => () => { const dir = path.join(datadir(), dirname); ensureDirectory(dir, clean); @@ -52,6 +56,7 @@ const pluginsdir = dirFunc('plugins'); const archivedir = dirFunc('archive'); const appdir = dirFunc('apps'); const filesdir = dirFunc('files'); +const logsdir = dirFunc('logs', 3600 * 24 * 7); function packagedPluginsDir() { // console.log('CALL DIR FROM', new Error('xxx').stack); @@ -127,11 +132,19 @@ function migrateDataDir() { if (fs.existsSync(oldDir) && !fs.existsSync(newDir)) { fs.renameSync(oldDir, newDir); } - } catch (e) { - console.log('Error migrating data dir:', e.message); + } catch (err) { + getLogger('directories').error({ err }, 'Error migrating data dir'); } } +function setLogsFilePath(value) { + logsFilePath = value; +} + +function getLogsFilePath() { + return logsFilePath; +} + migrateDataDir(); module.exports = { @@ -144,9 +157,12 @@ module.exports = { ensureDirectory, pluginsdir, filesdir, + logsdir, packagedPluginsDir, packagedPluginList, getPluginBackendPath, resolveArchiveFolder, clearArchiveLinksCache, + getLogsFilePath, + setLogsFilePath, }; diff --git a/packages/api/src/utility/pipeForkLogs.js b/packages/api/src/utility/pipeForkLogs.js new file mode 100644 index 000000000..a43c1929c --- /dev/null +++ b/packages/api/src/utility/pipeForkLogs.js @@ -0,0 +1,28 @@ +const byline = require('byline'); +const { safeJsonParse, getLogger } = require('dbgate-tools'); +const logger = getLogger(); + +const levelNames = { + 10: 'trace', + 20: 'debug', + 30: 'info', + 40: 'warn', + 50: 'error', + 60: 'fatal', +}; + +const logDispatcher = method => data => { + const json = safeJsonParse(data.toString()); + if (json && json.level && levelNames[json.level]) { + logger[levelNames[json.level]](json || data.toString()); + } else { + logger[method](json || data.toString()); + } +}; + +function pipeForkLogs(subprocess) { + byline(subprocess.stdout).on('data', logDispatcher('info')); + byline(subprocess.stderr).on('data', logDispatcher('error')); +} + +module.exports = pipeForkLogs; diff --git a/packages/api/src/utility/processArgs.js b/packages/api/src/utility/processArgs.js index 68514c79f..6bf866512 100644 --- a/packages/api/src/utility/processArgs.js +++ b/packages/api/src/utility/processArgs.js @@ -11,6 +11,7 @@ const startProcess = getNamedArg('--start-process'); const isForkedApi = process.argv.includes('--is-forked-api'); const pluginsDir = getNamedArg('--plugins-dir'); const workspaceDir = getNamedArg('--workspace-dir'); +const processDisplayName = getNamedArg('--process-display-name'); const listenApi = process.argv.includes('--listen-api'); const listenApiChild = process.argv.includes('--listen-api-child') || listenApi; @@ -37,4 +38,5 @@ module.exports = { workspaceDir, listenApi, listenApiChild, + processDisplayName, }; diff --git a/packages/api/src/utility/sshTunnel.js b/packages/api/src/utility/sshTunnel.js index f97abb808..45aafea73 100644 --- a/packages/api/src/utility/sshTunnel.js +++ b/packages/api/src/utility/sshTunnel.js @@ -5,6 +5,9 @@ const AsyncLock = require('async-lock'); const lock = new AsyncLock(); const { fork } = require('child_process'); const processArgs = require('../utility/processArgs'); +const { getLogger } = require('dbgate-tools'); +const pipeForkLogs = require('./pipeForkLogs'); +const logger = getLogger('sshTunnel'); const sshTunnelCache = {}; @@ -21,12 +24,14 @@ const CONNECTION_FIELDS = [ const TUNNEL_FIELDS = [...CONNECTION_FIELDS, 'server', 'port']; function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) { - let subprocess = fork(global['API_PACKAGE'] || process.argv[1], [ - '--is-forked-api', - '--start-process', - 'sshForwardProcess', - ...processArgs.getPassArgs(), - ]); + let subprocess = fork( + global['API_PACKAGE'] || process.argv[1], + ['--is-forked-api', '--start-process', 'sshForwardProcess', ...processArgs.getPassArgs()], + { + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + } + ); + pipeForkLogs(subprocess); subprocess.send({ msgtype: 'connect', @@ -45,7 +50,7 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) { } }); subprocess.on('exit', code => { - console.log('SSH forward process exited'); + logger.info('SSH forward process exited'); delete sshTunnelCache[tunnelCacheKey]; }); }); @@ -65,13 +70,13 @@ async function getSshTunnel(connection) { toHost: connection.server, }; try { - console.log( + logger.info( `Creating SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}` ); const subprocess = await callForwardProcess(connection, tunnelConfig, tunnelCacheKey); - console.log( + logger.info( `Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}` ); diff --git a/packages/api/src/utility/useController.js b/packages/api/src/utility/useController.js index 1c3e562c8..3ff9afc82 100644 --- a/packages/api/src/utility/useController.js +++ b/packages/api/src/utility/useController.js @@ -2,7 +2,9 @@ const _ = require('lodash'); const express = require('express'); const getExpressPath = require('./getExpressPath'); const { MissingCredentialsError } = require('./exceptions'); +const { getLogger } = require('dbgate-tools'); +const logger = getLogger('useController'); /** * @param {string} route */ @@ -10,11 +12,11 @@ module.exports = function useController(app, electron, route, controller) { const router = express.Router(); if (controller._init) { - console.log(`Calling init controller for controller ${route}`); + logger.info(`Calling init controller for controller ${route}`); try { controller._init(); } catch (err) { - console.log(`Error initializing controller, exiting application`, err); + logger.error({ err }, `Error initializing controller, exiting application`); process.exit(1); } } @@ -75,16 +77,16 @@ module.exports = function useController(app, electron, route, controller) { try { const data = await controller[key]({ ...req.body, ...req.query }, req); res.json(data); - } catch (e) { - console.log(e); - if (e instanceof MissingCredentialsError) { + } catch (err) { + logger.error({ err }, `Error when processing route ${route}/${key}`); + if (err instanceof MissingCredentialsError) { res.json({ missingCredentials: true, apiErrorMessage: 'Missing credentials', - detail: e.detail, + detail: err.detail, }); } else { - res.status(500).json({ apiErrorMessage: e.message }); + res.status(500).json({ apiErrorMessage: err.message }); } } }); diff --git a/packages/tools/package.json b/packages/tools/package.json index eeff6185b..50fe774a0 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -36,6 +36,7 @@ "debug": "^4.3.4", "json-stable-stringify": "^1.0.1", "lodash": "^4.17.21", + "pino": "^8.8.0", "uuid": "^3.4.0" } } diff --git a/packages/tools/src/DatabaseAnalyser.ts b/packages/tools/src/DatabaseAnalyser.ts index 126c77ab3..c3e1f24aa 100644 --- a/packages/tools/src/DatabaseAnalyser.ts +++ b/packages/tools/src/DatabaseAnalyser.ts @@ -3,6 +3,9 @@ import _sortBy from 'lodash/sortBy'; import _groupBy from 'lodash/groupBy'; import _pick from 'lodash/pick'; import _compact from 'lodash/compact'; +import { getLogger } from './getLogger'; + +const logger = getLogger('dbAnalyser'); const STRUCTURE_FIELDS = ['tables', 'collections', 'views', 'matviews', 'functions', 'procedures', 'triggers']; @@ -107,7 +110,7 @@ export class DatabaseAnalyser { this.modifications = structureModifications; if (structureWithRowCounts) this.structure = structureWithRowCounts; - console.log('DB modifications detected:', this.modifications); + logger.info({ modifications: this.modifications }, 'DB modifications detected:'); return this.addEngineField(this.mergeAnalyseResult(await this._runAnalysis())); } @@ -304,7 +307,7 @@ export class DatabaseAnalyser { try { return await this.driver.query(this.pool, sql); } catch (err) { - console.log('Error running analyser query', err.message); + logger.error({ err }, 'Error running analyser query'); return { rows: [], }; diff --git a/packages/tools/src/SqlGenerator.ts b/packages/tools/src/SqlGenerator.ts index edcefa6b8..42d180fe9 100644 --- a/packages/tools/src/SqlGenerator.ts +++ b/packages/tools/src/SqlGenerator.ts @@ -8,10 +8,13 @@ import type { ViewInfo, } from 'dbgate-types'; import _flatten from 'lodash/flatten'; -import _uniqBy from 'lodash/uniqBy' +import _uniqBy from 'lodash/uniqBy'; +import { getLogger } from './getLogger'; import { SqlDumper } from './SqlDumper'; import { extendDatabaseInfo } from './structureTools'; +const logger = getLogger('sqlGenerator'); + interface SqlGeneratorOptions { dropTables: boolean; checkIfTableExists: boolean; @@ -82,7 +85,7 @@ export class SqlGenerator { } private handleException = error => { - console.log('Unhandled error', error); + logger.error({ error }, 'Unhandled error'); this.isUnhandledException = true; }; diff --git a/packages/tools/src/createBulkInsertStreamBase.ts b/packages/tools/src/createBulkInsertStreamBase.ts index 9a8812226..d64165288 100644 --- a/packages/tools/src/createBulkInsertStreamBase.ts +++ b/packages/tools/src/createBulkInsertStreamBase.ts @@ -1,6 +1,9 @@ import _intersection from 'lodash/intersection'; +import { getLogger } from './getLogger'; import { prepareTableForImport } from './tableTransforms'; +const logger = getLogger('bulkStreamBase'); + export function createBulkInsertStreamBase(driver, stream, pool, name, options): any { const fullNameQuoted = name.schemaName ? `${driver.dialect.quoteIdentifier(name.schemaName)}.${driver.dialect.quoteIdentifier(name.pureName)}` @@ -28,14 +31,13 @@ export function createBulkInsertStreamBase(driver, stream, pool, name, options): let structure = await driver.analyseSingleTable(pool, name); // console.log('ANALYSING', name, structure); if (structure && options.dropIfExists) { - console.log(`Dropping table ${fullNameQuoted}`); + logger.info(`Dropping table ${fullNameQuoted}`); await driver.script(pool, `DROP TABLE ${fullNameQuoted}`); } if (options.createIfNotExists && (!structure || options.dropIfExists)) { - console.log(`Creating table ${fullNameQuoted}`); const dmp = driver.createDumper(); dmp.createTable(prepareTableForImport({ ...writable.structure, ...name })); - console.log(dmp.s); + logger.info({ sql: dmp.s }, `Creating table ${fullNameQuoted}`); await driver.script(pool, dmp.s); structure = await driver.analyseSingleTable(pool, name); } diff --git a/packages/tools/src/getLogger.ts b/packages/tools/src/getLogger.ts new file mode 100644 index 000000000..ef35df6a7 --- /dev/null +++ b/packages/tools/src/getLogger.ts @@ -0,0 +1,25 @@ +import pino, { Logger } from 'pino'; + +let _logger: Logger; +let _name: string = null; +const defaultLogger: Logger = pino(); + +export function setLogger(value: Logger) { + _logger = value; +} + +export function getLogger(caller?: string): Logger { + let res = _logger || defaultLogger; + if (caller) { + const props = { caller }; + if (_name) { + props['name'] = _name; + } + res = res.child(props); + } + return res; +} + +export function setLoggerName(value) { + _name = value; +} diff --git a/packages/tools/src/index.ts b/packages/tools/src/index.ts index 71f0d0833..b98ea267d 100644 --- a/packages/tools/src/index.ts +++ b/packages/tools/src/index.ts @@ -18,3 +18,4 @@ export * from './stringTools'; export * from './computeDiffRows'; export * from './preloadedRowsTools'; export * from './ScriptWriter'; +export * from './getLogger'; diff --git a/packages/web/src/commands/stdCommands.ts b/packages/web/src/commands/stdCommands.ts index 039cb005d..62302dfc2 100644 --- a/packages/web/src/commands/stdCommands.ts +++ b/packages/web/src/commands/stdCommands.ts @@ -451,6 +451,22 @@ registerCommand({ onClick: openArchiveFolder, }); +registerCommand({ + id: 'folder.showLogs', + category: 'Folder', + name: 'Open logs', + testEnabled: () => getElectron() != null, + onClick: () => electron.showItemInFolder(getCurrentConfig().logsFilePath), +}); + +registerCommand({ + id: 'folder.showData', + category: 'Folder', + name: 'Open data folder', + testEnabled: () => getElectron() != null, + onClick: () => electron.showItemInFolder(getCurrentConfig().connectionsFilePath), +}); + registerCommand({ id: 'file.import', category: 'File', diff --git a/plugins/dbgate-plugin-csv/src/backend/writer.js b/plugins/dbgate-plugin-csv/src/backend/writer.js index c8989d86a..bd8e10076 100644 --- a/plugins/dbgate-plugin-csv/src/backend/writer.js +++ b/plugins/dbgate-plugin-csv/src/backend/writer.js @@ -2,6 +2,10 @@ const csv = require('csv'); const fs = require('fs'); const stream = require('stream'); +const { getLogger } = global.DBGATE_TOOLS; + +const logger = getLogger('csvWriter'); + class CsvPrepareStream extends stream.Transform { constructor({ header }) { super({ objectMode: true }); @@ -23,7 +27,7 @@ class CsvPrepareStream extends stream.Transform { } async function writer({ fileName, encoding = 'utf-8', header = true, delimiter, quoted }) { - console.log(`Writing file ${fileName}`); + logger.info(`Writing file ${fileName}`); const csvPrepare = new CsvPrepareStream({ header }); const csvStream = csv.stringify({ delimiter, quoted }); const fileStream = fs.createWriteStream(fileName, encoding); diff --git a/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js b/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js index c54afbf2d..e554d9a05 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js +++ b/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js @@ -1,4 +1,8 @@ const ObjectId = require('mongodb').ObjectId; +const { getLogger } = global.DBGATE_TOOLS; + +const logger = getLogger('mongoBulkInsert'); + function createBulkInsertStream(driver, stream, pool, name, options) { const collectionName = name.pureName; @@ -27,11 +31,11 @@ function createBulkInsertStream(driver, stream, pool, name, options) { writable.checkStructure = async () => { if (options.dropIfExists) { - console.log(`Dropping collection ${collectionName}`); + logger.info(`Dropping collection ${collectionName}`); await db.collection(collectionName).drop(); } if (options.truncate) { - console.log(`Truncating collection ${collectionName}`); + logger.info(`Truncating collection ${collectionName}`); await db.collection(collectionName).deleteMany({}); } }; diff --git a/plugins/dbgate-plugin-mysql/src/backend/drivers.js b/plugins/dbgate-plugin-mysql/src/backend/drivers.js index 587db33f9..22c7fe6e9 100644 --- a/plugins/dbgate-plugin-mysql/src/backend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/backend/drivers.js @@ -5,6 +5,9 @@ const Analyser = require('./Analyser'); const mysql2 = require('mysql2'); const { createBulkInsertStreamBase, makeUniqueColumnNames } = require('dbgate-tools'); const { MySqlDumper } = require('antares-mysql-dumper'); +const { getLogger } = global.DBGATE_TOOLS; + +const logger = getLogger('mysqlDriver'); function extractColumns(fields) { if (fields) { @@ -111,7 +114,7 @@ const drivers = driverBases.map(driverBase => ({ }; const handleError = error => { - console.log('ERROR', error); + logger.error({ error }, 'Stream error'); const { message } = error; options.info({ message, diff --git a/plugins/dbgate-plugin-postgres/src/backend/drivers.js b/plugins/dbgate-plugin-postgres/src/backend/drivers.js index 8165c7191..edf5d1bba 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/drivers.js +++ b/plugins/dbgate-plugin-postgres/src/backend/drivers.js @@ -5,6 +5,9 @@ const driverBases = require('../frontend/drivers'); const Analyser = require('./Analyser'); const pg = require('pg'); const { createBulkInsertStreamBase, makeUniqueColumnNames } = require('dbgate-tools'); +const { getLogger } = global.DBGATE_TOOLS; + +const logger = getLogger('postreDriver'); pg.types.setTypeParser(1082, 'text', val => val); // date pg.types.setTypeParser(1114, 'text', val => val); // timestamp without timezone @@ -144,7 +147,7 @@ const drivers = driverBases.map(driverBase => ({ }); query.on('error', error => { - console.log('ERROR', error); + logger.error({ error }, 'Stream error'); const { message, position, procName } = error; let line = null; if (position) { diff --git a/plugins/dbgate-plugin-sqlite/src/backend/driver.js b/plugins/dbgate-plugin-sqlite/src/backend/driver.js index 1e4ee9e38..c75f919e1 100644 --- a/plugins/dbgate-plugin-sqlite/src/backend/driver.js +++ b/plugins/dbgate-plugin-sqlite/src/backend/driver.js @@ -4,6 +4,9 @@ const driverBase = require('../frontend/driver'); const Analyser = require('./Analyser'); const { splitQuery, sqliteSplitterOptions } = require('dbgate-query-splitter'); const { createBulkInsertStreamBase, makeUniqueColumnNames } = require('dbgate-tools'); +const { getLogger } = global.DBGATE_TOOLS; + +const logger = getLogger('sqliteDriver'); let Database; @@ -103,7 +106,7 @@ const driver = { try { inTransaction(); } catch (error) { - console.log('ERROR', error); + logger.error({ error }, 'Stream error'); const { message, procName } = error; options.info({ message, diff --git a/plugins/dbgate-plugin-xml/src/backend/reader.js b/plugins/dbgate-plugin-xml/src/backend/reader.js index 00f39932b..4b035b7e3 100644 --- a/plugins/dbgate-plugin-xml/src/backend/reader.js +++ b/plugins/dbgate-plugin-xml/src/backend/reader.js @@ -1,6 +1,9 @@ const fs = require('fs'); const stream = require('stream'); const NodeXmlStream = require('node-xml-stream-parser'); +const { getLogger } = global.DBGATE_TOOLS; + +const logger = getLogger('xmlReader'); class ParseStream extends stream.Transform { constructor({ itemElementName }) { @@ -56,7 +59,7 @@ class ParseStream extends stream.Transform { } async function reader({ fileName, encoding = 'utf-8', itemElementName }) { - console.log(`Reading file ${fileName}`); + logger.info(`Reading file ${fileName}`); const fileStream = fs.createReadStream(fileName, encoding); const parser = new ParseStream({ itemElementName }); diff --git a/plugins/dbgate-plugin-xml/src/backend/writer.js b/plugins/dbgate-plugin-xml/src/backend/writer.js index 8b87016d8..29009da94 100644 --- a/plugins/dbgate-plugin-xml/src/backend/writer.js +++ b/plugins/dbgate-plugin-xml/src/backend/writer.js @@ -1,5 +1,8 @@ const fs = require('fs'); const stream = require('stream'); +const { getLogger } = global.DBGATE_TOOLS; + +const logger = getLogger('xmlWriter'); function escapeXml(value) { return value.replace(/[<>&'"]/g, function (c) { @@ -67,7 +70,7 @@ class StringifyStream extends stream.Transform { } async function writer({ fileName, encoding = 'utf-8', itemElementName, rootElementName }) { - console.log(`Writing file ${fileName}`); + logger.info(`Writing file ${fileName}`); const stringify = new StringifyStream({ itemElementName, rootElementName }); const fileStream = fs.createWriteStream(fileName, encoding); stringify.pipe(fileStream); diff --git a/yarn.lock b/yarn.lock index f12ce2154..6f4aca331 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1701,6 +1701,13 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + abstract-logging@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/abstract-logging/-/abstract-logging-2.0.1.tgz#6b0c371df212db7129b57d2e7fcf282b8bf1c839" @@ -1925,7 +1932,7 @@ are-we-there-yet@~1.1.2: delegates "^1.0.0" readable-stream "^2.0.6" -argparse@^1.0.7: +argparse@^1.0.10, argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== @@ -2093,6 +2100,11 @@ atob@^2.1.2: resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== +atomic-sleep@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/atomic-sleep/-/atomic-sleep-1.0.0.tgz#eb85b77a601fc932cfe432c5acd364a9e2c9075b" + integrity sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ== + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -2373,6 +2385,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + braces@^2.3.1, braces@^2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729" @@ -2960,6 +2979,11 @@ colorette@^1.1.0: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.4.0.tgz#5190fbb87276259a86ad700bff2c6d6faa3fca40" integrity sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g== +colorette@^2.0.7: + version "2.0.19" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798" + integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ== + combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -3317,6 +3341,11 @@ date-fns@^2.0.1, date-fns@^2.28.0: resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.29.1.tgz#9667c2615525e552b5135a3116b95b1961456e60" integrity sha512-dlLD5rKaKxpFdnjrs+5azHDFOPEu4ANy/LTh04A1DTzMM7qoajmKCBc8pkKRFT41CNzw+4gQh79X5C+Jq27HAw== +dateformat@^4.6.3: + version "4.6.3" + resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5" + integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA== + dbgate-plugin-tools@^1.0.4, dbgate-plugin-tools@^1.0.7, dbgate-plugin-tools@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/dbgate-plugin-tools/-/dbgate-plugin-tools-1.0.8.tgz#bb16f38ff7161d9b57c08f0918e152593e7ac695" @@ -3635,6 +3664,16 @@ duplexify@^3.4.2, duplexify@^3.6.0: readable-stream "^2.0.0" stream-shift "^1.0.0" +duplexify@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.2.tgz#18b4f8d28289132fa0b9573c898d9f903f81c7b0" + integrity sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw== + dependencies: + end-of-stream "^1.4.1" + inherits "^2.0.3" + readable-stream "^3.1.1" + stream-shift "^1.0.0" + ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -3997,7 +4036,12 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== -events@^3.0.0: +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + +events@^3.0.0, events@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== @@ -4215,6 +4259,11 @@ extsprintf@^1.2.0: resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== +fast-copy@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/fast-copy/-/fast-copy-3.0.0.tgz#875ebf33b13948ae012b6e51d33da5e6e7571ab8" + integrity sha512-4HzS+9pQ5Yxtv13Lhs1Z1unMXamBdn5nA4bEi1abYpDNSpSp7ODYQ1KPMF6nTatfEzgH6/zPvXKU1zvHiUjWlA== + fast-deep-equal@^3.1.1: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" @@ -4253,6 +4302,16 @@ fast-levenshtein@~2.0.6: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== +fast-redact@^3.0.0, fast-redact@^3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.1.2.tgz#d58e69e9084ce9fa4c1a6fa98a3e1ecf5d7839aa" + integrity sha512-+0em+Iya9fKGfEQGcd62Yv6onjBmmhV1uh86XVfOU8VwAe6kaFdQCWI9s0/Nnugx5Vd9tdbZ7e6gE2tR9dzXdw== + +fast-safe-stringify@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884" + integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA== + fastq@^1.6.0: version "1.13.0" resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c" @@ -4718,6 +4777,17 @@ glob@^7.0.5, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^8.0.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-8.1.0.tgz#d388f656593ef708ee3e34640fdfb99a9fd1c33e" + integrity sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^5.0.1" + once "^1.3.0" + global-modules@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-1.0.0.tgz#6d770f0eb523ac78164d72b5e71a8877265cc3ea" @@ -4928,6 +4998,14 @@ hash.js@^1.0.0, hash.js@^1.0.3: inherits "^2.0.3" minimalistic-assert "^1.0.1" +help-me@^4.0.1: + version "4.2.0" + resolved "https://registry.yarnpkg.com/help-me/-/help-me-4.2.0.tgz#50712bfd799ff1854ae1d312c36eafcea85b0563" + integrity sha512-TAOnTB8Tz5Dw8penUuzHVrKNKlCIbwwbHnXraNJxPwf8LRtE2HlM84RYuezMFcwOJmoYOCWVDyJ8TQGxn9PgxA== + dependencies: + glob "^8.0.0" + readable-stream "^3.6.0" + highlight.js@11.6.0: version "11.6.0" resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-11.6.0.tgz#a50e9da05763f1bb0c1322c8f4f755242cff3f5a" @@ -6819,6 +6897,11 @@ jest@^28.1.3: import-local "^3.0.2" jest-cli "^28.1.3" +joycon@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/joycon/-/joycon-3.1.1.tgz#bce8596d6ae808f8b68168f5fc69280996894f03" + integrity sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw== + js-md4@^0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/js-md4/-/js-md4-0.3.2.tgz#cd3b3dc045b0c404556c81ddb5756c23e59d7cf5" @@ -7291,7 +7374,7 @@ lodash.sortby@^4.7.0: resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" integrity sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA== -lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0: +lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.21, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -7576,6 +7659,13 @@ minimatch@^3.0.3, minimatch@^3.0.4, minimatch@^3.1.1: dependencies: brace-expansion "^1.1.7" +minimatch@^5.0.1: + version "5.1.6" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.6.tgz#1cfcb8cf5522ea69952cd2af95ae09477f122a96" + integrity sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g== + dependencies: + brace-expansion "^2.0.1" + minimist-options@^4.0.2: version "4.1.0" resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619" @@ -8214,6 +8304,16 @@ object.pick@^1.3.0: dependencies: isobject "^3.0.1" +on-exit-leak-free@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-0.2.0.tgz#b39c9e3bf7690d890f4861558b0d7b90a442d209" + integrity sha512-dqaz3u44QbRXQooZLTUKU41ZrzYrcvLISVgbrzbyCMxpmSLJvZ3ZamIJIZ29P6OhZIkNIQKosdeM6t1LYbA9hg== + +on-exit-leak-free@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz#5c703c968f7e7f851885f6459bf8a8a57edc9cc4" + integrity sha512-VuCaZZAjReZ3vUwgOB8LxAosIurDiAW0s13rI1YwmaP++jvcxP77AWoQvenZebpCA2m8WC1/EosPYPMjnRAp/w== + on-finished@2.4.1, on-finished@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" @@ -8629,6 +8729,93 @@ pify@^4.0.1: resolved "https://registry.yarnpkg.com/pify/-/pify-4.0.1.tgz#4b2cd25c50d598735c50292224fd8c6df41e3231" integrity sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g== +pino-abstract-transport@^1.0.0, pino-abstract-transport@v1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-1.0.0.tgz#cc0d6955fffcadb91b7b49ef220a6cc111d48bb3" + integrity sha512-c7vo5OpW4wIS42hUVcT5REsL8ZljsUfBjqV/e2sFxmFEFZiq1XLUp5EYLtuDH6PEHq9W1egWqRbnLUP5FuZmOA== + dependencies: + readable-stream "^4.0.0" + split2 "^4.0.0" + +pino-abstract-transport@v0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-0.5.0.tgz#4b54348d8f73713bfd14e3dc44228739aa13d9c0" + integrity sha512-+KAgmVeqXYbTtU2FScx1XS3kNyfZ5TrXY07V96QnUSFqo2gAqlvmaxH67Lj7SWazqsMabf+58ctdTcBgnOLUOQ== + dependencies: + duplexify "^4.1.2" + split2 "^4.0.0" + +pino-multi-stream@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/pino-multi-stream/-/pino-multi-stream-6.0.0.tgz#2116bca740cb5eb606f430b20fd480f4944b2b99" + integrity sha512-oCuTtaDSUB5xK1S45r9oWE0Dj8RWdHVvaGTft5pO/rmzgIqQRkilf5Ooilz3uRm0IYj8sPRho3lVx48LCmXjvQ== + dependencies: + pino "^7.0.0" + +pino-pretty@^9.1.1: + version "9.1.1" + resolved "https://registry.yarnpkg.com/pino-pretty/-/pino-pretty-9.1.1.tgz#e7d64c1db98266ca428ab56567b844ba780cd0e1" + integrity sha512-iJrnjgR4FWQIXZkUF48oNgoRI9BpyMhaEmihonHeCnZ6F50ZHAS4YGfGBT/ZVNsPmd+hzkIPGzjKdY08+/yAXw== + dependencies: + colorette "^2.0.7" + dateformat "^4.6.3" + fast-copy "^3.0.0" + fast-safe-stringify "^2.1.1" + help-me "^4.0.1" + joycon "^3.1.1" + minimist "^1.2.6" + on-exit-leak-free "^2.1.0" + pino-abstract-transport "^1.0.0" + pump "^3.0.0" + readable-stream "^4.0.0" + secure-json-parse "^2.4.0" + sonic-boom "^3.0.0" + strip-json-comments "^3.1.1" + +pino-std-serializers@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-4.0.0.tgz#1791ccd2539c091ae49ce9993205e2cd5dbba1e2" + integrity sha512-cK0pekc1Kjy5w9V2/n+8MkZwusa6EyyxfeQCB799CQRhRt/CqYKiWs5adeu8Shve2ZNffvfC/7J64A2PJo1W/Q== + +pino-std-serializers@^6.0.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-6.1.0.tgz#307490fd426eefc95e06067e85d8558603e8e844" + integrity sha512-KO0m2f1HkrPe9S0ldjx7za9BJjeHqBku5Ch8JyxETxT8dEFGz1PwgrHaOQupVYitpzbFSYm7nnljxD8dik2c+g== + +pino@^7.0.0: + version "7.11.0" + resolved "https://registry.yarnpkg.com/pino/-/pino-7.11.0.tgz#0f0ea5c4683dc91388081d44bff10c83125066f6" + integrity sha512-dMACeu63HtRLmCG8VKdy4cShCPKaYDR4youZqoSWLxl5Gu99HUw8bw75thbPv9Nip+H+QYX8o3ZJbTdVZZ2TVg== + dependencies: + atomic-sleep "^1.0.0" + fast-redact "^3.0.0" + on-exit-leak-free "^0.2.0" + pino-abstract-transport v0.5.0 + pino-std-serializers "^4.0.0" + process-warning "^1.0.0" + quick-format-unescaped "^4.0.3" + real-require "^0.1.0" + safe-stable-stringify "^2.1.0" + sonic-boom "^2.2.1" + thread-stream "^0.15.1" + +pino@^8.8.0: + version "8.8.0" + resolved "https://registry.yarnpkg.com/pino/-/pino-8.8.0.tgz#1f0d6695a224aa06afc7ad60f2ccc4772d3b9233" + integrity sha512-cF8iGYeu2ODg2gIwgAHcPrtR63ILJz3f7gkogaHC/TXVVXxZgInmNYiIpDYEwgEkxZti2Se6P2W2DxlBIZe6eQ== + dependencies: + atomic-sleep "^1.0.0" + fast-redact "^3.1.1" + on-exit-leak-free "^2.1.0" + pino-abstract-transport v1.0.0 + pino-std-serializers "^6.0.0" + process-warning "^2.0.0" + quick-format-unescaped "^4.0.3" + real-require "^0.2.0" + safe-stable-stringify "^2.3.1" + sonic-boom "^3.1.0" + thread-stream "^2.0.0" + pirates@^4.0.1, pirates@^4.0.4: version "4.0.5" resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.5.tgz#feec352ea5c3268fb23a37c702ab1699f35a5f3b" @@ -8776,6 +8963,16 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== +process-warning@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-1.0.0.tgz#980a0b25dc38cd6034181be4b7726d89066b4616" + integrity sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q== + +process-warning@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-2.1.0.tgz#1e60e3bfe8183033bbc1e702c2da74f099422d1a" + integrity sha512-9C20RLxrZU/rFnxWncDkuF6O999NdIf3E1ws4B0ZeY3sRVPzWBMsYDE2lxjxhiXxg464cQTgKUGm8/i6y2YGXg== + process@^0.11.10: version "0.11.10" resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" @@ -8914,6 +9111,11 @@ queue-microtask@^1.2.2: resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== +quick-format-unescaped@^4.0.3: + version "4.0.4" + resolved "https://registry.yarnpkg.com/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz#93ef6dd8d3453cbc7970dd614fad4c5954d6b5a7" + integrity sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg== + quick-lru@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" @@ -9054,6 +9256,16 @@ readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: string_decoder "^1.1.1" util-deprecate "^1.0.1" +readable-stream@^4.0.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.3.0.tgz#0914d0c72db03b316c9733bb3461d64a3cc50cba" + integrity sha512-MuEnA0lbSi7JS8XM+WNJlWZkHAAdm7gETHdFK//Q/mChGyj2akEFtdLZh32jSdkWGbRwCW9pn6g3LWDdDeZnBQ== + dependencies: + abort-controller "^3.0.0" + buffer "^6.0.3" + events "^3.3.0" + process "^0.11.10" + readable-stream@~1.0.17, readable-stream@~1.0.31: version "1.0.34" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" @@ -9102,6 +9314,16 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +real-require@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/real-require/-/real-require-0.1.0.tgz#736ac214caa20632847b7ca8c1056a0767df9381" + integrity sha512-r/H9MzAWtrv8aSVjPCMFpDMl5q66GqtmmRkRjpHTsp4zBAa+snZyiQNlMONiUmEJcsnaw0wCauJ2GWODr/aFkg== + +real-require@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/real-require/-/real-require-0.2.0.tgz#209632dea1810be2ae063a6ac084fee7e33fba78" + integrity sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg== + realpath-native@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/realpath-native/-/realpath-native-1.1.0.tgz#2003294fea23fb0672f2476ebe22fcf498a2d65c" @@ -9458,6 +9680,11 @@ safe-regex@^1.1.0: dependencies: ret "~0.1.10" +safe-stable-stringify@^2.1.0, safe-stable-stringify@^2.3.1: + version "2.4.2" + resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.2.tgz#ec7b037768098bf65310d1d64370de0dc02353aa" + integrity sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA== + "safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" @@ -9530,6 +9757,11 @@ scmp@2.0.0: resolved "https://registry.yarnpkg.com/scmp/-/scmp-2.0.0.tgz#247110ef22ccf897b13a3f0abddb52782393cd6a" integrity sha512-FaHoAk75AYhT+rnBmMpkvHSIcQma4OHzYXOhn1XXtgNomi0FTV8YEXYuh2EIdCg5IKMVyFbXeJT4Cn96+fzABg== +secure-json-parse@^2.4.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862" + integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw== + semiver@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/semiver/-/semiver-1.1.0.tgz#9c97fb02c21c7ce4fcf1b73e2c7a24324bdddd5f" @@ -9801,6 +10033,20 @@ socks@^2.6.1, socks@^2.6.2: ip "^2.0.0" smart-buffer "^4.2.0" +sonic-boom@^2.2.1: + version "2.8.0" + resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-2.8.0.tgz#c1def62a77425090e6ad7516aad8eb402e047611" + integrity sha512-kuonw1YOYYNOve5iHdSahXPOK49GqwA+LZhI6Wz/l0rP57iKyXXIHaRagOBHAPmGwJC6od2Z9zgvZ5loSgMlVg== + dependencies: + atomic-sleep "^1.0.0" + +sonic-boom@^3.0.0, sonic-boom@^3.1.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-3.2.1.tgz#972ceab831b5840a08a002fa95a672008bda1c38" + integrity sha512-iITeTHxy3B9FGu8aVdiDXUVAcHMF9Ss0cCsAOo2HfCrmVGT3/DT5oYaeu0M/YKZDlKTvChEyPq0zI9Hf33EX6A== + dependencies: + atomic-sleep "^1.0.0" + sorcery@^0.10.0: version "0.10.0" resolved "https://registry.yarnpkg.com/sorcery/-/sorcery-0.10.0.tgz#8ae90ad7d7cb05fc59f1ab0c637845d5c15a52b7" @@ -9920,7 +10166,7 @@ split2@^0.1.2: dependencies: through2 "~0.4.1" -split2@^4.1.0: +split2@^4.0.0, split2@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/split2/-/split2-4.1.0.tgz#101907a24370f85bb782f08adaabe4e281ecf809" integrity sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ== @@ -10456,6 +10702,20 @@ text-table@^0.2.0: resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== +thread-stream@^0.15.1: + version "0.15.2" + resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-0.15.2.tgz#fb95ad87d2f1e28f07116eb23d85aba3bc0425f4" + integrity sha512-UkEhKIg2pD+fjkHQKyJO3yoIvAP3N6RlNFt2dUhcS1FGvCD1cQa1M/PGknCLFIyZdtJOWQjejp7bdNqmN7zwdA== + dependencies: + real-require "^0.1.0" + +thread-stream@^2.0.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.3.0.tgz#4fc07fb39eff32ae7bad803cb7dd9598349fed33" + integrity sha512-kaDqm1DET9pp3NXwR8382WHbnpXnRkN9xGN9dQt3B2+dmXiW8X1SOwmFOxAErEQ47ObhZ96J6yhZNXuyCOL7KA== + dependencies: + real-require "^0.2.0" + throat@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/throat/-/throat-4.1.0.tgz#89037cbc92c56ab18926e6ba4cbb200e15672a6a"