diff --git a/packages/api/src/controllers/connections.js b/packages/api/src/controllers/connections.js index 13a452a64..e221e8411 100644 --- a/packages/api/src/controllers/connections.js +++ b/packages/api/src/controllers/connections.js @@ -239,6 +239,19 @@ module.exports = { return (await this.datastore.find()).filter(x => connectionHasPermission(x, req)); }, + async getUsedEngines() { + const storage = require('./storage'); + + const storageEngines = await storage.getUsedEngines(); + if (storageEngines) { + return storageEngines; + } + if (portalConnections) { + return _.uniq(_.compact(portalConnections.map(x => x.engine))); + } + return _.uniq((await this.datastore.find()).map(x => x.engine)); + }, + test_meta: true, test({ connection, requestDbList = false }) { const subprocess = fork( diff --git a/packages/api/src/controllers/storage.js b/packages/api/src/controllers/storage.js index 6d498f869..f7066eb22 100644 --- a/packages/api/src/controllers/storage.js +++ b/packages/api/src/controllers/storage.js @@ -32,4 +32,8 @@ module.exports = { }, startRefreshLicense() {}, + + async getUsedEngines() { + return null; + }, }; diff --git a/packages/api/src/main.js b/packages/api/src/main.js index ac0c33ef5..3304d6a08 100644 --- a/packages/api/src/main.js +++ b/packages/api/src/main.js @@ -39,6 +39,7 @@ const { getDefaultAuthProvider } = require('./auth/authProvider'); const startCloudUpgradeTimer = require('./utility/cloudUpgrade'); const { isProApp } = require('./utility/checkLicense'); const { getHealthStatus, getHealthStatusSprinx } = require('./utility/healthStatus'); +const { startCloudFiles } = require('./utility/cloudIntf'); const logger = getLogger('main'); @@ -200,6 +201,8 @@ function start() { if (process.env.CLOUD_UPGRADE_FILE) { startCloudUpgradeTimer(); } + + startCloudFiles(); } function useAllControllers(app, electron) { diff --git a/packages/api/src/utility/cloudIntf.js b/packages/api/src/utility/cloudIntf.js index ee2b8301f..9d56f8edf 100644 --- a/packages/api/src/utility/cloudIntf.js +++ b/packages/api/src/utility/cloudIntf.js @@ -1,8 +1,16 @@ const axios = require('axios'); +const fs = require('fs-extra'); +const _ = require('lodash'); +const path = require('path'); const { getExternalParamsWithLicense } = require('./authProxy'); -const { getLogger, extractErrorLogData } = require('dbgate-tools'); +const { getLogger, extractErrorLogData, jsonLinesParse } = require('dbgate-tools'); +const { datadir } = require('./directories'); +const platformInfo = require('./platformInfo'); +const connections = require('../controllers/connections'); +const { isProApp } = require('./checkLicense'); const logger = getLogger('cloudIntf'); +let cloudFiles = null; const DBGATE_IDENTITY_URL = process.env.LOCAL_DBGATE_IDENTITY ? 'http://localhost:3103' @@ -54,7 +62,100 @@ function startCloudTokenChecking(sid, callback) { }, 500); } +async function loadCloudFiles() { + try { + const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files.jsonl'), 'utf-8'); + const parsedJson = jsonLinesParse(fileContent); + cloudFiles = parsedJson; + } catch (err) { + cloudFiles = []; + } +} + +async function collectCloudFilesSearchTags() { + const res = []; + if (platformInfo.isElectron) { + res.push('app'); + } else { + res.push('web'); + } + if (platformInfo.isWindows) { + res.push('windows'); + } + if (platformInfo.isMac) { + res.push('mac'); + } + if (platformInfo.isLinux) { + res.push('linux'); + } + if (platformInfo.isAwsUbuntuLayout) { + res.push('aws'); + } + if (platformInfo.isAzureUbuntuLayout) { + res.push('azure'); + } + if (platformInfo.isSnap) { + res.push('snap'); + } + const engines = await connections.getUsedEngines(); + const engineTags = engines.map(engine => engine.split('@')[0]); + res.push(...engineTags); + + // team-premium and trials will return the same cloud files as premium - no need to check + res.push(isProApp() ? 'premium' : 'community'); + + return res; +} + +async function updateCloudFiles() { + let lastCloudFilesTags; + try { + const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files-tags.json'), 'utf-8'); + cloudFiles = JSON.parse(fileContent); + } catch (err) { + lastCloudFilesTags = []; + } + + let lastCheckedTm = 0; + if (_.isEqual(cloudFiles, lastCloudFilesTags) && cloudFiles.length > 0) { + lastCheckedTm = _.max(cloudFiles.map(x => x.modifiedTm)); + } + const tags = await collectCloudFilesSearchTags(); + + const resp = await axios.default.post( + `${DBGATE_CLOUD_URL}/public-cloud-updates`, + { + lastCheckedTm, + tags, + }, + getExternalParamsWithLicense() + ); + + const filesByPath = _.keyBy(cloudFiles, 'path'); + for(const file of resp.data) { + filesByPath[file.path] = file; + } + + cloudFiles = Object.values(filesByPath); + + await fs.writeFile( + path.join(datadir(), 'cloud-files.jsonl'), + cloudFiles.map(x => JSON.stringify(x)).join('\n') + ); + + await fs.writeFile( + path.join(datadir(), 'cloud-files-tags.json'), + JSON.stringify(tags) + ); +} + +async function startCloudFiles() { + await loadCloudFiles(); + await updateCloudFiles(); +} + module.exports = { createDbGateIdentitySession, startCloudTokenChecking, + startCloudFiles, };