update cloud files

packages/api/src/controllers/connections.js

@@ -239,6 +239,19 @@ module.exports = {
     return (await this.datastore.find()).filter(x => connectionHasPermission(x, req));
   },
 
+  async getUsedEngines() {
+    const storage = require('./storage');
+
+    const storageEngines = await storage.getUsedEngines();
+    if (storageEngines) {
+      return storageEngines;
+    }
+    if (portalConnections) {
+      return _.uniq(_.compact(portalConnections.map(x => x.engine)));
+    }
+    return _.uniq((await this.datastore.find()).map(x => x.engine));
+  },
+
   test_meta: true,
   test({ connection, requestDbList = false }) {
     const subprocess = fork(

packages/api/src/controllers/storage.js

@@ -32,4 +32,8 @@ module.exports = {
   },
 
   startRefreshLicense() {},
+
+  async getUsedEngines() {
+    return null;
+  },
 };

packages/api/src/main.js

@@ -39,6 +39,7 @@ const { getDefaultAuthProvider } = require('./auth/authProvider');
 const startCloudUpgradeTimer = require('./utility/cloudUpgrade');
 const { isProApp } = require('./utility/checkLicense');
 const { getHealthStatus, getHealthStatusSprinx } = require('./utility/healthStatus');
+const { startCloudFiles } = require('./utility/cloudIntf');
 
 const logger = getLogger('main');
 
@@ -200,6 +201,8 @@ function start() {
   if (process.env.CLOUD_UPGRADE_FILE) {
     startCloudUpgradeTimer();
   }
+
+  startCloudFiles();
 }
 
 function useAllControllers(app, electron) {

packages/api/src/utility/cloudIntf.js

@@ -1,8 +1,16 @@
 const axios = require('axios');
+const fs = require('fs-extra');
+const _ = require('lodash');
+const path = require('path');
 const { getExternalParamsWithLicense } = require('./authProxy');
-const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const { getLogger, extractErrorLogData, jsonLinesParse } = require('dbgate-tools');
+const { datadir } = require('./directories');
+const platformInfo = require('./platformInfo');
+const connections = require('../controllers/connections');
+const { isProApp } = require('./checkLicense');
 
 const logger = getLogger('cloudIntf');
+let cloudFiles = null;
 
 const DBGATE_IDENTITY_URL = process.env.LOCAL_DBGATE_IDENTITY
   ? 'http://localhost:3103'
@@ -54,7 +62,100 @@ function startCloudTokenChecking(sid, callback) {
   }, 500);
 }
 
+async function loadCloudFiles() {
+  try {
+    const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files.jsonl'), 'utf-8');
+    const parsedJson = jsonLinesParse(fileContent);
+    cloudFiles = parsedJson;
+  } catch (err) {
+    cloudFiles = [];
+  }
+}
+
+async function collectCloudFilesSearchTags() {
+  const res = [];
+  if (platformInfo.isElectron) {
+    res.push('app');
+  } else {
+    res.push('web');
+  }
+  if (platformInfo.isWindows) {
+    res.push('windows');
+  }
+  if (platformInfo.isMac) {
+    res.push('mac');
+  }
+  if (platformInfo.isLinux) {
+    res.push('linux');
+  }
+  if (platformInfo.isAwsUbuntuLayout) {
+    res.push('aws');
+  }
+  if (platformInfo.isAzureUbuntuLayout) {
+    res.push('azure');
+  }
+  if (platformInfo.isSnap) {
+    res.push('snap');
+  }
+  const engines = await connections.getUsedEngines();
+  const engineTags = engines.map(engine => engine.split('@')[0]);
+  res.push(...engineTags);
+
+  // team-premium and trials will return the same cloud files as premium - no need to check
+  res.push(isProApp() ? 'premium' : 'community');
+
+  return res;
+}
+
+async function updateCloudFiles() {
+  let lastCloudFilesTags;
+  try {
+    const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files-tags.json'), 'utf-8');
+    cloudFiles = JSON.parse(fileContent);
+  } catch (err) {
+    lastCloudFilesTags = [];
+  }
+
+  let lastCheckedTm = 0;
+  if (_.isEqual(cloudFiles, lastCloudFilesTags) && cloudFiles.length > 0) {
+    lastCheckedTm = _.max(cloudFiles.map(x => x.modifiedTm));
+  }
+  const tags = await collectCloudFilesSearchTags();
+
+  const resp = await axios.default.post(
+    `${DBGATE_CLOUD_URL}/public-cloud-updates`,
+    {
+      lastCheckedTm,
+      tags,
+    },
+    getExternalParamsWithLicense()
+  );
+
+  const filesByPath = _.keyBy(cloudFiles, 'path');
+  for(const file of resp.data) {
+    filesByPath[file.path] = file;
+  }
+
+  cloudFiles = Object.values(filesByPath);
+
+  await fs.writeFile(
+    path.join(datadir(), 'cloud-files.jsonl'),
+    cloudFiles.map(x => JSON.stringify(x)).join('\n')
+  );
+
+  await fs.writeFile(
+    path.join(datadir(), 'cloud-files-tags.json'),
+    JSON.stringify(tags)
+  );
+}
+
+async function startCloudFiles() {
+  await loadCloudFiles();
+  await updateCloudFiles();
+}
+
 module.exports = {
   createDbGateIdentitySession,
   startCloudTokenChecking,
+  startCloudFiles,
 };