update cloud files

packages/api/src/utility/cloudIntf.js

@@ -1,8 +1,16 @@
 const axios = require('axios');
+const fs = require('fs-extra');
+const _ = require('lodash');
+const path = require('path');
 const { getExternalParamsWithLicense } = require('./authProxy');
-const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const { getLogger, extractErrorLogData, jsonLinesParse } = require('dbgate-tools');
+const { datadir } = require('./directories');
+const platformInfo = require('./platformInfo');
+const connections = require('../controllers/connections');
+const { isProApp } = require('./checkLicense');
 
 const logger = getLogger('cloudIntf');
+let cloudFiles = null;
 
 const DBGATE_IDENTITY_URL = process.env.LOCAL_DBGATE_IDENTITY
   ? 'http://localhost:3103'
@@ -54,7 +62,100 @@ function startCloudTokenChecking(sid, callback) {
   }, 500);
 }
 
+async function loadCloudFiles() {
+  try {
+    const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files.jsonl'), 'utf-8');
+    const parsedJson = jsonLinesParse(fileContent);
+    cloudFiles = parsedJson;
+  } catch (err) {
+    cloudFiles = [];
+  }
+}
+
+async function collectCloudFilesSearchTags() {
+  const res = [];
+  if (platformInfo.isElectron) {
+    res.push('app');
+  } else {
+    res.push('web');
+  }
+  if (platformInfo.isWindows) {
+    res.push('windows');
+  }
+  if (platformInfo.isMac) {
+    res.push('mac');
+  }
+  if (platformInfo.isLinux) {
+    res.push('linux');
+  }
+  if (platformInfo.isAwsUbuntuLayout) {
+    res.push('aws');
+  }
+  if (platformInfo.isAzureUbuntuLayout) {
+    res.push('azure');
+  }
+  if (platformInfo.isSnap) {
+    res.push('snap');
+  }
+  const engines = await connections.getUsedEngines();
+  const engineTags = engines.map(engine => engine.split('@')[0]);
+  res.push(...engineTags);
+
+  // team-premium and trials will return the same cloud files as premium - no need to check
+  res.push(isProApp() ? 'premium' : 'community');
+
+  return res;
+}
+
+async function updateCloudFiles() {
+  let lastCloudFilesTags;
+  try {
+    const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files-tags.json'), 'utf-8');
+    cloudFiles = JSON.parse(fileContent);
+  } catch (err) {
+    lastCloudFilesTags = [];
+  }
+
+  let lastCheckedTm = 0;
+  if (_.isEqual(cloudFiles, lastCloudFilesTags) && cloudFiles.length > 0) {
+    lastCheckedTm = _.max(cloudFiles.map(x => x.modifiedTm));
+  }
+  const tags = await collectCloudFilesSearchTags();
+
+  const resp = await axios.default.post(
+    `${DBGATE_CLOUD_URL}/public-cloud-updates`,
+    {
+      lastCheckedTm,
+      tags,
+    },
+    getExternalParamsWithLicense()
+  );
+
+  const filesByPath = _.keyBy(cloudFiles, 'path');
+  for(const file of resp.data) {
+    filesByPath[file.path] = file;
+  }
+
+  cloudFiles = Object.values(filesByPath);
+
+  await fs.writeFile(
+    path.join(datadir(), 'cloud-files.jsonl'),
+    cloudFiles.map(x => JSON.stringify(x)).join('\n')
+  );
+
+  await fs.writeFile(
+    path.join(datadir(), 'cloud-files-tags.json'),
+    JSON.stringify(tags)
+  );
+}
+
+async function startCloudFiles() {
+  await loadCloudFiles();
+  await updateCloudFiles();
+}
+
 module.exports = {
   createDbGateIdentitySession,
   startCloudTokenChecking,
+  startCloudFiles,
 };