public cloud search

packages/api/src/controllers/cloud.js

@@ -1,4 +1,4 @@
-const { getPublicCloudFiles, getPublicFileData } = require('../utility/cloudIntf');
+const { getPublicCloudFiles, getPublicFileData, refreshPublicFiles } = require('../utility/cloudIntf');
 
 module.exports = {
   publicFiles_meta: true,
@@ -12,4 +12,12 @@ module.exports = {
     const res = getPublicFileData(path);
     return res;
   },
+
+  refreshPublicFiles_meta: true,
+  async refreshPublicFiles() {
+    await refreshPublicFiles();
+    return {
+      status: 'ok',
+    };
+  },
 };

packages/api/src/utility/authProxy.js

@@ -36,14 +36,8 @@ async function callRefactorSqlQueryApi(query, task, structure, dialect) {
   return null;
 }
 
-function getExternalParamsWithLicense(isPost = false) {
+function getLicenseHttpHeaders() {
-  return {
+  return {};
-    headers: isPost
-      ? {
-          'Content-Type': 'application/json',
-        }
-      : {},
-  };
 }
 
 module.exports = {
@@ -57,5 +51,5 @@ module.exports = {
   callTextToSqlApi,
   callCompleteOnCursorApi,
   callRefactorSqlQueryApi,
-  getExternalParamsWithLicense,
+  getLicenseHttpHeaders,
 };

packages/api/src/utility/cloudIntf.js

@@ -2,13 +2,14 @@ const axios = require('axios');
 const fs = require('fs-extra');
 const _ = require('lodash');
 const path = require('path');
-const { getExternalParamsWithLicense } = require('./authProxy');
+const { getLicenseHttpHeaders } = require('./authProxy');
 const { getLogger, extractErrorLogData, jsonLinesParse } = require('dbgate-tools');
 const { datadir } = require('./directories');
 const platformInfo = require('./platformInfo');
 const connections = require('../controllers/connections');
 const { isProApp } = require('./checkLicense');
 const socket = require('./socket');
+const config = require('../controllers/config');
 
 const logger = getLogger('cloudIntf');
 
@@ -32,7 +33,12 @@ async function createDbGateIdentitySession(client) {
     {
       client,
     },
-    getExternalParamsWithLicense(true)
+    {
+      headers: {
+        ...getLicenseHttpHeaders(),
+        'Content-Type': 'application/json',
+      },
+    }
   );
   return {
     sid: resp.data.sid,
@@ -49,10 +55,11 @@ function startCloudTokenChecking(sid, callback) {
     }
 
     try {
-      const resp = await axios.default.get(
+      const resp = await axios.default.get(`${DBGATE_IDENTITY_URL}/api/get-token/${sid}`, {
-        `${DBGATE_IDENTITY_URL}/api/get-token/${sid}`,
+        headers: {
-        getExternalParamsWithLicense(false)
+          ...getLicenseHttpHeaders(),
-      );
+        },
+      });
 
       if (resp.data.status == 'ok') {
         clearInterval(interval);
@@ -68,7 +75,7 @@ async function loadCloudFiles() {
   try {
     const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files.jsonl'), 'utf-8');
     const parsedJson = jsonLinesParse(fileContent);
-    cloudFiles = parsedJson;
+    cloudFiles = _.sortBy(parsedJson, x => `${x.folder}/${x.title}`);
   } catch (err) {
     cloudFiles = [];
   }
@@ -99,6 +106,12 @@ async function collectCloudFilesSearchTags() {
   if (platformInfo.isSnap) {
     res.push('snap');
   }
+  if (platformInfo.isDocker) {
+    res.push('docker');
+  }
+  if (platformInfo.isNpmDist) {
+    res.push('npm');
+  }
   const engines = await connections.getUsedEngines();
   const engineTags = engines.map(engine => engine.split('@')[0]);
   res.push(...engineTags);
@@ -109,6 +122,17 @@ async function collectCloudFilesSearchTags() {
   return res;
 }
 
+async function getCloudSigninHeaders() {
+  const settingsValue = await config.getSettings();
+  const value = settingsValue['cloudSigninToken'];
+  if (value) {
+    return {
+      'x-cloud-login': value,
+    };
+  }
+  return {};
+}
+
 async function updateCloudFiles() {
   let lastCloudFilesTags;
   try {
@@ -127,14 +151,23 @@ async function updateCloudFiles() {
 
   const resp = await axios.default.get(
     `${DBGATE_CLOUD_URL}/public-cloud-updates?lastCheckedTm=${lastCheckedTm}&tags=${tags}`,
-    getExternalParamsWithLicense(false)
+    {
+      headers: {
+        ...getLicenseHttpHeaders(),
+        ...(await getCloudSigninHeaders()),
+      },
+    }
   );
 
   logger.info(`Downloaded ${resp.data.length} cloud files`);
 
-  const filesByPath = _.keyBy(cloudFiles, 'path');
+  const filesByPath = lastCheckedTm == 0 ? {} : _.keyBy(cloudFiles, 'path');
   for (const file of resp.data) {
-    filesByPath[file.path] = file;
+    if (file.isDeleted) {
+      delete filesByPath[file.path];
+    } else {
+      filesByPath[file.path] = file;
+    }
   }
 
   cloudFiles = Object.values(filesByPath);
@@ -146,12 +179,7 @@ async function updateCloudFiles() {
 }
 
 async function startCloudFiles() {
-  await loadCloudFiles();
+  refreshPublicFiles();
-  try {
-    await updateCloudFiles();
-  } catch (err) {
-    logger.error(extractErrorLogData(err), 'Error updating cloud files');
-  }
 }
 
 async function getPublicCloudFiles() {
@@ -162,14 +190,30 @@ async function getPublicCloudFiles() {
 }
 
 async function getPublicFileData(path) {
-  const resp = await axios.default.get(`${DBGATE_CLOUD_URL}/public/${path}`, getExternalParamsWithLicense(false));
+  const resp = await axios.default.get(`${DBGATE_CLOUD_URL}/public/${path}`, {
+    headers: {
+      ...getLicenseHttpHeaders(),
+    },
+  });
   return resp.data;
 }
 
+async function refreshPublicFiles() {
+  if (!cloudFiles) {
+    await loadCloudFiles();
+  }
+  try {
+    await updateCloudFiles();
+  } catch (err) {
+    logger.error(extractErrorLogData(err), 'Error updating cloud files');
+  }
+}
+
 module.exports = {
   createDbGateIdentitySession,
   startCloudTokenChecking,
   startCloudFiles,
   getPublicCloudFiles,
   getPublicFileData,
+  refreshPublicFiles,
 };

packages/web/src/appobj/CloudFileAppObject.svelte

@@ -2,11 +2,16 @@
   import AppObjectCore from './AppObjectCore.svelte';
 
   export const extractKey = data => data.path;
+  export const createMatcher =
+    filter =>
+    ({ title, description }) =>
+      filterName(filter, title, description);
 </script>
 
 <script lang="ts">
   import { apiCall } from '../utility/api';
   import newQuery from '../query/newQuery';
+  import { filterName } from 'dbgate-tools';
 
   export let data;
 

packages/web/src/stores.ts

@@ -182,7 +182,9 @@ export const focusedConnectionOrDatabase = writable<{ conid: string; database?:
 
 export const focusedTreeDbKey = writable<{ key: string; root: string; type: string; text: string }>(null);
 
-export const cloudSigninToken = writableWithStorage(null, 'cloudSigninToken');
+export const cloudSigninToken = getElectron()
+  ? writableSettingsValue(null, 'cloudSigninToken')
+  : writableWithStorage(null, 'cloudSigninToken');
 
 export const DEFAULT_OBJECT_SEARCH_SETTINGS = {
   pureName: true,

packages/web/src/widgets/CloudItemsWidget.svelte

@@ -10,17 +10,38 @@
   import { _t } from '../translations';
 
   import WidgetsInnerContainer from './WidgetsInnerContainer.svelte';
+  import SearchBoxWrapper from '../elements/SearchBoxWrapper.svelte';
+  import SearchInput from '../elements/SearchInput.svelte';
+  import CloseSearchButton from '../buttons/CloseSearchButton.svelte';
+  import InlineButton from '../buttons/InlineButton.svelte';
+  import FontIcon from '../icons/FontIcon.svelte';
+  import { apiCall } from '../utility/api';
+
+  let filter = '';
 
   $: publicFiles = usePublicCloudFiles();
+
+  async function handleRefreshPublic() {
+    await apiCall('cloud/refresh-public-files');
+  }
 </script>
 
 <WidgetColumnBar>
   <WidgetColumnBarItem title="Public cloud" name="cloud" height="70%" storageName="publicCloudItems">
     <WidgetsInnerContainer>
+      <SearchBoxWrapper>
+        <SearchInput placeholder="Search cloud files" bind:value={filter} />
+        <CloseSearchButton bind:filter />
+        <InlineButton on:click={handleRefreshPublic} title="Refresh files" data-testid="SavedFileList_buttonRefresh">
+          <FontIcon icon="icon refresh" />
+        </InlineButton>
+      </SearchBoxWrapper>
+
       <AppObjectList
         list={$publicFiles || []}
         module={cloudFileAppObject}
         groupFunc={data => data.folder || undefined}
+        {filter}
       />
     </WidgetsInnerContainer>
   </WidgetColumnBarItem>