diff --git a/packages/api/src/controllers/cloud.js b/packages/api/src/controllers/cloud.js new file mode 100644 index 000000000..3ecc8a3ae --- /dev/null +++ b/packages/api/src/controllers/cloud.js @@ -0,0 +1,15 @@ +const fs = require('fs-extra'); +const _ = require('lodash'); +const path = require('path'); +const { appdir } = require('../utility/directories'); +const socket = require('../utility/socket'); +const connections = require('./connections'); +const { getPublicCloudFiles } = require('../utility/cloudIntf'); + +module.exports = { + publicFiles_meta: true, + async publicFiles() { + const res = await getPublicCloudFiles(); + return res; + }, +}; diff --git a/packages/api/src/main.js b/packages/api/src/main.js index 3304d6a08..571593bd1 100644 --- a/packages/api/src/main.js +++ b/packages/api/src/main.js @@ -27,6 +27,7 @@ const plugins = require('./controllers/plugins'); const files = require('./controllers/files'); const scheduler = require('./controllers/scheduler'); const queryHistory = require('./controllers/queryHistory'); +const cloud = require('./controllers/cloud'); const onFinished = require('on-finished'); const processArgs = require('./utility/processArgs'); @@ -223,6 +224,7 @@ function useAllControllers(app, electron) { useController(app, electron, '/query-history', queryHistory); useController(app, electron, '/apps', apps); useController(app, electron, '/auth', auth); + useController(app, electron, '/cloud', cloud); } function setElectronSender(electronSender) { diff --git a/packages/api/src/utility/authProxy.js b/packages/api/src/utility/authProxy.js index 744536177..913c8c82f 100644 --- a/packages/api/src/utility/authProxy.js +++ b/packages/api/src/utility/authProxy.js @@ -36,11 +36,13 @@ async function callRefactorSqlQueryApi(query, task, structure, dialect) { return null; } -function getExternalParamsWithLicense() { +function getExternalParamsWithLicense(isPost = false) { return { - headers: { - 'Content-Type': 'application/json', - }, + headers: isPost + ? { + 'Content-Type': 'application/json', + } + : {}, }; } diff --git a/packages/api/src/utility/cloudIntf.js b/packages/api/src/utility/cloudIntf.js index 9d56f8edf..b39fab441 100644 --- a/packages/api/src/utility/cloudIntf.js +++ b/packages/api/src/utility/cloudIntf.js @@ -8,8 +8,10 @@ const { datadir } = require('./directories'); const platformInfo = require('./platformInfo'); const connections = require('../controllers/connections'); const { isProApp } = require('./checkLicense'); +const socket = require('./socket'); const logger = getLogger('cloudIntf'); + let cloudFiles = null; const DBGATE_IDENTITY_URL = process.env.LOCAL_DBGATE_IDENTITY @@ -30,7 +32,7 @@ async function createDbGateIdentitySession(client) { { client, }, - getExternalParamsWithLicense() + getExternalParamsWithLicense(true) ); return { sid: resp.data.sid, @@ -49,7 +51,7 @@ function startCloudTokenChecking(sid, callback) { try { const resp = await axios.default.get( `${DBGATE_IDENTITY_URL}/api/get-token/${sid}`, - getExternalParamsWithLicense() + getExternalParamsWithLicense(false) ); if (resp.data.status == 'ok') { @@ -110,52 +112,58 @@ async function collectCloudFilesSearchTags() { async function updateCloudFiles() { let lastCloudFilesTags; try { - const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files-tags.json'), 'utf-8'); - cloudFiles = JSON.parse(fileContent); + lastCloudFilesTags = await fs.readFile(path.join(datadir(), 'cloud-files-tags.txt'), 'utf-8'); } catch (err) { - lastCloudFilesTags = []; + lastCloudFilesTags = ''; } + const tags = (await collectCloudFilesSearchTags()).join(','); let lastCheckedTm = 0; - if (_.isEqual(cloudFiles, lastCloudFilesTags) && cloudFiles.length > 0) { - lastCheckedTm = _.max(cloudFiles.map(x => x.modifiedTm)); + if (tags == lastCloudFilesTags && cloudFiles.length > 0) { + lastCheckedTm = _.max(cloudFiles.map(x => parseInt(x.modifiedTm))); } - const tags = await collectCloudFilesSearchTags(); - const resp = await axios.default.post( - `${DBGATE_CLOUD_URL}/public-cloud-updates`, - { - lastCheckedTm, - tags, - }, - getExternalParamsWithLicense() + logger.info({ tags, lastCheckedTm }, 'Downloading cloud files'); + + const resp = await axios.default.get( + `${DBGATE_CLOUD_URL}/public-cloud-updates?lastCheckedTm=${lastCheckedTm}&tags=${tags}`, + getExternalParamsWithLicense(false) ); + logger.info(`Downloaded ${resp.data.length} cloud files`); + const filesByPath = _.keyBy(cloudFiles, 'path'); - for(const file of resp.data) { + for (const file of resp.data) { filesByPath[file.path] = file; } cloudFiles = Object.values(filesByPath); - await fs.writeFile( - path.join(datadir(), 'cloud-files.jsonl'), - cloudFiles.map(x => JSON.stringify(x)).join('\n') - ); + await fs.writeFile(path.join(datadir(), 'cloud-files.jsonl'), cloudFiles.map(x => JSON.stringify(x)).join('\n')); + await fs.writeFile(path.join(datadir(), 'cloud-files-tags.txt'), tags); - await fs.writeFile( - path.join(datadir(), 'cloud-files-tags.json'), - JSON.stringify(tags) - ); + socket.emitChanged(`public-cloud-changed`); } async function startCloudFiles() { await loadCloudFiles(); - await updateCloudFiles(); + try { + await updateCloudFiles(); + } catch (err) { + logger.error(extractErrorLogData(err), 'Error updating cloud files'); + } +} + +async function getPublicCloudFiles() { + if (!loadCloudFiles) { + await loadCloudFiles(); + } + return cloudFiles; } module.exports = { createDbGateIdentitySession, startCloudTokenChecking, startCloudFiles, + getPublicCloudFiles, }; diff --git a/packages/web/src/appobj/CloudFileAppObject.svelte b/packages/web/src/appobj/CloudFileAppObject.svelte new file mode 100644 index 000000000..de547b2a2 --- /dev/null +++ b/packages/web/src/appobj/CloudFileAppObject.svelte @@ -0,0 +1,29 @@ + + + + + diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index 22dc64d7b..babcd45ed 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -40,6 +40,8 @@ 'icon invisible-box': 'mdi mdi-minus-box-outline icon-invisible', 'icon cloud-upload': 'mdi mdi-cloud-upload', 'icon cloud': 'mdi mdi-cloud', + 'icon cloud-public': 'mdi mdi-cloud-search', + 'icon cloud-logged': 'mdi mdi-cloud-key', 'icon import': 'mdi mdi-application-import', 'icon export': 'mdi mdi-application-export', 'icon new-connection': 'mdi mdi-database-plus', diff --git a/packages/web/src/utility/metadataLoaders.ts b/packages/web/src/utility/metadataLoaders.ts index 2fc374a02..fd3fd2a21 100644 --- a/packages/web/src/utility/metadataLoaders.ts +++ b/packages/web/src/utility/metadataLoaders.ts @@ -166,6 +166,12 @@ const authTypesLoader = ({ engine }) => ({ errorValue: null, }); +const publicCloudFilesLoader = () => ({ + url: 'cloud/public-files', + params: {}, + reloadTrigger: { key: `public-cloud-changed` }, +}); + async function getCore(loader, args) { const { url, params, reloadTrigger, transform, onLoaded, errorValue } = loader(args); const key = stableStringify({ url, ...params }); @@ -456,3 +462,10 @@ export function getSchemaList(args) { export function useSchemaList(args) { return useCore(schemaListLoader, args); } + +export function getPublicCloudFiles(args) { + return getCore(publicCloudFilesLoader, args); +} +export function usePublicCloudFiles(args = {}) { + return useCore(publicCloudFilesLoader, args); +} diff --git a/packages/web/src/widgets/CloudItemsWidget.svelte b/packages/web/src/widgets/CloudItemsWidget.svelte new file mode 100644 index 000000000..6c6a2036e --- /dev/null +++ b/packages/web/src/widgets/CloudItemsWidget.svelte @@ -0,0 +1,25 @@ + + + + + + data.folder} /> + + + + + diff --git a/packages/web/src/widgets/WidgetContainer.svelte b/packages/web/src/widgets/WidgetContainer.svelte index 814a60c96..38c1de040 100644 --- a/packages/web/src/widgets/WidgetContainer.svelte +++ b/packages/web/src/widgets/WidgetContainer.svelte @@ -9,6 +9,7 @@ import AppWidget from './AppWidget.svelte'; import AdminMenuWidget from './AdminMenuWidget.svelte'; import AdminPremiumPromoWidget from './AdminPremiumPromoWidget.svelte'; + import CloudItemsWidget from './CloudItemsWidget.svelte';