Merge branch 'master' into feature/mongosh

.github/workflows/build-app-pro-beta.yaml

@@ -39,7 +39,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 8a4dc2732a7097b5c4c48b4feb62609111cdf3e0
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-app-pro.yaml

@@ -39,7 +39,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 8a4dc2732a7097b5c4c48b4feb62609111cdf3e0
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-cloud-pro.yaml

@@ -39,7 +39,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 8a4dc2732a7097b5c4c48b4feb62609111cdf3e0
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-docker-pro.yaml

@@ -44,7 +44,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 8a4dc2732a7097b5c4c48b4feb62609111cdf3e0
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-npm-pro.yaml

@@ -32,7 +32,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 8a4dc2732a7097b5c4c48b4feb62609111cdf3e0
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/e2e-pro.yaml

@@ -26,7 +26,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 8a4dc2732a7097b5c4c48b4feb62609111cdf3e0
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

CHANGELOG.md

@@ -8,6 +8,12 @@ Builds:
  - linux - application for linux
  - win - application for Windows
 
+## 6.6.0
+- ADDED: Database chat - AI powered chatbot, which knows your database (Premium)
+- ADDED: Firestore support (Premium)
+- REMOVED: Query AI assistant (replaced by Database Chat)  (Premium)
+- FIXED: Chart permissions were ignored  (Premium)
+
 ## 6.5.6
 - ADDED: New object window - quick access to most common functions
 - ADDED: Possibility to disable split query by empty line #1162

README.md

@@ -39,6 +39,7 @@ DbGate is licensed under GPL-3.0 license and is free to use for any purpose.
 * libSQL/Turso (Premium)
 * DuckDB
 * Firebird
+* Firestore (Premium)
 
 
 <a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot1.png">
@@ -81,6 +82,7 @@ DbGate is licensed under GPL-3.0 license and is free to use for any purpose.
 * Archives - backup your data in NDJSON files on local filesystem (or on DbGate server, when using web application)
 * NDJSON data viewer and editor - browse NDJSON data, edit data and structure directly on NDJSON files. Works also for big NDSON files
 * Charts, export chart to HTML page
+* AI powered database chat
 * Show GEO data on map, export map to HTML page
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture

e2e-tests/cypress/e2e/browse-data.cy.js

@@ -390,7 +390,7 @@ describe('Data browser data', () => {
     cy.get('body').realType('find most popular artist');
     cy.get('body').realPress('{enter}');
     cy.testid('DatabaseChatTab_executeAllQueries', { timeout: 20000 }).click();
-    cy.wait(4000);
+    cy.wait(20000);
     // cy.contains('Iron Maiden');
     cy.themeshot('database-chat');
 

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "6.5.7-beta.4",
+  "version": "6.6.0",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",

packages/api/src/storageModel.js

@@ -674,6 +674,12 @@ module.exports = {
           "columnName": "awsRegion",
           "dataType": "varchar(250)",
           "notNull": false
+        },
+        {
+          "pureName": "connections",
+          "columnName": "connectionDefinition",
+          "dataType": "text",
+          "notNull": false
         }
       ],
       "foreignKeys": [],

packages/api/src/utility/crypting.js

@@ -101,24 +101,26 @@ function decryptObjectPasswordField(obj, field, encryptor = null) {
   return obj;
 }
 
+const fieldsToEncrypt = ['password', 'sshPassword', 'sshKeyfilePassword', 'connectionDefinition'];
+
 function encryptConnection(connection, encryptor = null) {
   if (connection.passwordMode != 'saveRaw') {
-    connection = encryptObjectPasswordField(connection, 'password', encryptor);
-    connection = encryptObjectPasswordField(connection, 'sshPassword', encryptor);
-    connection = encryptObjectPasswordField(connection, 'sshKeyfilePassword', encryptor);
+    for (const field of fieldsToEncrypt) {
+      connection = encryptObjectPasswordField(connection, field, encryptor);
+    }
   }
   return connection;
 }
 
 function maskConnection(connection) {
   if (!connection) return connection;
-  return _.omit(connection, ['password', 'sshPassword', 'sshKeyfilePassword']);
+  return _.omit(connection, fieldsToEncrypt);
 }
 
-function decryptConnection(connection, encryptor = null) {
-  connection = decryptObjectPasswordField(connection, 'password', encryptor);
-  connection = decryptObjectPasswordField(connection, 'sshPassword', encryptor);
-  connection = decryptObjectPasswordField(connection, 'sshKeyfilePassword', encryptor);
+function decryptConnection(connection) {
+  for (const field of fieldsToEncrypt) {
+    connection = decryptObjectPasswordField(connection, field);
+  }
   return connection;
 }
 
@@ -188,9 +190,9 @@ function recryptObjectPasswordFieldInPlace(obj, field, decryptEncryptor, encrypt
 }
 
 function recryptConnection(connection, decryptEncryptor, encryptEncryptor) {
-  connection = recryptObjectPasswordField(connection, 'password', decryptEncryptor, encryptEncryptor);
-  connection = recryptObjectPasswordField(connection, 'sshPassword', decryptEncryptor, encryptEncryptor);
-  connection = recryptObjectPasswordField(connection, 'sshKeyfilePassword', decryptEncryptor, encryptEncryptor);
+  for (const field of fieldsToEncrypt) {
+    connection = recryptObjectPasswordField(connection, field, decryptEncryptor, encryptEncryptor);
+  }
   return connection;
 }
 

packages/types/engines.d.ts

@@ -241,7 +241,7 @@ export interface EngineDriver<TClient = any> extends FilterBehaviourProvider {
   defaultSocketPath?: string;
   authTypeLabel?: string;
   importExportArgs?: any[];
-  connect({ server, port, user, password, database, certificateJson }): Promise<DatabaseHandle<TClient>>;
+  connect({ server, port, user, password, database, connectionDefinition }): Promise<DatabaseHandle<TClient>>;
   close(dbhan: DatabaseHandle<TClient>): Promise<any>;
   query(dbhan: DatabaseHandle<TClient>, sql: string, options?: QueryOptions): Promise<QueryResult>;
   stream(dbhan: DatabaseHandle<TClient>, sql: string, options: StreamOptions);

packages/web/src/appobj/DatabaseAppObject.svelte

@@ -511,7 +511,8 @@ await dbgateApi.executeQuery(${JSON.stringify(
           text: _t('database.exportDbModel', { defaultMessage: 'Export DB model' }),
         },
       isProApp() &&
-        driver?.databaseEngineTypes?.includes('sql') && {
+        driver?.databaseEngineTypes?.includes('sql') &&
+        hasPermission('dbops/chat') && {
           onClick: handleDatabaseChat,
           text: _t('database.databaseChat', { defaultMessage: 'Database chat' }),
         },

packages/web/src/commands/stdCommands.ts

@@ -723,7 +723,8 @@ if (isProApp()) {
     icon: 'icon ai',
     testEnabled: () =>
       getCurrentDatabase() != null &&
-      findEngineDriver(getCurrentDatabase()?.connection, getExtensions())?.databaseEngineTypes?.includes('sql'),
+      findEngineDriver(getCurrentDatabase()?.connection, getExtensions())?.databaseEngineTypes?.includes('sql') &&
+      hasPermission('dbops/chat'),
     onClick: () => {
       openNewTab({
         title: 'Chat',

packages/web/src/datagrid/DataGridCore.svelte

@@ -424,6 +424,7 @@
   import { _t } from '../translations';
   import { isProApp } from '../utility/proTools';
   import SaveArchiveModal from '../modals/SaveArchiveModal.svelte';
+  import hasPermission from '../utility/hasPermission';
 
   export let onLoadNextData = undefined;
   export let grider = undefined;
@@ -1848,6 +1849,7 @@
     // },
     isProApp() && { command: 'dataGrid.sendToDataDeploy' },
     isProApp() &&
+      hasPermission('dbops/charts') &&
       onOpenChart && {
         text: 'Open chart',
         onClick: () => onOpenChart(),

packages/web/src/forms/FormFileInputField.svelte

@@ -0,0 +1,58 @@
+<script lang="ts">
+  import SimpleFilesInput, { ProcessedFile } from '../impexp/SimpleFilesInput.svelte';
+  import { parseFileAsString } from '../utility/parseFileAsString';
+  import { getFormContext } from './FormProviderCore.svelte';
+  import { createEventDispatcher } from 'svelte';
+
+  export let label: string;
+  export let buttonLabel: string = 'Choose File';
+  export let name: string;
+  export let disabled: boolean = false;
+  export let accept: string = '.json,application/json';
+  export let templateProps = {};
+
+  const { template, setFieldValue, values } = getFormContext();
+  const dispatch = createEventDispatcher();
+
+  let fileName: string | null = null;
+  $: hasValue = $values?.[name] != null;
+  $: displayLabel = getDisplayLabel(buttonLabel, hasValue, fileName);
+
+  async function handleFileChange(fileData: ProcessedFile): Promise<void> {
+    const parseResult = await parseFileAsString(fileData.file);
+
+    if (parseResult.success) {
+      fileName = fileData.name;
+      setFieldValue(name, parseResult.data);
+      dispatch('change', {
+        success: true,
+        data: parseResult.data,
+        fileName: fileData.name,
+      });
+    } else {
+      fileName = null;
+      setFieldValue(name, null);
+      dispatch('change', {
+        success: false,
+        error: parseResult.error,
+        fileName: fileData.name,
+      });
+    }
+  }
+
+  function getDisplayLabel(baseLabel: string, hasValue: boolean, fileName: string | null): string {
+    if (!hasValue) {
+      return baseLabel;
+    }
+
+    if (fileName) {
+      return `${baseLabel} (${fileName})`;
+    }
+
+    return `${baseLabel} (JSON loaded)`;
+  }
+</script>
+
+<svelte:component this={template} type="file" {label} {disabled} {...templateProps}>
+  <SimpleFilesInput label={displayLabel} {accept} {disabled} onChange={handleFileChange} {...$$restProps} />
+</svelte:component>

packages/web/src/settings/ConnectionDriverFields.svelte

@@ -18,9 +18,7 @@
   import FormDropDownTextField from '../forms/FormDropDownTextField.svelte';
   import { getConnectionLabel } from 'dbgate-tools';
   import { _t } from '../translations';
-  import FilesInput from '../impexp/FilesInput.svelte';
-  import SimpleFilesInput from '../impexp/SimpleFilesInput.svelte';
-  import FormJsonFileInputField from '../forms/FormJsonFileInputField.svelte';
+  import FormFileInputField from '../forms/FormFileInputField.svelte';
 
   export let getDatabaseList;
   export let currentConnection;
@@ -465,8 +463,8 @@
   />
 {/if}
 
-{#if driver?.showConnectionField('certificateJson', $values, showConnectionFieldArgs)}
-  <FormJsonFileInputField disabled={isConnected} label="Service account key JSON" name="certificateJson" />
+{#if driver?.showConnectionField('connectionDefinition', $values, showConnectionFieldArgs)}
+  <FormFileInputField disabled={isConnected} label="Service account key JSON" name="connectionDefinition" />
 {/if}
 
 {#if driver}

packages/web/src/tabs/QueryTab.svelte

@@ -154,6 +154,7 @@
   import RowsLimitModal from '../modals/RowsLimitModal.svelte';
   import _ from 'lodash';
   import FontIcon from '../icons/FontIcon.svelte';
+  import hasPermission from '../utility/hasPermission';
 
   export let tabid;
   export let conid;
@@ -794,7 +795,7 @@
       hideDisabled
     />
 
-    {#if isProApp() && visibleResultTabs && !busy}
+    {#if isProApp() && visibleResultTabs && !busy && hasPermission('dbops/charts')}
       <ToolStripButton
         icon="icon chart"
         data-testid="QueryTab_openChartButton"
@@ -805,7 +806,7 @@
         Open chart</ToolStripButton
       >
     {/if}
-    {#if isProApp() && !visibleResultTabs}
+    {#if isProApp() && !visibleResultTabs && hasPermission('dbops/charts')}
       <ToolStripButton
         icon="icon chart"
         data-testid="QueryTab_detectChartButton"

packages/web/src/utility/parseFileAsJson.js

@@ -1,26 +1,3 @@
-/**
- * @template [T = any]
- * @typedef {Object} FileParseResultSuccess
- * @property {true} success
- * @property {T} data
- */
-
-/**
- * @typedef {Object} FileParseResultError
- * @property {false} success
- * @property {string} error
- */
-
-/**
- * @template [T = any]
- * @typedef {FileParseResultSuccess<T> | FileParseResultError} FileParseResult
- */
-
-/**
- * @template [T = any]
- * @param {File} file
- * @returns {Promise<FileParseResult<T>>}
- */
 export async function parseFileAsJson(file) {
   try {
     const text = await file.text();

packages/web/src/utility/parseFileAsString.js

@@ -0,0 +1,15 @@
+export async function parseFileAsString(file) {
+  try {
+    const text = await file.text();
+    const data = text;
+    return {
+      success: true,
+      data,
+    };
+  } catch (error) {
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : 'Unknown parsing error',
+    };
+  }
+}

packages/web/src/widgets/AdminPremiumPromoWidget.svelte

@@ -11,6 +11,7 @@
 
   <ul>
     <li>Query designer</li>
+    <li>AI powered database chat</li>
     <li>Unlimited DbGate Cloud storage</li>
     <li>Shared cloud folders</li>
     <li>Charts from query result</li>
@@ -19,8 +20,7 @@
     <li>Backup &amp; restore database</li>
     <li>Advanced ER diagram settings</li>
     <li>Export database model</li>
-    <li>AI assistant</li>
-    <li>libSQL, Turso, CosmosDB, Redshift support</li>
+    <li>Firestore, libSQL, Turso, CosmosDB, Redshift support</li>
     <li>Amazon and Azure identity providers</li>
     <li>E-mail support</li>
   </ul>

workflow-templates/includes.tpl.yaml

@@ -7,7 +7,7 @@ checkout-and-merge-pro:
       repository: dbgate/dbgate-pro
       token: ${{ secrets.GH_TOKEN }}
       path: dbgate-pro
-      ref: 8a4dc2732a7097b5c4c48b4feb62609111cdf3e0
+      ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
   - name: Merge dbgate/dbgate-pro
     run: |
       mkdir ../dbgate-pro