Merge branch 'master' of github.com:dbgate/dbgate

.github/workflows/build-app-pro-beta.yaml

@@ -39,7 +39,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: e4e157db83a2785057ef4778c3297abe758a7505
+          ref: 32bb976b7a07547641d1e8517c86e7c4d70bd088
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-app-pro.yaml

@@ -39,7 +39,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: e4e157db83a2785057ef4778c3297abe758a7505
+          ref: 32bb976b7a07547641d1e8517c86e7c4d70bd088
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-cloud-pro.yaml

@@ -39,7 +39,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: e4e157db83a2785057ef4778c3297abe758a7505
+          ref: 32bb976b7a07547641d1e8517c86e7c4d70bd088
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-docker-pro.yaml

@@ -44,7 +44,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: e4e157db83a2785057ef4778c3297abe758a7505
+          ref: 32bb976b7a07547641d1e8517c86e7c4d70bd088
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-npm-pro.yaml

@@ -32,7 +32,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: e4e157db83a2785057ef4778c3297abe758a7505
+          ref: 32bb976b7a07547641d1e8517c86e7c4d70bd088
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/e2e-pro.yaml

@@ -26,7 +26,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: e4e157db83a2785057ef4778c3297abe758a7505
+          ref: 32bb976b7a07547641d1e8517c86e7c4d70bd088
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

CHANGELOG.md

@@ -8,6 +8,17 @@ Builds:
  - linux - application for linux
  - win - application for Windows
 
+### 6.3.0
+- ADDED: Support for libSQL and Turso (Premium)
+- ADDED: Native backup and restore database for MySQL and PostgreSQL (Premium)
+- REMOVED: DbGate internal dump export for MySQL (replaced with call of mysqldump)
+- REMOVED: Import SQL dump with internal DbGate capabilities (replaced by calling of mysql and psql utilities)
+- FIXED: Many fixes in stream processing (imoprt/export), especialy for MongoDB
+- ADDED: Indicating progress of import/export tasks, better error reporting
+- CHANGED: #1060 - Changed shortcut for AI assistant
+- ADDED: /health endpoint with diagnostic info
+- FIXED: Linux Appimage crash => A JavaScript error occurred in the main process #1065 , #1067
+
 ### 6.2.1
 - ADDED: Commit/rollback and autocommit in scripts #1039
 - FIXED: Doesn't import all the records from MongoDB #1044

e2e-tests/cypress/e2e/browse-data.cy.js

@@ -15,11 +15,16 @@ beforeEach(() => {
 describe('Data browser data', () => {
   it('Export window', () => {
     cy.contains('MySql-connection').click();
-    cy.contains('MyChinook').rightclick();
+    cy.contains('MyChinook').click();
+    cy.contains('Album').rightclick();
     cy.contains('Export').click();
+    cy.contains('Export advanced').click();
     cy.wait(1000);
     // cy.testid('SourceTargetConfig_buttonCurrentArchive_target').click();
-    cy.testid('FormTablesSelect_buttonAll_tables').click();
+    // cy.testid('FormTablesSelect_buttonAll_tables').click();
+    // cy.testid('SourceTargetConfig_tablesSelect_source').click();
+    // cy.find('.listContainer').contains('Album').click();
+    // cy.find('.listContainer').contains('Track').click();
     // cy.wait(4000);
     // cy.contains('All tables').click();
     cy.contains('Run').click();
@@ -76,6 +81,20 @@ describe('Data browser data', () => {
     cy.contains('Aerosmith').should('not.exist');
   });
 
+  it('Data filter', () => {
+    cy.contains('MySql-connection').click();
+    cy.contains('MyChinook').click();
+    cy.contains('Album').click();
+    cy.testid('DataFilterControl_input_Title').type('Rock{enter}');
+    cy.contains('Rows: 7');
+    cy.testid('DataFilterControl_input_AlbumId').type('>10{enter}');
+    cy.contains('Rows: 5');
+    cy.testid('DataFilterControl_filtermenu_Title').click();
+    cy.themeshot('filter');
+    cy.testid('DataGridCore_button_clearFilters').click();
+    cy.contains('Rows: 347');
+  });
+
   it('Data grid screenshots', () => {
     cy.contains('MySql-connection').click();
     cy.window().then(win => {
@@ -350,7 +369,7 @@ describe('Data browser data', () => {
     cy.themeshot('comparesettings');
   });
 
-  it.skip('Query editor - AI assistant', () => {
+  it('Query editor - AI assistant', () => {
     cy.contains('MySql-connection').click();
     cy.contains('MyChinook').click();
     cy.testid('TabsPanel_buttonNewQuery').click();

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "6.2.2-beta.10",
+  "version": "6.3.1",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",

packages/api/src/controllers/auth.js

@@ -43,6 +43,7 @@ function authMiddleware(req, res, next) {
     '/connections/dblogin-app',
     '/connections/dblogin-auth',
     '/connections/dblogin-auth-token',
+    '/health',
   ];
 
   // console.log('********************* getAuthProvider()', getAuthProvider());

packages/api/src/controllers/runners.js

@@ -171,6 +171,7 @@ module.exports = {
       this.rejectRequest(runid, { message: 'No data returned, maybe input data source is too big' });
       logger.info({ code, pid: subprocess.pid }, 'Exited process');
       socket.emit(`runner-done-${runid}`, code);
+      this.opened = this.opened.filter(x => x.runid != runid);
     });
     subprocess.on('error', error => {
       // console.log('... ERROR subprocess', error);
@@ -180,6 +181,7 @@ module.exports = {
         severity: 'error',
         message: error.toString(),
       });
+      this.opened = this.opened.filter(x => x.runid != runid);
     });
     const newOpened = {
       runid,
@@ -224,6 +226,7 @@ module.exports = {
       if (onFinished) {
         onFinished();
       }
+      this.opened = this.opened.filter(x => x.runid != runid);
     });
     subprocess.on('spawn', () => {
       this.dispatchMessage(runid, `Started external process ${command}`);
@@ -241,6 +244,7 @@ module.exports = {
         });
       }
       socket.emit(`runner-done-${runid}`);
+      this.opened = this.opened.filter(x => x.runid != runid);
     });
 
     if (stdinFilePath) {

packages/api/src/controllers/sessions.js

@@ -127,6 +127,9 @@ module.exports = {
       this.dispatchMessage(sesid, 'Query session closed');
       socket.emit(`session-closed-${sesid}`);
     });
+    subprocess.on('error', () => {
+      this.opened = this.opened.filter(x => x.sesid != sesid);
+    });
 
     subprocess.send({
       msgtype: 'connect',

packages/api/src/main.js

@@ -38,6 +38,7 @@ const { getLogger } = require('dbgate-tools');
 const { getDefaultAuthProvider } = require('./auth/authProvider');
 const startCloudUpgradeTimer = require('./utility/cloudUpgrade');
 const { isProApp } = require('./utility/checkLicense');
+const getHealthStatus = require('./utility/healthStatus');
 
 const logger = getLogger('main');
 
@@ -117,6 +118,12 @@ function start() {
     });
   });
 
+  app.get(getExpressPath('/health'), async function (req, res) {
+    res.setHeader('Content-Type', 'application/json');
+    const health = await getHealthStatus();
+    res.end(JSON.stringify(health, null, 2));
+  });
+
   app.use(bodyParser.json({ limit: '50mb' }));
 
   app.use(

packages/api/src/utility/healthStatus.js

@@ -0,0 +1,27 @@
+const os = require('os');
+
+const databaseConnections = require('../controllers/databaseConnections');
+const serverConnections = require('../controllers/serverConnections');
+const sessions = require('../controllers/sessions');
+const runners = require('../controllers/runners');
+
+async function getHealthStatus() {
+  const memory = process.memoryUsage();
+  const cpuUsage = process.cpuUsage();
+
+  return {
+    status: 'ok',
+    databaseConnectionCount: databaseConnections.opened.length,
+    serverConnectionCount: serverConnections.opened.length,
+    sessionCount: sessions.opened.length,
+    runProcessCount: runners.opened.length,
+    memory,
+    cpuUsage,
+    systemMemory: {
+      total: os.totalmem(),
+      free: os.freemem(),
+    },
+  };
+}
+
+module.exports = getHealthStatus;

packages/web/src/datagrid/DataFilterControl.svelte

@@ -296,11 +296,21 @@
   {/if}
   {#if conid && database && driver}
     {#if driver?.databaseEngineTypes?.includes('sql') && foreignKey}
-      <InlineButton on:click={handleShowDictionary} narrow square>
+      <InlineButton
+        on:click={handleShowDictionary}
+        narrow
+        square
+        data-testid={`DataFilterControl_choosevalues_${uniqueName}`}
+      >
         <FontIcon icon="icon dots-horizontal" />
       </InlineButton>
     {:else if (pureName && columnName) || (pureName && uniqueName && driver?.databaseEngineTypes?.includes('document'))}
-      <InlineButton on:click={handleShowValuesModal} narrow square>
+      <InlineButton
+        on:click={handleShowValuesModal}
+        narrow
+        square
+        data-testid={`DataFilterControl_choosevalues_${uniqueName}`}
+      >
         <FontIcon icon="icon dots-vertical" />
       </InlineButton>
     {/if}
@@ -309,7 +319,12 @@
       <FontIcon icon="icon dots-vertical" />
     </InlineButton>
   {/if}
-  <DropDownButton icon="icon filter" menu={createMenu} narrow />
+  <DropDownButton
+    icon="icon filter"
+    menu={createMenu}
+    narrow
+    data-testid={`DataFilterControl_filtermenu_${uniqueName}`}
+  />
   {#if showResizeSplitter}
     <div class="horizontal-split-handle resizeHandleControl" use:splitterDrag={'clientX'} on:resizeSplitter />
   {/if}

packages/web/src/datagrid/DataGridCore.svelte

@@ -1946,7 +1946,11 @@
               style={`width:${headerColWidth}px; min-width:${headerColWidth}px; max-width:${headerColWidth}px`}
             >
               {#if display.filterCount > 0}
-                <InlineButton on:click={() => display.clearFilters()} square>
+                <InlineButton
+                  on:click={() => display.clearFilters()}
+                  square
+                  data-testid="DataGridCore_button_clearFilters"
+                >
                   <FontIcon icon="icon filter-off" />
                 </InlineButton>
               {/if}

packages/web/src/forms/SelectField.svelte

@@ -64,6 +64,7 @@
       {isMulti}
       bind:listOpen
       bind:isFocused
+      class={$$props['data-testid'] ? 'select-testid-' + $$props['data-testid'] : undefined}
     />
   </div>
 {/if}

packages/web/src/impexp/SourceTargetConfig.svelte

@@ -141,6 +141,9 @@
         schemaName={schemaNameField}
         databaseName={databaseNameField}
         name={tablesField}
+        data-testid={direction == 'source'
+          ? 'SourceTargetConfig_tablesSelect_source'
+          : 'SourceTargetConfig_tablesSelect_target'}
         label={_t('importExport.tablesViewsCollections', { defaultMessage: 'Tables / views / collections' })}
       />
     {/if}

packages/web/src/stores.ts

@@ -26,6 +26,10 @@ export interface TabDefinition {
   focused?: boolean;
 }
 
+function getSystemTheme() {
+  return window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches ? 'theme-dark' : 'theme-light';
+}
+
 export function writableWithStorage<T>(defaultValue: T, storageName) {
   const init = localStorage.getItem(storageName);
   const res = writable<T>(init ? safeJsonParse(init, defaultValue, true) : defaultValue);
@@ -100,8 +104,8 @@ export const extensions = writable<ExtensionsDirectory>(null);
 export const visibleCommandPalette = writable(null);
 export const commands = writable({});
 export const currentTheme = getElectron()
-  ? writableSettingsValue('theme-light', 'currentTheme')
-  : writableWithStorage('theme-light', 'currentTheme');
+  ? writableSettingsValue(getSystemTheme(), 'currentTheme')
+  : writableWithStorage(getSystemTheme(), 'currentTheme');
 export const currentEditorTheme = getElectron()
   ? writableSettingsValue(null, 'currentEditorTheme')
   : writableWithStorage(null, 'currentEditorTheme');
@@ -194,8 +198,13 @@ export const connectionAppObjectSearchSettings = writableWithStorage(
 );
 
 export const currentThemeDefinition = derived([currentTheme, extensions], ([$currentTheme, $extensions]) =>
-  $extensions.themes.find(x => x.themeClassName == $currentTheme)
+  $extensions?.themes?.find(x => x.themeClassName == $currentTheme)
 );
+currentThemeDefinition.subscribe(value => {
+  if (value?.themeType) {
+    localStorage.setItem('currentThemeType', value?.themeType);
+  }
+});
 export const openedConnectionsWithTemporary = derived(
   [openedConnections, temporaryOpenedConnections, openedSingleDatabaseConnections],
   ([$openedConnections, $temporaryOpenedConnections, $openedSingleDatabaseConnections]) =>

packages/web/src/widgets/AppStartInfo.svelte

@@ -1,8 +1,14 @@
 <script lang="ts">
   export let message;
+
+  const isDark =
+    localStorage.getItem('currentThemeType') === 'dark' ||
+    (!localStorage.getItem('currentThemeType') &&
+      window.matchMedia &&
+      window.matchMedia('(prefers-color-scheme: dark)').matches);
 </script>
 
-<div class="starting-dbgate">
+<div class="starting-dbgate" class:isDark>
   <div class="inner-flex">
     <div class="lds-ellipsis">
       <div />
@@ -15,21 +21,34 @@
 </div>
 
 <style>
+  @media (prefers-color-scheme: dark) {
+    #starting-dbgate {
+      background-color: #111;
+      color: #e3e3e3;
+    }
+  }
   .lds-ellipsis {
     display: inline-block;
     position: relative;
     width: 80px;
     height: 80px;
   }
+  .isDark {
+    background-color: #111;
+    color: #e3e3e3;
+  }
   .lds-ellipsis div {
     position: absolute;
     top: 33px;
     width: 13px;
     height: 13px;
     border-radius: 50%;
-    background: #000;
+    background: #262626;
     animation-timing-function: cubic-bezier(0, 1, 1, 0);
   }
+  .isDark .lds-ellipsis div {
+    background: #e3e3e3;
+  }
   .lds-ellipsis div:nth-child(1) {
     left: 8px;
     animation: lds-ellipsis1 0.6s infinite;

setCurrentVersion.js

@@ -47,6 +47,11 @@ changePackageFile('packages/serve', json.version);
 changePackageFile('packages/filterparser', json.version);
 changePackageFile('packages/dbmodel', json.version);
 
+if (fs.existsSync('packer/azure-ubuntu.pkr.hcl')) {
+  const text = fs.readFileSync('packer/azure-ubuntu.pkr.hcl', { encoding: 'utf-8' });
+  fs.writeFileSync('packer/azure-ubuntu.pkr.hcl', text.replace(/image_version\s*=\s*"[^"]+"/, `image_version = "${json.version}"`), { encoding: 'utf-8' });
+}
+
 for (const package of fs.readdirSync('plugins')) {
   if (!package.startsWith('dbgate-plugin-')) continue;
 

workflow-templates/includes.tpl.yaml

@@ -7,7 +7,7 @@ checkout-and-merge-pro:
       repository: dbgate/dbgate-pro
       token: ${{ secrets.GH_TOKEN }}
       path: dbgate-pro
-      ref: e4e157db83a2785057ef4778c3297abe758a7505
+      ref: 32bb976b7a07547641d1e8517c86e7c4d70bd088
   - name: Merge dbgate/dbgate-pro
     run: |
       mkdir ../dbgate-pro