schemaList moved from dbinfo to separate request

integration-tests/__tests__/schema-tests.spec.js

@@ -23,17 +23,16 @@ describe('Schema tests', () => {
     testWrapper(async (conn, driver, engine) => {
       await baseStructure(conn, driver);
       const structure1 = await driver.analyseFull(conn);
-      expect(structure1.schemas.find(x => x.schemaName == 'myschema')).toBeFalsy();
+      const schemas1 = await driver.listSchemas(conn);
-      const count = structure1.schemas.length;
+      expect(schemas1.find(x => x.schemaName == 'myschema')).toBeFalsy();
+      const count = schemas1.length;
       expect(structure1.tables.length).toEqual(2);
       await runCommandOnDriver(conn, driver, dmp => dmp.createSchema('myschema'));
       const structure2 = await driver.analyseIncremental(conn, structure1);
-      expect(structure2.schemas.find(x => x.schemaName == 'myschema')).toBeTruthy();
+      const schemas2 = await driver.listSchemas(conn);
-      expect(structure2.tables.length).toEqual(2);
+      expect(schemas2.find(x => x.schemaName == 'myschema')).toBeTruthy();
-      expect(structure2.schemas.length).toEqual(count + 1);
+      expect(schemas2.length).toEqual(count + 1);
-
+      expect(structure2).toBeNull();
-      const structure3 = await driver.analyseIncremental(conn, structure2);
-      expect(structure3).toBeNull();
     })
   );
 
@@ -44,29 +43,14 @@ describe('Schema tests', () => {
       await runCommandOnDriver(conn, driver, dmp => dmp.createSchema('myschema'));
 
       const structure1 = await driver.analyseFull(conn);
-      expect(structure1.schemas.find(x => x.schemaName == 'myschema')).toBeTruthy();
+      const schemas1 = await driver.listSchemas(conn);
+      expect(schemas1.find(x => x.schemaName == 'myschema')).toBeTruthy();
       expect(structure1.tables.length).toEqual(2);
       await runCommandOnDriver(conn, driver, dmp => dmp.dropSchema('myschema'));
       const structure2 = await driver.analyseIncremental(conn, structure1);
-      expect(structure2.schemas.find(x => x.schemaName == 'myschema')).toBeFalsy();
+      const schemas2 = await driver.listSchemas(conn);
-      expect(structure2.tables.length).toEqual(2);
+      expect(schemas2.find(x => x.schemaName == 'myschema')).toBeFalsy();
-
+      expect(structure2).toBeNull();
-      const structure3 = await driver.analyseIncremental(conn, structure2);
-      expect(structure3).toBeNull();
-    })
-  );
-
-  test.each(engines.filter(x => x.supportSchemas).map(engine => [engine.label, engine]))(
-    'Create table - keep schemas - %s',
-    testWrapper(async (conn, driver, engine) => {
-      await baseStructure(conn, driver);
-      const structure1 = await driver.analyseFull(conn);
-      const count = structure1.schemas.length;
-      expect(structure1.tables.length).toEqual(2);
-      await driver.query(conn, `create table t3 (id int not null primary key)`);
-      const structure2 = await driver.analyseIncremental(conn, structure1);
-      expect(structure2.tables.length).toEqual(3);
-      expect(structure2.schemas.length).toEqual(count);
     })
   );
 });

packages/api/src/controllers/databaseConnections.js

@@ -199,7 +199,7 @@ module.exports = {
     return res.result || null;
   },
 
-  async loadDataCore(msgtype, { conid, database, ...args }, req) {
+    async loadDataCore(msgtype, { conid, database, ...args }, req) {
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype, ...args });
@@ -213,6 +213,12 @@ module.exports = {
     return res.result || null;
   },
 
+  schemaList_meta: true,
+  async schemaList({ conid, database }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('schemaList', { conid, database });
+  },
+
   loadKeys_meta: true,
   async loadKeys({ conid, database, root, filter }, req) {
     testConnectionPermission(conid, req);

packages/api/src/proc/databaseConnectionProcess.js

@@ -213,6 +213,10 @@ async function handleDriverDataCore(msgid, callMethod) {
   }
 }
 
+async function handleSchemaList({ msgid }) {
+  return handleDriverDataCore(msgid, driver => driver.listSchemas(systemConnection));
+}
+
 async function handleCollectionData({ msgid, options }) {
   return handleDriverDataCore(msgid, driver => driver.readCollection(systemConnection, options));
 }
@@ -337,6 +341,7 @@ const messageHandlers = {
   loadFieldValues: handleLoadFieldValues,
   sqlSelect: handleSqlSelect,
   exportKeys: handleExportKeys,
+  schemaList: handleSchemaList,
   // runCommand: handleRunCommand,
 };
 

packages/tools/src/DatabaseAnalyser.ts

@@ -5,7 +5,6 @@ import _pick from 'lodash/pick';
 import _compact from 'lodash/compact';
 import { getLogger } from './getLogger';
 import { type Logger } from 'pinomin';
-import stableStringify from 'json-stable-stringify';
 
 const logger = getLogger('dbAnalyser');
 
@@ -71,10 +70,7 @@ export class DatabaseAnalyser {
   async fullAnalysis() {
     const res = this.addEngineField(await this._runAnalysis());
     // console.log('FULL ANALYSIS', res);
-    return {
+    return res;
-      ...res,
-      schemas: await this.readSchemaList(),
-    };
   }
 
   async singleObjectAnalysis(name, typeField) {
@@ -91,10 +87,6 @@ export class DatabaseAnalyser {
     return obj;
   }
 
-  async readSchemaList() {
-    return undefined;
-  }
-
   async incrementalAnalysis(structure) {
     this.structure = structure;
 
@@ -107,35 +99,22 @@ export class DatabaseAnalyser {
     const structureModifications = modifications.filter(x => x.action != 'setTableRowCounts');
     const setTableRowCounts = modifications.find(x => x.action == 'setTableRowCounts');
 
-    let structureUpdated = null;
+    let structureWithRowCounts = null;
     if (setTableRowCounts) {
       const newStructure = mergeTableRowCounts(structure, setTableRowCounts.rowCounts);
       if (areDifferentRowCounts(structure, newStructure)) {
-        structureUpdated = newStructure;
+        structureWithRowCounts = newStructure;
       }
     }
 
-    const schemas = await this.readSchemaList();
-    const areSchemasDifferent = stableStringify(schemas) != stableStringify(this.structure.schemas);
-    if (areSchemasDifferent) {
-      structureUpdated = {
-        ...structure,
-        ...structureUpdated,
-        schemas,
-      };
-    }
-
     if (structureModifications.length == 0) {
-      return structureUpdated ? this.addEngineField(structureUpdated) : null;
+      return structureWithRowCounts ? this.addEngineField(structureWithRowCounts) : null;
     }
 
     this.modifications = structureModifications;
-    if (structureUpdated) this.structure = structureUpdated;
+    if (structureWithRowCounts) this.structure = structureWithRowCounts;
     logger.info({ modifications: this.modifications }, 'DB modifications detected:');
-    return {
+    return this.addEngineField(this.mergeAnalyseResult(await this._runAnalysis()));
-      ...this.addEngineField(this.mergeAnalyseResult(await this._runAnalysis())),
-      schemas,
-    };
   }
 
   mergeAnalyseResult(newlyAnalysed) {
@@ -359,7 +338,6 @@ export class DatabaseAnalyser {
       functions: [],
       procedures: [],
       triggers: [],
-      schemas: [],
     };
   }
 

packages/types/dbinfo.d.ts

@@ -139,7 +139,6 @@ export interface DatabaseInfoObjects {
 }
 
 export interface DatabaseInfo extends DatabaseInfoObjects {
-  schemas?: SchemaInfo[];
   engine?: string;
   defaultSchema?: string;
 }

packages/types/engines.d.ts

@@ -11,6 +11,7 @@ import {
   FunctionInfo,
   TriggerInfo,
   CollectionInfo,
+  SchemaInfo,
 } from './dbinfo';
 import { FilterBehaviour } from './filter-type';
 
@@ -230,6 +231,7 @@ export interface EngineDriver extends FilterBehaviourProvider {
   ): any[];
   // adapts table info from different source (import, other database) to be suitable for this database
   adaptTableInfo(table: TableInfo): TableInfo;
+  async listSchemas(pool): SchemaInfo[];
 
   analyserClass?: any;
   dumperClass?: any;

packages/web/src/impexp/FormSchemaSelect.svelte

@@ -1,15 +1,15 @@
 <script lang="ts">
   import { getFormContext } from '../forms/FormProviderCore.svelte';
   import FormSelectField from '../forms/FormSelectField.svelte';
-  import { useDatabaseInfo, useDatabaseList } from '../utility/metadataLoaders';
+  import { useSchemaList } from '../utility/metadataLoaders';
 
   export let conidName;
   export let databaseName;
 
   const { values } = getFormContext();
-  $: dbinfo = useDatabaseInfo({ conid: $values[conidName], database: values[databaseName] });
+  $: schemaList = useSchemaList({ conid: $values[conidName], database: values[databaseName] });
 
-  $: schemaOptions = (($dbinfo && $dbinfo.schemas) || []).map(schema => ({
+  $: schemaOptions = ($schemaList || []).map(schema => ({
     value: schema.schemaName,
     label: schema.schemaName,
   }));

packages/web/src/query/codeCompletion.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import { addCompleter, setCompleters } from 'ace-builds/src-noconflict/ext-language_tools';
-import { getDatabaseInfo } from '../utility/metadataLoaders';
+import { getDatabaseInfo, getSchemaList } from '../utility/metadataLoaders';
 import analyseQuerySources from './analyseQuerySources';
 import { getStringSettingsValue } from '../settings/settingsTools';
 
@@ -24,9 +24,9 @@ const COMMON_KEYWORDS = [
   'go',
 ];
 
-function createTableLikeList(dbinfo, schemaCondition) {
+function createTableLikeList(schemaList, dbinfo, schemaCondition) {
   return [
-    ...(dbinfo.schemas?.map(x => ({
+    ...(schemaList?.map(x => ({
       name: x.schemaName,
       value: x.schemaName,
       caption: x.schemaName,
@@ -78,6 +78,7 @@ export function mountCodeCompletion({ conid, database, editor, getText }) {
       const cursor = session.selection.cursor;
       const line = session.getLine(cursor.row).slice(0, cursor.column);
       const dbinfo = await getDatabaseInfo({ conid, database });
+      const schemaList = await getSchemaList({ conid, database });
 
       const convertUpper = getStringSettingsValue('sqlEditor.sqlCommandsCase', 'upperCase') == 'upperCase';
 
@@ -147,9 +148,9 @@ export function mountCodeCompletion({ conid, database, editor, getText }) {
               ];
             }
           } else {
-            const schema = (dbinfo.schemas || []).find(x => x.schemaName == colMatch[1]);
+            const schema = (schemaList || []).find(x => x.schemaName == colMatch[1]);
             if (schema) {
-              list = createTableLikeList(dbinfo, x => x.schemaName == schema.schemaName);
+              list = createTableLikeList(schemaList, dbinfo, x => x.schemaName == schema.schemaName);
             }
           }
         } else {
@@ -167,7 +168,11 @@ export function mountCodeCompletion({ conid, database, editor, getText }) {
           } else {
             list = [
               ...(onlyTables ? [] : list),
-              ...createTableLikeList(dbinfo, x => !dbinfo.defaultSchema || dbinfo.defaultSchema == x.schemaName),
+              ...createTableLikeList(
+                schemaList,
+                dbinfo,
+                x => !dbinfo.defaultSchema || dbinfo.defaultSchema == x.schemaName
+              ),
 
               ...(onlyTables
                 ? []

packages/web/src/settings/ConnectionDriverFields.svelte

@@ -249,6 +249,10 @@
   <FormCheckboxField label={`Use only database ${defaultDatabase}`} name="singleDatabase" disabled={isConnected} />
 {/if}
 
+{#if driver?.showConnectionField('useSeparateSchemas', $values, showConnectionFieldArgs)}
+  <FormCheckboxField label={`Use schemas separately (use this if you have many large schemas)`} name="useSeparateSchemas" disabled={isConnected} />
+{/if}
+
 {#if driver}
   <div class="row">
     <div class="col-6 mr-1">

packages/web/src/tableeditor/TableEditor.svelte

@@ -92,6 +92,7 @@
   export let driver;
   export let resetCounter;
   export let isCreateTable;
+  export let schemaList;
 
   $: isWritable = !!setTableInfo;
 
@@ -172,7 +173,7 @@
         title="Table properties"
         fieldDefinitions={tableFormOptions ?? []}
         pureNameTitle={isCreateTable ? 'Table name' : null}
-        schemaList={isCreateTable && dbInfo?.schemas?.length >= 0 ? dbInfo?.schemas : null}
+        schemaList={isCreateTable && schemaList?.length >= 0 ? schemaList : null}
         values={_.pick(tableInfo, ['schemaName', 'pureName', ...(tableFormOptions ?? []).map(x => x.name)])}
         onChangeValues={vals => {
           if (!_.isEmpty(vals)) {

packages/web/src/tabs/TableStructureTab.svelte

@@ -44,7 +44,7 @@
   import TableEditor from '../tableeditor/TableEditor.svelte';
   import createActivator, { getActiveComponent } from '../utility/createActivator';
 
-  import { useConnectionInfo, useDatabaseInfo, useDbCore } from '../utility/metadataLoaders';
+  import { useConnectionInfo, useDatabaseInfo, useDbCore, useSchemaList } from '../utility/metadataLoaders';
   import { showModal } from '../modals/modalTools';
   import ConfirmSqlModal from '../modals/ConfirmSqlModal.svelte';
   import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
@@ -75,6 +75,7 @@
   $: tableInfoWithPairingId = $tableInfo ? generateTablePairingId($tableInfo) : null;
   $: connection = useConnectionInfo({ conid });
   $: driver = findEngineDriver($connection, $extensions);
+  $: schemaList = useSchemaList({ conid, database });
 
   const { editorState, editorValue, setEditorData, clearEditorData } = useEditorData({ tabid });
 
@@ -146,6 +147,7 @@
     bind:this={domEditor}
     tableInfo={showTable}
     dbInfo={$dbInfo}
+    schemaList={$schemaList}
     {driver}
     {resetCounter}
     isCreateTable={objectTypeField == 'tables' && !$editorValue?.base}

packages/web/src/utility/metadataLoaders.ts

@@ -13,6 +13,12 @@ const databaseInfoLoader = ({ conid, database }) => ({
   transform: extendDatabaseInfo,
 });
 
+const schemaListLoader = ({ conid, database }) => ({
+  url: 'database-connections/schema-list',
+  params: { conid, database },
+  reloadTrigger: { key: `schema-list-changed`, conid, database },
+});
+
 // const tableInfoLoader = ({ conid, database, schemaName, pureName }) => ({
 //   url: 'metadata/table-info',
 //   params: { conid, database, schemaName, pureName },
@@ -449,3 +455,9 @@ export function useAuthTypes(args) {
 // export function useDatabaseKeys(args) {
 //   return useCore(databaseKeysLoader, args);
 // }
+export function getSchemaList(args) {
+  return getCore(schemaListLoader, args);
+}
+export function useSchemaList(args) {
+  return useCore(schemaListLoader, args);
+}

packages/web/src/widgets/SchemaSelector.svelte

@@ -4,14 +4,13 @@
 
   import _ from 'lodash';
   import FontIcon from '../icons/FontIcon.svelte';
-  import { DatabaseInfo } from 'dbgate-types';
   import { showModal } from '../modals/modalTools';
   import ConfirmModal from '../modals/ConfirmModal.svelte';
   import { runOperationOnDatabase } from '../modals/ConfirmSqlModal.svelte';
   import InputTextModal from '../modals/InputTextModal.svelte';
   import { appliedCurrentSchema } from '../stores';
 
-  export let dbinfo: DatabaseInfo;
+  export let schemaList;
   export let selectedSchema;
   export let objectList;
 
@@ -44,7 +43,7 @@
   }
 
   $: schemaList = _.uniq(
-    _.compact([selectedSchema, ...Object.keys(countBySchema), ...(dbinfo?.schemas?.map(x => x.schemaName) ?? [])])
+    _.compact([selectedSchema, ...Object.keys(countBySchema), ...(schemaList?.map(x => x.schemaName) ?? [])])
   );
   $: countBySchema = computeCountBySchema(objectList ?? []);
 

packages/web/src/widgets/SqlObjectList.svelte

@@ -16,7 +16,7 @@
   import InlineButton from '../buttons/InlineButton.svelte';
   import SearchInput from '../elements/SearchInput.svelte';
   import WidgetsInnerContainer from './WidgetsInnerContainer.svelte';
-  import { useConnectionInfo, useDatabaseInfo, useDatabaseStatus, useUsedApps } from '../utility/metadataLoaders';
+  import { useConnectionInfo, useDatabaseInfo, useDatabaseStatus, useSchemaList, useUsedApps } from '../utility/metadataLoaders';
   import SearchBoxWrapper from '../elements/SearchBoxWrapper.svelte';
   import AppObjectList from '../appobj/AppObjectList.svelte';
   import _ from 'lodash';
@@ -46,6 +46,7 @@
 
   $: objects = useDatabaseInfo({ conid, database });
   $: status = useDatabaseStatus({ conid, database });
+  $: schemaList = useSchemaList({ conid, database });
 
   $: connection = useConnectionInfo({ conid });
   $: driver = findEngineDriver($connection, $extensions);
@@ -144,7 +145,7 @@
     </InlineButton>
   </SearchBoxWrapper>
   <SchemaSelector
-    dbinfo={$objects}
+    schemaList={$schemaList}
     bind:selectedSchema
     objectList={flatFilteredList}
     valueStorageKey={`sql-object-list-schema-${conid}-${database}`}

plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js

@@ -79,12 +79,6 @@ class MsSqlAnalyser extends DatabaseAnalyser {
     this.singleObjectId = resId.rows[0].id;
   }
 
-  async readSchemaList() {
-    const schemaRows = await this.analyserQuery('getSchemas');
-    const schemas = schemaRows.rows;
-    return schemas;
-  }
-
   async _runAnalysis() {
     this.feedback({ analysingMessage: 'Loading tables' });
     const tablesRows = await this.analyserQuery('tables', ['tables']);

plugins/dbgate-plugin-mssql/src/backend/driver.js

@@ -150,6 +150,10 @@ const driver = {
   getAccessTokenFromAuth: (connection, req) => {
     return req?.user?.msentraToken;
   },
+  async listSchemas(pool) {
+    const { rows } = await this.query(pool, 'select schema_id as objectId, name as schemaName from sys.schemas');
+    return rows;
+  },
 };
 
 driver.initialize = dbgateEnv => {

plugins/dbgate-plugin-mssql/src/backend/sql/getSchemas.js

@@ -1 +0,0 @@
-module.exports = `select schema_id as objectId, name as schemaName from sys.schemas`;

plugins/dbgate-plugin-mssql/src/backend/sql/index.js

@@ -8,7 +8,6 @@ const loadSqlCode = require('./loadSqlCode');
 const views = require('./views');
 const programmables = require('./programmables');
 const viewColumns = require('./viewColumns');
-const getSchemas = require('./getSchemas');
 const indexes = require('./indexes');
 const indexcols = require('./indexcols');
 
@@ -22,7 +21,6 @@ module.exports = {
   views,
   programmables,
   viewColumns,
-  getSchemas,
   indexes,
   indexcols,
   tableSizes,

plugins/dbgate-plugin-postgres/src/backend/Analyser.js

@@ -312,17 +312,6 @@ class Analyser extends DatabaseAnalyser {
     return res;
   }
 
-  async readSchemaList() {
-    const schemaRows = await this.analyserQuery('getSchemas');
-
-    const schemas = schemaRows.rows.map(x => ({
-      schemaName: x.schema_name,
-      objectId: `schemas:${x.schema_name}`,
-    }));
-
-    return schemas;
-  }
-
   async _getFastSnapshot() {
     const tableModificationsQueryData = this.driver.dialect.stringAgg
       ? await this.analyserQuery('tableModifications')

plugins/dbgate-plugin-postgres/src/backend/drivers.js

@@ -4,7 +4,7 @@ const stream = require('stream');
 const driverBases = require('../frontend/drivers');
 const Analyser = require('./Analyser');
 const pg = require('pg');
-const { getLogger, createBulkInsertStreamBase, makeUniqueColumnNames } = global.DBGATE_PACKAGES['dbgate-tools'];;
+const { getLogger, createBulkInsertStreamBase, makeUniqueColumnNames } = global.DBGATE_PACKAGES['dbgate-tools'];
 
 const logger = getLogger('postreDriver');
 
@@ -267,6 +267,20 @@ const drivers = driverBases.map(driverBase => ({
       },
     ];
   },
+
+  async listSchemas(pool) {
+    const schemaRows = await this.query(
+      pool,
+      'select oid as "object_id", nspname as "schema_name" from pg_catalog.pg_namespace'
+    );
+
+    const schemas = schemaRows.rows.map(x => ({
+      schemaName: x.schema_name,
+      objectId: x.object_id,
+    }));
+
+    return schemas;
+  },
 }));
 
 module.exports = drivers;

plugins/dbgate-plugin-postgres/src/backend/sql/getSchemas.js

@@ -1 +0,0 @@
-module.exports = `select oid as "object_id", nspname as "schema_name" from pg_catalog.pg_namespace`;

plugins/dbgate-plugin-postgres/src/backend/sql/index.js

@@ -12,7 +12,6 @@ const matviewColumns = require('./matviewColumns');
 const indexes = require('./indexes');
 const indexcols = require('./indexcols');
 const uniqueNames = require('./uniqueNames');
-const getSchemas = require('./getSchemas');
 const geometryColumns = require('./geometryColumns');
 const geographyColumns = require('./geographyColumns');
 
@@ -40,5 +39,4 @@ module.exports = {
   uniqueNames,
   geometryColumns,
   geographyColumns,
-  getSchemas,
 };

plugins/dbgate-plugin-postgres/src/frontend/drivers.js

@@ -136,7 +136,9 @@ const postgresDriverBase = {
     }
 
     return (
-      ['authType', 'user', 'password', 'defaultDatabase', 'singleDatabase', 'isReadOnly'].includes(field) ||
+      ['authType', 'user', 'password', 'defaultDatabase', 'singleDatabase', 'isReadOnly', 'useSeparateSchemas'].includes(
+        field
+      ) ||
       (values.authType == 'socket' && ['socketPath'].includes(field)) ||
       (values.authType != 'socket' && ['server', 'port'].includes(field))
     );
@@ -242,7 +244,8 @@ const redshiftDriver = {
   title: 'Amazon Redshift',
   defaultPort: 5439,
   databaseUrlPlaceholder: 'e.g. redshift-cluster-1.xxxx.redshift.amazonaws.com:5439/dev',
-  showConnectionField: (field, values) => ['databaseUrl', 'user', 'password', 'isReadOnly'].includes(field),
+  showConnectionField: (field, values) =>
+    ['databaseUrl', 'user', 'password', 'isReadOnly', 'useSeparateSchemas'].includes(field),
   beforeConnectionSave: connection => {
     const { databaseUrl } = connection;
     if (databaseUrl) {