postgresql materialized views #123

integration-tests/__tests__/object-analyse.spec.js

@@ -32,7 +32,7 @@ describe('Object analyse', () => {
       const structure = await driver.analyseFull(conn);
 
       expect(structure[type].length).toEqual(1);
-      expect(structure[type][0]).toEqual(type == 'views' ? view1Match : obj1Match);
+      expect(structure[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
     })
   );
 
@@ -47,7 +47,7 @@ describe('Object analyse', () => {
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
       expect(structure2[type].length).toEqual(2);
-      expect(structure2[type].find(x => x.pureName == 'obj1')).toEqual(type == 'views' ? view1Match : obj1Match);
+      expect(structure2[type].find(x => x.pureName == 'obj1')).toEqual(type.includes('views') ? view1Match : obj1Match);
     })
   );
 
@@ -63,7 +63,7 @@ describe('Object analyse', () => {
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
       expect(structure2[type].length).toEqual(1);
-      expect(structure2[type][0]).toEqual(type == 'views' ? view1Match : obj1Match);
+      expect(structure2[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
     })
   );
 
@@ -83,7 +83,7 @@ describe('Object analyse', () => {
       const structure3 = await driver.analyseIncremental(conn, structure2);
 
       expect(structure3[type].length).toEqual(1);
-      expect(structure3[type][0]).toEqual(type == 'views' ? view1Match : obj1Match);
+      expect(structure3[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
     })
   );
 });

integration-tests/engines.js

@@ -5,6 +5,13 @@ const views = {
   drop1: 'DROP VIEW obj1',
   drop2: 'DROP VIEW obj2',
 };
+const matviews = {
+  type: 'matviews',
+  create1: 'CREATE MATERIALIZED VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE MATERIALIZED VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP MATERIALIZED VIEW obj1',
+  drop2: 'DROP MATERIALIZED VIEW obj2',
+};
 
 const engines = [
   {
@@ -39,6 +46,7 @@ const engines = [
     },
     objects: [
       views,
+      matviews,
       {
         type: 'procedures',
         create1: 'CREATE PROCEDURE obj1() LANGUAGE SQL AS $$  select * from t1 $$',

packages/api/src/shell/requirePlugin.js

@@ -22,7 +22,7 @@ function requirePlugin(packageName, requiredPlugin = null) {
       // @ts-ignore
       module = __non_webpack_require__(modulePath);
     } catch (err) {
-      console.log('Failed load webpacked module', err.message);
+      // console.log('Failed load webpacked module', err.message);
       module = require(modulePath);
     }
     requiredPlugin = module.__esModule ? module.default : module;

packages/tools/src/DatabaseAnalyser.ts

@@ -66,7 +66,7 @@ export class DatabaseAnalyser {
     }
 
     const res = {};
-    for (const field of ['tables', 'collections', 'views', 'functions', 'procedures', 'triggers']) {
+    for (const field of ['tables', 'collections', 'views', 'matviews', 'functions', 'procedures', 'triggers']) {
       const removedIds = this.modifications
         .filter(x => x.action == 'remove' && x.objectTypeField == field)
         .map(x => x.objectId);
@@ -159,6 +159,7 @@ export class DatabaseAnalyser {
       ...this.getDeletedObjectsForField(snapshot, 'tables'),
       ...this.getDeletedObjectsForField(snapshot, 'collections'),
       ...this.getDeletedObjectsForField(snapshot, 'views'),
+      ...this.getDeletedObjectsForField(snapshot, 'matviews'),
       ...this.getDeletedObjectsForField(snapshot, 'procedures'),
       ...this.getDeletedObjectsForField(snapshot, 'functions'),
       ...this.getDeletedObjectsForField(snapshot, 'triggers'),
@@ -179,6 +180,10 @@ export class DatabaseAnalyser {
         res.push({ objectTypeField: field, action: 'all' });
         continue;
       }
+      if (items === undefined) {
+        // skip - undefined meens, that field is not supported
+        continue;
+      }
       for (const item of items) {
         const { objectId, schemaName, pureName, contentHash } = item;
         const obj = this.structure[field].find(x => x.objectId == objectId);
@@ -211,6 +216,7 @@ export class DatabaseAnalyser {
       tables: [],
       collections: [],
       views: [],
+      matviews: [],
       functions: [],
       procedures: [],
       triggers: [],

packages/tools/src/structureTools.ts

@@ -64,6 +64,10 @@ function fillTableExtendedInfo(db: DatabaseInfo): DatabaseInfo {
       ...obj,
       objectTypeField: 'views',
     })),
+    matviews: (db.matviews || []).map(obj => ({
+      ...obj,
+      objectTypeField: 'matviews',
+    })),
     procedures: (db.procedures || []).map(obj => ({
       ...obj,
       objectTypeField: 'procedures',

packages/types/dbinfo.d.ts

@@ -95,6 +95,7 @@ export interface DatabaseInfoObjects {
   tables: TableInfo[];
   collections: CollectionInfo[];
   views: ViewInfo[];
+  matviews: ViewInfo[];
   procedures: ProcedureInfo[];
   functions: FunctionInfo[];
   triggers: TriggerInfo[];

packages/web/src/appobj/DatabaseObjectAppObject.svelte

@@ -6,6 +6,7 @@
     tables: 'img table',
     collections: 'img collection',
     views: 'img view',
+    matviews: 'img view',
     procedures: 'img procedure',
     functions: 'img function',
   };
@@ -14,6 +15,7 @@
     tables: 'TableDataTab',
     collections: 'CollectionDataTab',
     views: 'ViewDataTab',
+    matviews: 'ViewDataTab',
   };
 
   const menus = {
@@ -146,6 +148,63 @@
         },
       },
     ],
+    matviews: [
+      {
+        label: 'Open data',
+        tab: 'ViewDataTab',
+        forceNewTab: true,
+      },
+      {
+        label: 'Open structure',
+        tab: 'TableStructureTab',
+      },
+      {
+        label: 'Query designer',
+        isQueryDesigner: true,
+      },
+      {
+        divider: true,
+      },
+      {
+        label: 'Export',
+        isExport: true,
+      },
+      {
+        label: 'Open in free table editor',
+        isOpenFreeTable: true,
+      },
+      {
+        label: 'Open active chart',
+        isActiveChart: true,
+      },
+      {
+        divider: true,
+      },
+      {
+        label: 'SQL: CREATE MATERIALIZED VIEW',
+        scriptTemplate: 'CREATE OBJECT',
+      },
+      {
+        label: 'SQL: CREATE TABLE',
+        scriptTemplate: 'CREATE TABLE',
+      },
+      {
+        label: 'SQL: SELECT',
+        scriptTemplate: 'SELECT',
+      },
+      {
+        label: 'SQL Generator: CREATE MATERIALIZED VIEW',
+        sqlGeneratorProps: {
+          createMatviews: true,
+        },
+      },
+      {
+        label: 'SQL Generator: DROP MATERIALIZED VIEW',
+        sqlGeneratorProps: {
+          dropMatviews: true,
+        },
+      },
+    ],
     procedures: [
       {
         label: 'SQL: CREATE PROCEDURE',

packages/web/src/modals/SqlGeneratorModal.svelte

@@ -41,6 +41,7 @@
     createTables: true,
     createForeignKeys: true,
     createViews: true,
+    createMatviews: true,
     createProcedures: true,
     createFunctions: true,
     createTriggers: true,
@@ -48,6 +49,8 @@
 
   export let initialObjects = null;
 
+  const OBJ_TYPE_LABELS = { Matview: 'Materialized view' };
+
   let busy = false;
   let managerSize;
   let objectsFilter = '';
@@ -72,7 +75,7 @@
   $: generatePreview($valuesStore, $checkedObjectsStore);
 
   $: objectList = _.flatten(
-    ['tables', 'views', 'procedures', 'functions'].map(objectTypeField =>
+    ['tables', 'views', 'matviews', 'procedures', 'functions'].map(objectTypeField =>
       _.sortBy(
         (($dbinfo || {})[objectTypeField] || []).map(obj => ({ ...obj, objectTypeField })),
         ['schemaName', 'pureName']
@@ -125,6 +128,7 @@
     );
     closeCurrentModal();
   }
+
 </script>
 
 <FormProviderCore values={valuesStore} template={FormFieldTemplateTiny}>
@@ -211,8 +215,8 @@
 
                   <FormCheckboxField label="Truncate tables (delete all rows)" name="truncate" />
 
-                  {#each ['View', 'Procedure', 'Function', 'Trigger'] as objtype}
+                  {#each ['View', 'MatView', 'Procedure', 'Function', 'Trigger'] as objtype}
-                    <div class="obj-heading">{objtype}s</div>
+                    <div class="obj-heading">{OBJ_TYPE_LABELS[objtype] || objtype}s</div>
                     <FormCheckboxField label="Create" name={`create${objtype}s`} />
                     <FormCheckboxField label="Drop" name={`drop${objtype}s`} />
                     {#if values[`drop${objtype}s`]}
@@ -254,4 +258,5 @@
   .dbname {
     color: var(--theme-font-3);
   }
+
 </style>

packages/web/src/utility/metadataLoaders.ts

@@ -260,9 +260,18 @@ export function useDbCore(args, objectTypeField = undefined) {
   if (!dbStore) return null;
   return derived(dbStore, db => {
     if (!db) return null;
+    if (_.isArray(objectTypeField)) {
+      for (const field of objectTypeField) {
+        const res = db[field || args.objectTypeField].find(
+          x => x.pureName == args.pureName && x.schemaName == args.schemaName
+        );
+        if (res) return res;
+      }
+    } else {
       return db[objectTypeField || args.objectTypeField].find(
         x => x.pureName == args.pureName && x.schemaName == args.schemaName
       );
+    }
   });
 }
 
@@ -283,7 +292,7 @@ export function getViewInfo(args) {
 
 /** @returns {import('dbgate-types').ViewInfo} */
 export function useViewInfo(args) {
-  return useDbCore(args, 'views');
+  return useDbCore(args, ['views', 'matviews']);
 }
 
 /** @returns {import('dbgate-types').CollectionInfo} */
@@ -344,7 +353,6 @@ export function useDatabaseServerVersion(args) {
   return useCore(databaseServerVersionLoader, args);
 }
 
-
 export function getServerStatus() {
   return getCore(serverStatusLoader, {});
 }

packages/web/src/widgets/SqlObjectList.svelte

@@ -21,10 +21,10 @@
   $: objects = useDatabaseInfo({ conid, database });
   $: status = useDatabaseStatus({ conid, database });
 
-  // $: console.log('objects', $objects);
+  // $: console.log('OBJECTS', $objects);
 
   $: objectList = _.flatten(
-    ['tables', 'collections', 'views', 'procedures', 'functions'].map(objectTypeField =>
+    ['tables', 'collections', 'views', 'matviews', 'procedures', 'functions'].map(objectTypeField =>
       _.sortBy(
         (($objects || {})[objectTypeField] || []).map(obj => ({ ...obj, objectTypeField })),
         ['schemaName', 'pureName']
@@ -35,6 +35,9 @@
   const handleRefreshDatabase = () => {
     axiosInstance.post('database-connections/refresh', { conid, database });
   };
+
+  const OBJECT_TYPE_LABELS = { matviews: 'Materialized views' };
+
 </script>
 
 {#if $status && $status.name == 'error'}
@@ -62,9 +65,10 @@
       <AppObjectList
         list={objectList.map(x => ({ ...x, conid, database }))}
         module={databaseObjectAppObject}
-        groupFunc={data => _.startCase(data.objectTypeField)}
+        groupFunc={data => OBJECT_TYPE_LABELS[data.objectTypeField] || _.startCase(data.objectTypeField)}
         subItemsComponent={SubColumnParamList}
-        isExpandable={data => data.objectTypeField == 'tables' || data.objectTypeField == 'views'}
+        isExpandable={data =>
+          data.objectTypeField == 'tables' || data.objectTypeField == 'views' || data.objectTypeField == 'matviews'}
         expandIconFunc={chevronExpandIcon}
         {filter}
       />

plugins/dbgate-plugin-postgres/src/backend/Analyser.js

@@ -56,6 +56,12 @@ class Analyser extends DatabaseAnalyser {
     const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables']));
     const fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables']));
     const views = await this.driver.query(this.pool, this.createQuery('views', ['views']));
+    const matviews = this.driver.dialect.materializedViews
+      ? await this.driver.query(this.pool, this.createQuery('matviews', ['matviews']))
+      : null;
+    const matviewColumns = this.driver.dialect.materializedViews
+      ? await this.driver.query(this.pool, this.createQuery('matviewColumns', ['matviews']))
+      : null;
     const routines = await this.driver.query(this.pool, this.createQuery('routines', ['procedures', 'functions']));
 
     return {
@@ -108,6 +114,18 @@ class Analyser extends DatabaseAnalyser {
           .filter(col => col.pure_name == view.pure_name && col.schema_name == view.schema_name)
           .map(getColumnInfo),
       })),
+      matviews: matviews
+        ? matviews.rows.map(matview => ({
+            objectId: `matviews:${matview.schema_name}.${matview.pure_name}`,
+            pureName: matview.pure_name,
+            schemaName: matview.schema_name,
+            contentHash: matview.hash_code,
+            createSql: `CREATE MATERIALIZED VIEW "${matview.schema_name}"."${matview.pure_name}"\nAS\n${matview.definition}`,
+            columns: matviewColumns.rows
+              .filter(col => col.pure_name == matview.pure_name && col.schema_name == matview.schema_name)
+              .map(getColumnInfo),
+          }))
+        : undefined,
       procedures: routines.rows
         .filter(x => x.object_type == 'PROCEDURE')
         .map(proc => ({
@@ -133,6 +151,9 @@ class Analyser extends DatabaseAnalyser {
       ? await this.driver.query(this.pool, this.createQuery('tableModifications'))
       : null;
     const viewModificationsQueryData = await this.driver.query(this.pool, this.createQuery('viewModifications'));
+    const matviewModificationsQueryData = this.driver.dialect.materializedViews
+      ? await this.driver.query(this.pool, this.createQuery('matviewModifications'))
+      : null;
     const routineModificationsQueryData = await this.driver.query(this.pool, this.createQuery('routineModifications'));
 
     return {
@@ -150,6 +171,14 @@ class Analyser extends DatabaseAnalyser {
         schemaName: x.schema_name,
         contentHash: x.hash_code,
       })),
+      matviews: matviewModificationsQueryData
+        ? matviewModificationsQueryData.rows.map(x => ({
+            objectId: `matviews:${x.schema_name}.${x.pure_name}`,
+            pureName: x.pure_name,
+            schemaName: x.schema_name,
+            contentHash: x.hash_code,
+          }))
+        : undefined,
       procedures: routineModificationsQueryData.rows
         .filter(x => x.object_type == 'PROCEDURE')
         .map(x => ({

plugins/dbgate-plugin-postgres/src/backend/sql/index.js

@@ -2,11 +2,14 @@ const columns = require('./columns');
 const tableModifications = require('./tableModifications');
 const tableList = require('./tableList');
 const viewModifications = require('./viewModifications');
+const matviewModifications = require('./matviewModifications');
 const primaryKeys = require('./primaryKeys');
 const foreignKeys = require('./foreignKeys');
 const views = require('./views');
+const matviews = require('./matviews');
 const routines = require('./routines');
 const routineModifications = require('./routineModifications');
+const matviewColumns = require('./matviewColumns');
 
 module.exports = {
   columns,
@@ -18,4 +21,7 @@ module.exports = {
   views,
   routines,
   routineModifications,
+  matviews,
+  matviewModifications,
+  matviewColumns,
 };

plugins/dbgate-plugin-postgres/src/backend/sql/matviewColumns.js

@@ -0,0 +1,17 @@
+module.exports = `
+SELECT pg_namespace.nspname AS "schema_name"
+    , pg_class.relname AS "pure_name"
+    , pg_attribute.attname AS "column_name"
+    , pg_catalog.format_type(pg_attribute.atttypid, pg_attribute.atttypmod) AS "data_type"
+FROM pg_catalog.pg_class
+    INNER JOIN pg_catalog.pg_namespace
+        ON pg_class.relnamespace = pg_namespace.oid
+    INNER JOIN pg_catalog.pg_attribute
+        ON pg_class.oid = pg_attribute.attrelid
+-- Keeps only materialized views, and non-db/catalog/index columns 
+WHERE pg_class.relkind = 'm'
+    AND pg_attribute.attnum >= 1
+    AND ('matviews:' || pg_namespace.nspname || '.' || pg_class.relname) =OBJECT_ID_CONDITION
+
+ORDER BY pg_attribute.attnum
+`;

plugins/dbgate-plugin-postgres/src/backend/sql/matviewModifications.js

@@ -0,0 +1,8 @@
+module.exports = `
+select 
+  matviewname as "pure_name",
+  schemaname as "schema_name",
+  md5(definition) as "hash_code"
+from
+  pg_catalog.pg_matviews WHERE schemaname NOT LIKE 'pg_%' 
+`;

plugins/dbgate-plugin-postgres/src/backend/sql/matviews.js

@@ -0,0 +1,10 @@
+module.exports = `
+select 
+  matviewname as "pure_name",
+  schemaname as "schema_name",
+  definition as "definition",
+  md5(definition) as "hash_code"
+from
+  pg_catalog.pg_matviews WHERE schemaname NOT LIKE 'pg_%' 
+  and ('matviews:' || schemaname || '.' ||  matviewname) =OBJECT_ID_CONDITION
+`;

plugins/dbgate-plugin-postgres/src/frontend/drivers.js

@@ -29,6 +29,10 @@ const postgresDriver = {
   engine: 'postgres@dbgate-plugin-postgres',
   title: 'Postgre SQL',
   defaultPort: 5432,
+  dialect: {
+    ...dialect,
+    materializedViews: true,
+  },
 };
 
 /** @type {import('dbgate-types').EngineDriver} */