postgresql analyse optyimalization #273

integration-tests/docker-compose.yaml

@@ -8,30 +8,39 @@ services:
     ports: 
       - 15000:5432
 
-  mysql:
+  mariadb:
-    image: mysql:8.0.18
+    image: mariadb
     command: --default-authentication-plugin=mysql_native_password
     restart: always
     ports: 
-      - 15001:3306
+      - 15004:3306
     environment:
       - MYSQL_ROOT_PASSWORD=Pwd2020Db
 
-  mssql:
+  # mysql:
-    image: mcr.microsoft.com/mssql/server
+  #   image: mysql:8.0.18
-    restart: always
+  #   command: --default-authentication-plugin=mysql_native_password
-    ports: 
+  #   restart: always
-      - 15002:1433
+  #   ports: 
-    environment:
+  #     - 15001:3306
-      - ACCEPT_EULA=Y
+  #   environment:
-      - SA_PASSWORD=Pwd2020Db
+  #     - MYSQL_ROOT_PASSWORD=Pwd2020Db
-      - MSSQL_PID=Express
 
-  cockroachdb:
+  # mssql:
-    image: cockroachdb/cockroach
+  #   image: mcr.microsoft.com/mssql/server
-    ports:
+  #   restart: always
-      - 15003:26257
+  #   ports: 
-    command: start-single-node --insecure
+  #     - 15002:1433
+  #   environment:
+  #     - ACCEPT_EULA=Y
+  #     - SA_PASSWORD=Pwd2020Db
+  #     - MSSQL_PID=Express
+      
+  # cockroachdb:
+  #   image: cockroachdb/cockroach
+  #   ports:
+  #     - 15003:26257
+  #   command: start-single-node --insecure
 
 #   mongodb:
 #     image: mongo:4.0.12

integration-tests/engines.js

@@ -31,6 +31,23 @@ const engines = [
     objects: [views],
     dbSnapshotBySeconds: true,
   },
+  {
+    label: 'MariaDB',
+    connection: {
+      engine: 'mariadb@dbgate-plugin-mysql',
+      password: 'Pwd2020Db',
+      user: 'root',
+      server: 'mysql',
+      port: 3306,
+    },
+    local: {
+      server: 'localhost',
+      port: 15004,
+    },
+    skipOnCI: true,
+    objects: [views],
+    dbSnapshotBySeconds: true,
+  },
   {
     label: 'PostgreSQL',
     connection: {
@@ -117,9 +134,10 @@ const engines = [
 const filterLocal = [
   // filter local testing
   '-MySQL',
-  '-PostgreSQL',
+  '-MariaDB',
+  'PostgreSQL',
   '-SQL Server',
-  'SQLite',
+  '-SQLite',
   '-CockroachDB',
 ];
 

packages/tools/src/DatabaseAnalyser.ts

@@ -170,7 +170,15 @@ export class DatabaseAnalyser {
   //   return this.structure.tables.find((x) => x.objectId == id);
   // }
 
+  containsObjectIdCondition(typeFields) {
+    return this.createQueryCore('=OBJECT_ID_CONDITION', typeFields) != ' is not null';
+  }
+
   createQuery(template, typeFields) {
+    return this.createQueryCore(template, typeFields);
+  }
+
+  createQueryCore(template, typeFields) {
     // let res = template;
     if (this.singleObjectFilter) {
       const { typeField } = this.singleObjectFilter;

plugins/dbgate-plugin-postgres/src/backend/Analyser.js

@@ -61,8 +61,68 @@ class Analyser extends DatabaseAnalyser {
     const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables', 'views']));
     this.feedback({ analysingMessage: 'Loading primary keys' });
     const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables']));
+
+    let fkColumns = null;
+
+    // if (true) {
+    if (this.containsObjectIdCondition(['tables']) || this.driver.__analyserInternals.refTableCond) {
       this.feedback({ analysingMessage: 'Loading foreign keys' });
-    const fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables']));
+      fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables']));
+    } else {
+      this.feedback({ analysingMessage: 'Loading foreign key constraints' });
+      const fk_tableConstraints = await this.driver.query(
+        this.pool,
+        this.createQuery('fk_tableConstraints', ['tables'])
+      );
+
+      this.feedback({ analysingMessage: 'Loading foreign key refs' });
+      const fk_referentialConstraints = await this.driver.query(
+        this.pool,
+        this.createQuery('fk_referentialConstraints', ['tables'])
+      );
+
+      this.feedback({ analysingMessage: 'Loading foreign key columns' });
+      const fk_keyColumnUsage = await this.driver.query(this.pool, this.createQuery('fk_keyColumnUsage', ['tables']));
+
+      const cntKey = x => `${x.constraint_name}|${x.constraint_schema}`;
+      const rows = [];
+      const constraintDct = _.keyBy(fk_tableConstraints.rows, cntKey);
+      for (const fkRef of fk_referentialConstraints.rows) {
+        const cntBase = constraintDct[cntKey(fkRef)];
+        const cntRef = constraintDct[`${fkRef.unique_constraint_name}|${fkRef.unique_constraint_schema}`];
+        if (!cntBase || !cntRef) continue;
+        const baseCols = _.sortBy(
+          fk_keyColumnUsage.rows.filter(
+            x => x.table_name == cntBase.table_name && x.constraint_name == cntBase.constraint_name
+          ),
+          'ordinal_position'
+        );
+        const refCols = _.sortBy(
+          fk_keyColumnUsage.rows.filter(
+            x => x.table_name == cntRef.table_name && x.constraint_name == cntRef.constraint_name
+          ),
+          'ordinal_position'
+        );
+        if (baseCols.length != refCols.length) continue;
+
+        for (let i = 0; i < baseCols.length; i++) {
+          const baseCol = baseCols[i];
+          const refCol = refCols[i];
+
+          rows.push({
+            ...fkRef,
+            pure_name: cntBase.table_name,
+            schema_name: cntBase.table_schema,
+            ref_table_name: cntRef.table_name,
+            ref_schema_name: cntRef.table_schema,
+            column_name: baseCol.column_name,
+            ref_column_name: refCol.column_name,
+          });
+        }
+      }
+      fkColumns = { rows };
+    }
+
     this.feedback({ analysingMessage: 'Loading views' });
     const views = await this.driver.query(this.pool, this.createQuery('views', ['views']));
     this.feedback({ analysingMessage: 'Loading materialized views' });
@@ -85,9 +145,9 @@ class Analyser extends DatabaseAnalyser {
       : await this.driver.query(this.pool, this.createQuery('indexcols', ['tables']));
     this.feedback({ analysingMessage: 'Loading unique names' });
     const uniqueNames = await this.driver.query(this.pool, this.createQuery('uniqueNames', ['tables']));
-    this.feedback({ analysingMessage: null });
+    this.feedback({ analysingMessage: 'Finalizing DB structure' });
 
-    return {
+    const res = {
       tables: tables.rows.map(table => {
         const newTable = {
           pureName: table.pure_name,
@@ -207,6 +267,10 @@ class Analyser extends DatabaseAnalyser {
           contentHash: func.hash_code,
         })),
     };
+
+    this.feedback({ analysingMessage: null });
+
+    return res;
   }
 
   async _getFastSnapshot() {

plugins/dbgate-plugin-postgres/src/backend/sql/fk_key_column_usage.js

@@ -0,0 +1,10 @@
+module.exports = `
+select 
+	basecol.constraint_name,
+	basecol.constraint_schema,
+	basecol.column_name as "column_name",
+	basecol.table_schema,
+	basecol.table_name,
+	basecol.ordinal_position
+from  information_schema.key_column_usage basecol
+`;

plugins/dbgate-plugin-postgres/src/backend/sql/fk_referential_constraints.js

@@ -0,0 +1,10 @@
+module.exports = `
+select 
+	fk.constraint_name as "constraint_name",
+	fk.constraint_schema as "constraint_schema",
+	fk.update_rule as "update_action",
+	fk.delete_rule as "delete_action",
+    fk.unique_constraint_name as "unique_constraint_name",
+    fk.unique_constraint_schema as "unique_constraint_schema" 
+from information_schema.referential_constraints fk
+`;

plugins/dbgate-plugin-postgres/src/backend/sql/fk_table_constraints.js

@@ -0,0 +1,8 @@
+module.exports = `
+select 
+	base.table_name as "table_name",
+	base.table_schema as "table_schema",
+	base.constraint_name as "constraint_name",
+	base.constraint_schema as "constraint_schema"
+from information_schema.table_constraints base
+`;

plugins/dbgate-plugin-postgres/src/backend/sql/index.js

@@ -14,6 +14,10 @@ const indexes = require('./indexes');
 const indexcols = require('./indexcols');
 const uniqueNames = require('./uniqueNames');
 
+const fk_keyColumnUsage = require('./fk_key_column_usage');
+const fk_referentialConstraints = require('./fk_referential_constraints');
+const fk_tableConstraints = require('./fk_table_constraints');
+
 module.exports = {
   columns,
   tableModifications,
@@ -21,6 +25,9 @@ module.exports = {
   viewModifications,
   primaryKeys,
   foreignKeys,
+  fk_keyColumnUsage,
+  fk_referentialConstraints,
+  fk_tableConstraints,
   views,
   routines,
   routineModifications,