correct handle big data in sql preview

packages/api/src/controllers/databaseConnections.js

@@ -160,7 +160,7 @@ module.exports = {
 
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'sqlPreview', objects, options });
-    return res.sql;
+    return res;
   },
 
 

packages/api/src/proc/databaseConnectionProcess.js

@@ -3,7 +3,7 @@ const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
-const { SqlGenerator } = require('dbgate-tools')
+const { SqlGenerator } = require('dbgate-tools');
 
 let systemConnection;
 let storedConnection;
@@ -94,15 +94,25 @@ async function handleQueryData({ msgid, sql }) {
   }
 }
 
-
 async function handleSqlPreview({ msgid, objects, options }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
-  const dmp = driver.createDumper();
+  try {
-  const generator = new SqlGenerator(analysedStructure, options, objects, dmp, driver, systemConnection);
+    const dmp = driver.createDumper();
-  await generator.dump();
+    const generator = new SqlGenerator(analysedStructure, options, objects, dmp, driver, systemConnection);
-  process.send({ msgtype: 'response', msgid, sql: dmp.s });
+
+    await generator.dump();
+    process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
+    if (generator.isUnhandledException) {
+      setTimeout(() => {
+        console.log('Exiting because of unhandled exception');
+        process.exit(0);
+      }, 500);
+    }
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, isError: true, errorMessage: err.message });
+  }
 }
 
 // async function handleRunCommand({ msgid, sql }) {

packages/tools/src/SqlGenerator.ts

@@ -55,6 +55,8 @@ export class SqlGenerator {
   private functions: FunctionInfo[];
   private triggers: TriggerInfo[];
   public dbinfo: DatabaseInfo;
+  public isTruncated = false;
+  public isUnhandledException = false;
 
   constructor(
     dbinfo: DatabaseInfo,
@@ -72,39 +74,50 @@ export class SqlGenerator {
     this.triggers = this.extract('triggers');
   }
 
+  private handleException = error => {
+    console.log('Unhandled error', error);
+    this.isUnhandledException = true;
+  };
+
   async dump() {
-    this.dropObjects(this.procedures, 'Procedure');
+    try {
-    if (this.checkDumper()) return;
+      process.on('uncaughtException', this.handleException);
-    this.dropObjects(this.functions, 'Function');
-    if (this.checkDumper()) return;
-    this.dropObjects(this.views, 'View');
-    if (this.checkDumper()) return;
-    this.dropObjects(this.triggers, 'Trigger');
-    if (this.checkDumper()) return;
 
-    this.dropTables();
+      this.dropObjects(this.procedures, 'Procedure');
-    if (this.checkDumper()) return;
+      if (this.checkDumper()) return;
+      this.dropObjects(this.functions, 'Function');
+      if (this.checkDumper()) return;
+      this.dropObjects(this.views, 'View');
+      if (this.checkDumper()) return;
+      this.dropObjects(this.triggers, 'Trigger');
+      if (this.checkDumper()) return;
 
-    this.createTables();
+      this.dropTables();
-    if (this.checkDumper()) return;
+      if (this.checkDumper()) return;
 
-    this.truncateTables();
+      this.createTables();
-    if (this.checkDumper()) return;
+      if (this.checkDumper()) return;
 
-    await this.insertData();
+      this.truncateTables();
-    if (this.checkDumper()) return;
+      if (this.checkDumper()) return;
 
-    this.createForeignKeys();
+      await this.insertData();
-    if (this.checkDumper()) return;
+      if (this.checkDumper()) return;
 
-    this.createObjects(this.procedures, 'Procedure');
+      this.createForeignKeys();
-    if (this.checkDumper()) return;
+      if (this.checkDumper()) return;
-    this.createObjects(this.functions, 'Function');
+
-    if (this.checkDumper()) return;
+      this.createObjects(this.procedures, 'Procedure');
-    this.createObjects(this.views, 'View');
+      if (this.checkDumper()) return;
-    if (this.checkDumper()) return;
+      this.createObjects(this.functions, 'Function');
-    this.createObjects(this.triggers, 'Trigger');
+      if (this.checkDumper()) return;
-    if (this.checkDumper()) return;
+      this.createObjects(this.views, 'View');
+      if (this.checkDumper()) return;
+      this.createObjects(this.triggers, 'Trigger');
+      if (this.checkDumper()) return;
+    } finally {
+      process.off('uncaughtException', this.handleException);
+    }
   }
 
   createForeignKeys() {
@@ -159,6 +172,15 @@ export class SqlGenerator {
   }
 
   checkDumper() {
+    if (this.dmp.s.length > 4000000) {
+      if (!this.isTruncated) {
+        this.dmp.putRaw('\n');
+        this.dmp.comment(' *************** SQL is truncated ******************');
+        this.dmp.putRaw('\n');
+      }
+      this.isTruncated = true;
+      return true;
+    }
     return false;
   }
 
@@ -216,8 +238,19 @@ export class SqlGenerator {
       ? table.columns.filter(x => !x.autoIncrement)
       : table.columns;
     const columnNames = columnsFiltered.map(x => x.columnName);
+    let isClosed = false;
+
     return new Promise(resolve => {
       readable.on('data', chunk => {
+        if (isClosed) return;
+
+        if (this.checkDumper()) {
+          isClosed = true;
+          resolve(undefined);
+          readable.destroy();
+          return;
+        }
+
         const columnNamesCopy = this.options.omitNulls ? columnNames.filter(col => chunk[col] != null) : columnNames;
         this.dmp.put(
           '^insert ^into %f (%,i) ^values (%,v);&n',

packages/web/public/global.css

@@ -51,6 +51,9 @@ body {
 .flex1 {
   flex: 1;
 }
+.relative {
+  position: relative;
+}
 
 .col-9 {
   flex-basis: 75%;

packages/web/src/modals/SqlGeneratorModal.svelte

@@ -29,6 +29,8 @@
   import { closeCurrentModal } from './modalTools';
   import WidgetTitle from '../widgets/WidgetTitle.svelte';
   import openNewTab from '../utility/openNewTab';
+  import ErrorInfo from '../elements/ErrorInfo.svelte';
+  import LoadingInfo from '../elements/LoadingInfo.svelte';
 
   export let conid;
   export let database;
@@ -51,6 +53,8 @@
   let objectsFilter = '';
   let sqlPreview = '';
   let initialized = false;
+  let error = null;
+  let truncated = false;
 
   $: dbinfo = useDatabaseInfo({ conid, database });
 
@@ -90,8 +94,16 @@
       // newer load exists
       return;
     }
-    if (_.isString(response.data)) {
+    const { sql, isTruncated, isError, errorMessage } = response.data || {};
-      sqlPreview = response.data;
+
+    truncated = isTruncated;
+    if (isError) {
+      error = errorMessage || 'Unknown error';
+    } else {
+      error = null;
+      if (_.isString(sql)) {
+        sqlPreview = sql;
+      }
     }
     busy = false;
   }
@@ -148,7 +160,21 @@
       <svelte:fragment slot="2">
         <HorizontalSplitter initialValue="~300px">
           <svelte:fragment slot="1">
-            <SqlEditor readOnly value={sqlPreview} />
+            {#if error}
+              <ErrorInfo message={error} />
+            {:else}
+              <div class="flexcol flex1">
+                {#if truncated}
+                  <ErrorInfo icon="img warn" message="SQL truncated, file size limit exceed" />
+                {/if}
+                <div class="relative flex1">
+                  <SqlEditor readOnly value={sqlPreview} />
+                </div>
+              </div>
+              {#if busy}
+                <LoadingInfo wrapper message="Loading SQL preview" />
+              {/if}
+            {/if}
           </svelte:fragment>
           <svelte:fragment slot="2">
             <div class="flexcol flex1">