Merge branch 'feature/duplicator-weak-refs'

integration-tests/__tests__/data-duplicator.spec.js

@@ -91,4 +91,68 @@ describe('Data duplicator', () => {
       expect(res2.rows[0].cnt.toString()).toEqual('6');
     })
   );
+
+  test.each(engines.filter(x => !x.skipDataDuplicator).map(engine => [engine.label, engine]))(
+    'Skip nullable weak refs - %s',
+    testWrapper(async (conn, driver, engine) => {
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't1',
+          columns: [
+            { columnName: 'id', dataType: 'int', notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+        })
+      );
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't2',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+            { columnName: 'valfk', dataType: 'int', notNull: false },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+          foreignKeys: [{ refTableName: 't1', columns: [{ columnName: 'valfk', refColumnName: 'id' }] }],
+        })
+      );
+      runCommandOnDriver(conn, driver, dmp => dmp.put("insert into ~t1 (~id, ~val) values (1, 'first')"));
+
+      const gett2 = () =>
+        stream.Readable.from([
+          { __isStreamHeader: true, __isDynamicStructure: true },
+          { id: 1, val: 'v1', valfk: 1 },
+          { id: 2, val: 'v2', valfk: 2 },
+        ]);
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: gett2,
+          },
+        ],
+        options: {
+          setNullForUnresolvedNullableRefs: true,
+        },
+      });
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('1');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('2');
+
+      const res3 = await driver.query(conn, `select count(*) as cnt from t2 where valfk is not null`);
+      expect(res3.rows[0].cnt.toString()).toEqual('1');
+    })
+  );
 });

integration-tests/engines.js

@@ -186,9 +186,9 @@ const engines = [
 
 const filterLocal = [
   // filter local testing
-  '-MySQL',
+  'MySQL',
   '-MariaDB',
-  'PostgreSQL',
+  '-PostgreSQL',
   '-SQL Server',
   '-SQLite',
   '-CockroachDB',

packages/datalib/src/DataDuplicator.ts

@@ -21,6 +21,7 @@ export interface DataDuplicatorItem {
 export interface DataDuplicatorOptions {
   rollbackAfterFinish?: boolean;
   skipRowsWithUnresolvedRefs?: boolean;
+  setNullForUnresolvedNullableRefs?: boolean;
 }
 
 class DuplicatorReference {
@@ -36,9 +37,19 @@ class DuplicatorReference {
   }
 }
 
+class DuplicatorWeakReference {
+  constructor(public base: DuplicatorItemHolder, public ref: TableInfo, public foreignKey: ForeignKeyInfo) {}
+
+  get columnName() {
+    return this.foreignKey.columns[0].columnName;
+  }
+}
+
 class DuplicatorItemHolder {
   references: DuplicatorReference[] = [];
   backReferences: DuplicatorReference[] = [];
+  // not mandatory references to entities out of the model
+  weakReferences: DuplicatorWeakReference[] = [];
   table: TableInfo;
   isPlanned = false;
   idMap = {};
@@ -65,23 +76,33 @@ class DuplicatorItemHolder {
     for (const fk of this.table.foreignKeys) {
       if (fk.columns?.length != 1) continue;
       const refHolder = this.duplicator.itemHolders.find(y => y.name.toUpperCase() == fk.refTableName.toUpperCase());
-      if (refHolder == null) continue;
       const isMandatory = this.table.columns.find(x => x.columnName == fk.columns[0]?.columnName)?.notNull;
-      const newref = new DuplicatorReference(this, refHolder, isMandatory, fk);
+      if (refHolder == null) {
-      this.references.push(newref);
+        if (!isMandatory) {
-      this.refByColumn[newref.columnName] = newref;
+          const weakref = new DuplicatorWeakReference(
+            this,
+            this.duplicator.db.tables.find(x => x.pureName == fk.refTableName),
+            fk
+          );
+          this.weakReferences.push(weakref);
+        }
+      } else {
+        const newref = new DuplicatorReference(this, refHolder, isMandatory, fk);
+        this.references.push(newref);
+        this.refByColumn[newref.columnName] = newref;
 
-      refHolder.isReferenced = true;
+        refHolder.isReferenced = true;
+      }
     }
   }
 
-  createInsertObject(chunk) {
+  createInsertObject(chunk, weakrefcols: string[]) {
     const res = _omit(
       _pick(
         chunk,
         this.table.columns.map(x => x.columnName)
       ),
-      [this.autoColumn, ...this.backReferences.map(x => x.columnName)]
+      [this.autoColumn, ...this.backReferences.map(x => x.columnName), ...weakrefcols]
     );
 
     for (const key in res) {
@@ -102,6 +123,28 @@ class DuplicatorItemHolder {
     return res;
   }
 
+  // returns list of columns that are weak references and are not resolved
+  async getMissingWeakRefsForRow(row): Promise<string[]> {
+    if (!this.duplicator.options.setNullForUnresolvedNullableRefs || !this.weakReferences?.length) {
+      return [];
+    }
+
+    const qres = await runQueryOnDriver(this.duplicator.pool, this.duplicator.driver, dmp => {
+      dmp.put('^select ');
+      dmp.putCollection(',', this.weakReferences, weakref => {
+        dmp.put(
+          '(^case ^when ^exists (^select * ^from %f where %i = %v) ^then 1 ^else 0 ^end) as %i',
+          weakref.ref,
+          weakref.foreignKey.columns[0].refColumnName,
+          row[weakref.foreignKey.columns[0].columnName],
+          weakref.foreignKey.columns[0].columnName
+        );
+      });
+    });
+    const qrow = qres.rows[0];
+    return this.weakReferences.filter(x => qrow[x.columnName] == 0).map(x => x.columnName);
+  }
+
   async runImport() {
     const readStream = await this.item.openStream();
     const driver = this.duplicator.driver;
@@ -112,6 +155,8 @@ class DuplicatorItemHolder {
     let skipped = 0;
     let lastLogged = new Date();
 
+    const existingWeakRefs = {};
+
     const writeStream = createAsyncWriteStream(this.duplicator.stream, {
       processItem: async chunk => {
         if (chunk.__isStreamHeader) {
@@ -120,7 +165,8 @@ class DuplicatorItemHolder {
 
         const doCopy = async () => {
           // console.log('chunk', this.name, JSON.stringify(chunk));
-          const insertedObj = this.createInsertObject(chunk);
+          const weakrefcols = await this.getMissingWeakRefsForRow(chunk);
+          const insertedObj = this.createInsertObject(chunk, weakrefcols);
           // console.log('insertedObj', this.name, JSON.stringify(insertedObj));
           if (insertedObj == null) {
             skipped += 1;

packages/tools/src/ScriptWriter.ts

@@ -58,7 +58,7 @@ export class ScriptWriter {
   }
 
   dataDuplicator(options) {
-    this._put(`await dbgateApi.dataDuplicator(${JSON.stringify(options)});`);
+    this._put(`await dbgateApi.dataDuplicator(${JSON.stringify(options, null, 2)});`);
   }
 
   comment(s) {

packages/web/src/appobj/ArchiveFolderAppObject.svelte

@@ -19,6 +19,7 @@
   import { apiCall } from '../utility/api';
   import hasPermission from '../utility/hasPermission';
   import { isProApp } from '../utility/proTools';
+  import { extractShellConnection } from '../impexp/createImpExpScript';
 
   export let data;
 
@@ -66,10 +67,7 @@
         
 await dbgateApi.deployDb(${JSON.stringify(
           {
-            connection: {
+            connection: extractShellConnection($currentDatabase.connection, $currentDatabase.name),
-              ..._.omit($currentDatabase.connection, '_id', 'displayName'),
-              database: $currentDatabase.name,
-            },
             modelFolder: `archive:${data.name}`,
           },
           undefined,

packages/web/src/appobj/DatabaseAppObject.svelte

@@ -297,10 +297,7 @@
         
 await dbgateApi.dropAllDbObjects(${JSON.stringify(
                 {
-                  connection: {
+                  connection: extractShellConnection(connection, name),
-                    ..._.omit(connection, '_id', 'displayName'),
-                    database: name,
-                  },
                 },
                 undefined,
                 2
@@ -311,6 +308,30 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify(
       });
     };
 
+    const handleImportWithDbDuplicator = () => {
+      showModal(ChooseArchiveFolderModal, {
+        message: 'Choose archive folder for import from',
+        onConfirm: archiveFolder => {
+          openNewTab(
+            {
+              title: archiveFolder,
+              icon: 'img duplicator',
+              tabComponent: 'DataDuplicatorTab',
+              props: {
+                conid: connection?._id,
+                database: name,
+              },
+            },
+            {
+              editor: {
+                archiveFolder,
+              },
+            }
+          );
+        },
+      });
+    };
+
     const driver = findEngineDriver(connection, getExtensions());
 
     const commands = _.flatten((apps || []).map(x => x.commands || []));
@@ -390,6 +411,14 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify(
           text: 'Shell: Drop all objects',
         },
 
+      driver?.databaseEngineTypes?.includes('sql') &&
+        hasPermission(`dbops/import`) && {
+          onClick: handleImportWithDbDuplicator,
+          text: 'Import with DB duplicator',
+        },
+
+      { divider: true },
+
       commands.length > 0 && [
         commands.map((cmd: any) => ({
           text: cmd.name,
@@ -451,6 +480,8 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify(
   import { loadSchemaList, switchCurrentDatabase } from '../utility/common';
   import { isProApp } from '../utility/proTools';
   import ExportDbModelModal from '../modals/ExportDbModelModal.svelte';
+  import ChooseArchiveFolderModal from '../modals/ChooseArchiveFolderModal.svelte';
+  import { extractShellConnection } from '../impexp/createImpExpScript';
 
   export let data;
   export let passProps;

packages/web/src/impexp/createImpExpScript.ts

@@ -39,7 +39,10 @@ export function extractShellConnection(connection, database) {
 
   return config.allowShellConnection
     ? {
-        ..._.omit(connection, ['_id', 'displayName', 'databases', 'connectionColor']),
+        ..._.omitBy(
+          _.omit(connection, ['_id', 'displayName', 'databases', 'connectionColor', 'status', 'unsaved']),
+          v => !v
+        ),
         database,
       }
     : {
@@ -192,7 +195,7 @@ export function normalizeExportColumnMap(colmap) {
   return null;
 }
 
-export default async function createImpExpScript(extensions, values,  forceScript = false) {
+export default async function createImpExpScript(extensions, values, forceScript = false) {
   const config = getCurrentConfig();
   const script =
     config.allowShellScripting || forceScript

packages/web/src/modals/ChooseArchiveFolderModal.svelte

@@ -0,0 +1,33 @@
+<script lang="ts">
+  import FormStyledButton from '../buttons/FormStyledButton.svelte';
+  import FormArchiveFolderSelect from '../forms/FormArchiveFolderSelect.svelte';
+
+  import FormProvider from '../forms/FormProvider.svelte';
+  import FormSubmit from '../forms/FormSubmit.svelte';
+  import ModalBase from './ModalBase.svelte';
+  import { closeCurrentModal } from './modalTools';
+
+  export let message = '';
+  export let onConfirm;
+</script>
+
+<FormProvider>
+  <ModalBase {...$$restProps}>
+    <svelte:fragment slot="header">Choose archive folder</svelte:fragment>
+
+    <div>{message}</div>
+
+    <FormArchiveFolderSelect label="Archive folder" name="archiveFolder" isNative />
+
+    <svelte:fragment slot="footer">
+      <FormSubmit
+        value="OK"
+        on:click={e => {
+          closeCurrentModal();
+          onConfirm(e.detail.archiveFolder);
+        }}
+      />
+      <FormStyledButton type="button" value="Close" on:click={closeCurrentModal} />
+    </svelte:fragment>
+  </ModalBase>
+</FormProvider>

packages/web/src/tabs/DataDuplicatorTab.svelte

@@ -22,6 +22,16 @@
     testEnabled: () => getCurrentEditor()?.canKill(),
     onClick: () => getCurrentEditor().kill(),
   });
+  registerCommand({
+    id: 'dataDuplicator.generateScript',
+    category: 'Data duplicator',
+    icon: 'img shell',
+    name: 'Generate Script',
+    toolbar: true,
+    isRelatedToTab: true,
+    testEnabled: () => getCurrentEditor()?.canRun(),
+    onClick: () => getCurrentEditor().generateScript(),
+  });
 </script>
 
 <script lang="ts">
@@ -52,7 +62,6 @@
   import useEffect from '../utility/useEffect';
   import useTimerLabel from '../utility/useTimerLabel';
   import appObjectTypes from '../appobj';
-  import RowHeaderCell from '../datagrid/RowHeaderCell.svelte';
 
   export let conid;
   export let database;
@@ -124,6 +133,7 @@
       options: {
         rollbackAfterFinish: !!$editorState.value?.rollbackAfterFinish,
         skipRowsWithUnresolvedRefs: !!$editorState.value?.skipRowsWithUnresolvedRefs,
+        setNullForUnresolvedNullableRefs: !!$editorState.value?.setNullForUnresolvedNullableRefs,
       },
     });
     return script.getScript();
@@ -145,6 +155,18 @@
     timerLabel.start();
   }
 
+  export async function generateScript() {
+    const code = await createScript();
+    openNewTab(
+      {
+        title: 'Shell #',
+        icon: 'img shell',
+        tabComponent: 'ShellTab',
+      },
+      { editor: code }
+    );
+  }
+
   $: effect = useEffect(() => registerRunnerDone(runnerId));
 
   function registerRunnerDone(rid) {
@@ -286,6 +308,29 @@
               }}
             />
           </FormFieldTemplateLarge>
+
+          <FormFieldTemplateLarge
+            label="Set NULL for nullable unresolved references"
+            type="checkbox"
+            labelProps={{
+              onClick: () => {
+                setEditorData(old => ({
+                  ...old,
+                  setNullForUnresolvedNullableRefs: !$editorState.value?.setNullForUnresolvedNullableRefs,
+                }));
+              },
+            }}
+          >
+            <CheckboxField
+              checked={$editorState.value?.setNullForUnresolvedNullableRefs}
+              on:change={e => {
+                setEditorData(old => ({
+                  ...old,
+                  setNullForUnresolvedNullableRefs: e.target.checked,
+                }));
+              }}
+            />
+          </FormFieldTemplateLarge>
         </ObjectConfigurationControl>
 
         <ObjectConfigurationControl title="Imported files">
@@ -398,6 +443,7 @@
   <svelte:fragment slot="toolstrip">
     <ToolStripCommandButton command="dataDuplicator.run" />
     <ToolStripCommandButton command="dataDuplicator.kill" />
+    <ToolStripCommandButton command="dataDuplicator.generateScript" />
   </svelte:fragment>
 </ToolStripContainer>
 

packages/web/src/tabs/ImportExportTab.svelte

@@ -275,7 +275,7 @@
     {:else}
       <ToolStripButton on:click={handleExecute} icon="icon run">Run</ToolStripButton>
     {/if}
-    <ToolStripButton icon="img sql-file" on:click={handleGenerateScript}>Generate script</ToolStripButton>
+    <ToolStripButton icon="img shell" on:click={handleGenerateScript}>Generate script</ToolStripButton>
     <ToolStripSaveButton idPrefix="job" />
   </svelte:fragment>
 </ToolStripContainer>