Merge branch 'feature/duplicator-weak-refs'

2026-04-26 17:55:59 +00:00 · 2024-11-07 16:56:24 +01:00
parent 7aaf6bb024 c19c69266a
commit 3665a0d064
10 changed files with 244 additions and 23 deletions
--- a/integration-tests/tests/data-duplicator.spec.js
+++ b/integration-tests/tests/data-duplicator.spec.js
@@ -91,4 +91,68 @@ describe('Data duplicator', () => {
      expect(res2.rows[0].cnt.toString()).toEqual('6');
    })
  );
+
+  test.each(engines.filter(x => !x.skipDataDuplicator).map(engine => [engine.label, engine]))(
+    'Skip nullable weak refs - %s',
+    testWrapper(async (conn, driver, engine) => {
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't1',
+          columns: [
+            { columnName: 'id', dataType: 'int', notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+        })
+      );
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't2',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+            { columnName: 'valfk', dataType: 'int', notNull: false },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+          foreignKeys: [{ refTableName: 't1', columns: [{ columnName: 'valfk', refColumnName: 'id' }] }],
+        })
+      );
+      runCommandOnDriver(conn, driver, dmp => dmp.put("insert into ~t1 (~id, ~val) values (1, 'first')"));
+
+      const gett2 = () =>
+        stream.Readable.from([
+          { __isStreamHeader: true, __isDynamicStructure: true },
+          { id: 1, val: 'v1', valfk: 1 },
+          { id: 2, val: 'v2', valfk: 2 },
+        ]);
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: gett2,
+          },
+        ],
+        options: {
+          setNullForUnresolvedNullableRefs: true,
+        },
+      });
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('1');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('2');
+
+      const res3 = await driver.query(conn, `select count(*) as cnt from t2 where valfk is not null`);
+      expect(res3.rows[0].cnt.toString()).toEqual('1');
+    })
+  );
 });
--- a/integration-tests/engines.js
+++ b/integration-tests/engines.js
@@ -186,9 +186,9 @@ const engines = [

 const filterLocal = [
  // filter local testing
-  '-MySQL',
+  'MySQL',
  '-MariaDB',
-  'PostgreSQL',
+  '-PostgreSQL',
  '-SQL Server',
  '-SQLite',
  '-CockroachDB',
--- a/packages/datalib/src/DataDuplicator.ts
+++ b/packages/datalib/src/DataDuplicator.ts
@@ -21,6 +21,7 @@ export interface DataDuplicatorItem {
 export interface DataDuplicatorOptions {
  rollbackAfterFinish?: boolean;
  skipRowsWithUnresolvedRefs?: boolean;
+  setNullForUnresolvedNullableRefs?: boolean;
 }

 class DuplicatorReference {
@@ -36,9 +37,19 @@ class DuplicatorReference {
  }
 }

+class DuplicatorWeakReference {
+  constructor(public base: DuplicatorItemHolder, public ref: TableInfo, public foreignKey: ForeignKeyInfo) {}
+
+  get columnName() {
+    return this.foreignKey.columns[0].columnName;
+  }
+}
+
 class DuplicatorItemHolder {
  references: DuplicatorReference[] = [];
  backReferences: DuplicatorReference[] = [];
+  // not mandatory references to entities out of the model
+  weakReferences: DuplicatorWeakReference[] = [];
  table: TableInfo;
  isPlanned = false;
  idMap = {};
@@ -65,23 +76,33 @@ class DuplicatorItemHolder {
    for (const fk of this.table.foreignKeys) {
      if (fk.columns?.length != 1) continue;
      const refHolder = this.duplicator.itemHolders.find(y => y.name.toUpperCase() == fk.refTableName.toUpperCase());
-      if (refHolder == null) continue;
      const isMandatory = this.table.columns.find(x => x.columnName == fk.columns[0]?.columnName)?.notNull;
-      const newref = new DuplicatorReference(this, refHolder, isMandatory, fk);
-      this.references.push(newref);
-      this.refByColumn[newref.columnName] = newref;
+      if (refHolder == null) {
+        if (!isMandatory) {
+          const weakref = new DuplicatorWeakReference(
+            this,
+            this.duplicator.db.tables.find(x => x.pureName == fk.refTableName),
+            fk
+          );
+          this.weakReferences.push(weakref);
+        }
+      } else {
+        const newref = new DuplicatorReference(this, refHolder, isMandatory, fk);
+        this.references.push(newref);
+        this.refByColumn[newref.columnName] = newref;

-      refHolder.isReferenced = true;
+        refHolder.isReferenced = true;
+      }
    }
  }

-  createInsertObject(chunk) {
+  createInsertObject(chunk, weakrefcols: string[]) {
    const res = _omit(
      _pick(
        chunk,
        this.table.columns.map(x => x.columnName)
      ),
-      [this.autoColumn, ...this.backReferences.map(x => x.columnName)]
+      [this.autoColumn, ...this.backReferences.map(x => x.columnName), ...weakrefcols]
    );

    for (const key in res) {
@@ -102,6 +123,28 @@ class DuplicatorItemHolder {
    return res;
  }

+  // returns list of columns that are weak references and are not resolved
+  async getMissingWeakRefsForRow(row): Promise<string[]> {
+    if (!this.duplicator.options.setNullForUnresolvedNullableRefs || !this.weakReferences?.length) {
+      return [];
+    }
+
+    const qres = await runQueryOnDriver(this.duplicator.pool, this.duplicator.driver, dmp => {
+      dmp.put('^select ');
+      dmp.putCollection(',', this.weakReferences, weakref => {
+        dmp.put(
+          '(^case ^when ^exists (^select * ^from %f where %i = %v) ^then 1 ^else 0 ^end) as %i',
+          weakref.ref,
+          weakref.foreignKey.columns[0].refColumnName,
+          row[weakref.foreignKey.columns[0].columnName],
+          weakref.foreignKey.columns[0].columnName
+        );
+      });
+    });
+    const qrow = qres.rows[0];
+    return this.weakReferences.filter(x => qrow[x.columnName] == 0).map(x => x.columnName);
+  }
+
  async runImport() {
    const readStream = await this.item.openStream();
    const driver = this.duplicator.driver;
@@ -112,6 +155,8 @@ class DuplicatorItemHolder {
    let skipped = 0;
    let lastLogged = new Date();

+    const existingWeakRefs = {};
+
    const writeStream = createAsyncWriteStream(this.duplicator.stream, {
      processItem: async chunk => {
        if (chunk.__isStreamHeader) {
@@ -120,7 +165,8 @@ class DuplicatorItemHolder {

        const doCopy = async () => {
          // console.log('chunk', this.name, JSON.stringify(chunk));
-          const insertedObj = this.createInsertObject(chunk);
+          const weakrefcols = await this.getMissingWeakRefsForRow(chunk);
+          const insertedObj = this.createInsertObject(chunk, weakrefcols);
          // console.log('insertedObj', this.name, JSON.stringify(insertedObj));
          if (insertedObj == null) {
            skipped += 1;
--- a/packages/tools/src/ScriptWriter.ts
+++ b/packages/tools/src/ScriptWriter.ts
@@ -58,7 +58,7 @@ export class ScriptWriter {
  }

  dataDuplicator(options) {
-    this._put(`await dbgateApi.dataDuplicator(${JSON.stringify(options)});`);
+    this._put(`await dbgateApi.dataDuplicator(${JSON.stringify(options, null, 2)});`);
  }

  comment(s) {
--- a/packages/web/src/appobj/ArchiveFolderAppObject.svelte
+++ b/packages/web/src/appobj/ArchiveFolderAppObject.svelte
@@ -19,6 +19,7 @@
  import { apiCall } from '../utility/api';
  import hasPermission from '../utility/hasPermission';
  import { isProApp } from '../utility/proTools';
+  import { extractShellConnection } from '../impexp/createImpExpScript';

  export let data;

@@ -66,10 +67,7 @@
        
 await dbgateApi.deployDb(${JSON.stringify(
          {
-            connection: {
-              ..._.omit($currentDatabase.connection, '_id', 'displayName'),
-              database: $currentDatabase.name,
-            },
+            connection: extractShellConnection($currentDatabase.connection, $currentDatabase.name),
            modelFolder: `archive:${data.name}`,
          },
          undefined,
--- a/packages/web/src/appobj/DatabaseAppObject.svelte
+++ b/packages/web/src/appobj/DatabaseAppObject.svelte
@@ -297,10 +297,7 @@
        
 await dbgateApi.dropAllDbObjects(${JSON.stringify(
                {
-                  connection: {
-                    ..._.omit(connection, '_id', 'displayName'),
-                    database: name,
-                  },
+                  connection: extractShellConnection(connection, name),
                },
                undefined,
                2
@@ -311,6 +308,30 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify(
      });
    };

+    const handleImportWithDbDuplicator = () => {
+      showModal(ChooseArchiveFolderModal, {
+        message: 'Choose archive folder for import from',
+        onConfirm: archiveFolder => {
+          openNewTab(
+            {
+              title: archiveFolder,
+              icon: 'img duplicator',
+              tabComponent: 'DataDuplicatorTab',
+              props: {
+                conid: connection?._id,
+                database: name,
+              },
+            },
+            {
+              editor: {
+                archiveFolder,
+              },
+            }
+          );
+        },
+      });
+    };
+
    const driver = findEngineDriver(connection, getExtensions());

    const commands = _.flatten((apps || []).map(x => x.commands || []));
@@ -390,6 +411,14 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify(
          text: 'Shell: Drop all objects',
        },

+      driver?.databaseEngineTypes?.includes('sql') &&
+        hasPermission(`dbops/import`) && {
+          onClick: handleImportWithDbDuplicator,
+          text: 'Import with DB duplicator',
+        },
+
+      { divider: true },
+
      commands.length > 0 && [
        commands.map((cmd: any) => ({
          text: cmd.name,
@@ -451,6 +480,8 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify(
  import { loadSchemaList, switchCurrentDatabase } from '../utility/common';
  import { isProApp } from '../utility/proTools';
  import ExportDbModelModal from '../modals/ExportDbModelModal.svelte';
+  import ChooseArchiveFolderModal from '../modals/ChooseArchiveFolderModal.svelte';
+  import { extractShellConnection } from '../impexp/createImpExpScript';

  export let data;
  export let passProps;
--- a/packages/web/src/impexp/createImpExpScript.ts
+++ b/packages/web/src/impexp/createImpExpScript.ts
@@ -39,7 +39,10 @@ export function extractShellConnection(connection, database) {

  return config.allowShellConnection
    ? {
-        ..._.omit(connection, ['_id', 'displayName', 'databases', 'connectionColor']),
+        ..._.omitBy(
+          _.omit(connection, ['_id', 'displayName', 'databases', 'connectionColor', 'status', 'unsaved']),
+          v => !v
+        ),
        database,
      }
    : {
@@ -192,7 +195,7 @@ export function normalizeExportColumnMap(colmap) {
  return null;
 }

-export default async function createImpExpScript(extensions, values,  forceScript = false) {
+export default async function createImpExpScript(extensions, values, forceScript = false) {
  const config = getCurrentConfig();
  const script =
    config.allowShellScripting || forceScript
--- a/packages/web/src/modals/ChooseArchiveFolderModal.svelte
+++ b/packages/web/src/modals/ChooseArchiveFolderModal.svelte
@@ -0,0 +1,33 @@
+<script lang="ts">
+  import FormStyledButton from '../buttons/FormStyledButton.svelte';
+  import FormArchiveFolderSelect from '../forms/FormArchiveFolderSelect.svelte';
+
+  import FormProvider from '../forms/FormProvider.svelte';
+  import FormSubmit from '../forms/FormSubmit.svelte';
+  import ModalBase from './ModalBase.svelte';
+  import { closeCurrentModal } from './modalTools';
+
+  export let message = '';
+  export let onConfirm;
+</script>
+
+<FormProvider>
+  <ModalBase {...$$restProps}>
+    <svelte:fragment slot="header">Choose archive folder</svelte:fragment>
+
+    <div>{message}</div>
+
+    <FormArchiveFolderSelect label="Archive folder" name="archiveFolder" isNative />
+
+    <svelte:fragment slot="footer">
+      <FormSubmit
+        value="OK"
+        on:click={e => {
+          closeCurrentModal();
+          onConfirm(e.detail.archiveFolder);
+        }}
+      />
+      <FormStyledButton type="button" value="Close" on:click={closeCurrentModal} />
+    </svelte:fragment>
+  </ModalBase>
+</FormProvider>
--- a/packages/web/src/tabs/DataDuplicatorTab.svelte
+++ b/packages/web/src/tabs/DataDuplicatorTab.svelte
@@ -22,6 +22,16 @@
    testEnabled: () => getCurrentEditor()?.canKill(),
    onClick: () => getCurrentEditor().kill(),
  });
+  registerCommand({
+    id: 'dataDuplicator.generateScript',
+    category: 'Data duplicator',
+    icon: 'img shell',
+    name: 'Generate Script',
+    toolbar: true,
+    isRelatedToTab: true,
+    testEnabled: () => getCurrentEditor()?.canRun(),
+    onClick: () => getCurrentEditor().generateScript(),
+  });
 </script>

 <script lang="ts">
@@ -52,7 +62,6 @@
  import useEffect from '../utility/useEffect';
  import useTimerLabel from '../utility/useTimerLabel';
  import appObjectTypes from '../appobj';
-  import RowHeaderCell from '../datagrid/RowHeaderCell.svelte';

  export let conid;
  export let database;
@@ -124,6 +133,7 @@
      options: {
        rollbackAfterFinish: !!$editorState.value?.rollbackAfterFinish,
        skipRowsWithUnresolvedRefs: !!$editorState.value?.skipRowsWithUnresolvedRefs,
+        setNullForUnresolvedNullableRefs: !!$editorState.value?.setNullForUnresolvedNullableRefs,
      },
    });
    return script.getScript();
@@ -145,6 +155,18 @@
    timerLabel.start();
  }

+  export async function generateScript() {
+    const code = await createScript();
+    openNewTab(
+      {
+        title: 'Shell #',
+        icon: 'img shell',
+        tabComponent: 'ShellTab',
+      },
+      { editor: code }
+    );
+  }
+
  $: effect = useEffect(() => registerRunnerDone(runnerId));

  function registerRunnerDone(rid) {
@@ -286,6 +308,29 @@
              }}
            />
          </FormFieldTemplateLarge>
+
+          <FormFieldTemplateLarge
+            label="Set NULL for nullable unresolved references"
+            type="checkbox"
+            labelProps={{
+              onClick: () => {
+                setEditorData(old => ({
+                  ...old,
+                  setNullForUnresolvedNullableRefs: !$editorState.value?.setNullForUnresolvedNullableRefs,
+                }));
+              },
+            }}
+          >
+            <CheckboxField
+              checked={$editorState.value?.setNullForUnresolvedNullableRefs}
+              on:change={e => {
+                setEditorData(old => ({
+                  ...old,
+                  setNullForUnresolvedNullableRefs: e.target.checked,
+                }));
+              }}
+            />
+          </FormFieldTemplateLarge>
        </ObjectConfigurationControl>

        <ObjectConfigurationControl title="Imported files">
@@ -398,6 +443,7 @@
  <svelte:fragment slot="toolstrip">
    <ToolStripCommandButton command="dataDuplicator.run" />
    <ToolStripCommandButton command="dataDuplicator.kill" />
+    <ToolStripCommandButton command="dataDuplicator.generateScript" />
  </svelte:fragment>
 </ToolStripContainer>

--- a/packages/web/src/tabs/ImportExportTab.svelte
+++ b/packages/web/src/tabs/ImportExportTab.svelte
@@ -275,7 +275,7 @@
    {:else}
      <ToolStripButton on:click={handleExecute} icon="icon run">Run</ToolStripButton>
    {/if}
-    <ToolStripButton icon="img sql-file" on:click={handleGenerateScript}>Generate script</ToolStripButton>
+    <ToolStripButton icon="img shell" on:click={handleGenerateScript}>Generate script</ToolStripButton>
    <ToolStripSaveButton idPrefix="job" />
  </svelte:fragment>
 </ToolStripContainer>