diff --git a/integration-tests/__tests__/data-duplicator.spec.js b/integration-tests/__tests__/data-duplicator.spec.js index 7d5c532e5..67023aa50 100644 --- a/integration-tests/__tests__/data-duplicator.spec.js +++ b/integration-tests/__tests__/data-duplicator.spec.js @@ -91,4 +91,68 @@ describe('Data duplicator', () => { expect(res2.rows[0].cnt.toString()).toEqual('6'); }) ); + + test.each(engines.filter(x => !x.skipDataDuplicator).map(engine => [engine.label, engine]))( + 'Skip nullable weak refs - %s', + testWrapper(async (conn, driver, engine) => { + runCommandOnDriver(conn, driver, dmp => + dmp.createTable({ + pureName: 't1', + columns: [ + { columnName: 'id', dataType: 'int', notNull: true }, + { columnName: 'val', dataType: 'varchar(50)' }, + ], + primaryKey: { + columns: [{ columnName: 'id' }], + }, + }) + ); + runCommandOnDriver(conn, driver, dmp => + dmp.createTable({ + pureName: 't2', + columns: [ + { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true }, + { columnName: 'val', dataType: 'varchar(50)' }, + { columnName: 'valfk', dataType: 'int', notNull: false }, + ], + primaryKey: { + columns: [{ columnName: 'id' }], + }, + foreignKeys: [{ refTableName: 't1', columns: [{ columnName: 'valfk', refColumnName: 'id' }] }], + }) + ); + runCommandOnDriver(conn, driver, dmp => dmp.put("insert into ~t1 (~id, ~val) values (1, 'first')")); + + const gett2 = () => + stream.Readable.from([ + { __isStreamHeader: true, __isDynamicStructure: true }, + { id: 1, val: 'v1', valfk: 1 }, + { id: 2, val: 'v2', valfk: 2 }, + ]); + + await dataDuplicator({ + systemConnection: conn, + driver, + items: [ + { + name: 't2', + operation: 'copy', + openStream: gett2, + }, + ], + options: { + setNullForUnresolvedNullableRefs: true, + }, + }); + + const res1 = await driver.query(conn, `select count(*) as cnt from t1`); + expect(res1.rows[0].cnt.toString()).toEqual('1'); + + const res2 = await driver.query(conn, `select count(*) as cnt from t2`); + expect(res2.rows[0].cnt.toString()).toEqual('2'); + + const res3 = await driver.query(conn, `select count(*) as cnt from t2 where valfk is not null`); + expect(res3.rows[0].cnt.toString()).toEqual('1'); + }) + ); }); diff --git a/integration-tests/engines.js b/integration-tests/engines.js index eb14d8c4e..77acf4629 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -186,9 +186,9 @@ const engines = [ const filterLocal = [ // filter local testing - '-MySQL', + 'MySQL', '-MariaDB', - 'PostgreSQL', + '-PostgreSQL', '-SQL Server', '-SQLite', '-CockroachDB', diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index 421ada4ee..50c0b6571 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -21,6 +21,7 @@ export interface DataDuplicatorItem { export interface DataDuplicatorOptions { rollbackAfterFinish?: boolean; skipRowsWithUnresolvedRefs?: boolean; + setNullForUnresolvedNullableRefs?: boolean; } class DuplicatorReference { @@ -36,9 +37,19 @@ class DuplicatorReference { } } +class DuplicatorWeakReference { + constructor(public base: DuplicatorItemHolder, public ref: TableInfo, public foreignKey: ForeignKeyInfo) {} + + get columnName() { + return this.foreignKey.columns[0].columnName; + } +} + class DuplicatorItemHolder { references: DuplicatorReference[] = []; backReferences: DuplicatorReference[] = []; + // not mandatory references to entities out of the model + weakReferences: DuplicatorWeakReference[] = []; table: TableInfo; isPlanned = false; idMap = {}; @@ -65,23 +76,33 @@ class DuplicatorItemHolder { for (const fk of this.table.foreignKeys) { if (fk.columns?.length != 1) continue; const refHolder = this.duplicator.itemHolders.find(y => y.name.toUpperCase() == fk.refTableName.toUpperCase()); - if (refHolder == null) continue; const isMandatory = this.table.columns.find(x => x.columnName == fk.columns[0]?.columnName)?.notNull; - const newref = new DuplicatorReference(this, refHolder, isMandatory, fk); - this.references.push(newref); - this.refByColumn[newref.columnName] = newref; + if (refHolder == null) { + if (!isMandatory) { + const weakref = new DuplicatorWeakReference( + this, + this.duplicator.db.tables.find(x => x.pureName == fk.refTableName), + fk + ); + this.weakReferences.push(weakref); + } + } else { + const newref = new DuplicatorReference(this, refHolder, isMandatory, fk); + this.references.push(newref); + this.refByColumn[newref.columnName] = newref; - refHolder.isReferenced = true; + refHolder.isReferenced = true; + } } } - createInsertObject(chunk) { + createInsertObject(chunk, weakrefcols: string[]) { const res = _omit( _pick( chunk, this.table.columns.map(x => x.columnName) ), - [this.autoColumn, ...this.backReferences.map(x => x.columnName)] + [this.autoColumn, ...this.backReferences.map(x => x.columnName), ...weakrefcols] ); for (const key in res) { @@ -102,6 +123,28 @@ class DuplicatorItemHolder { return res; } + // returns list of columns that are weak references and are not resolved + async getMissingWeakRefsForRow(row): Promise { + if (!this.duplicator.options.setNullForUnresolvedNullableRefs || !this.weakReferences?.length) { + return []; + } + + const qres = await runQueryOnDriver(this.duplicator.pool, this.duplicator.driver, dmp => { + dmp.put('^select '); + dmp.putCollection(',', this.weakReferences, weakref => { + dmp.put( + '(^case ^when ^exists (^select * ^from %f where %i = %v) ^then 1 ^else 0 ^end) as %i', + weakref.ref, + weakref.foreignKey.columns[0].refColumnName, + row[weakref.foreignKey.columns[0].columnName], + weakref.foreignKey.columns[0].columnName + ); + }); + }); + const qrow = qres.rows[0]; + return this.weakReferences.filter(x => qrow[x.columnName] == 0).map(x => x.columnName); + } + async runImport() { const readStream = await this.item.openStream(); const driver = this.duplicator.driver; @@ -112,6 +155,8 @@ class DuplicatorItemHolder { let skipped = 0; let lastLogged = new Date(); + const existingWeakRefs = {}; + const writeStream = createAsyncWriteStream(this.duplicator.stream, { processItem: async chunk => { if (chunk.__isStreamHeader) { @@ -120,7 +165,8 @@ class DuplicatorItemHolder { const doCopy = async () => { // console.log('chunk', this.name, JSON.stringify(chunk)); - const insertedObj = this.createInsertObject(chunk); + const weakrefcols = await this.getMissingWeakRefsForRow(chunk); + const insertedObj = this.createInsertObject(chunk, weakrefcols); // console.log('insertedObj', this.name, JSON.stringify(insertedObj)); if (insertedObj == null) { skipped += 1; diff --git a/packages/tools/src/ScriptWriter.ts b/packages/tools/src/ScriptWriter.ts index 84ab4bdb1..7c4f93b09 100644 --- a/packages/tools/src/ScriptWriter.ts +++ b/packages/tools/src/ScriptWriter.ts @@ -58,7 +58,7 @@ export class ScriptWriter { } dataDuplicator(options) { - this._put(`await dbgateApi.dataDuplicator(${JSON.stringify(options)});`); + this._put(`await dbgateApi.dataDuplicator(${JSON.stringify(options, null, 2)});`); } comment(s) { diff --git a/packages/web/src/appobj/ArchiveFolderAppObject.svelte b/packages/web/src/appobj/ArchiveFolderAppObject.svelte index 2e005264d..86e7e6394 100644 --- a/packages/web/src/appobj/ArchiveFolderAppObject.svelte +++ b/packages/web/src/appobj/ArchiveFolderAppObject.svelte @@ -19,6 +19,7 @@ import { apiCall } from '../utility/api'; import hasPermission from '../utility/hasPermission'; import { isProApp } from '../utility/proTools'; + import { extractShellConnection } from '../impexp/createImpExpScript'; export let data; @@ -66,10 +67,7 @@ await dbgateApi.deployDb(${JSON.stringify( { - connection: { - ..._.omit($currentDatabase.connection, '_id', 'displayName'), - database: $currentDatabase.name, - }, + connection: extractShellConnection($currentDatabase.connection, $currentDatabase.name), modelFolder: `archive:${data.name}`, }, undefined, diff --git a/packages/web/src/appobj/DatabaseAppObject.svelte b/packages/web/src/appobj/DatabaseAppObject.svelte index 094b3d91f..87bbbe4be 100644 --- a/packages/web/src/appobj/DatabaseAppObject.svelte +++ b/packages/web/src/appobj/DatabaseAppObject.svelte @@ -297,10 +297,7 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify( { - connection: { - ..._.omit(connection, '_id', 'displayName'), - database: name, - }, + connection: extractShellConnection(connection, name), }, undefined, 2 @@ -311,6 +308,30 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify( }); }; + const handleImportWithDbDuplicator = () => { + showModal(ChooseArchiveFolderModal, { + message: 'Choose archive folder for import from', + onConfirm: archiveFolder => { + openNewTab( + { + title: archiveFolder, + icon: 'img duplicator', + tabComponent: 'DataDuplicatorTab', + props: { + conid: connection?._id, + database: name, + }, + }, + { + editor: { + archiveFolder, + }, + } + ); + }, + }); + }; + const driver = findEngineDriver(connection, getExtensions()); const commands = _.flatten((apps || []).map(x => x.commands || [])); @@ -390,6 +411,14 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify( text: 'Shell: Drop all objects', }, + driver?.databaseEngineTypes?.includes('sql') && + hasPermission(`dbops/import`) && { + onClick: handleImportWithDbDuplicator, + text: 'Import with DB duplicator', + }, + + { divider: true }, + commands.length > 0 && [ commands.map((cmd: any) => ({ text: cmd.name, @@ -451,6 +480,8 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify( import { loadSchemaList, switchCurrentDatabase } from '../utility/common'; import { isProApp } from '../utility/proTools'; import ExportDbModelModal from '../modals/ExportDbModelModal.svelte'; + import ChooseArchiveFolderModal from '../modals/ChooseArchiveFolderModal.svelte'; + import { extractShellConnection } from '../impexp/createImpExpScript'; export let data; export let passProps; diff --git a/packages/web/src/impexp/createImpExpScript.ts b/packages/web/src/impexp/createImpExpScript.ts index cb001994f..0fa3c076c 100644 --- a/packages/web/src/impexp/createImpExpScript.ts +++ b/packages/web/src/impexp/createImpExpScript.ts @@ -39,7 +39,10 @@ export function extractShellConnection(connection, database) { return config.allowShellConnection ? { - ..._.omit(connection, ['_id', 'displayName', 'databases', 'connectionColor']), + ..._.omitBy( + _.omit(connection, ['_id', 'displayName', 'databases', 'connectionColor', 'status', 'unsaved']), + v => !v + ), database, } : { @@ -192,7 +195,7 @@ export function normalizeExportColumnMap(colmap) { return null; } -export default async function createImpExpScript(extensions, values, forceScript = false) { +export default async function createImpExpScript(extensions, values, forceScript = false) { const config = getCurrentConfig(); const script = config.allowShellScripting || forceScript diff --git a/packages/web/src/modals/ChooseArchiveFolderModal.svelte b/packages/web/src/modals/ChooseArchiveFolderModal.svelte new file mode 100644 index 000000000..8374f10fc --- /dev/null +++ b/packages/web/src/modals/ChooseArchiveFolderModal.svelte @@ -0,0 +1,33 @@ + + + + + Choose archive folder + +
{message}
+ + + + + { + closeCurrentModal(); + onConfirm(e.detail.archiveFolder); + }} + /> + + +
+
diff --git a/packages/web/src/tabs/DataDuplicatorTab.svelte b/packages/web/src/tabs/DataDuplicatorTab.svelte index 562eaa5ae..424e125d4 100644 --- a/packages/web/src/tabs/DataDuplicatorTab.svelte +++ b/packages/web/src/tabs/DataDuplicatorTab.svelte @@ -22,6 +22,16 @@ testEnabled: () => getCurrentEditor()?.canKill(), onClick: () => getCurrentEditor().kill(), }); + registerCommand({ + id: 'dataDuplicator.generateScript', + category: 'Data duplicator', + icon: 'img shell', + name: 'Generate Script', + toolbar: true, + isRelatedToTab: true, + testEnabled: () => getCurrentEditor()?.canRun(), + onClick: () => getCurrentEditor().generateScript(), + });