diff --git a/integration-tests/__tests__/deploy-database.spec.js b/integration-tests/__tests__/deploy-database.spec.js index 781cb8228..d3c109bf1 100644 --- a/integration-tests/__tests__/deploy-database.spec.js +++ b/integration-tests/__tests__/deploy-database.spec.js @@ -221,4 +221,33 @@ describe('Deploy database', () => { ); }) ); + + test.each(engines.map(engine => [engine.label, engine]))( + 'Deploy preloaded data - %s', + testWrapper(async (conn, driver, engine) => { + await testDatabaseDeploy(conn, driver, [ + [ + { + name: 't1.table.yaml', + json: { + name: 't1', + columns: [ + { name: 'id', type: 'int' }, + { name: 'value', type: 'int' }, + ], + primaryKey: ['id'], + data: [ + { id: 1, value: 1 }, + { id: 2, value: 2 }, + { id: 3, value: 3 }, + ], + }, + }, + ], + ]); + + const res = await driver.query(conn, `select count(*) as cnt from t1`); + expect(res.rows[0].cnt.toString()).toEqual('3'); + }) + ); }); diff --git a/integration-tests/engines.js b/integration-tests/engines.js index eebc11757..f73f0c98e 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -118,7 +118,7 @@ const filterLocal = [ // filter local testing 'MySQL', 'PostgreSQL', - 'SQL Server', + '-SQL Server', 'SQLite', '-CockroachDB', ]; diff --git a/packages/api/src/shell/generateDeploySql.js b/packages/api/src/shell/generateDeploySql.js index 9e4e428a7..2364b1191 100644 --- a/packages/api/src/shell/generateDeploySql.js +++ b/packages/api/src/shell/generateDeploySql.js @@ -5,6 +5,7 @@ const { databaseInfoFromYamlModel, extendDatabaseInfo, modelCompareDbDiffOptions, + enrichWithPreloadedRows, } = require('dbgate-tools'); const importDbModel = require('../utility/importDbModel'); const requireEngineDriver = require('../utility/requireEngineDriver'); @@ -19,8 +20,9 @@ async function generateDeploySql({ loadedDbModel = undefined, }) { if (!driver) driver = requireEngineDriver(connection); + + const pool = systemConnection || (await connectUtility(driver, connection)); if (!analysedStructure) { - const pool = systemConnection || (await connectUtility(driver, connection)); analysedStructure = await driver.analyseFull(pool); } @@ -39,6 +41,12 @@ async function generateDeploySql({ noRenameColumn: true, }; const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts); + const currentModelPairedPreloaded = await enrichWithPreloadedRows( + deployedModel, + currentModelPaired, + pool, + driver + ); // console.log('deployedModel', deployedModel.tables[0]); // console.log('currentModel', currentModel.tables[0]); // console.log('currentModelPaired', currentModelPaired.tables[0]); @@ -46,7 +54,7 @@ async function generateDeploySql({ currentModelPaired, deployedModel, opts, - currentModelPaired, + currentModelPairedPreloaded, deployedModel, driver ); diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index 5f7dd9bd0..88427a2b3 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -1,3 +1,4 @@ +import _ from 'lodash'; import { ColumnInfo, ConstraintInfo, @@ -604,4 +605,40 @@ export class SqlDumper implements AlterProcessor { dropSqlObject(obj: SqlObjectInfo) { this.putCmd('^drop %s %f', this.getSqlObjectSqlName(obj.objectTypeField), obj); } + + fillPreloadedRows(table: NamedObjectInfo, oldRows: any[], newRows: any[], key: string[]) { + let was = false; + for (const row of newRows) { + const old = oldRows?.find(r => key.every(col => r[col] == row[col])); + const rowKeys = _.keys(row); + if (old) { + const updated = []; + for (const col of rowKeys) { + if (row[col] != old[col]) { + updated.push(col); + } + } + if (updated.length > 0) { + if (was) this.put(';\n'); + was = true; + this.put('^update %f ^set ', table); + this.putCollection(', ', updated, col => this.put('%i=%v', col, row[col])); + this.put(' ^ where '); + this.putCollection(' ^and ', key, col => this.put('%i=%v', col, row[col])); + } + } else { + if (was) this.put(';\n'); + was = true; + this.put( + '^insert ^into %f (%,i) ^values (%,v)', + table, + rowKeys, + rowKeys.map(x => row[x]) + ); + } + } + if (was) { + this.endCommand(); + } + } } diff --git a/packages/tools/src/alterPlan.ts b/packages/tools/src/alterPlan.ts index 5cb64fa91..858138f89 100644 --- a/packages/tools/src/alterPlan.ts +++ b/packages/tools/src/alterPlan.ts @@ -8,6 +8,7 @@ import { SqlObjectInfo, SqlDialect, TableInfo, + NamedObjectInfo, } from '../../types'; import { DatabaseInfoAlterProcessor } from './database-info-alter-processor'; import { DatabaseAnalyser } from './DatabaseAnalyser'; @@ -86,6 +87,13 @@ interface AlterOperation_RecreateTable { table: TableInfo; operations: AlterOperation[]; } +interface AlterOperation_FillPreloadedRows { + operationType: 'fillPreloadedRows'; + table: NamedObjectInfo; + oldRows: any[]; + newRows: any[]; + key: string[]; +} type AlterOperation = | AlterOperation_CreateColumn @@ -101,7 +109,8 @@ type AlterOperation = | AlterOperation_RenameConstraint | AlterOperation_CreateSqlObject | AlterOperation_DropSqlObject - | AlterOperation_RecreateTable; + | AlterOperation_RecreateTable + | AlterOperation_FillPreloadedRows; export class AlterPlan { recreates = { @@ -223,6 +232,16 @@ export class AlterPlan { this.recreates.tables += 1; } + fillPreloadedRows(table: NamedObjectInfo, oldRows: any[], newRows: any[], key: string[]) { + this.operations.push({ + operationType: 'fillPreloadedRows', + table, + oldRows, + newRows, + key, + }); + } + run(processor: AlterProcessor) { for (const op of this.operations) { runAlterOperation(op, processor); @@ -545,6 +564,9 @@ export function runAlterOperation(op: AlterOperation, processor: AlterProcessor) case 'dropSqlObject': processor.dropSqlObject(op.oldObject); break; + case 'fillPreloadedRows': + processor.fillPreloadedRows(op.table, op.oldRows, op.newRows, op.key); + break; case 'recreateTable': { const oldTable = generateTablePairingId(op.table); diff --git a/packages/tools/src/database-info-alter-processor.ts b/packages/tools/src/database-info-alter-processor.ts index e95909131..d0e0fdaa4 100644 --- a/packages/tools/src/database-info-alter-processor.ts +++ b/packages/tools/src/database-info-alter-processor.ts @@ -10,6 +10,7 @@ import { CheckInfo, UniqueInfo, SqlObjectInfo, + NamedObjectInfo, } from '../../types'; export class DatabaseInfoAlterProcessor { @@ -114,4 +115,10 @@ export class DatabaseInfoAlterProcessor { recreateTable(oldTable: TableInfo, newTable: TableInfo) { throw new Error('recreateTable not implemented for DatabaseInfoAlterProcessor'); } + + fillPreloadedRows(table: NamedObjectInfo, oldRows: any[], newRows: any[], key: string[]) { + const tableInfo = this.db.tables.find(x => x.pureName == table.pureName && x.schemaName == table.schemaName); + tableInfo.preloadedRows = newRows; + tableInfo.preloadedRowsKey = key; + } } diff --git a/packages/tools/src/diffTools.ts b/packages/tools/src/diffTools.ts index 14231b062..b7c218ab6 100644 --- a/packages/tools/src/diffTools.ts +++ b/packages/tools/src/diffTools.ts @@ -325,6 +325,13 @@ function createPairs(oldList, newList, additionalCondition = null) { return res; } +function planTablePreload(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo) { + const key = newTable.preloadedRowsKey || newTable.primaryKey?.columns?.map(x => x.columnName); + if (newTable.preloadedRows?.length > 0 && key?.length > 0) { + plan.fillPreloadedRows(newTable, oldTable?.preloadedRows, newTable.preloadedRows, key); + } +} + function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo, opts: DbDiffOptions) { // if (oldTable.primaryKey) @@ -374,6 +381,8 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf }); constraintPairs.filter(x => x[0] == null).forEach(x => plan.createConstraint(x[1])); + + planTablePreload(plan, oldTable, newTable); } export function testEqualTables( @@ -405,6 +414,7 @@ export function createAlterTablePlan( const plan = new AlterPlan(wholeOldDb, wholeNewDb, driver.dialect, opts); if (oldTable == null) { plan.createTable(newTable); + planTablePreload(plan, null, newTable); } else if (newTable == null) { plan.dropTable(oldTable); } else { @@ -452,7 +462,10 @@ export function createAlterDatabasePlan( for (const newobj of newDb[objectTypeField] || []) { const oldobj = (oldDb[objectTypeField] || []).find(x => x.pairingId == newobj.pairingId); if (objectTypeField == 'tables') { - if (oldobj == null) plan.createTable(newobj); + if (oldobj == null) { + plan.createTable(newobj); + planTablePreload(plan, null, newobj); + } } else { if (oldobj == null) plan.createSqlObject(newobj); } diff --git a/packages/tools/src/index.ts b/packages/tools/src/index.ts index c0e5ee9e2..2ce79b5c1 100644 --- a/packages/tools/src/index.ts +++ b/packages/tools/src/index.ts @@ -16,3 +16,4 @@ export * from './schemaEditorTools'; export * from './yamlModelConv'; export * from './stringTools'; export * from './computeDiffRows'; +export * from './preloadedRowsTools'; diff --git a/packages/tools/src/preloadedRowsTools.ts b/packages/tools/src/preloadedRowsTools.ts new file mode 100644 index 000000000..882bc8a9c --- /dev/null +++ b/packages/tools/src/preloadedRowsTools.ts @@ -0,0 +1,47 @@ +import _ from 'lodash'; +import { DatabaseInfo, EngineDriver } from 'dbgate-types'; + +export async function enrichWithPreloadedRows( + dbModel: DatabaseInfo, + dbTarget: DatabaseInfo, + conn, + driver: EngineDriver +): Promise { + // const res = { ...dbTarget, tables: [...(dbTarget.tables || [])] }; + const repl = {}; + for (const tableTarget of dbTarget.tables) { + const tableModel = dbModel.tables.find(x => x.pairingId == tableTarget.pairingId); + if (tableModel.preloadedRows?.length || 0 == 0) continue; + const keyColumns = tableModel.preloadedRowsKey || tableModel.primaryKey?.columns?.map(x => x.columnName); + if (keyColumns?.length || 0 == 0) continue; + const dmp = driver.createDumper(); + if (keyColumns.length == 1) { + dmp.putCmd( + '^select * ^from %f ^where %i ^in (%,v)', + tableTarget, + keyColumns[0], + tableModel.preloadedRows.map(x => x[keyColumns[0]]) + ); + } else { + dmp.put('^select * ^from %f ^where', tableTarget); + dmp.putCollection(' ^or ', tableTarget.preloadedRows, row => { + dmp.put('('); + dmp.putCollection(' ^and ', keyColumns, col => dmp.put('%i=%v', col, row[col])); + dmp.put(')'); + }); + dmp.endCommand(); + } + const resp = await driver.query(conn, dmp.s); + repl[tableTarget.pairingId] = { + ...tableTarget, + preloadedRows: resp.rows, + preloadedRowsKey: keyColumns, + }; + } + + if (_.isEmpty(repl)) return dbTarget; + return { + ...dbTarget, + tables: dbTarget.tables.map(x => repl[x.pairingId] || x), + }; +} diff --git a/packages/tools/src/yamlModelConv.ts b/packages/tools/src/yamlModelConv.ts index 24a51aea2..ad1e2fff9 100644 --- a/packages/tools/src/yamlModelConv.ts +++ b/packages/tools/src/yamlModelConv.ts @@ -22,6 +22,9 @@ export interface TableInfoYaml { // schema?: string; columns: ColumnInfoYaml[]; primaryKey?: string[]; + + insertKey?: string[]; + data?: any[]; } export interface ForeignKeyInfoYaml { @@ -119,6 +122,8 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml columns: table.primaryKey.map(columnName => ({ columnName })), }; } + res.preloadedRows = table.data; + res.preloadedRowsKey = table.insertKey; return res; } diff --git a/packages/types/alter-processor.d.ts b/packages/types/alter-processor.d.ts index 4cb7aad63..25de4885d 100644 --- a/packages/types/alter-processor.d.ts +++ b/packages/types/alter-processor.d.ts @@ -15,4 +15,5 @@ export interface AlterProcessor { recreateTable(oldTable: TableInfo, newTable: TableInfo); createSqlObject(obj: SqlObjectInfo); dropSqlObject(obj: SqlObjectInfo); + fillPreloadedRows(table: NamedObjectInfo, oldRows: any[], newRows: any[], key: string[]); } diff --git a/packages/types/dbinfo.d.ts b/packages/types/dbinfo.d.ts index 3f5f655e9..abe23ac5f 100644 --- a/packages/types/dbinfo.d.ts +++ b/packages/types/dbinfo.d.ts @@ -78,6 +78,8 @@ export interface TableInfo extends DatabaseObjectInfo { indexes?: IndexInfo[]; uniques?: UniqueInfo[]; checks?: CheckInfo[]; + preloadedRows?: any[]; + preloadedRowsKey?: string[]; } export interface CollectionInfo extends DatabaseObjectInfo {}