preloaded data works

integration-tests/__tests__/deploy-database.spec.js

@@ -221,4 +221,33 @@ describe('Deploy database', () => {
       );
     })
   );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy preloaded data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'value', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, value: 1 },
+                { id: 2, value: 2 },
+                { id: 3, value: 3 },
+              ],
+            },
+          },
+        ],
+      ]);
+
+      const res = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res.rows[0].cnt.toString()).toEqual('3');
+    })
+  );
 });

integration-tests/engines.js

@@ -118,7 +118,7 @@ const filterLocal = [
   // filter local testing
   'MySQL',
   'PostgreSQL',
-  'SQL Server',
+  '-SQL Server',
   'SQLite',
   '-CockroachDB',
 ];

packages/api/src/shell/generateDeploySql.js

@@ -5,6 +5,7 @@ const {
   databaseInfoFromYamlModel,
   extendDatabaseInfo,
   modelCompareDbDiffOptions,
+  enrichWithPreloadedRows,
 } = require('dbgate-tools');
 const importDbModel = require('../utility/importDbModel');
 const requireEngineDriver = require('../utility/requireEngineDriver');
@@ -19,8 +20,9 @@ async function generateDeploySql({
   loadedDbModel = undefined,
 }) {
   if (!driver) driver = requireEngineDriver(connection);
+
+  const pool = systemConnection || (await connectUtility(driver, connection));
   if (!analysedStructure) {
-    const pool = systemConnection || (await connectUtility(driver, connection));
     analysedStructure = await driver.analyseFull(pool);
   }
 
@@ -39,6 +41,12 @@ async function generateDeploySql({
     noRenameColumn: true,
   };
   const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
+  const currentModelPairedPreloaded = await enrichWithPreloadedRows(
+    deployedModel,
+    currentModelPaired,
+    pool,
+    driver
+  );
   // console.log('deployedModel', deployedModel.tables[0]);
   // console.log('currentModel', currentModel.tables[0]);
   // console.log('currentModelPaired', currentModelPaired.tables[0]);
@@ -46,7 +54,7 @@ async function generateDeploySql({
     currentModelPaired,
     deployedModel,
     opts,
-    currentModelPaired,
+    currentModelPairedPreloaded,
     deployedModel,
     driver
   );

packages/tools/src/SqlDumper.ts

@@ -1,3 +1,4 @@
+import _ from 'lodash';
 import {
   ColumnInfo,
   ConstraintInfo,
@@ -604,4 +605,40 @@ export class SqlDumper implements AlterProcessor {
   dropSqlObject(obj: SqlObjectInfo) {
     this.putCmd('^drop %s %f', this.getSqlObjectSqlName(obj.objectTypeField), obj);
   }
+
+  fillPreloadedRows(table: NamedObjectInfo, oldRows: any[], newRows: any[], key: string[]) {
+    let was = false;
+    for (const row of newRows) {
+      const old = oldRows?.find(r => key.every(col => r[col] == row[col]));
+      const rowKeys = _.keys(row);
+      if (old) {
+        const updated = [];
+        for (const col of rowKeys) {
+          if (row[col] != old[col]) {
+            updated.push(col);
+          }
+        }
+        if (updated.length > 0) {
+          if (was) this.put(';\n');
+          was = true;
+          this.put('^update %f ^set ', table);
+          this.putCollection(', ', updated, col => this.put('%i=%v', col, row[col]));
+          this.put(' ^ where ');
+          this.putCollection(' ^and ', key, col => this.put('%i=%v', col, row[col]));
+        }
+      } else {
+        if (was) this.put(';\n');
+        was = true;
+        this.put(
+          '^insert ^into %f (%,i) ^values (%,v)',
+          table,
+          rowKeys,
+          rowKeys.map(x => row[x])
+        );
+      }
+    }
+    if (was) {
+      this.endCommand();
+    }
+  }
 }

packages/tools/src/alterPlan.ts

@@ -8,6 +8,7 @@ import {
   SqlObjectInfo,
   SqlDialect,
   TableInfo,
+  NamedObjectInfo,
 } from '../../types';
 import { DatabaseInfoAlterProcessor } from './database-info-alter-processor';
 import { DatabaseAnalyser } from './DatabaseAnalyser';
@@ -86,6 +87,13 @@ interface AlterOperation_RecreateTable {
   table: TableInfo;
   operations: AlterOperation[];
 }
+interface AlterOperation_FillPreloadedRows {
+  operationType: 'fillPreloadedRows';
+  table: NamedObjectInfo;
+  oldRows: any[];
+  newRows: any[];
+  key: string[];
+}
 
 type AlterOperation =
   | AlterOperation_CreateColumn
@@ -101,7 +109,8 @@ type AlterOperation =
   | AlterOperation_RenameConstraint
   | AlterOperation_CreateSqlObject
   | AlterOperation_DropSqlObject
-  | AlterOperation_RecreateTable;
+  | AlterOperation_RecreateTable
+  | AlterOperation_FillPreloadedRows;
 
 export class AlterPlan {
   recreates = {
@@ -223,6 +232,16 @@ export class AlterPlan {
     this.recreates.tables += 1;
   }
 
+  fillPreloadedRows(table: NamedObjectInfo, oldRows: any[], newRows: any[], key: string[]) {
+    this.operations.push({
+      operationType: 'fillPreloadedRows',
+      table,
+      oldRows,
+      newRows,
+      key,
+    });
+  }
+
   run(processor: AlterProcessor) {
     for (const op of this.operations) {
       runAlterOperation(op, processor);
@@ -545,6 +564,9 @@ export function runAlterOperation(op: AlterOperation, processor: AlterProcessor)
     case 'dropSqlObject':
       processor.dropSqlObject(op.oldObject);
       break;
+    case 'fillPreloadedRows':
+      processor.fillPreloadedRows(op.table, op.oldRows, op.newRows, op.key);
+      break;
     case 'recreateTable':
       {
         const oldTable = generateTablePairingId(op.table);

packages/tools/src/database-info-alter-processor.ts

@@ -10,6 +10,7 @@ import {
   CheckInfo,
   UniqueInfo,
   SqlObjectInfo,
+  NamedObjectInfo,
 } from '../../types';
 
 export class DatabaseInfoAlterProcessor {
@@ -114,4 +115,10 @@ export class DatabaseInfoAlterProcessor {
   recreateTable(oldTable: TableInfo, newTable: TableInfo) {
     throw new Error('recreateTable not implemented for DatabaseInfoAlterProcessor');
   }
+
+  fillPreloadedRows(table: NamedObjectInfo, oldRows: any[], newRows: any[], key: string[]) {
+    const tableInfo = this.db.tables.find(x => x.pureName == table.pureName && x.schemaName == table.schemaName);
+    tableInfo.preloadedRows = newRows;
+    tableInfo.preloadedRowsKey = key;
+  }
 }

packages/tools/src/diffTools.ts

@@ -325,6 +325,13 @@ function createPairs(oldList, newList, additionalCondition = null) {
   return res;
 }
 
+function planTablePreload(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo) {
+  const key = newTable.preloadedRowsKey || newTable.primaryKey?.columns?.map(x => x.columnName);
+  if (newTable.preloadedRows?.length > 0 && key?.length > 0) {
+    plan.fillPreloadedRows(newTable, oldTable?.preloadedRows, newTable.preloadedRows, key);
+  }
+}
+
 function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo, opts: DbDiffOptions) {
   // if (oldTable.primaryKey)
 
@@ -374,6 +381,8 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
     });
 
   constraintPairs.filter(x => x[0] == null).forEach(x => plan.createConstraint(x[1]));
+
+  planTablePreload(plan, oldTable, newTable);
 }
 
 export function testEqualTables(
@@ -405,6 +414,7 @@ export function createAlterTablePlan(
   const plan = new AlterPlan(wholeOldDb, wholeNewDb, driver.dialect, opts);
   if (oldTable == null) {
     plan.createTable(newTable);
+    planTablePreload(plan, null, newTable);
   } else if (newTable == null) {
     plan.dropTable(oldTable);
   } else {
@@ -452,7 +462,10 @@ export function createAlterDatabasePlan(
     for (const newobj of newDb[objectTypeField] || []) {
       const oldobj = (oldDb[objectTypeField] || []).find(x => x.pairingId == newobj.pairingId);
       if (objectTypeField == 'tables') {
-        if (oldobj == null) plan.createTable(newobj);
+        if (oldobj == null) {
+          plan.createTable(newobj);
+          planTablePreload(plan, null, newobj);
+        }
       } else {
         if (oldobj == null) plan.createSqlObject(newobj);
       }

packages/tools/src/index.ts

@@ -16,3 +16,4 @@ export * from './schemaEditorTools';
 export * from './yamlModelConv';
 export * from './stringTools';
 export * from './computeDiffRows';
+export * from './preloadedRowsTools';

packages/tools/src/preloadedRowsTools.ts

@@ -0,0 +1,47 @@
+import _ from 'lodash';
+import { DatabaseInfo, EngineDriver } from 'dbgate-types';
+
+export async function enrichWithPreloadedRows(
+  dbModel: DatabaseInfo,
+  dbTarget: DatabaseInfo,
+  conn,
+  driver: EngineDriver
+): Promise<DatabaseInfo> {
+  // const res = { ...dbTarget, tables: [...(dbTarget.tables || [])] };
+  const repl = {};
+  for (const tableTarget of dbTarget.tables) {
+    const tableModel = dbModel.tables.find(x => x.pairingId == tableTarget.pairingId);
+    if (tableModel.preloadedRows?.length || 0 == 0) continue;
+    const keyColumns = tableModel.preloadedRowsKey || tableModel.primaryKey?.columns?.map(x => x.columnName);
+    if (keyColumns?.length || 0 == 0) continue;
+    const dmp = driver.createDumper();
+    if (keyColumns.length == 1) {
+      dmp.putCmd(
+        '^select * ^from %f ^where %i ^in (%,v)',
+        tableTarget,
+        keyColumns[0],
+        tableModel.preloadedRows.map(x => x[keyColumns[0]])
+      );
+    } else {
+      dmp.put('^select * ^from %f ^where', tableTarget);
+      dmp.putCollection(' ^or ', tableTarget.preloadedRows, row => {
+        dmp.put('(');
+        dmp.putCollection(' ^and ', keyColumns, col => dmp.put('%i=%v', col, row[col]));
+        dmp.put(')');
+      });
+      dmp.endCommand();
+    }
+    const resp = await driver.query(conn, dmp.s);
+    repl[tableTarget.pairingId] = {
+      ...tableTarget,
+      preloadedRows: resp.rows,
+      preloadedRowsKey: keyColumns,
+    };
+  }
+
+  if (_.isEmpty(repl)) return dbTarget;
+  return {
+    ...dbTarget,
+    tables: dbTarget.tables.map(x => repl[x.pairingId] || x),
+  };
+}

packages/tools/src/yamlModelConv.ts

@@ -22,6 +22,9 @@ export interface TableInfoYaml {
   // schema?: string;
   columns: ColumnInfoYaml[];
   primaryKey?: string[];
+
+  insertKey?: string[];
+  data?: any[];
 }
 
 export interface ForeignKeyInfoYaml {
@@ -119,6 +122,8 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
       columns: table.primaryKey.map(columnName => ({ columnName })),
     };
   }
+  res.preloadedRows = table.data;
+  res.preloadedRowsKey = table.insertKey;
   return res;
 }
 

packages/types/alter-processor.d.ts

@@ -15,4 +15,5 @@ export interface AlterProcessor {
   recreateTable(oldTable: TableInfo, newTable: TableInfo);
   createSqlObject(obj: SqlObjectInfo);
   dropSqlObject(obj: SqlObjectInfo);
+  fillPreloadedRows(table: NamedObjectInfo, oldRows: any[], newRows: any[], key: string[]);
 }

packages/types/dbinfo.d.ts

@@ -78,6 +78,8 @@ export interface TableInfo extends DatabaseObjectInfo {
   indexes?: IndexInfo[];
   uniques?: UniqueInfo[];
   checks?: CheckInfo[];
+  preloadedRows?: any[];
+  preloadedRowsKey?: string[];
 }
 
 export interface CollectionInfo extends DatabaseObjectInfo {}