data duplicator works in simple case

packages/tools/src/ScriptWriter.ts

@@ -57,6 +57,10 @@ export class ScriptWriter {
     this._put(`await dbgateApi.importDatabase(${JSON.stringify(options)});`);
   }
 
+  dataDuplicator(options) {
+    this._put(`await dbgateApi.dataDuplicator(${JSON.stringify(options)});`);
+  }
+
   comment(s) {
     this._put(`// ${s}`);
   }
@@ -143,6 +147,13 @@ export class ScriptWriterJson {
     });
   }
 
+  dataDuplicator(options) {
+    this.commands.push({
+      type: 'dataDuplicator',
+      options,
+    });
+  }
+
   getScript(schedule = null) {
     return {
       type: 'json',
@@ -186,6 +197,9 @@ export function jsonScriptToJavascript(json) {
       case 'importDatabase':
         script.importDatabase(cmd.options);
         break;
+      case 'dataDuplicator':
+        script.dataDuplicator(cmd.options);
+        break;
     }
   }
 

packages/tools/src/SqlDumper.ts

@@ -197,6 +197,8 @@ export class SqlDumper implements AlterProcessor {
 
   specialColumnOptions(column) {}
 
+  selectScopeIdentity(table: TableInfo) {}
+
   columnDefinition(column: ColumnInfo, { includeDefault = true, includeNullable = true, includeCollate = true } = {}) {
     if (column.computedExpression) {
       this.put('^as %s', column.computedExpression);

packages/tools/src/createAsyncWriteStream.ts

@@ -0,0 +1,41 @@
+import _intersection from 'lodash/intersection';
+import _isArray from 'lodash/isArray';
+import { getLogger } from './getLogger';
+
+const logger = getLogger('asyncWriteStream');
+
+export interface AsyncWriteStreamOptions {
+  processItem: (chunk: any) => Promise<void>;
+}
+
+export function createAsyncWriteStream(stream, options: AsyncWriteStreamOptions): any {
+  const writable = new stream.Writable({
+    objectMode: true,
+  });
+
+  writable._write = async (chunk, encoding, callback) => {
+    await options.processItem(chunk);
+
+    // const { sql, id, newIdSql } = chunk;
+    // if (_isArray(sql)) {
+    //   for (const item of sql) await driver.query(pool, item, { discardResult: true });
+    // } else {
+    //   await driver.query(pool, sql, { discardResult: true });
+    // }
+    // if (newIdSql) {
+    //   const res = await driver.query(pool, newIdSql);
+    //   const resId = Object.entries(res?.rows?.[0])?.[0]?.[1];
+
+    //   if (options?.mapResultId) {
+    //     options?.mapResultId(id, resId as string);
+    //   }
+    // }
+    callback();
+  };
+
+  // writable._final = async callback => {
+  //   callback();
+  // };
+
+  return writable;
+}

packages/tools/src/createBulkInsertStreamBase.ts

@@ -1,10 +1,11 @@
+import { EngineDriver, WriteTableOptions } from 'dbgate-types';
 import _intersection from 'lodash/intersection';
 import { getLogger } from './getLogger';
 import { prepareTableForImport } from './tableTransforms';
 
 const logger = getLogger('bulkStreamBase');
 
-export function createBulkInsertStreamBase(driver, stream, pool, name, options): any {
+export function createBulkInsertStreamBase(driver: EngineDriver, stream, pool, name, options: WriteTableOptions): any {
   const fullNameQuoted = name.schemaName
     ? `${driver.dialect.quoteIdentifier(name.schemaName)}.${driver.dialect.quoteIdentifier(name.pureName)}`
     : driver.dialect.quoteIdentifier(name.pureName);
@@ -58,21 +59,21 @@ export function createBulkInsertStreamBase(driver, stream, pool, name, options):
     const dmp = driver.createDumper();
 
     dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`);
-    dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col)));
+    dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col as string)));
     dmp.putRaw(')\n VALUES\n');
 
     let wasRow = false;
     for (const row of rows) {
       if (wasRow) dmp.putRaw(',\n');
       dmp.putRaw('(');
-      dmp.putCollection(',', writable.columnNames, col => dmp.putValue(row[col]));
+      dmp.putCollection(',', writable.columnNames, col => dmp.putValue(row[col as string]));
       dmp.putRaw(')');
       wasRow = true;
     }
     dmp.putRaw(';');
     // require('fs').writeFileSync('/home/jena/test.sql', dmp.s);
     // console.log(dmp.s);
-    await driver.query(pool, dmp.s);
+    await driver.query(pool, dmp.s, { discardResult: true });
   };
 
   writable.sendIfFull = async () => {

packages/tools/src/driverBase.ts

@@ -2,7 +2,7 @@ import _compact from 'lodash/compact';
 import { SqlDumper } from './SqlDumper';
 import { splitQuery } from 'dbgate-query-splitter';
 import { dumpSqlSelect } from 'dbgate-sqltree';
-import { EngineDriver, RunScriptOptions } from 'dbgate-types';
+import { EngineDriver, QueryResult, RunScriptOptions } from 'dbgate-types';
 
 const dialect = {
   limitSelect: true,
@@ -20,12 +20,22 @@ const dialect = {
   defaultSchemaName: null,
 };
 
-export async function runCommandOnDriver(pool, driver: EngineDriver, cmd: (dmp: SqlDumper) => void) {
+export async function runCommandOnDriver(pool, driver: EngineDriver, cmd: (dmp: SqlDumper) => void): Promise<void> {
   const dmp = driver.createDumper();
   cmd(dmp as any);
   await driver.query(pool, dmp.s, { discardResult: true });
 }
 
+export async function runQueryOnDriver(
+  pool,
+  driver: EngineDriver,
+  cmd: (dmp: SqlDumper) => void
+): Promise<QueryResult> {
+  const dmp = driver.createDumper();
+  cmd(dmp as any);
+  return await driver.query(pool, dmp.s);
+}
+
 export const driverBase = {
   analyserClass: null,
   dumperClass: SqlDumper,

packages/tools/src/index.ts

@@ -3,6 +3,7 @@ export * from './nameTools';
 export * from './tableTransforms';
 export * from './packageTools';
 export * from './createBulkInsertStreamBase';
+export * from './createAsyncWriteStream';
 export * from './DatabaseAnalyser';
 export * from './driverBase';
 export * from './SqlDumper';