From 82ae588d9aa9860aebd9d375a0c8cc999cd188a5 Mon Sep 17 00:00:00 2001
From: Jan Prochazka <jan.prochazka@gmail.com>
Date: Thu, 9 Dec 2021 09:10:07 +0100
Subject: [PATCH] ability to create string mongo IDs when importing

---
 packages/types/engines.d.ts                     |  1 +
 .../web/src/impexp/SourceTargetConfig.svelte    | 15 +++++++++++++--
 packages/web/src/impexp/createImpExpScript.ts   | 17 ++++++++++++++---
 .../web/src/modals/ImportExportModal.svelte     |  2 --
 .../src/backend/createBulkInsertStream.js       | 10 +++++++++-
 .../dbgate-plugin-mongo/src/backend/driver.js   |  4 +++-
 .../dbgate-plugin-mongo/src/frontend/driver.js  | 10 ++++++++++
 7 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts
index a1ea74766..5c8bc6050 100644
--- a/packages/types/engines.d.ts
+++ b/packages/types/engines.d.ts
@@ -48,6 +48,7 @@ export interface EngineDriver {
   beforeConnectionSave?: (values: any) => any;
   databaseUrlPlaceholder?: string;
   defaultAuthTypeName?: string;
+  importExportArgs?: any[];
   connect({ server, port, user, password, database }): Promise<any>;
   close(pool): Promise<any>;
   query(pool: any, sql: string, options?: QueryOptions): Promise<QueryResult>;
diff --git a/packages/web/src/impexp/SourceTargetConfig.svelte b/packages/web/src/impexp/SourceTargetConfig.svelte
index eb7511d9b..3ba270ca3 100644
--- a/packages/web/src/impexp/SourceTargetConfig.svelte
+++ b/packages/web/src/impexp/SourceTargetConfig.svelte
@@ -14,12 +14,13 @@
   import { findFileFormat, getFileFormatDirections } from '../plugins/fileformats';
   import SqlEditor from '../query/SqlEditor.svelte';
   import { currentArchive, currentDatabase, extensions } from '../stores';
-  import { useArchiveFiles, useDatabaseInfo } from '../utility/metadataLoaders';
+  import { useArchiveFiles, useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders';
   import FilesInput from './FilesInput.svelte';
   import FormConnectionSelect from './FormConnectionSelect.svelte';
   import FormDatabaseSelect from './FormDatabaseSelect.svelte';
   import FormSchemaSelect from './FormSchemaSelect.svelte';
   import FormTablesSelect from './FormTablesSelect.svelte';
+  import { findEngineDriver } from 'dbgate-tools';
 
   export let direction;
   export let storageTypeField;
@@ -52,6 +53,8 @@
   $: dbinfo = useDatabaseInfo({ conid: $values[connectionIdField], database: $values[databaseNameField] });
   $: archiveFiles = useArchiveFiles({ folder: $values[archiveFolderField] });
   $: format = findFileFormat($extensions, storageType);
+  $: connectionInfo = useConnectionInfo({ conid: $values[connectionIdField] });
+  $: driver = findEngineDriver($connectionInfo, $extensions);
 </script>
 
 <div class="column">
@@ -159,12 +162,20 @@
     <FilesInput {setPreviewSource} />
   {/if}
 
-  {#if format && format.args}
+  {#if format?.args}
     <FormArgumentList
       args={format.args.filter(arg => !arg.direction || arg.direction == direction)}
       namePrefix={`${direction}_${format.storageType}_`}
     />
   {/if}
+
+  {#if driver?.importExportArgs}
+  <FormArgumentList
+    args={driver?.importExportArgs.filter(arg => !arg.direction || arg.direction == direction)}
+    namePrefix={`${direction}_${driver.engine}_`}
+  />
+{/if}
+
 </div>
 
 <style>
diff --git a/packages/web/src/impexp/createImpExpScript.ts b/packages/web/src/impexp/createImpExpScript.ts
index 015fa7376..0f35bcc73 100644
--- a/packages/web/src/impexp/createImpExpScript.ts
+++ b/packages/web/src/impexp/createImpExpScript.ts
@@ -17,7 +17,7 @@ export function getTargetName(extensions, source, values) {
   return source;
 }
 
-function extractApiParameters(values, direction, format) {
+function extractFormatApiParameters(values, direction, format) {
   const pairs = (format.args || [])
     .filter(arg => arg.apiName)
     .map(arg => [arg.apiName, values[`${direction}_${format.storageType}_${arg.name}`]])
@@ -25,6 +25,14 @@ function extractApiParameters(values, direction, format) {
   return _.fromPairs(pairs);
 }
 
+function extractDriverApiParameters(values, direction, driver) {
+  const pairs = (driver.importExportArgs || [])
+    .filter(arg => arg.apiName)
+    .map(arg => [arg.apiName, values[`${direction}_${driver.engine}_${arg.name}`]])
+    .filter(x => x[1] != null);
+  return _.fromPairs(pairs);
+}
+
 async function getConnection(extensions, storageType, conid, database) {
   if (storageType == 'database' || storageType == 'query') {
     const conn = await getConnectionInfo({ conid });
@@ -48,6 +56,7 @@ function getSourceExpr(extensions, sourceName, values, sourceConnection, sourceD
       'tableReader',
       {
         connection: sourceConnection,
+        ...extractDriverApiParameters(values, 'source', sourceDriver),
         ...fullName,
       },
     ];
@@ -57,6 +66,7 @@ function getSourceExpr(extensions, sourceName, values, sourceConnection, sourceD
       'queryReader',
       {
         connection: sourceConnection,
+        ...extractDriverApiParameters(values, 'source', sourceDriver),
         sql: values.sourceSql,
       },
     ];
@@ -69,7 +79,7 @@ function getSourceExpr(extensions, sourceName, values, sourceConnection, sourceD
         format.readerFunc,
         {
           ..._.omit(sourceFile, ['isDownload']),
-          ...extractApiParameters(values, 'source', format),
+          ...extractFormatApiParameters(values, 'source', format),
         },
       ];
     }
@@ -121,7 +131,7 @@ function getTargetExpr(extensions, sourceName, values, targetConnection, targetD
           : {
               fileName: getTargetName(extensions, sourceName, values),
             }),
-        ...extractApiParameters(values, 'target', format),
+        ...extractFormatApiParameters(values, 'target', format),
       },
     ];
   }
@@ -132,6 +142,7 @@ function getTargetExpr(extensions, sourceName, values, targetConnection, targetD
         connection: targetConnection,
         schemaName: values.targetSchemaName,
         pureName: getTargetName(extensions, sourceName, values),
+        ...extractDriverApiParameters(values, 'target', targetDriver),
         ...getFlagsFroAction(values[`actionType_${sourceName}`]),
       },
     ];
diff --git a/packages/web/src/modals/ImportExportModal.svelte b/packages/web/src/modals/ImportExportModal.svelte
index 6f98a2820..20a6e7142 100644
--- a/packages/web/src/modals/ImportExportModal.svelte
+++ b/packages/web/src/modals/ImportExportModal.svelte
@@ -63,8 +63,6 @@
     }
   }
 
-  // $: targetArchiveFolder = importToArchive ? `import-${moment().format('YYYY-MM-DD-hh-mm-ss')}` : $currentArchive;
-
   $: effect = useEffect(() => registerRunnerDone(runnerId));
 
   function registerRunnerDone(rid) {
diff --git a/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js b/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js
index a26b66096..abb1811cf 100644
--- a/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js
+++ b/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js
@@ -1,3 +1,5 @@
+const ObjectId = require('mongodb').ObjectId;
+
 function createBulkInsertStream(driver, stream, pool, name, options) {
   const collectionName = name.pureName;
   const db = pool.__getDatabase();
@@ -19,11 +21,17 @@ function createBulkInsertStream(driver, stream, pool, name, options) {
       )
         return;
     }
+    if (options.createStringId) {
+      row = {
+        _id: new ObjectId().toString(),
+        ...row,
+      }
+    }
     writable.buffer.push(row);
   };
 
   writable.checkStructure = async () => {
-    if (options.dropIfExists || options.truncate) {
+    if (options.dropIfExists) {
       console.log(`Dropping collection ${collectionName}`);
       await db.collection(collectionName).drop();
     }
diff --git a/plugins/dbgate-plugin-mongo/src/backend/driver.js b/plugins/dbgate-plugin-mongo/src/backend/driver.js
index 6c3b2809a..d13f8c8cd 100644
--- a/plugins/dbgate-plugin-mongo/src/backend/driver.js
+++ b/plugins/dbgate-plugin-mongo/src/backend/driver.js
@@ -59,7 +59,9 @@ const driver = {
       ? `mongodb://${user}:${password}@${server}:${port}`
       : `mongodb://${server}:${port}`;
 
-    const options = {};
+    const options = {
+      useUnifiedTopology: true,
+    };
     if (ssl) {
       options.tls = true;
       options.tlsCAFile = ssl.ca;
diff --git a/plugins/dbgate-plugin-mongo/src/frontend/driver.js b/plugins/dbgate-plugin-mongo/src/frontend/driver.js
index ac23a53bb..15381d9c2 100644
--- a/plugins/dbgate-plugin-mongo/src/frontend/driver.js
+++ b/plugins/dbgate-plugin-mongo/src/frontend/driver.js
@@ -44,6 +44,16 @@ const driver = {
     return ['server', 'port', 'user', 'password', 'defaultDatabase', 'singleDatabase'].includes(field);
   },
 
+  importExportArgs: [
+    {
+      type: 'checkbox',
+      name: 'createStringId',
+      label: 'Create string _id attribute',
+      apiName: 'createStringId',
+      direction: 'target',
+    },
+  ],
+
   getCollectionUpdateScript(changeSet) {
     let res = '';
     for (const insert of changeSet.inserts) {