SYNC: mongo import tests

e2e-tests/.localconfig.js

@@ -1,7 +1,8 @@
 module.exports = {
-  mysql: true,
-  postgres: true,
-  mssql: true,
-  oracle: true,
-  sqlite: true,
+  // mysql: true,
+  // postgres: true,
+  // mssql: true,
+  // oracle: true,
+  // sqlite: true,
+  mongo: true
 };

e2e-tests/cypress/e2e/multi-sql.cy.js

@@ -28,32 +28,32 @@ beforeEach(() => {
   cy.viewport(1250, 900);
 });
 
-function multiTest(testName, testDefinition) {
+function multiTest(testProps, testDefinition) {
   if (localconfig.mysql) {
-    it(testName + ' MySQL', () => testDefinition('MySql-connection', 'my_guitar_shop', 'mysql@dbgate-plugin-mysql'));
+    it('MySQL', () => testDefinition('MySql-connection', 'my_guitar_shop', 'mysql@dbgate-plugin-mysql'));
   }
   if (localconfig.postgres) {
-    it(testName + ' Postgres', () =>
-      testDefinition('Postgres-connection', 'my_guitar_shop', 'postgres@dbgate-plugin-postgres')
-    );
+    it('Postgres', () => testDefinition('Postgres-connection', 'my_guitar_shop', 'postgres@dbgate-plugin-postgres'));
   }
   if (localconfig.mssql) {
-    it(testName + ' Mssql', () => testDefinition('Mssql-connection', 'my_guitar_shop', 'mssql@dbgate-plugin-mssql'));
+    it('Mssql', () => testDefinition('Mssql-connection', 'my_guitar_shop', 'mssql@dbgate-plugin-mssql'));
   }
   if (localconfig.oracle) {
-    it(testName + ' Oracle', () =>
+    it('Oracle', () =>
       testDefinition('Oracle-connection', 'C##MY_GUITAR_SHOP', 'oracle@dbgate-plugin-oracle', {
         implicitTransactions: true,
-      })
-    );
+      }));
   }
   if (localconfig.sqlite) {
-    it(testName + ' Sqlite', () => testDefinition('Sqlite-connection', null, 'sqlite@dbgate-plugin-sqlite'));
+    it('Sqlite', () => testDefinition('Sqlite-connection', null, 'sqlite@dbgate-plugin-sqlite'));
+  }
+  if (localconfig.mongo && !testProps.skipMongo) {
+    it('MongoDB', () => testDefinition('Mongo-connection', 'my_guitar_shop', 'mongo@dbgate-plugin-mongo'));
   }
 }
 
 describe('Transactions', () => {
-  multiTest('Transactions', (connectionName, databaseName, engine, options = {}) => {
+  multiTest({ skipMongo: true }, (connectionName, databaseName, engine, options = {}) => {
     const driver = requireEngineDriver(engine);
     const implicitTransactions = options.implicitTransactions ?? false;
 
@@ -101,7 +101,7 @@ describe('Transactions', () => {
 });
 
 describe('Backup table', () => {
-  multiTest('Backup table', (connectionName, databaseName, engine, options = {}) => {
+  multiTest({ skipMongo: true }, (connectionName, databaseName, engine, options = {}) => {
     cy.contains(connectionName).click();
     if (databaseName) cy.contains(databaseName).click();
     cy.contains('customers').rightclick();
@@ -113,7 +113,7 @@ describe('Backup table', () => {
 });
 
 describe('Truncate table', () => {
-  multiTest('Truncate table', (connectionName, databaseName, engine, options = {}) => {
+  multiTest({ skipMongo: true }, (connectionName, databaseName, engine, options = {}) => {
     cy.contains(connectionName).click();
     if (databaseName) cy.contains(databaseName).click();
     cy.contains('order_items').rightclick();
@@ -125,7 +125,7 @@ describe('Truncate table', () => {
 });
 
 describe('Drop table', () => {
-  multiTest('Drop table', (connectionName, databaseName, engine, options = {}) => {
+  multiTest({ skipMongo: true }, (connectionName, databaseName, engine, options = {}) => {
     cy.contains(connectionName).click();
     if (databaseName) cy.contains(databaseName).click();
     cy.contains('order_items').rightclick();
@@ -136,7 +136,7 @@ describe('Drop table', () => {
 });
 
 describe('Import CSV', () => {
-  multiTest('Import CSV', (connectionName, databaseName, engine, options = {}) => {
+  multiTest({}, (connectionName, databaseName, engine, options = {}) => {
     cy.contains(connectionName).click();
     if (databaseName) cy.contains(databaseName).click();
     cy.testid('ConnectionList_container')
@@ -146,7 +146,7 @@ describe('Import CSV', () => {
 
     cy.get('input[type=file]').selectFile('cypress/fixtures/customers-20.csv', { force: true });
     cy.contains('customers-20');
-    cy.testid('ImportExportTab_preview').contains('Jessica').should('be.visible');
+    cy.testid('ImportExportTab_preview_content').contains('50ddd99fAdF48B3').should('be.visible');
 
     cy.testid('ImportExportTab_executeButton').click();
     cy.contains('20 rows written').should('be.visible');
@@ -164,7 +164,7 @@ describe('Import CSV', () => {
 });
 
 describe('Import CSV - source error', () => {
-  multiTest('Import CSV - source error', (connectionName, databaseName, engine, options = {}) => {
+  multiTest({}, (connectionName, databaseName, engine, options = {}) => {
     cy.contains(connectionName).click();
     if (databaseName) cy.contains(databaseName).click();
     cy.testid('ConnectionList_container')
@@ -184,7 +184,7 @@ describe('Import CSV - source error', () => {
 });
 
 describe('Import CSV - target error', () => {
-  multiTest('Import CSV - target error', (connectionName, databaseName, engine, options = {}) => {
+  multiTest({ skipMongo: true }, (connectionName, databaseName, engine, options = {}) => {
     cy.contains(connectionName).click();
     if (databaseName) cy.contains(databaseName).click();
     cy.testid('ConnectionList_container')

e2e-tests/env/multi-sql/.env

@@ -1,4 +1,4 @@
-CONNECTIONS=mysql,postgres,mssql,oracle,sqlite
+CONNECTIONS=mysql,postgres,mssql,oracle,sqlite,mongo
 LOG_CONNECTION_SENSITIVE_VALUES=true
 
 LABEL_mysql=MySql-connection
@@ -35,3 +35,11 @@ ENGINE_mssql=mssql@dbgate-plugin-mssql
 LABEL_sqlite=Sqlite-connection
 FILE_sqlite=%%E2E_TEST_DATA_DIRECTORY%%/my_guitar_shop.db
 ENGINE_sqlite=sqlite@dbgate-plugin-sqlite
+
+LABEL_mongo=Mongo-connection
+SERVER_mongo=localhost
+USER_mongo=root
+PASSWORD_mongo=Pwd2020Db
+PORT_mongo=16010
+ENGINE_mongo=mongo@dbgate-plugin-mongo
+

e2e-tests/init/multi-sql.js

@@ -8,12 +8,13 @@ dbgateApi.registerPlugins(dbgatePluginMysql);
 const dbgatePluginPostgres = require('dbgate-plugin-postgres');
 dbgateApi.registerPlugins(dbgatePluginPostgres);
 
-async function createDb(connection, dropDbSql, createDbSql, database = 'my_guitar_shop') {
+async function createDb(connection, dropDbSql, createDbSql, database = 'my_guitar_shop', { dropDatabaseName }) {
   if (dropDbSql) {
     try {
       await dbgateApi.executeQuery({
         connection,
         sql: dropDbSql,
+        database: dropDatabaseName,
       });
     } catch (err) {
       console.error('Failed to drop database', err);
@@ -108,6 +109,22 @@ async function run() {
       null
     );
   }
+
+  if (localconfig.mongo) {
+    await createDb(
+      {
+        server: process.env.SERVER_mongo,
+        user: process.env.USER_mongo,
+        password: process.env.PASSWORD_mongo,
+        port: process.env.PORT_mongo,
+        engine: 'mongo@dbgate-plugin-mongo',
+      },
+      'db.dropDatabase()',
+      null,
+      'my_guitar_shop',
+      { dropDatabaseName: 'my_guitar_shop' }
+    );
+  }
 }
 
 dbgateApi.runScript(run);

packages/api/src/shell/importDbFromFolder.js

@@ -23,82 +23,97 @@ async function importDbFromFolder({ connection, systemConnection, driver, folder
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
 
   try {
-    const model = await importDbModel(folder);
+    if (driver?.databaseEngineTypes?.includes('sql')) {
+      const model = await importDbModel(folder);
 
-    let modelAdapted = {
-      ...model,
-      tables: model.tables.map(table => driver.adaptTableInfo(table)),
-    };
-    for (const transform of modelTransforms || []) {
-      modelAdapted = transform(modelAdapted);
-    }
-
-    const modelNoFk = {
-      ...modelAdapted,
-      tables: modelAdapted.tables.map(table => ({
-        ...table,
-        foreignKeys: [],
-      })),
-    };
-
-    // const plan = createAlterDatabasePlan(
-    //   DatabaseAnalyser.createEmptyStructure(),
-    //   driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
-    //   {},
-    //   DatabaseAnalyser.createEmptyStructure(),
-    //   driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
-    //   driver
-    // );
-    // const dmp1 = driver.createDumper({ useHardSeparator: true });
-    // if (driver.dialect.enableAllForeignKeys) {
-    //   dmp1.enableAllForeignKeys(false);
-    // }
-    // plan.run(dmp1);
-    // if (driver.dialect.enableAllForeignKeys) {
-    //   dmp1.enableAllForeignKeys(true);
-    // }
-
-    const { sql } = getAlterDatabaseScript(
-      DatabaseAnalyser.createEmptyStructure(),
-      driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
-      {},
-      DatabaseAnalyser.createEmptyStructure(),
-      driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
-      driver
-    );
-    // console.log('CREATING STRUCTURE:', sql);
-    await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });
-
-    if (driver.dialect.enableAllForeignKeys) {
-      await runCommandOnDriver(dbhan, driver, dmp => dmp.enableAllForeignKeys(false));
-    }
-
-    for (const table of modelAdapted.tables) {
-      const fileName = path.join(folder, `${table.pureName}.jsonl`);
-      if (await fs.exists(fileName)) {
-        const src = await jsonLinesReader({ fileName });
-        const dst = await tableWriter({
-          systemConnection: dbhan,
-          pureName: table.pureName,
-          driver,
-          targetTableStructure: table,
-        });
-        await copyStream(src, dst);
+      let modelAdapted = {
+        ...model,
+        tables: model.tables.map(table => driver.adaptTableInfo(table)),
+      };
+      for (const transform of modelTransforms || []) {
+        modelAdapted = transform(modelAdapted);
+      }
+
+      const modelNoFk = {
+        ...modelAdapted,
+        tables: modelAdapted.tables.map(table => ({
+          ...table,
+          foreignKeys: [],
+        })),
+      };
+
+      // const plan = createAlterDatabasePlan(
+      //   DatabaseAnalyser.createEmptyStructure(),
+      //   driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+      //   {},
+      //   DatabaseAnalyser.createEmptyStructure(),
+      //   driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+      //   driver
+      // );
+      // const dmp1 = driver.createDumper({ useHardSeparator: true });
+      // if (driver.dialect.enableAllForeignKeys) {
+      //   dmp1.enableAllForeignKeys(false);
+      // }
+      // plan.run(dmp1);
+      // if (driver.dialect.enableAllForeignKeys) {
+      //   dmp1.enableAllForeignKeys(true);
+      // }
+
+      const { sql } = getAlterDatabaseScript(
+        DatabaseAnalyser.createEmptyStructure(),
+        driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+        {},
+        DatabaseAnalyser.createEmptyStructure(),
+        driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+        driver
+      );
+      // console.log('CREATING STRUCTURE:', sql);
+      await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });
+
+      if (driver.dialect.enableAllForeignKeys) {
+        await runCommandOnDriver(dbhan, driver, dmp => dmp.enableAllForeignKeys(false));
       }
-    }
 
-    if (driver.dialect.enableAllForeignKeys) {
-      await runCommandOnDriver(dbhan, driver, dmp => dmp.enableAllForeignKeys(true));
-    } else if (driver.dialect.createForeignKey) {
-      const dmp = driver.createDumper();
       for (const table of modelAdapted.tables) {
-        for (const fk of table.foreignKeys) {
-          dmp.createForeignKey(fk);
+        const fileName = path.join(folder, `${table.pureName}.jsonl`);
+        if (await fs.exists(fileName)) {
+          const src = await jsonLinesReader({ fileName });
+          const dst = await tableWriter({
+            systemConnection: dbhan,
+            pureName: table.pureName,
+            driver,
+            targetTableStructure: table,
+          });
+          await copyStream(src, dst);
         }
       }
 
-      // create foreign keys
-      await executeQuery({ connection, systemConnection: dbhan, driver, sql: dmp.s, logScriptItems: true });
+      if (driver.dialect.enableAllForeignKeys) {
+        await runCommandOnDriver(dbhan, driver, dmp => dmp.enableAllForeignKeys(true));
+      } else if (driver.dialect.createForeignKey) {
+        const dmp = driver.createDumper();
+        for (const table of modelAdapted.tables) {
+          for (const fk of table.foreignKeys) {
+            dmp.createForeignKey(fk);
+          }
+        }
+
+        // create foreign keys
+        await executeQuery({ connection, systemConnection: dbhan, driver, sql: dmp.s, logScriptItems: true });
+      }
+    } else if (driver?.databaseEngineTypes?.includes('document')) {
+      for (const file of fs.readdirSync(folder)) {
+        if (!file.endsWith('.jsonl')) continue;
+        const pureName = path.parse(file).name;
+        const src = await jsonLinesReader({ fileName: path.join(folder, file) });
+        const dst = await tableWriter({
+          systemConnection: dbhan,
+          pureName,
+          driver,
+          createIfNotExists: true,
+        });
+        await copyStream(src, dst);
+      }
     }
   } finally {
     if (!systemConnection) {