Merge branch 'feature/copy-stdin-import'

integration-tests/__tests__/db-import.spec.js

@@ -3,6 +3,7 @@ const stream = require('stream');
 const { testWrapper } = require('../tools');
 const tableWriter = require('dbgate-api/src/shell/tableWriter');
 const copyStream = require('dbgate-api/src/shell/copyStream');
+const importDatabase = require('dbgate-api/src/shell/importDatabase');
 const fakeObjectReader = require('dbgate-api/src/shell/fakeObjectReader');
 
 function createImportStream() {
@@ -72,4 +73,29 @@ describe('DB Import', () => {
     })
   );
 
+  test.each(engines.filter(x => x.dumpFile).map(engine => [engine.label, engine]))(
+    'Import SQL dump - %s',
+    testWrapper(async (conn, driver, engine) => {
+      // const reader = await fakeObjectReader({ delay: 10 });
+      // const reader = await fakeObjectReader();
+      await importDatabase({
+        systemConnection: conn,
+        driver,
+        inputFile: engine.dumpFile,
+      });
+
+      const structure = await driver.analyseFull(conn);
+
+      for (const check of engine.dumpChecks || []) {
+        const res = await driver.query(conn, check.sql);
+        expect(res.rows[0].res.toString()).toEqual(check.res);
+      }
+
+      // const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      // expect(res1.rows[0].cnt.toString()).toEqual('6');
+
+      // const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      // expect(res2.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
 });

integration-tests/docker-compose.yaml

@@ -8,14 +8,14 @@ services:
     ports: 
       - 15000:5432
 
-  # mariadb:
-  #   image: mariadb
-  #   command: --default-authentication-plugin=mysql_native_password
-  #   restart: always
-  #   ports: 
-  #     - 15004:3306
-  #   environment:
-  #     - MYSQL_ROOT_PASSWORD=Pwd2020Db
+  mariadb:
+    image: mariadb
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    ports: 
+      - 15004:3306
+    environment:
+      - MYSQL_ROOT_PASSWORD=Pwd2020Db
 
   # mysql:
   #   image: mysql:8.0.18

integration-tests/engines.js

@@ -30,6 +30,13 @@ const engines = [
     // skipOnCI: true,
     objects: [views],
     dbSnapshotBySeconds: true,
+    dumpFile: 'data/chinook-mysql.sql',
+    dumpChecks: [
+      {
+        sql: 'select count(*) as res from genre',
+        res: '25',
+      },
+    ],
   },
   {
     label: 'MariaDB',
@@ -47,6 +54,13 @@ const engines = [
     skipOnCI: true,
     objects: [views],
     dbSnapshotBySeconds: true,
+    dumpFile: 'data/chinook-mysql.sql',
+    dumpChecks: [
+      {
+        sql: 'select count(*) as res from genre',
+        res: '25',
+      },
+    ],
   },
   {
     label: 'PostgreSQL',
@@ -83,6 +97,13 @@ const engines = [
     ],
     supportSchemas: true,
     defaultSchemaName: 'public',
+    dumpFile: 'data/chinook-postgre.sql',
+    dumpChecks: [
+      {
+        sql: 'select count(*) as res from "public"."Genre"',
+        res: '25',
+      },
+    ],
   },
   {
     label: 'SQL Server',
@@ -164,10 +185,10 @@ const engines = [
 const filterLocal = [
   // filter local testing
   '-MySQL',
-  '-MariaDB',
-  '-PostgreSQL',
+  'MariaDB',
+  'PostgreSQL',
   '-SQL Server',
-  'SQLite',
+  '-SQLite',
   '-CockroachDB',
   '-ClickHouse',
 ];

packages/api/package.json

@@ -27,7 +27,7 @@
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
     "dbgate-datalib": "^5.0.0-alpha.1",
-    "dbgate-query-splitter": "^4.10.3",
+    "dbgate-query-splitter": "^4.10.5",
     "dbgate-sqltree": "^5.0.0-alpha.1",
     "dbgate-tools": "^5.0.0-alpha.1",
     "debug": "^4.3.4",

packages/api/src/shell/importDatabase.js

@@ -9,14 +9,28 @@ const { getLogger } = require('dbgate-tools');
 const logger = getLogger('importDb');
 
 class ImportStream extends stream.Transform {
-  constructor(pool, driver) {
+  constructor(dbhan, driver) {
     super({ objectMode: true });
-    this.pool = pool;
+    this.dbhan = dbhan;
     this.driver = driver;
+    this.writeQueryStream = null;
   }
   async _transform(chunk, encoding, cb) {
     try {
-      await this.driver.script(this.pool, chunk, { queryOptions: { importSqlDump: true } });
+      if (chunk.specialMarker == 'copy_stdin_start') {
+        this.writeQueryStream = await this.driver.writeQueryFromStream(this.dbhan, chunk.text);
+      } else if (chunk.specialMarker == 'copy_stdin_line') {
+        this.writeQueryStream.write(chunk.text);
+      } else if (chunk.specialMarker == 'copy_stdin_end') {
+        this.writeQueryStream.end();
+        await new Promise((resolve, reject) => {
+          this.writeQueryStream.on('finish', resolve);
+          this.writeQueryStream.on('error', reject);
+        });
+        this.writeQueryStream = null;
+      } else {
+        await this.driver.script(this.dbhan, chunk.text, { queryOptions: { importSqlDump: true } });
+      }
     } catch (err) {
       this.emit('error', err.message);
     }
@@ -44,7 +58,7 @@ async function importDatabase({ connection = undefined, systemConnection = undef
   logger.info(`Importing database`);
 
   if (!driver) driver = requireEngineDriver(connection);
-  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
   logger.info(`Connected.`);
 
   logger.info(`Input file: ${inputFile}`);
@@ -52,8 +66,11 @@ async function importDatabase({ connection = undefined, systemConnection = undef
   logger.info(`Downloaded file: ${downloadedFile}`);
 
   const fileStream = fs.createReadStream(downloadedFile, 'utf-8');
-  const splittedStream = splitQueryStream(fileStream, driver.getQuerySplitterOptions('script'));
-  const importStream = new ImportStream(pool, driver);
+  const splittedStream = splitQueryStream(fileStream, {
+    ...driver.getQuerySplitterOptions('import'),
+    returnRichInfo: true,
+  });
+  const importStream = new ImportStream(dbhan, driver);
   // @ts-ignore
   splittedStream.pipe(importStream);
   await awaitStreamEnd(importStream);

packages/tools/package.json

@@ -31,7 +31,7 @@
     "typescript": "^4.4.3"
   },
   "dependencies": {
-    "dbgate-query-splitter": "^4.10.3",
+    "dbgate-query-splitter": "^4.10.5",
     "dbgate-sqltree": "^5.0.0-alpha.1",
     "debug": "^4.3.4",
     "json-stable-stringify": "^1.0.1",

packages/tools/src/driverBase.ts

@@ -195,4 +195,8 @@ export const driverBase = {
   async listSchemas(pool) {
     return null;
   },
+
+  async writeQueryFromStream(dbhan, sql) {
+    return null;
+  },
 };

packages/types/engines.d.ts

@@ -188,6 +188,8 @@ export interface EngineDriver extends FilterBehaviourProvider {
   stream(dbhan: DatabaseHandle, sql: string, options: StreamOptions);
   readQuery(dbhan: DatabaseHandle, sql: string, structure?: TableInfo): Promise<stream.Readable>;
   readJsonQuery(dbhan: DatabaseHandle, query: any, structure?: TableInfo): Promise<stream.Readable>;
+  // eg. PostgreSQL COPY FROM stdin
+  writeQueryFromStream(dbhan: DatabaseHandle, sql: string): Promise<stream.Writable>;
   writeTable(dbhan: DatabaseHandle, name: NamedObjectInfo, options: WriteTableOptions): Promise<stream.Writable>;
   analyseSingleObject(
     dbhan: DatabaseHandle,
@@ -218,7 +220,7 @@ export interface EngineDriver extends FilterBehaviourProvider {
   getCollectionUpdateScript(changeSet: any, collectionInfo: CollectionInfo): string;
   createDatabase(dbhan: DatabaseHandle, name: string): Promise;
   dropDatabase(dbhan: DatabaseHandle, name: string): Promise;
-  getQuerySplitterOptions(usage: 'stream' | 'script' | 'editor'): any;
+  getQuerySplitterOptions(usage: 'stream' | 'script' | 'editor' | 'import'): any;
   script(dbhan: DatabaseHandle, sql: string, options?: RunScriptOptions): Promise;
   operation(dbhan: DatabaseHandle, operation: {}, options?: RunScriptOptions): Promise;
   getNewObjectTemplates(): NewObjectTemplate[];

packages/web/package.json

@@ -25,7 +25,7 @@
     "chartjs-adapter-moment": "^1.0.0",
     "cross-env": "^7.0.3",
     "dbgate-datalib": "^5.0.0-alpha.1",
-    "dbgate-query-splitter": "^4.10.3",
+    "dbgate-query-splitter": "^4.10.5",
     "dbgate-sqltree": "^5.0.0-alpha.1",
     "dbgate-tools": "^5.0.0-alpha.1",
     "dbgate-types": "^5.0.0-alpha.1",

plugins/dbgate-plugin-mongo/package.json

@@ -33,7 +33,7 @@
   "devDependencies": {
     "bson": "^6.8.0",
     "dbgate-plugin-tools": "^1.0.7",
-    "dbgate-query-splitter": "^4.10.3",
+    "dbgate-query-splitter": "^4.10.5",
     "dbgate-tools": "^5.0.0-alpha.1",
     "is-promise": "^4.0.0",
     "lodash": "^4.17.21",

plugins/dbgate-plugin-mssql/package.json

@@ -33,7 +33,7 @@
   "devDependencies": {
     "async-lock": "^1.2.6",
     "dbgate-plugin-tools": "^1.0.7",
-    "dbgate-query-splitter": "^4.10.3",
+    "dbgate-query-splitter": "^4.10.5",
     "dbgate-tools": "^5.0.0-alpha.1",
     "tedious": "^18.2.0",
     "webpack": "^5.91.0",

plugins/dbgate-plugin-mysql/package.json

@@ -33,7 +33,7 @@
   "devDependencies": {
     "antares-mysql-dumper": "^0.0.1",
     "dbgate-plugin-tools": "^1.0.7",
-    "dbgate-query-splitter": "^4.10.3",
+    "dbgate-query-splitter": "^4.10.5",
     "dbgate-tools": "^5.0.0-alpha.1",
     "mysql2": "^3.11.0",
     "webpack": "^5.91.0",

plugins/dbgate-plugin-mysql/src/backend/drivers.js

@@ -52,7 +52,7 @@ const drivers = driverBases.map(driverBase => ({
     const dbhan = {
       client,
       database,
-    }
+    };
     if (isReadOnly) {
       await this.query(dbhan, 'SET SESSION TRANSACTION READ ONLY');
     }
@@ -69,7 +69,11 @@ const drivers = driverBases.map(driverBase => ({
       };
     }
 
-    if (options?.importSqlDump && sql.trim().startsWith('/*!') && sql.includes('character_set_client')) {
+    if (
+      options?.importSqlDump &&
+      (sql.trim().startsWith('/*!') || sql.trim().startsWith('/*M!')) &&
+      (sql.includes('character_set_client') || sql.includes('NOTE_VERBOSITY'))
+    ) {
       // skip this in SQL dumps
       return {
         rows: [],

plugins/dbgate-plugin-oracle/package.json

@@ -31,7 +31,7 @@
   },
   "devDependencies": {
     "dbgate-plugin-tools": "^1.0.8",
-    "dbgate-query-splitter": "^4.10.3",
+    "dbgate-query-splitter": "^4.10.5",
     "dbgate-tools": "^5.0.0-alpha.1",
     "lodash": "^4.17.21",
     "webpack": "^5.91.0",

plugins/dbgate-plugin-postgres/package.json

@@ -31,11 +31,14 @@
   },
   "devDependencies": {
     "dbgate-plugin-tools": "^1.0.7",
-    "dbgate-query-splitter": "^4.10.3",
+    "dbgate-query-splitter": "^4.10.5",
     "dbgate-tools": "^5.0.0-alpha.1",
     "lodash": "^4.17.21",
     "pg": "^8.11.5",
     "webpack": "^5.91.0",
     "webpack-cli": "^5.1.4"
+  },
+  "dependencies": {
+    "pg-copy-streams": "^6.0.6"
   }
 }

plugins/dbgate-plugin-postgres/src/backend/drivers.js

@@ -4,6 +4,7 @@ const stream = require('stream');
 const driverBases = require('../frontend/drivers');
 const Analyser = require('./Analyser');
 const pg = require('pg');
+const pgCopyStreams = require('pg-copy-streams');
 const { getLogger, createBulkInsertStreamBase, makeUniqueColumnNames, extractDbNameFromComposite } =
   global.DBGATE_PACKAGES['dbgate-tools'];
 
@@ -292,6 +293,11 @@ const drivers = driverBases.map(driverBase => ({
 
     return schemas;
   },
+
+  writeQueryFromStream(dbhan, sql) {
+    const stream = dbhan.client.query(pgCopyStreams.from(sql));
+    return stream;
+  },
 }));
 
 module.exports = drivers;

plugins/dbgate-plugin-postgres/src/frontend/drivers.js

@@ -124,6 +124,11 @@ const postgresDriverBase = {
   getQuerySplitterOptions: usage =>
     usage == 'editor'
       ? { ...postgreSplitterOptions, ignoreComments: true, preventSingleLineSplit: true }
+      : usage == 'import'
+      ? {
+          ...postgreSplitterOptions,
+          copyFromStdin: true,
+        }
       : postgreSplitterOptions,
   readOnlySessions: true,
 

plugins/dbgate-plugin-redis/package.json

@@ -30,7 +30,7 @@
   },
   "devDependencies": {
     "dbgate-plugin-tools": "^1.0.7",
-    "dbgate-query-splitter": "^4.10.3",
+    "dbgate-query-splitter": "^4.10.5",
     "dbgate-tools": "^5.0.0-alpha.1",
     "lodash": "^4.17.21",
     "webpack": "^5.91.0",

plugins/dbgate-plugin-sqlite/package.json

@@ -32,7 +32,7 @@
     "devDependencies": {
       "dbgate-tools": "^5.0.0-alpha.1",
       "dbgate-plugin-tools": "^1.0.4",
-      "dbgate-query-splitter": "^4.10.3",
+      "dbgate-query-splitter": "^4.10.5",
       "byline": "^5.0.0",
       "webpack": "^5.91.0",
       "webpack-cli": "^5.1.4"

yarn.lock

@@ -3224,10 +3224,10 @@ dbgate-plugin-xml@^5.0.0-alpha.1:
   resolved "https://registry.yarnpkg.com/dbgate-plugin-xml/-/dbgate-plugin-xml-5.2.7.tgz#0762af51ba6f100e75a63907ea6c679e827c9f7c"
   integrity sha512-gBXy4qetf7eJQW6lM01B+OKLnKB8MKesojdYKysD9oZ+YpQCX8Tq7aHJCrN14FiyIDinpX61kmFH1+LGJ2RkxQ==
 
-dbgate-query-splitter@^4.10.3:
-  version "4.10.3"
-  resolved "https://registry.yarnpkg.com/dbgate-query-splitter/-/dbgate-query-splitter-4.10.3.tgz#8aad658cc1f942acfbd661de402a90da5ffb6cfd"
-  integrity sha512-PDkt4RFeMEgqnUQK3XlyN77KtWTcf+WlcPcIBzpRscqBAYqlk4gFXDTb/gY8t0mMi28SzawqQ4H1eitbRc7C7A==
+dbgate-query-splitter@^4.10.5:
+  version "4.10.5"
+  resolved "https://registry.yarnpkg.com/dbgate-query-splitter/-/dbgate-query-splitter-4.10.5.tgz#bef9f2c2232b14aab6d3a0d490739ad216a1b45b"
+  integrity sha512-AWcKcU3hbS8rAYrA52bwzdwtLv1llZgJ7Ut8AVPVWm5i38J4EdRFA2nMnx4Y5GLPqA1SRSUFsjBsNWWaHa/BAg==
 
 debug@2.6.9, debug@^2.2.0, debug@^2.3.3:
   version "2.6.9"
@@ -7936,6 +7936,11 @@ object.pick@^1.3.0:
   dependencies:
     isobject "^3.0.1"
 
+obuf@^1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/obuf/-/obuf-1.1.2.tgz#09bea3343d41859ebd446292d11c9d4db619084e"
+  integrity sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==
+
 on-exit-leak-free@^2.1.0:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz#fed195c9ebddb7d9e4c3842f93f281ac8dadd3b8"
@@ -8235,6 +8240,13 @@ pg-connection-string@^2.6.4:
   resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.4.tgz#f543862adfa49fa4e14bc8a8892d2a84d754246d"
   integrity sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==
 
+pg-copy-streams@^6.0.6:
+  version "6.0.6"
+  resolved "https://registry.yarnpkg.com/pg-copy-streams/-/pg-copy-streams-6.0.6.tgz#a24dbcbd2999f4ecd4ab919f92a1b256656e4f3c"
+  integrity sha512-Z+Dd2C2NIDTsjyFKmc6a9QLlpM8tjpERx+43RSx0WmL7j3uNChERi3xSvZUL0hWJ1oRUn4S3fhyt3apdSrTyKQ==
+  dependencies:
+    obuf "^1.1.2"
+
 pg-int8@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"