bundling refactor

adjustPackageJson.js

@@ -1,5 +1,6 @@
 const fs = require('fs');
 const path = require('path');
+const volatilePackages = require('./common/volatilePackages');
 
 function adjustFile(file) {
   const json = JSON.parse(fs.readFileSync(file, { encoding: 'utf-8' }));
@@ -11,6 +12,10 @@ function adjustFile(file) {
     );
     for (const depkey of ['dependencies', 'optionalDependencies']) {
       for (const dependency of Object.keys(pluginJson[depkey] || {})) {
+        if (!volatilePackages.includes(dependency)) {
+          // add only voletile packages
+          continue;
+        }
         if (!json[depkey]) {
           json[depkey] = {};
         }
@@ -34,3 +39,5 @@ function adjustFile(file) {
 
 adjustFile('packages/api/package.json');
 adjustFile('app/package.json');
+
+fs.writeFileSync('common/useBundleExternals.js', "module.exports = 'true';", 'utf-8');

app/src/nativeModulesContent.js

@@ -0,0 +1,9 @@
+
+// this file is generated automatically by script fillNativeModules.js, do not edit it manually
+const content = {};
+
+content['better-sqlite3'] = () => require('better-sqlite3');
+content['oracledb'] = () => require('oracledb');
+
+
+module.exports = content;

common/buildPluginExternals.js

@@ -0,0 +1,19 @@
+const useBundleExternals = require('./useBundleExternals');
+const getBundleExternals = require('./getBundleExternals');
+
+function buildExternalsFromDependencies(packageJson) {
+  if (useBundleExternals == 'true') {
+    return getBundleExternals();
+  }
+  const { dependencies, optionalDependencies } = packageJson;
+  const externals = {};
+  for (let dep in dependencies || {}) {
+    externals[dep] = `commonjs ${dep}`;
+  }
+  for (let dep in optionalDependencies || {}) {
+    externals[dep] = `commonjs ${dep}`;
+  }
+  return externals;
+}
+
+module.exports = buildExternalsFromDependencies;

common/getBundleExternals.js

@@ -0,0 +1,10 @@
+const volatilePackages = require('./volatilePackages');
+
+function getBundleExternals() {
+  return volatilePackages.reduce((acc, item) => {
+    acc[item] = `commonjs ${item}`;
+    return acc;
+  }, {});
+}
+
+module.exports = getBundleExternals;

common/useBundleExternals.js

@@ -0,0 +1 @@
+module.exports = 'false';

common/volatilePackages.js

@@ -0,0 +1,25 @@
+// these packages will be never bundled with webpack
+
+const volatilePackages = [
+  '@clickhouse/client',
+  'bson', // this package is already bundled and is used in mongodb
+  'mongodb',
+  'mongodb-client-encryption',
+  'tedious',
+  'msnodesqlv8',
+  'mysql2',
+  'oracledb',
+  'pg-copy-streams',
+  'pg',
+  'ioredis',
+  'node-redis-dump2',
+  'better-sqlite3',
+  '@azure/cosmos',
+  '@aws-sdk/rds-signer',
+  'activedirectory2',
+  'axios',
+  'bufferutil',
+  'ssh2',
+];
+
+module.exports = volatilePackages;

packages/api/c:\test\upg\upgrade.zip.version

@@ -0,0 +1 @@
+5.5.6

packages/api/package.json

@@ -16,6 +16,9 @@
     "export",
     "dbgate"
   ],
+  "files": [
+    "src"
+  ],
   "dependencies": {
     "@aws-sdk/rds-signer": "^3.665.0",
     "activedirectory2": "^2.1.0",

packages/api/src/nativeModulesContent.js

@@ -0,0 +1,9 @@
+
+// this file is generated automatically by script fillNativeModules.js, do not edit it manually
+const content = {};
+
+content['better-sqlite3'] = () => require('better-sqlite3');
+content['oracledb'] = () => require('oracledb');
+
+
+module.exports = content;

packages/api/webpack.config.js

@@ -1,5 +1,8 @@
 var webpack = require('webpack');
 var path = require('path');
+var getBundleExternals = require('../../common/getBundleExternals');
+
+var externals = getBundleExternals();
 
 var config = {
   context: __dirname + '/src',
@@ -29,27 +32,28 @@ var config = {
       },
     ],
   },
-  plugins: [
+  // plugins: [
-    new webpack.IgnorePlugin({
+  //   new webpack.IgnorePlugin({
-      checkResource(resource) {
+  //     checkResource(resource) {
-        const lazyImports = ['uws'];
+  //       const lazyImports = ['uws'];
-        if (!lazyImports.includes(resource)) {
+  //       if (!lazyImports.includes(resource)) {
-          return false;
+  //         return false;
-        }
+  //       }
-        try {
+  //       try {
-          require.resolve(resource);
+  //         require.resolve(resource);
-        } catch (err) {
+  //       } catch (err) {
-          return true;
+  //         return true;
-        }
+  //       }
-        return false;
+  //       return false;
-      },
+  //     },
-    }),
+  //   }),
-  ],
+  // ],
-  externals: {
+  // externals: {
-    'better-sqlite3': 'commonjs better-sqlite3',
+  //   'better-sqlite3': 'commonjs better-sqlite3',
-    'oracledb': 'commonjs oracledb',
+  //   'oracledb': 'commonjs oracledb',
-    'msnodesqlv8': 'commonjs msnodesqlv8',
+  //   'msnodesqlv8': 'commonjs msnodesqlv8',
-  },
+  // },
+  externals,
 };
 
 module.exports = config;

plugins/dbgate-plugin-clickhouse/webpack-backend.config.js

@@ -1,6 +1,10 @@
 var webpack = require('webpack');
 var path = require('path');
 
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
 var config = {
   context: __dirname + '/src/backend',
 
@@ -19,12 +23,7 @@ var config = {
   //     minimize: false,
   //   },
 
-  externals: {
+  externals,
-    '@clickhouse/client': 'commonjs @clickhouse/client',
-    'json-stable-stringify': 'commonjs json-stable-stringify',
-    'dbgate-tools': 'commonjs dbgate-tools',
-    lodash: 'commonjs lodash',
-  },
 };
 
 module.exports = config;

plugins/dbgate-plugin-csv/webpack-backend.config.js

@@ -1,6 +1,10 @@
 var webpack = require('webpack');
 var path = require('path');
 
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
 var config = {
   context: __dirname + '/src/backend',
 
@@ -19,11 +23,7 @@ var config = {
   //     minimize: false,
   //   },
 
-  externals: {
+  externals,
-    csv: 'commonjs csv',
-    'line-reader': 'commonjs line-reader',
-    lodash: 'commonjs lodash',
-  },
 };
 
 module.exports = config;

plugins/dbgate-plugin-excel/webpack-backend.config.js

@@ -1,6 +1,10 @@
 var webpack = require('webpack');
 var path = require('path');
 
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
 var config = {
   context: __dirname + '/src/backend',
 
@@ -19,10 +23,7 @@ var config = {
   //     minimize: false,
   //   },
 
-  externals: {
+  externals,
-    xlsx: 'commonjs xlsx',
-    lodash: 'commonjs lodash',
-  },
 };
 
 module.exports = config;

plugins/dbgate-plugin-mongo/webpack-backend.config.js

@@ -1,6 +1,10 @@
 var webpack = require('webpack');
 var path = require('path');
 
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
 var config = {
   context: __dirname + '/src/backend',
 
@@ -19,15 +23,7 @@ var config = {
   //     minimize: false,
   //   },
 
-  externals: {
+  externals,
-    'dbgate-tools': 'commonjs dbgate-tools',
-    'dbgate-query-splitter': 'commonjs dbgate-query-splitter',
-    lodash: 'commonjs lodash',
-    mongodb: 'commonjs mongodb',
-    'mongodb-client-encryption': 'commonjs mongodb-client-encryption',
-    bson: 'commonjs bson',
-    'is-promise': 'commonjs is-promise',
-  },
 };
 
 module.exports = config;

plugins/dbgate-plugin-mssql/webpack-backend.config.js

@@ -1,6 +1,10 @@
 var webpack = require('webpack');
 var path = require('path');
 
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
 var config = {
   context: __dirname + '/src/backend',
 
@@ -18,14 +22,7 @@ var config = {
   //   minimize: false,
   // },
 
-  externals: {
+  externals,
-    msnodesqlv8: 'commonjs msnodesqlv8',
-    'async-lock': 'commonjs async-lock',
-    'dbgate-query-splitter': 'commonjs dbgate-query-splitter',
-    'dbgate-tools': 'commonjs dbgate-tools',
-    lodash: 'commonjs lodash',
-    tedious: 'commonjs tedious',
-  },
 };
 
 module.exports = config;

plugins/dbgate-plugin-mysql/webpack-backend.config.js

@@ -1,6 +1,10 @@
 var webpack = require('webpack');
 var path = require('path');
 
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
 var config = {
   context: __dirname + '/src/backend',
 
@@ -14,13 +18,7 @@ var config = {
     libraryTarget: 'commonjs2',
   },
 
-  externals: {
+  externals,
-    'dbgate-tools': 'commonjs dbgate-tools',
-    'dbgate-query-splitter': 'commonjs dbgate-query-splitter',
-    'antares-mysql-dumper': 'commonjs antares-mysql-dumper',
-    lodash: 'commonjs lodash',
-    mysql2: 'commonjs mysql2',
-  },
 
   // uncomment for disable minimalization
   // optimization: {

plugins/dbgate-plugin-oracle/webpack-backend.config.js

@@ -1,6 +1,10 @@
 var webpack = require('webpack');
 var path = require('path');
 
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
 var config = {
   context: __dirname + '/src/backend',
 
@@ -36,12 +40,7 @@ var config = {
     }),
   ],
 
-  externals: {
+  externals,
-    'dbgate-tools': 'commonjs dbgate-tools',
-    'dbgate-query-splitter': 'commonjs dbgate-query-splitter',
-    lodash: 'commonjs lodash',
-    oracledb: 'commonjs oracledb',
-  },
 };
 
 module.exports = config;

plugins/dbgate-plugin-postgres/webpack-backend.config.js

@@ -1,6 +1,10 @@
 var webpack = require('webpack');
 var path = require('path');
 
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
 var config = {
   context: __dirname + '/src/backend',
 
@@ -36,13 +40,7 @@ var config = {
     }),
   ],
 
-  externals: {
+  externals,
-    'dbgate-tools': 'commonjs dbgate-tools',
-    'dbgate-query-splitter': 'commonjs dbgate-query-splitter',
-    lodash: 'commonjs lodash',
-    pg: 'commonjs pg',
-    'pg-copy-streams': 'commonjs pg-copy-streams',
-  }
 };
 
 module.exports = config;

plugins/dbgate-plugin-redis/webpack-backend.config.js

@@ -1,6 +1,10 @@
 var webpack = require('webpack');
 var path = require('path');
 
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
 var config = {
   context: __dirname + '/src/backend',
 
@@ -18,14 +22,7 @@ var config = {
   //   optimization: {
   //     minimize: false,
   //   },
-  externals: {
+  externals,
-    'dbgate-tools': 'commmonjs dbgate-tools',
-    'dbgate-query-splitter': 'commmonjs dbgate-query-splitter',
-    lodash: 'commmonjs lodash',
-    async: 'commmonjs async',
-    ioredis: 'commmonjs ioredis',
-    'node-redis-dump2': 'commmonjs node-redis-dump2',
-  },
 };
 
 module.exports = config;

plugins/dbgate-plugin-sqlite/webpack-backend.config.js

@@ -1,6 +1,10 @@
 var webpack = require('webpack');
 var path = require('path');
 
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
 var config = {
   context: __dirname + '/src/backend',
 
@@ -18,12 +22,7 @@ var config = {
   //   optimization: {
   //     minimize: false,
   //   },
-  externals: {
+  externals,
-    'dbgate-tools': 'commonjs dbgate-tools',
-    'dbgate-query-splitter': 'commonjs dbgate-query-splitter',
-    'better-sqlite3': 'commonjs better-sqlite3',
-    lodash: 'commonjs lodash',
-  },
 };
 
 module.exports = config;

plugins/dbgate-plugin-xml/webpack-backend.config.js

@@ -1,6 +1,10 @@
 var webpack = require('webpack');
 var path = require('path');
 
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
 var config = {
   context: __dirname + '/src/backend',
 
@@ -18,9 +22,7 @@ var config = {
   //   optimization: {
   //     minimize: false,
   //   },
-  externals: {
+  externals,
-    'node-xml-stream-parser': 'commonjs node-xml-stream-parser',
-  },
 };
 
 module.exports = config;