removed MS excel support, moved to plugin

packages/api/src/controllers/files.js

@@ -1,12 +0,0 @@
-const xlsx = require('xlsx');
-const _ = require('lodash');
-
-module.exports = {
-  openedReaders: {},
-
-  analyseExcel_meta: 'get',
-  async analyseExcel({ filePath }) {
-    const workbook = xlsx.readFile(filePath, { bookSheets: true });
-    return workbook.SheetNames;
-  },
-};

packages/api/src/controllers/plugins.js

@@ -4,6 +4,7 @@ const path = require('path');
 const pacote = require('pacote');
 const { pluginstmpdir, pluginsdir } = require('../utility/directories');
 const socket = require('../utility/socket');
+const requirePlugin = require('../shell/requirePlugin');
 
 async function loadPackageInfo(dir) {
   const readmeFile = path.join(dir, 'README.md');
@@ -85,4 +86,10 @@ module.exports = {
     await fs.rmdir(dir, { recursive: true });
     socket.emitChanged(`installed-plugins-changed`);
   },
+
+  command_meta: 'post',
+  async command({ packageName, command, args }) {
+    const content = requirePlugin(packageName);
+    return content.commands[command](args);
+  },
 };

packages/api/src/controllers/runners.js

@@ -19,7 +19,7 @@ const requirePluginsTemplate = (plugins) =>
     .map(
       (packageName) => `const ${_.camelCase(packageName)} = require(process.env.PLUGIN_${_.camelCase(packageName)});\n`
     )
-    .join('');
+    .join('') + `dbgateApi.registerPlugins(${plugins.map((x) => _.camelCase(x)).join(',')});\n`;
 
 const scriptTemplate = (script) => `
 const dbgateApi = require(process.env.DBGATE_API);

packages/api/src/main.js

@@ -20,7 +20,6 @@ const sessions = require('./controllers/sessions');
 const runners = require('./controllers/runners');
 const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
-const files = require('./controllers/files');
 const archive = require('./controllers/archive');
 const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
@@ -65,7 +64,6 @@ function start(argument = null) {
   useController(app, '/runners', runners);
   useController(app, '/jsldata', jsldata);
   useController(app, '/config', config);
-  useController(app, '/files', files);
   useController(app, '/archive', archive);
   useController(app, '/uploads', uploads);
   useController(app, '/plugins', plugins);

packages/api/src/shell/excelSheetReader.js

@@ -1,41 +0,0 @@
-const xlsx = require('xlsx');
-const stream = require('stream');
-const _ = require('lodash');
-
-const loadedWorkbooks = {};
-
-async function loadWorkbook(fileName) {
-  let workbook = loadedWorkbooks[fileName];
-  if (workbook) return workbook;
-  console.log(`Loading excel ${fileName}`);
-  workbook = xlsx.readFile(fileName);
-  loadedWorkbooks[fileName] = workbook;
-  return workbook;
-}
-
-async function excelSheetReader({ fileName, sheetName, limitRows = undefined }) {
-  const workbook = await loadWorkbook(fileName);
-  const sheet = workbook.Sheets[sheetName];
-
-  const pass = new stream.PassThrough({
-    objectMode: true,
-  });
-  const rows = xlsx.utils.sheet_to_json(sheet, { header: 1 });
-  const header = rows[0];
-  const structure = {
-    columns: _.range(header.length).map((index) => ({ columnName: header[index] })),
-  };
-  pass.write(structure);
-  for (let rowIndex = 1; rowIndex < rows.length; rowIndex++) {
-    if (limitRows && rowIndex > limitRows) break;
-    const row = rows[rowIndex];
-    const rowData = _.fromPairs(structure.columns.map((col, index) => [col.columnName, row[index]]));
-    if (_.isEmpty(_.omitBy(rowData, (v) => v == null || v.toString().trim().length == 0))) continue;
-    pass.write(rowData);
-  }
-  pass.end();
-
-  return pass;
-}
-
-module.exports = excelSheetReader;

packages/api/src/shell/excelSheetWriter.js

@@ -1,56 +0,0 @@
-const xlsx = require('xlsx');
-const stream = require('stream');
-const finalizer = require('./finalizer');
-
-const writingWorkbooks = {};
-
-async function saveExcelFiles() {
-  for (const file in writingWorkbooks) {
-    xlsx.writeFile(writingWorkbooks[file], file);
-  }
-}
-
-finalizer.register(saveExcelFiles);
-
-function createWorkbook(fileName) {
-  let workbook = writingWorkbooks[fileName];
-  if (workbook) return workbook;
-  workbook = xlsx.utils.book_new();
-  writingWorkbooks[fileName] = workbook;
-  return workbook;
-}
-
-class ExcelSheetWriterStream extends stream.Writable {
-  constructor({ fileName, sheetName }) {
-    super({ objectMode: true });
-    this.rows = [];
-    this.structure = null;
-    this.fileName = fileName;
-    this.sheetName = sheetName;
-  }
-  _write(chunk, enc, next) {
-    if (this.structure) {
-      this.rows.push(this.structure.columns.map((col) => chunk[col.columnName]));
-    } else {
-      this.structure = chunk;
-      this.rows.push(chunk.columns.map((x) => x.columnName));
-    }
-
-    next();
-  }
-
-  _final(callback) {
-    const workbook = createWorkbook(this.fileName);
-    xlsx.utils.book_append_sheet(workbook, xlsx.utils.aoa_to_sheet(this.rows), this.sheetName || 'Sheet 1');
-    callback();
-  }
-}
-
-async function excelSheetWriter({ fileName, sheetName }) {
-  return new ExcelSheetWriterStream({
-    fileName,
-    sheetName,
-  });
-}
-
-module.exports = excelSheetWriter;

packages/api/src/shell/index.js

@@ -5,8 +5,6 @@ const tableReader = require('./tableReader');
 const copyStream = require('./copyStream');
 const fakeObjectReader = require('./fakeObjectReader');
 const consoleObjectWriter = require('./consoleObjectWriter');
-const excelSheetReader = require('./excelSheetReader');
-const excelSheetWriter = require('./excelSheetWriter');
 const jsonLinesWriter = require('./jsonLinesWriter');
 const jsonLinesReader = require('./jsonLinesReader');
 const jslDataReader = require('./jslDataReader');
@@ -14,14 +12,15 @@ const archiveWriter = require('./archiveWriter');
 const archiveReader = require('./archiveReader');
 const collectorWriter = require('./collectorWriter');
 const finalizer = require('./finalizer');
+const registerPlugins = require('./registerPlugins');
+const requirePlugin = require('./requirePlugin');
 
-module.exports = {
+const dbgateApi = {
   queryReader,
   runScript,
   tableWriter,
   tableReader,
   copyStream,
-  excelSheetReader,
   jsonLinesWriter,
   jsonLinesReader,
   fakeObjectReader,
@@ -30,6 +29,10 @@ module.exports = {
   archiveWriter,
   archiveReader,
   collectorWriter,
-  excelSheetWriter,
   finalizer,
+  registerPlugins,
 };
+
+requirePlugin.initialize(dbgateApi);
+
+module.exports = dbgateApi;

packages/api/src/shell/registerPlugins.js

@@ -0,0 +1,9 @@
+const requirePlugin = require('./requirePlugin');
+
+function registerPlugins(...plugins) {
+  for (const plugin of plugins) {
+    requirePlugin(plugin.packageName, plugin);
+  }
+}
+
+module.exports = registerPlugins;

packages/api/src/shell/requirePlugin.js

@@ -0,0 +1,29 @@
+const path = require('path');
+const { pluginsdir } = require('../utility/directories');
+
+const loadedPlugins = {};
+
+const dbgateEnv = {
+  dbgateApi: null,
+};
+
+function requirePlugin(packageName, requiredPlugin = null) {
+  if (!packageName) throw new Error('Missing packageName in plugin');
+  if (loadedPlugins[packageName]) return loadedPlugins[packageName];
+
+  if (requiredPlugin == null) {
+    console.log('Loading module', packageName);
+    const module = require(path.join(pluginsdir(), packageName, 'lib', 'backend.js'));
+    requiredPlugin = module.__esModule ? module.default : module;
+  }
+  loadedPlugins[packageName] = requiredPlugin;
+  if (requiredPlugin.initialize) requiredPlugin.initialize(dbgateEnv);
+
+  return requiredPlugin;
+}
+
+requirePlugin.initialize = (value) => {
+  dbgateEnv.dbgateApi = value;
+};
+
+module.exports = requirePlugin;