added plugins

plugins/dbgate-plugin-excel/src/backend/index.js

@@ -0,0 +1,26 @@
+const xlsx = require('xlsx');
+const reader = require('./reader');
+const writer = require('./writer');
+
+let dbgateApi;
+
+module.exports = {
+  packageName: 'dbgate-plugin-excel',
+  shellApi: {
+    reader,
+    writer,
+  },
+
+  commands: {
+    analyse: async ({ fileName }) => {
+      const downloadedFile = await dbgateApi.download(fileName);
+      const workbook = xlsx.readFile(downloadedFile, { bookSheets: true });
+      return workbook.SheetNames;
+    },
+  },
+  initialize(dbgateEnv) {
+    dbgateApi = dbgateEnv.dbgateApi;
+    writer.initialize(dbgateEnv);
+    reader.initialize(dbgateEnv);
+  },
+};

plugins/dbgate-plugin-excel/src/backend/reader.js

@@ -0,0 +1,67 @@
+const xlsx = require('xlsx');
+const stream = require('stream');
+const _ = require('lodash');
+
+const loadedWorkbooks = {};
+let dbgateApi;
+
+async function loadWorkbook(fileName) {
+  let workbook = loadedWorkbooks[fileName];
+  if (workbook) return workbook;
+  console.log(`Loading excel ${fileName}`);
+  const downloadedFile = await dbgateApi.download(fileName);
+  workbook = xlsx.readFile(downloadedFile);
+  loadedWorkbooks[fileName] = workbook;
+  return workbook;
+}
+
+async function waitForDrain(stream) {
+  return new Promise((resolve) => {
+    stream.once('drain', () => {
+      resolve();
+    });
+  });
+}
+
+async function reader({ fileName, sheetName, limitRows = undefined }) {
+  const pass = new stream.PassThrough({
+    objectMode: true,
+    highWaterMark: 100,
+  });
+
+  const workbook = await loadWorkbook(fileName);
+  const sheet = workbook.Sheets[sheetName];
+
+  const rows = xlsx.utils.sheet_to_json(sheet, {
+    header: 1,
+    blankrows: false,
+  });
+  const header = rows[0];
+  const structure = {
+    __isStreamHeader: true,
+    columns: _.range(header.length).map((index) => ({ columnName: header[index] })),
+  };
+  if (!pass.write(structure)) await waitForDrain(pass);
+
+  const sendAsync = async () => {
+    for (let rowIndex = 1; rowIndex < rows.length; rowIndex++) {
+      if (limitRows && rowIndex > limitRows) break;
+      const row = rows[rowIndex];
+      const rowData = _.fromPairs(structure.columns.map((col, index) => [col.columnName, row[index]]));
+      if (_.isEmpty(_.omitBy(rowData, (v) => v == null || v.toString().trim().length == 0))) continue;
+      if (!pass.write(rowData)) await waitForDrain(pass);
+    }
+    pass.end();
+  };
+
+  // don't wait for sending
+  sendAsync();
+
+  return pass;
+}
+
+reader.initialize = (dbgateEnv) => {
+  dbgateApi = dbgateEnv.dbgateApi;
+};
+
+module.exports = reader;

plugins/dbgate-plugin-excel/src/backend/writer.js

@@ -0,0 +1,57 @@
+const xlsx = require('xlsx');
+const stream = require('stream');
+
+const writingWorkbooks = {};
+
+async function saveExcelFiles() {
+  for (const file in writingWorkbooks) {
+    xlsx.writeFile(writingWorkbooks[file], file);
+  }
+}
+
+function createWorkbook(fileName) {
+  let workbook = writingWorkbooks[fileName];
+  if (workbook) return workbook;
+  workbook = xlsx.utils.book_new();
+  writingWorkbooks[fileName] = workbook;
+  return workbook;
+}
+
+class ExcelSheetWriterStream extends stream.Writable {
+  constructor({ fileName, sheetName }) {
+    super({ objectMode: true });
+    this.rows = [];
+    this.structure = null;
+    this.fileName = fileName;
+    this.sheetName = sheetName;
+  }
+  _write(chunk, enc, next) {
+    if (this.structure) {
+      this.rows.push(this.structure.columns.map((col) => chunk[col.columnName]));
+    } else {
+      this.structure = chunk;
+      this.rows.push(chunk.columns.map((x) => x.columnName));
+    }
+
+    next();
+  }
+
+  _final(callback) {
+    const workbook = createWorkbook(this.fileName);
+    xlsx.utils.book_append_sheet(workbook, xlsx.utils.aoa_to_sheet(this.rows), this.sheetName || 'Sheet 1');
+    callback();
+  }
+}
+
+async function writer({ fileName, sheetName }) {
+  return new ExcelSheetWriterStream({
+    fileName,
+    sheetName,
+  });
+}
+
+writer.initialize = ({ dbgateApi }) => {
+  dbgateApi.finalizer.register(saveExcelFiles);
+};
+
+module.exports = writer;

plugins/dbgate-plugin-excel/src/frontend/index.js

@@ -0,0 +1,68 @@
+let axios;
+
+function initialize(dbgateEnv) {
+  axios = dbgateEnv.axios;
+}
+
+const fileFormat = {
+  packageName: 'dbgate-plugin-excel',
+  // file format identifier
+  storageType: 'excel',
+  // file extension without leading dot
+  extension: 'xlsx',
+  // human readable file format name
+  name: 'MS Excel',
+  // function name from backend, which contains reader factory, postfixed by package name
+  readerFunc: 'reader@dbgate-plugin-excel',
+  // function name from backend, which contains writer factory, postfixed by package name
+  writerFunc: 'writer@dbgate-plugin-excel',
+
+  addFileToSourceList: async ({ fileName }, newSources, newValues) => {
+    const resp = await axios.post('plugins/command', {
+      command: 'analyse',
+      packageName: 'dbgate-plugin-excel',
+      args: {
+        fileName,
+      },
+    });
+    const sheetNames = resp.data;
+    for (const sheetName of sheetNames) {
+      newSources.push(sheetName);
+      newValues[`sourceFile_${sheetName}`] = {
+        fileName,
+        sheetName,
+      };
+    }
+  },
+
+  args: [
+    {
+      type: 'checkbox',
+      name: 'singleFile',
+      label: 'Create single file',
+      direction: 'target',
+    },
+  ],
+
+  getDefaultOutputName: (sourceName, values) => {
+    if (values.target_excel_singleFile) {
+      return sourceName;
+    }
+    return null;
+  },
+
+  getOutputParams: (sourceName, values) => {
+    if (values.target_excel_singleFile) {
+      return {
+        sheetName: values[`targetName_${sourceName}`] || sourceName,
+        fileName: 'data.xlsx',
+      };
+    }
+    return null;
+  },
+};
+
+export default {
+  fileFormats: [fileFormat],
+  initialize,
+};