using xlsx lib instead of exceljs, export excel files

packages/api/package.json

@@ -2,16 +2,14 @@
   "name": "dbgate-api",
   "main": "src/index.js",
   "version": "1.0.5",
-
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
     "url": "https://github.com/dbshell/dbgate.git"
-  },  
+  },
   "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
   "author": "Jan Prochazka",
   "license": "GPL",
-  
   "keywords": [
     "sql",
     "mssql",
@@ -24,11 +22,7 @@
     "export",
     "dbgate"
   ],
-
   "dependencies": {
-    "dbgate-engines": "^1.0.0",
-    "dbgate-sqltree": "^1.0.0",
-    "dbgate-tools": "^1.0.0",
     "async-lock": "^1.2.4",
     "axios": "^0.19.0",
     "body-parser": "^1.19.0",
@@ -37,8 +31,10 @@
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
     "csv": "^5.3.2",
+    "dbgate-engines": "^1.0.0",
+    "dbgate-sqltree": "^1.0.0",
+    "dbgate-tools": "^1.0.0",
     "eslint": "^6.8.0",
-    "exceljs": "^4.0.1",
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
     "express-fileupload": "^1.2.0",
@@ -51,7 +47,8 @@
     "mysql": "^2.17.1",
     "nedb-promises": "^4.0.1",
     "pg": "^7.17.0",
-    "pg-query-stream": "^3.1.1"
+    "pg-query-stream": "^3.1.1",
+    "xlsx": "^0.16.8"
   },
   "scripts": {
     "start": "nodemon src/index.js",
@@ -60,8 +57,8 @@
     "build": "webpack"
   },
   "devDependencies": {
-    "dbgate-types": "^1.0.0",
     "@types/lodash": "^4.14.149",
+    "dbgate-types": "^1.0.0",
     "env-cmd": "^10.1.0",
     "nodemon": "^2.0.2",
     "typescript": "^3.7.4",

packages/api/src/controllers/files.js

@@ -1,4 +1,4 @@
-const exceljs = require('exceljs');
+const xlsx = require('xlsx');
 const _ = require('lodash');
 
 module.exports = {
@@ -6,14 +6,7 @@ module.exports = {
 
   analyseExcel_meta: 'get',
   async analyseExcel({ filePath }) {
-    const workbook = new exceljs.Workbook();
-    await workbook.xlsx.readFile(filePath);
-    return {
-      tables: workbook.worksheets.map((sheet) => {
-        const header = sheet.getRow(1);
-        const columns = _.range(header.cellCount).map((index) => ({ columnName: header.getCell(index + 1).value }));
-        return { pureName: sheet.name, columns };
-      }),
-    };
+    const workbook = xlsx.readFile(filePath, { bookSheets: true });
+    return workbook.SheetNames;
   },
 };

packages/api/src/controllers/runners.js

@@ -12,6 +12,7 @@ const dbgateApi = require(process.env.DBGATE_API || "dbgate-api");
 require=null;
 async function run() {
 ${script}
+await dbgateApi.finalizer.run();
 console.log('Finished job script');
 }
 dbgateApi.runScript(run);

packages/api/src/shell/excelSheetReader.js

@@ -1,4 +1,4 @@
-const exceljs = require('exceljs');
+const xlsx = require('xlsx');
 const stream = require('stream');
 const _ = require('lodash');
 
@@ -8,28 +8,28 @@ async function loadWorkbook(fileName) {
   let workbook = loadedWorkbooks[fileName];
   if (workbook) return workbook;
   console.log(`Loading excel ${fileName}`);
-  workbook = new exceljs.Workbook();
-  await workbook.xlsx.readFile(fileName);
+  workbook = xlsx.readFile(fileName);
   loadedWorkbooks[fileName] = workbook;
   return workbook;
 }
 
 async function excelSheetReader({ fileName, sheetName, limitRows = undefined }) {
   const workbook = await loadWorkbook(fileName);
-  const sheet = workbook.getWorksheet(sheetName);
+  const sheet = workbook.Sheets[sheetName];
 
   const pass = new stream.PassThrough({
     objectMode: true,
   });
-  const header = sheet.getRow(1);
+  const rows = xlsx.utils.sheet_to_json(sheet, { header: 1 });
+  const header = rows[0];
   const structure = {
-    columns: _.range(header.cellCount).map((index) => ({ columnName: header.getCell(index + 1).value })),
+    columns: _.range(header.length).map((index) => ({ columnName: header[index] })),
   };
   pass.write(structure);
-  for (let rowIndex = 2; rowIndex <= sheet.rowCount; rowIndex++) {
-    if (limitRows && rowIndex > limitRows + 1) break;
-    const row = sheet.getRow(rowIndex);
-    const rowData = _.fromPairs(structure.columns.map((col, index) => [col.columnName, row.getCell(index + 1).value]));
+  for (let rowIndex = 1; rowIndex < rows.length; rowIndex++) {
+    if (limitRows && rowIndex > limitRows) break;
+    const row = rows[rowIndex];
+    const rowData = _.fromPairs(structure.columns.map((col, index) => [col.columnName, row[index]]));
     if (_.isEmpty(_.omitBy(rowData, (v) => v == null || v.toString().trim().length == 0))) continue;
     pass.write(rowData);
   }

packages/api/src/shell/excelSheetWriter.js

@@ -0,0 +1,56 @@
+const xlsx = require('xlsx');
+const stream = require('stream');
+const finalizer = require('./finalizer');
+
+const writingWorkbooks = {};
+
+async function saveExcelFiles() {
+  for (const file in writingWorkbooks) {
+    xlsx.writeFile(writingWorkbooks[file], file);
+  }
+}
+
+finalizer.register(saveExcelFiles);
+
+function createWorkbook(fileName) {
+  let workbook = writingWorkbooks[fileName];
+  if (workbook) return workbook;
+  workbook = xlsx.utils.book_new();
+  writingWorkbooks[fileName] = workbook;
+  return workbook;
+}
+
+class ExcelSheetWriterStream extends stream.Writable {
+  constructor({ fileName, sheetName }) {
+    super({ objectMode: true });
+    this.rows = [];
+    this.structure = null;
+    this.fileName = fileName;
+    this.sheetName = sheetName;
+  }
+  _write(chunk, enc, next) {
+    if (this.structure) {
+      this.rows.push(this.structure.columns.map((col) => chunk[col.columnName]));
+    } else {
+      this.structure = chunk;
+      this.rows.push(chunk.columns.map((x) => x.columnName));
+    }
+
+    next();
+  }
+
+  _final(callback) {
+    const workbook = createWorkbook(this.fileName);
+    xlsx.utils.book_append_sheet(workbook, xlsx.utils.aoa_to_sheet(this.rows), this.sheetName || 'Sheet 1');
+    callback();
+  }
+}
+
+async function excelSheetWriter({ fileName, sheetName }) {
+  return new ExcelSheetWriterStream({
+    fileName,
+    sheetName,
+  });
+}
+
+module.exports = excelSheetWriter;

packages/api/src/shell/finalizer.js

@@ -0,0 +1,12 @@
+const finalizers = [];
+
+module.exports = {
+  async run() {
+    for (const func of finalizers) {
+      await func();
+    }
+  },
+  register(func) {
+    finalizers.push(func);
+  },
+};

packages/api/src/shell/index.js

@@ -8,12 +8,14 @@ const copyStream = require('./copyStream');
 const fakeObjectReader = require('./fakeObjectReader');
 const consoleObjectWriter = require('./consoleObjectWriter');
 const excelSheetReader = require('./excelSheetReader');
+const excelSheetWriter = require('./excelSheetWriter');
 const jsonLinesWriter = require('./jsonLinesWriter');
 const jsonLinesReader = require('./jsonLinesReader');
 const jslDataReader = require('./jslDataReader');
 const archiveWriter = require('./archiveWriter');
 const archiveReader = require('./archiveReader');
 const collectorWriter = require('./collectorWriter');
+const finalizer = require('./finalizer');
 
 module.exports = {
   queryReader,
@@ -32,4 +34,6 @@ module.exports = {
   archiveWriter,
   archiveReader,
   collectorWriter,
+  excelSheetWriter,
+  finalizer,
 };