added plugins

plugins/dbgate-plugin-excel/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Jan Prochazka
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

plugins/dbgate-plugin-excel/README.md

@@ -0,0 +1,52 @@
+[![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
+[![Donate](https://img.shields.io/badge/donate-paypal-blue.svg)](https://paypal.me/JanProchazkaCz/30eur)
+[![NPM version](https://img.shields.io/npm/v/dbgate-plugin-excel.svg)](https://www.npmjs.com/package/dbgate-plugin-excel)
+
+# dbgate-plugin-excel
+
+MS Excel import/export plugin for DbGate
+
+
+## Usage without DbGate
+
+Export from fake object reader into MS Excel file. Fake object file can be replaced with other reader/writer factory functions, as described in 
+[dbgate-api package](https://www.npmjs.com/package/dbgate-api)
+
+```javascript
+const dbgateApi = require('dbgate-api');
+const dbgatePluginExcel = require("dbgate-plugin-excel");
+
+dbgateApi.registerPlugins(dbgatePluginExcel);
+
+
+async function run() {
+  const reader = await dbgateApi.fakeObjectReader();
+  const writer = await dbgatePluginExcel.shellApi.writer({ fileName: 'myfile1.xlsx', sheetName: 'Sheet 1' });
+  await dbgateApi.copyStream(reader, writer);
+  console.log('Finished job script');
+}
+dbgateApi.runScript(run);
+
+
+```
+
+## Factory functions
+
+### shellApi.reader
+Reads tabular data from one sheet in MS Excel file.
+```js
+  const reader = await dbgatePluginExcel.shellApi.reader({
+    fileName: 'test.xlsx',
+    sheetName: 'Album',
+    limitRows: null
+  });
+```
+
+### shellApi.writer
+Writes tabular data into MS excel file. There could be more writes into the some file in one script, if property sheetName is different.
+```js
+  const reader = await dbgatePluginExcel.shellApi.writer({
+    fileName: 'test.xlsx',
+    sheetName: 'Album',
+  });
+```

plugins/dbgate-plugin-excel/icon.svg

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 512 512" style="enable-background:new 0 0 512 512;" xml:space="preserve">
+<path style="fill:#ECEFF1;" d="M496,432.011H272c-8.832,0-16-7.168-16-16s0-311.168,0-320s7.168-16,16-16h224
+	c8.832,0,16,7.168,16,16v320C512,424.843,504.832,432.011,496,432.011z"/>
+<g>
+	<path style="fill:#388E3C;" d="M336,176.011h-64c-8.832,0-16-7.168-16-16s7.168-16,16-16h64c8.832,0,16,7.168,16,16
+		S344.832,176.011,336,176.011z"/>
+	<path style="fill:#388E3C;" d="M336,240.011h-64c-8.832,0-16-7.168-16-16s7.168-16,16-16h64c8.832,0,16,7.168,16,16
+		S344.832,240.011,336,240.011z"/>
+	<path style="fill:#388E3C;" d="M336,304.011h-64c-8.832,0-16-7.168-16-16s7.168-16,16-16h64c8.832,0,16,7.168,16,16
+		S344.832,304.011,336,304.011z"/>
+	<path style="fill:#388E3C;" d="M336,368.011h-64c-8.832,0-16-7.168-16-16s7.168-16,16-16h64c8.832,0,16,7.168,16,16
+		S344.832,368.011,336,368.011z"/>
+	<path style="fill:#388E3C;" d="M432,176.011h-32c-8.832,0-16-7.168-16-16s7.168-16,16-16h32c8.832,0,16,7.168,16,16
+		S440.832,176.011,432,176.011z"/>
+	<path style="fill:#388E3C;" d="M432,240.011h-32c-8.832,0-16-7.168-16-16s7.168-16,16-16h32c8.832,0,16,7.168,16,16
+		S440.832,240.011,432,240.011z"/>
+	<path style="fill:#388E3C;" d="M432,304.011h-32c-8.832,0-16-7.168-16-16s7.168-16,16-16h32c8.832,0,16,7.168,16,16
+		S440.832,304.011,432,304.011z"/>
+	<path style="fill:#388E3C;" d="M432,368.011h-32c-8.832,0-16-7.168-16-16s7.168-16,16-16h32c8.832,0,16,7.168,16,16
+		S440.832,368.011,432,368.011z"/>
+</g>
+<path style="fill:#2E7D32;" d="M282.208,19.691c-3.648-3.04-8.544-4.352-13.152-3.392l-256,48C5.472,65.707,0,72.299,0,80.011v352
+	c0,7.68,5.472,14.304,13.056,15.712l256,48c0.96,0.192,1.952,0.288,2.944,0.288c3.712,0,7.328-1.28,10.208-3.68
+	c3.68-3.04,5.792-7.584,5.792-12.32v-448C288,27.243,285.888,22.731,282.208,19.691z"/>
+<path style="fill:#FAFAFA;" d="M220.032,309.483l-50.592-57.824l51.168-65.792c5.44-6.976,4.16-17.024-2.784-22.464
+	c-6.944-5.44-16.992-4.16-22.464,2.784l-47.392,60.928l-39.936-45.632c-5.856-6.72-15.968-7.328-22.56-1.504
+	c-6.656,5.824-7.328,15.936-1.504,22.56l44,50.304L83.36,310.187c-5.44,6.976-4.16,17.024,2.784,22.464
+	c2.944,2.272,6.432,3.36,9.856,3.36c4.768,0,9.472-2.112,12.64-6.176l40.8-52.48l46.528,53.152
+	c3.168,3.648,7.584,5.504,12.032,5.504c3.744,0,7.488-1.312,10.528-3.968C225.184,326.219,225.856,316.107,220.032,309.483z"/>
+</svg>

plugins/dbgate-plugin-excel/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "dbgate-plugin-excel",
+  "main": "dist/backend.js",
+  "version": "1.0.8",
+  "homepage": "https://github.com/dbgate/dbgate-plugin-excel",
+  "description": "MS Excel import/export plugin for DbGate",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate-plugin-excel.git"
+  },
+  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
+  "author": "Jan Prochazka",
+  "license": "GPL",
+  "keywords": [
+    "excel",
+    "import",
+    "export",
+    "dbgate",
+    "dbgateplugin"
+  ],
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build:frontend": "webpack --config webpack-frontend.config",
+    "build:backend": "webpack --config webpack-backend.config.js",
+    "build": "yarn build:frontend && yarn build:backend",
+    "plugin": "yarn build && dbgate-plugin dbgate-plugin-excel",
+    "plugout": "dbgate-plugout dbgate-plugin-excel",
+    "prepublishOnly": "yarn build"
+  },
+  "devDependencies": {
+    "dbgate-plugin-tools": "^1.0.0",
+    "lodash": "^4.17.20",
+    "webpack": "^4.42.0",
+    "webpack-cli": "^3.3.11",
+    "xlsx": "^0.16.8"
+  }
+}

plugins/dbgate-plugin-excel/prettier.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  trailingComma: 'es5',
+  tabWidth: 2,
+  semi: true,
+  singleQuote: true,
+  arrowParen: 'avoid',
+  printWidth: 120,
+};

plugins/dbgate-plugin-excel/src/backend/index.js

@@ -0,0 +1,26 @@
+const xlsx = require('xlsx');
+const reader = require('./reader');
+const writer = require('./writer');
+
+let dbgateApi;
+
+module.exports = {
+  packageName: 'dbgate-plugin-excel',
+  shellApi: {
+    reader,
+    writer,
+  },
+
+  commands: {
+    analyse: async ({ fileName }) => {
+      const downloadedFile = await dbgateApi.download(fileName);
+      const workbook = xlsx.readFile(downloadedFile, { bookSheets: true });
+      return workbook.SheetNames;
+    },
+  },
+  initialize(dbgateEnv) {
+    dbgateApi = dbgateEnv.dbgateApi;
+    writer.initialize(dbgateEnv);
+    reader.initialize(dbgateEnv);
+  },
+};

plugins/dbgate-plugin-excel/src/backend/reader.js

@@ -0,0 +1,67 @@
+const xlsx = require('xlsx');
+const stream = require('stream');
+const _ = require('lodash');
+
+const loadedWorkbooks = {};
+let dbgateApi;
+
+async function loadWorkbook(fileName) {
+  let workbook = loadedWorkbooks[fileName];
+  if (workbook) return workbook;
+  console.log(`Loading excel ${fileName}`);
+  const downloadedFile = await dbgateApi.download(fileName);
+  workbook = xlsx.readFile(downloadedFile);
+  loadedWorkbooks[fileName] = workbook;
+  return workbook;
+}
+
+async function waitForDrain(stream) {
+  return new Promise((resolve) => {
+    stream.once('drain', () => {
+      resolve();
+    });
+  });
+}
+
+async function reader({ fileName, sheetName, limitRows = undefined }) {
+  const pass = new stream.PassThrough({
+    objectMode: true,
+    highWaterMark: 100,
+  });
+
+  const workbook = await loadWorkbook(fileName);
+  const sheet = workbook.Sheets[sheetName];
+
+  const rows = xlsx.utils.sheet_to_json(sheet, {
+    header: 1,
+    blankrows: false,
+  });
+  const header = rows[0];
+  const structure = {
+    __isStreamHeader: true,
+    columns: _.range(header.length).map((index) => ({ columnName: header[index] })),
+  };
+  if (!pass.write(structure)) await waitForDrain(pass);
+
+  const sendAsync = async () => {
+    for (let rowIndex = 1; rowIndex < rows.length; rowIndex++) {
+      if (limitRows && rowIndex > limitRows) break;
+      const row = rows[rowIndex];
+      const rowData = _.fromPairs(structure.columns.map((col, index) => [col.columnName, row[index]]));
+      if (_.isEmpty(_.omitBy(rowData, (v) => v == null || v.toString().trim().length == 0))) continue;
+      if (!pass.write(rowData)) await waitForDrain(pass);
+    }
+    pass.end();
+  };
+
+  // don't wait for sending
+  sendAsync();
+
+  return pass;
+}
+
+reader.initialize = (dbgateEnv) => {
+  dbgateApi = dbgateEnv.dbgateApi;
+};
+
+module.exports = reader;

plugins/dbgate-plugin-excel/src/backend/writer.js

@@ -0,0 +1,57 @@
+const xlsx = require('xlsx');
+const stream = require('stream');
+
+const writingWorkbooks = {};
+
+async function saveExcelFiles() {
+  for (const file in writingWorkbooks) {
+    xlsx.writeFile(writingWorkbooks[file], file);
+  }
+}
+
+function createWorkbook(fileName) {
+  let workbook = writingWorkbooks[fileName];
+  if (workbook) return workbook;
+  workbook = xlsx.utils.book_new();
+  writingWorkbooks[fileName] = workbook;
+  return workbook;
+}
+
+class ExcelSheetWriterStream extends stream.Writable {
+  constructor({ fileName, sheetName }) {
+    super({ objectMode: true });
+    this.rows = [];
+    this.structure = null;
+    this.fileName = fileName;
+    this.sheetName = sheetName;
+  }
+  _write(chunk, enc, next) {
+    if (this.structure) {
+      this.rows.push(this.structure.columns.map((col) => chunk[col.columnName]));
+    } else {
+      this.structure = chunk;
+      this.rows.push(chunk.columns.map((x) => x.columnName));
+    }
+
+    next();
+  }
+
+  _final(callback) {
+    const workbook = createWorkbook(this.fileName);
+    xlsx.utils.book_append_sheet(workbook, xlsx.utils.aoa_to_sheet(this.rows), this.sheetName || 'Sheet 1');
+    callback();
+  }
+}
+
+async function writer({ fileName, sheetName }) {
+  return new ExcelSheetWriterStream({
+    fileName,
+    sheetName,
+  });
+}
+
+writer.initialize = ({ dbgateApi }) => {
+  dbgateApi.finalizer.register(saveExcelFiles);
+};
+
+module.exports = writer;

plugins/dbgate-plugin-excel/src/frontend/index.js

@@ -0,0 +1,68 @@
+let axios;
+
+function initialize(dbgateEnv) {
+  axios = dbgateEnv.axios;
+}
+
+const fileFormat = {
+  packageName: 'dbgate-plugin-excel',
+  // file format identifier
+  storageType: 'excel',
+  // file extension without leading dot
+  extension: 'xlsx',
+  // human readable file format name
+  name: 'MS Excel',
+  // function name from backend, which contains reader factory, postfixed by package name
+  readerFunc: 'reader@dbgate-plugin-excel',
+  // function name from backend, which contains writer factory, postfixed by package name
+  writerFunc: 'writer@dbgate-plugin-excel',
+
+  addFileToSourceList: async ({ fileName }, newSources, newValues) => {
+    const resp = await axios.post('plugins/command', {
+      command: 'analyse',
+      packageName: 'dbgate-plugin-excel',
+      args: {
+        fileName,
+      },
+    });
+    const sheetNames = resp.data;
+    for (const sheetName of sheetNames) {
+      newSources.push(sheetName);
+      newValues[`sourceFile_${sheetName}`] = {
+        fileName,
+        sheetName,
+      };
+    }
+  },
+
+  args: [
+    {
+      type: 'checkbox',
+      name: 'singleFile',
+      label: 'Create single file',
+      direction: 'target',
+    },
+  ],
+
+  getDefaultOutputName: (sourceName, values) => {
+    if (values.target_excel_singleFile) {
+      return sourceName;
+    }
+    return null;
+  },
+
+  getOutputParams: (sourceName, values) => {
+    if (values.target_excel_singleFile) {
+      return {
+        sheetName: values[`targetName_${sourceName}`] || sourceName,
+        fileName: 'data.xlsx',
+      };
+    }
+    return null;
+  },
+};
+
+export default {
+  fileFormats: [fileFormat],
+  initialize,
+};

plugins/dbgate-plugin-excel/webpack-backend.config.js

@@ -0,0 +1,23 @@
+var webpack = require('webpack');
+var path = require('path');
+
+var config = {
+  context: __dirname + '/src/backend',
+
+  entry: {
+    app: './index.js',
+  },
+  target: 'node',
+  output: {
+    path: path.resolve(__dirname, 'dist'),
+    filename: 'backend.js',
+    libraryTarget: 'commonjs2',
+  },
+
+  // uncomment for disable minimalization
+  //   optimization: {
+  //     minimize: false,
+  //   },
+};
+
+module.exports = config;

plugins/dbgate-plugin-excel/webpack-frontend.config.js

@@ -0,0 +1,24 @@
+var webpack = require("webpack");
+var path = require("path");
+
+var config = {
+  context: __dirname + "/src/frontend",
+
+  entry: {
+    app: "./index.js",
+  },
+  target: "web",
+  output: {
+    path: path.resolve(__dirname, "dist"),
+    filename: "frontend.js",
+    libraryTarget: "var",
+    library: 'plugin',
+  },
+
+  // uncomment for disable minimalization
+  // optimization: {
+  //   minimize: false,
+  // },
+};
+
+module.exports = config;