dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-17 23:45:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

removed MS excel support, moved to plugin

Browse Source
This commit is contained in:
Jan Prochazka
2020-11-22 17:25:41 +01:00
parent e23e749cc5
commit a22320e141
13 changed files with 64 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/api/package.json
View File

@@ -47,8 +47,7 @@
"node-fetch": "^2.6.1",
"pacote": "^11.1.13",
"pg": "^7.17.0",
"pg-query-stream": "^3.1.1",
"xlsx": "^0.16.8"
"pg-query-stream": "^3.1.1"
},
"scripts": {
"start": "nodemon src/index.js",

12
packages/api/src/controllers/files.js
View File

@@ -1,12 +0,0 @@
const xlsx = require('xlsx');
const _ = require('lodash');
module.exports = {
openedReaders: {},
analyseExcel_meta: 'get',
async analyseExcel({ filePath }) {
const workbook = xlsx.readFile(filePath, { bookSheets: true });
return workbook.SheetNames;
},
};

7
packages/api/src/controllers/plugins.js
View File

@@ -4,6 +4,7 @@ const path = require('path');
const pacote = require('pacote');
const { pluginstmpdir, pluginsdir } = require('../utility/directories');
const socket = require('../utility/socket');
const requirePlugin = require('../shell/requirePlugin');
async function loadPackageInfo(dir) {
const readmeFile = path.join(dir, 'README.md');
@@ -85,4 +86,10 @@ module.exports = {
await fs.rmdir(dir, { recursive: true });
socket.emitChanged(`installed-plugins-changed`);
},
command_meta: 'post',
async command({ packageName, command, args }) {
const content = requirePlugin(packageName);
return content.commands[command](args);
},
};

2
packages/api/src/controllers/runners.js
View File

@@ -19,7 +19,7 @@ const requirePluginsTemplate = (plugins) =>
.map(
(packageName) => `const ${_.camelCase(packageName)} = require(process.env.PLUGIN_${_.camelCase(packageName)});\n`
)
.join('');
.join('') + `dbgateApi.registerPlugins(${plugins.map((x) => _.camelCase(x)).join(',')});\n`;
const scriptTemplate = (script) => `
const dbgateApi = require(process.env.DBGATE_API);

2
packages/api/src/main.js
View File

@@ -20,7 +20,6 @@ const sessions = require('./controllers/sessions');
const runners = require('./controllers/runners');
const jsldata = require('./controllers/jsldata');
const config = require('./controllers/config');
const files = require('./controllers/files');
const archive = require('./controllers/archive');
const uploads = require('./controllers/uploads');
const plugins = require('./controllers/plugins');
@@ -65,7 +64,6 @@ function start(argument = null) {
useController(app, '/runners', runners);
useController(app, '/jsldata', jsldata);
useController(app, '/config', config);
useController(app, '/files', files);
useController(app, '/archive', archive);
useController(app, '/uploads', uploads);
useController(app, '/plugins', plugins);

41
packages/api/src/shell/excelSheetReader.js
View File

@@ -1,41 +0,0 @@
const xlsx = require('xlsx');
const stream = require('stream');
const _ = require('lodash');
const loadedWorkbooks = {};
async function loadWorkbook(fileName) {
let workbook = loadedWorkbooks[fileName];
if (workbook) return workbook;
console.log(`Loading excel ${fileName}`);
workbook = xlsx.readFile(fileName);
loadedWorkbooks[fileName] = workbook;
return workbook;
}
async function excelSheetReader({ fileName, sheetName, limitRows = undefined }) {
const workbook = await loadWorkbook(fileName);
const sheet = workbook.Sheets[sheetName];
const pass = new stream.PassThrough({
objectMode: true,
});
const rows = xlsx.utils.sheet_to_json(sheet, { header: 1 });
const header = rows[0];
const structure = {
columns: _.range(header.length).map((index) => ({ columnName: header[index] })),
};
pass.write(structure);
for (let rowIndex = 1; rowIndex < rows.length; rowIndex++) {
if (limitRows && rowIndex > limitRows) break;
const row = rows[rowIndex];
const rowData = _.fromPairs(structure.columns.map((col, index) => [col.columnName, row[index]]));
if (_.isEmpty(_.omitBy(rowData, (v) => v == null || v.toString().trim().length == 0))) continue;
pass.write(rowData);
}
pass.end();
return pass;
}
module.exports = excelSheetReader;

56
packages/api/src/shell/excelSheetWriter.js
View File

@@ -1,56 +0,0 @@
const xlsx = require('xlsx');
const stream = require('stream');
const finalizer = require('./finalizer');
const writingWorkbooks = {};
async function saveExcelFiles() {
for (const file in writingWorkbooks) {
xlsx.writeFile(writingWorkbooks[file], file);
}
}
finalizer.register(saveExcelFiles);
function createWorkbook(fileName) {
let workbook = writingWorkbooks[fileName];
if (workbook) return workbook;
workbook = xlsx.utils.book_new();
writingWorkbooks[fileName] = workbook;
return workbook;
}
class ExcelSheetWriterStream extends stream.Writable {
constructor({ fileName, sheetName }) {
super({ objectMode: true });
this.rows = [];
this.structure = null;
this.fileName = fileName;
this.sheetName = sheetName;
}
_write(chunk, enc, next) {
if (this.structure) {
this.rows.push(this.structure.columns.map((col) => chunk[col.columnName]));
} else {
this.structure = chunk;
this.rows.push(chunk.columns.map((x) => x.columnName));
}
next();
}
_final(callback) {
const workbook = createWorkbook(this.fileName);
xlsx.utils.book_append_sheet(workbook, xlsx.utils.aoa_to_sheet(this.rows), this.sheetName || 'Sheet 1');
callback();
}
}
async function excelSheetWriter({ fileName, sheetName }) {
return new ExcelSheetWriterStream({
fileName,
sheetName,
});
}
module.exports = excelSheetWriter;

13
packages/api/src/shell/index.js
View File

@@ -5,8 +5,6 @@ const tableReader = require('./tableReader');
const copyStream = require('./copyStream');
const fakeObjectReader = require('./fakeObjectReader');
const consoleObjectWriter = require('./consoleObjectWriter');
const excelSheetReader = require('./excelSheetReader');
const excelSheetWriter = require('./excelSheetWriter');
const jsonLinesWriter = require('./jsonLinesWriter');
const jsonLinesReader = require('./jsonLinesReader');
const jslDataReader = require('./jslDataReader');
@@ -14,14 +12,15 @@ const archiveWriter = require('./archiveWriter');
const archiveReader = require('./archiveReader');
const collectorWriter = require('./collectorWriter');
const finalizer = require('./finalizer');
const registerPlugins = require('./registerPlugins');
const requirePlugin = require('./requirePlugin');
module.exports = {
const dbgateApi = {
queryReader,
runScript,
tableWriter,
tableReader,
copyStream,
excelSheetReader,
jsonLinesWriter,
jsonLinesReader,
fakeObjectReader,
@@ -30,6 +29,10 @@ module.exports = {
archiveWriter,
archiveReader,
collectorWriter,
excelSheetWriter,
finalizer,
registerPlugins,
};
requirePlugin.initialize(dbgateApi);
module.exports = dbgateApi;

9
packages/api/src/shell/registerPlugins.js Normal file
View File

@@ -0,0 +1,9 @@
const requirePlugin = require('./requirePlugin');
function registerPlugins(...plugins) {
for (const plugin of plugins) {
requirePlugin(plugin.packageName, plugin);
}
}
module.exports = registerPlugins;

29
packages/api/src/shell/requirePlugin.js Normal file
View File

@@ -0,0 +1,29 @@
const path = require('path');
const { pluginsdir } = require('../utility/directories');
const loadedPlugins = {};
const dbgateEnv = {
dbgateApi: null,
};
function requirePlugin(packageName, requiredPlugin = null) {
if (!packageName) throw new Error('Missing packageName in plugin');
if (loadedPlugins[packageName]) return loadedPlugins[packageName];
if (requiredPlugin == null) {
console.log('Loading module', packageName);
const module = require(path.join(pluginsdir(), packageName, 'lib', 'backend.js'));
requiredPlugin = module.__esModule ? module.default : module;
}
loadedPlugins[packageName] = requiredPlugin;
if (requiredPlugin.initialize) requiredPlugin.initialize(dbgateEnv);
return requiredPlugin;
}
requirePlugin.initialize = (value) => {
dbgateEnv.dbgateApi = value;
};
module.exports = requirePlugin;

8
packages/web/src/plugins/PluginsProvider.js
View File

@@ -5,6 +5,10 @@ import { useInstalledPlugins } from '../utility/metadataLoaders';
const PluginsContext = React.createContext(null);
const dbgateEnv = {
axios,
};
export default function PluginsProvider({ children }) {
const installedPlugins = useInstalledPlugins();
const [plugins, setPlugins] = React.useState({});
@@ -22,7 +26,9 @@ export default function PluginsProvider({ children }) {
});
const module = eval(resp.data);
console.log('Loaded plugin', module);
newPlugins[installed.name] = module.__esModule ? module.default : module;
const moduleContent = module.__esModule ? module.default : module;
if (moduleContent.initialize) moduleContent.initialize(dbgateEnv);
newPlugins[installed.name] = moduleContent;
}
}
setPlugins((x) =>

52
packages/web/src/utility/fileformats.js
View File

@@ -1,53 +1,3 @@
import { usePlugins } from '../plugins/PluginsProvider';
import axios from './axios';
import { FormSchemaSelect } from './forms';
const excelFormat = {
storageType: 'excel',
extension: 'xlsx',
name: 'MS Excel',
readerFunc: 'excelSheetReader',
writerFunc: 'excelSheetWriter',
addFilesToSourceList: async (file, newSources, newValues) => {
const resp = await axios.get(`files/analyse-excel?filePath=${encodeURIComponent(file.full)}`);
const sheetNames = resp.data;
for (const sheetName of sheetNames) {
newSources.push(sheetName);
newValues[`sourceFile_${sheetName}`] = {
fileName: file.full,
sheetName,
};
}
},
args: [
{
type: 'checkbox',
name: 'singleFile',
label: 'Create single file',
direction: 'target',
},
],
getDefaultOutputName: (sourceName, values) => {
if (values.target_excel_singleFile) {
return sourceName;
}
return null;
},
getOutputParams: (sourceName, values) => {
if (values.target_excel_singleFile) {
return {
sheetName: values[`targetName_${sourceName}`] || sourceName,
fileName: 'data.xlsx',
};
}
return null;
},
};
const jsonlFormat = {
storageType: 'jsonl',
extension: 'jsonl',
@@ -58,7 +8,7 @@ const jsonlFormat = {
/** @returns {import('dbgate-types').FileFormatDefinition[]} */
export function buildFileFormats(plugins) {
const res = [excelFormat, jsonlFormat];
const res = [ jsonlFormat];
for (const { content } of plugins) {
const { fileFormats } = content;
if (fileFormats) res.push(...fileFormats);

87
yarn.lock
View File

@@ -2018,14 +2018,6 @@ adjust-sourcemap-loader@2.0.0:
object-path "0.11.4"
regex-parser "2.2.10"
adler-32@~1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.2.0.tgz#6a3e6bf0a63900ba15652808cb15c6813d1a5f25"
integrity sha1-aj5r8KY5ALoVZSgIyxXGgT0aXyU=
dependencies:
exit-on-epipe "~1.0.1"
printj "~1.1.0"
after@0.8.2:
version "0.8.2"
resolved "https://registry.yarnpkg.com/after/-/after-0.8.2.tgz#fedb394f9f0e02aa9768e702bda23b505fae7e1f"
@@ -3154,15 +3146,6 @@ caseless@~0.12.0:
resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=
cfb@^1.1.4:
version "1.2.0"
resolved "https://registry.yarnpkg.com/cfb/-/cfb-1.2.0.tgz#6a4d0872b525ed60349e1ef51fb4b0bf73eca9a8"
integrity sha512-sXMvHsKCICVR3Naq+J556K+ExBo9n50iKl6LGarlnvuA2035uMlGA/qVrc0wQtow5P1vJEw9UyrKLCbtIKz+TQ==
dependencies:
adler-32 "~1.2.0"
crc-32 "~1.2.0"
printj "~1.1.2"
chalk@2.4.2, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.4.1, chalk@^2.4.2:
version "2.4.2"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
@@ -3376,14 +3359,6 @@ code-point-at@^1.0.0:
resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"
integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=
codepage@~1.14.0:
version "1.14.0"
resolved "https://registry.yarnpkg.com/codepage/-/codepage-1.14.0.tgz#8cbe25481323559d7d307571b0fff91e7a1d2f99"
integrity sha1-jL4lSBMjVZ19MHVxsP/5HnodL5k=
dependencies:
commander "~2.14.1"
exit-on-epipe "~1.0.1"
collection-visit@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0"
@@ -3439,7 +3414,7 @@ combined-stream@^1.0.6, combined-stream@~1.0.6:
dependencies:
delayed-stream "~1.0.0"
commander@2.17.x, commander@~2.17.1:
commander@2.17.x:
version "2.17.1"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf"
integrity sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==
@@ -3454,11 +3429,6 @@ commander@^4.0.0:
resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068"
integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==
commander@~2.14.1:
version "2.14.1"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.14.1.tgz#2235123e37af8ca3c65df45b026dbd357b01b9aa"
integrity sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw==
commander@~2.19.0:
version "2.19.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a"
@@ -3723,14 +3693,6 @@ cosmiconfig@^6.0.0:
path-type "^4.0.0"
yaml "^1.7.2"
crc-32@~1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208"
integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==
dependencies:
exit-on-epipe "~1.0.1"
printj "~1.1.0"
create-ecdh@^4.0.0:
version "4.0.3"
resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.3.tgz#c9111b6f33045c4697f144787f9254cdc77c45ff"
@@ -5050,11 +5012,6 @@ exenv@^1.2.0:
resolved "https://registry.yarnpkg.com/exenv/-/exenv-1.2.2.tgz#2ae78e85d9894158670b03d47bec1f03bd91bb9d"
integrity sha1-KueOhdmJQVhnCwPUe+wfA72Ru50=
exit-on-epipe@~1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692"
integrity sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==
exit@^0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c"
@@ -5551,11 +5508,6 @@ forwarded@~0.1.2:
resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84"
integrity sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=
frac@~1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/frac/-/frac-1.1.2.tgz#3d74f7f6478c88a1b5020306d747dc6313c74d0b"
integrity sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==
fragment-cache@^0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19"
@@ -10142,11 +10094,6 @@ pretty-format@^25.1.0:
ansi-styles "^4.0.0"
react-is "^16.12.0"
printj@~1.1.0, printj@~1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222"
integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==
private@^0.1.6:
version "0.1.8"
resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff"
@@ -11792,13 +11739,6 @@ sqlstring@2.3.1:
resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.1.tgz#475393ff9e91479aea62dcaf0ca3d14983a7fb40"
integrity sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=
ssf@~0.11.2:
version "0.11.2"
resolved "https://registry.yarnpkg.com/ssf/-/ssf-0.11.2.tgz#0b99698b237548d088fc43cdf2b70c1a7512c06c"
integrity sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==
dependencies:
frac "~1.1.2"
sshpk@^1.7.0:
version "1.16.1"
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"
@@ -13192,21 +13132,11 @@ widest-line@^2.0.0:
dependencies:
string-width "^2.1.1"
wmf@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/wmf/-/wmf-1.0.2.tgz#7d19d621071a08c2bdc6b7e688a9c435298cc2da"
integrity sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==
word-wrap@~1.2.3:
version "1.2.3"
resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==
word@~0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/word/-/word-0.3.0.tgz#8542157e4f8e849f4a363a288992d47612db9961"
integrity sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==
workbox-background-sync@^4.3.1:
version "4.3.1"
resolved "https://registry.yarnpkg.com/workbox-background-sync/-/workbox-background-sync-4.3.1.tgz#26821b9bf16e9e37fd1d640289edddc08afd1950"
@@ -13434,21 +13364,6 @@ xdg-basedir@^3.0.0:
resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4"
integrity sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=
xlsx@^0.16.8:
version "0.16.8"
resolved "https://registry.yarnpkg.com/xlsx/-/xlsx-0.16.8.tgz#5546de9b0ba15169b36770d4e43b24790d3ff1b8"
integrity sha512-qWub4YCn0xLEGHI7WWhk6IJ73MDu7sPSJQImxN6/LiI8wsHi0hUhICEDbyqBT+jgFgORZxrii0HvhNSwBNAPoQ==
dependencies:
adler-32 "~1.2.0"
cfb "^1.1.4"
codepage "~1.14.0"
commander "~2.17.1"
crc-32 "~1.2.0"
exit-on-epipe "~1.0.1"
ssf "~0.11.2"
wmf "~1.0.1"
word "~0.3.0"
xml-name-validator@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"