diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js index 3cf3b4ec3..f47a70661 100644 --- a/packages/api/src/controllers/runners.js +++ b/packages/api/src/controllers/runners.js @@ -107,6 +107,8 @@ module.exports = { }, startCore(runid, scriptText) { + // console.log('************* RUN SCRIPT *************'); + // console.log(scriptText); const directory = path.join(rundir(), runid); const scriptFile = path.join(uploadsdir(), runid + '.js'); fs.writeFileSync(`${scriptFile}`, scriptText); diff --git a/plugins/dbgate-plugin-dbf/README.md b/plugins/dbgate-plugin-dbf/README.md new file mode 100644 index 000000000..5c37c9cdb --- /dev/null +++ b/plugins/dbgate-plugin-dbf/README.md @@ -0,0 +1,4 @@ +[![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier) +[![NPM version](https://img.shields.io/npm/v/dbgate-plugin-dbf.svg)](https://www.npmjs.com/package/dbgate-plugin-dbf) + +# dbgate-plugin-dbf diff --git a/plugins/dbgate-plugin-dbf/icon.svg b/plugins/dbgate-plugin-dbf/icon.svg new file mode 100644 index 000000000..3418cc7d1 --- /dev/null +++ b/plugins/dbgate-plugin-dbf/icon.svg @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/plugins/dbgate-plugin-dbf/package.json b/plugins/dbgate-plugin-dbf/package.json new file mode 100644 index 000000000..e0db031dc --- /dev/null +++ b/plugins/dbgate-plugin-dbf/package.json @@ -0,0 +1,43 @@ +{ + "name": "dbgate-plugin-dbf", + "main": "dist/backend.js", + "version": "6.0.0-alpha.1", + "homepage": "https://dbgate.org", + "description": "DBF import/export plugin for DbGate", + "repository": { + "type": "git", + "url": "https://github.com/dbgate/dbgate" + }, + "author": "Jan Prochazka", + "license": "GPL-3.0", + "keywords": [ + "dbf", + "import", + "export", + "dbgate", + "dbgatebuiltin" + ], + "files": [ + "dist", + "icon.svg" + ], + "scripts": { + "build:frontend": "webpack --config webpack-frontend.config", + "build:frontend:watch": "webpack --watch --config webpack-frontend.config", + "build:backend": "webpack --config webpack-backend.config.js", + "build": "yarn build:frontend && yarn build:backend", + "plugin": "yarn build && yarn pack && dbgate-plugin dbgate-plugin-dbf", + "copydist": "yarn build && yarn pack && dbgate-copydist ../dist/dbgate-plugin-dbf", + "plugout": "dbgate-plugout dbgate-plugin-dbf", + "prepublishOnly": "yarn build" + }, + "devDependencies": { + "dbgate-plugin-tools": "^1.0.7", + "webpack": "^5.91.0", + "webpack-cli": "^5.1.4" + }, + "dependencies": { + "dbffile": "^1.12.0", + "lodash": "^4.17.21" + } +} \ No newline at end of file diff --git a/plugins/dbgate-plugin-dbf/prettier.config.js b/plugins/dbgate-plugin-dbf/prettier.config.js new file mode 100644 index 000000000..406484074 --- /dev/null +++ b/plugins/dbgate-plugin-dbf/prettier.config.js @@ -0,0 +1,8 @@ +module.exports = { + trailingComma: 'es5', + tabWidth: 2, + semi: true, + singleQuote: true, + arrowParen: 'avoid', + printWidth: 120, +}; diff --git a/plugins/dbgate-plugin-dbf/src/backend/index.js b/plugins/dbgate-plugin-dbf/src/backend/index.js new file mode 100644 index 000000000..8d7e6390b --- /dev/null +++ b/plugins/dbgate-plugin-dbf/src/backend/index.js @@ -0,0 +1,12 @@ +const reader = require('./reader'); +// const writer = require('./writer'); + +module.exports = { + packageName: 'dbgate-plugin-dbf', + shellApi: { + reader, + }, + initialize(dbgateEnv) { + reader.initialize(dbgateEnv); + }, +}; diff --git a/plugins/dbgate-plugin-dbf/src/backend/reader.js b/plugins/dbgate-plugin-dbf/src/backend/reader.js new file mode 100644 index 000000000..be26f53ab --- /dev/null +++ b/plugins/dbgate-plugin-dbf/src/backend/reader.js @@ -0,0 +1,85 @@ +const _ = require('lodash'); +const csv = require('csv'); +const fs = require('fs'); +const stream = require('stream'); +const { DBFFile } = require('dbffile'); + +let dbgateApi; + +function getFieldType(field) { + const { type, size } = field; + switch (type) { + case 'C': + return `varchar(${size})`; + case 'N': + return 'numeric'; + case 'F': + return 'float'; + case 'Y': + return 'money'; + case 'I': + return 'int'; + case 'L': + return 'boolean'; + case 'D': + return 'date'; + case 'T': + return 'datetime'; + case 'B': + return 'duouble'; + case 'M': + return 'varchar(max)'; + default: + return 'string'; + } +} + +async function reader({ fileName, encoding = 'ISO-8859-1', includeDeletedRecords = false, limitRows = undefined }) { + console.log(`Reading file ${fileName}`); + const downloadedFile = await dbgateApi.download(fileName); + + const pass = new stream.PassThrough({ + objectMode: true, + }); + + (async () => { + try { + // Open the DBF file + const dbf = await DBFFile.open(downloadedFile, { encoding, includeDeletedRecords }); + + const columns = dbf.fields.map((field) => ({ + columnName: field.name, + dataType: getFieldType(field), + })); + + pass.write({ + __isStreamHeader: true, + columns, + }) + + let readedRows = 0; + // Read each record and push it into the stream + for await (const record of dbf) { + // Emit the record as a chunk + pass.write(record); + readedRows++; + if (limitRows && readedRows >= limitRows) { + break; + } + } + + pass.end(); + } catch (error) { + // If any error occurs, destroy the stream with the error + pass.end(); + } + })(); + + return pass; +} + +reader.initialize = (dbgateEnv) => { + dbgateApi = dbgateEnv.dbgateApi; +}; + +module.exports = reader; diff --git a/plugins/dbgate-plugin-dbf/src/frontend/index.js b/plugins/dbgate-plugin-dbf/src/frontend/index.js new file mode 100644 index 000000000..87f2a6c42 --- /dev/null +++ b/plugins/dbgate-plugin-dbf/src/frontend/index.js @@ -0,0 +1,25 @@ +const fileFormat = { + packageName: 'dbgate-plugin-dbf', + // file format identifier + storageType: 'dbf', + // file extension without leading dot + extension: 'dbf', + // human readable file format name + name: 'DBF', + // function name from backend, which contains reader factory, postfixed by package name + readerFunc: 'reader@dbgate-plugin-dbf', + + args: [ + { + type: 'text', + name: 'encoding', + label: 'Encoding', + apiName: 'encoding', + }, + ], + +}; + +export default { + fileFormats: [fileFormat], +}; diff --git a/plugins/dbgate-plugin-dbf/webpack-backend.config.js b/plugins/dbgate-plugin-dbf/webpack-backend.config.js new file mode 100644 index 000000000..20c953370 --- /dev/null +++ b/plugins/dbgate-plugin-dbf/webpack-backend.config.js @@ -0,0 +1,29 @@ +var webpack = require('webpack'); +var path = require('path'); + +const packageJson = require('./package.json'); +const buildPluginExternals = require('../../common/buildPluginExternals'); +const externals = buildPluginExternals(packageJson); + +var config = { + context: __dirname + '/src/backend', + + entry: { + app: './index.js', + }, + target: 'node', + output: { + path: path.resolve(__dirname, 'dist'), + filename: 'backend.js', + libraryTarget: 'commonjs2', + }, + + // uncomment for disable minimalization + // optimization: { + // minimize: false, + // }, + + externals, +}; + +module.exports = config; diff --git a/plugins/dbgate-plugin-dbf/webpack-frontend.config.js b/plugins/dbgate-plugin-dbf/webpack-frontend.config.js new file mode 100644 index 000000000..379cb7081 --- /dev/null +++ b/plugins/dbgate-plugin-dbf/webpack-frontend.config.js @@ -0,0 +1,30 @@ +var webpack = require("webpack"); +var path = require("path"); + +var config = { + context: __dirname + "/src/frontend", + + entry: { + app: "./index.js", + }, + target: "web", + output: { + path: path.resolve(__dirname, "dist"), + filename: "frontend.js", + libraryTarget: "var", + library: 'plugin', + }, + + plugins: [ + new webpack.DefinePlugin({ + 'global.DBGATE_PACKAGES': 'window.DBGATE_PACKAGES', + }), + ], + + // uncomment for disable minimalization + // optimization: { + // minimize: false, + // }, +}; + +module.exports = config; diff --git a/yarn.lock b/yarn.lock index 8e3c0bce1..200b3c388 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4221,6 +4221,13 @@ dateformat@^4.6.3: resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5" integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA== +dbffile@^1.12.0: + version "1.12.0" + resolved "https://registry.yarnpkg.com/dbffile/-/dbffile-1.12.0.tgz#626593b5248b273b3194bbb57e21bb8f1661802c" + integrity sha512-repLNtp1jyODRyAbqSaCiComsLi9/2w+ljYC760TXf69ExFVoJzdDr5STBx21gunM59RipYK5zHeDIuGf1LA8w== + dependencies: + iconv-lite "^0.4.24" + dbgate-plugin-tools@^1.0.4, dbgate-plugin-tools@^1.0.7, dbgate-plugin-tools@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/dbgate-plugin-tools/-/dbgate-plugin-tools-1.0.8.tgz#bb16f38ff7161d9b57c08f0918e152593e7ac695"