DBF reader plugin

2026-04-17 23:45:59 +00:00 · 2025-01-07 17:24:56 +01:00
parent 79bf9016a3
commit 2521f05526
11 changed files with 322 additions and 0 deletions
--- a/packages/api/src/controllers/runners.js
+++ b/packages/api/src/controllers/runners.js
@@ -107,6 +107,8 @@ module.exports = {
  },

  startCore(runid, scriptText) {
+    // console.log('************* RUN SCRIPT *************');
+    // console.log(scriptText);
    const directory = path.join(rundir(), runid);
    const scriptFile = path.join(uploadsdir(), runid + '.js');
    fs.writeFileSync(`${scriptFile}`, scriptText);
--- a/plugins/dbgate-plugin-dbf/README.md
+++ b/plugins/dbgate-plugin-dbf/README.md
@@ -0,0 +1,4 @@
+[![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
+[![NPM version](https://img.shields.io/npm/v/dbgate-plugin-dbf.svg)](https://www.npmjs.com/package/dbgate-plugin-dbf)
+
+# dbgate-plugin-dbf
--- a/plugins/dbgate-plugin-dbf/icon.svg
+++ b/plugins/dbgate-plugin-dbf/icon.svg
@@ -0,0 +1,77 @@
+
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg fill="#000000" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" 
+	 width="800px" height="800px" viewBox="0 0 550.801 550.801"
+	 xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<path d="M283.948,460.825h-10.335v37.778c2.987,0.391,6.56,0.391,11.728,0.391c13.722,0,25.851-5.168,25.851-19.279
+				C311.191,466.198,299.062,460.825,283.948,460.825z"/>
+			<path d="M308.006,422.656c0-10.136-7.757-15.504-21.479-15.504c-6.561,0-10.333,0.39-12.92,0.785v31.214h9.945
+				C299.463,439.162,308.006,432.602,308.006,422.656z"/>
+			<path d="M149.347,408.344c-6.75,0-11.129,0.591-13.708,1.191v87.677c2.586,0.601,6.76,0.601,10.533,0.601
+				c27.433,0.201,45.333-14.913,45.333-46.923C191.706,423.052,175.403,408.344,149.347,408.344z"/>
+			<path d="M475.095,131.997c-0.031-2.526-0.828-5.021-2.562-6.992L366.325,3.694c-0.021-0.034-0.053-0.045-0.085-0.076
+				c-0.633-0.707-1.36-1.29-2.141-1.804c-0.232-0.15-0.464-0.287-0.707-0.422c-0.686-0.366-1.392-0.67-2.13-0.892
+				c-0.2-0.058-0.38-0.14-0.58-0.192C359.87,0.119,359.048,0,358.204,0H97.2c-11.907,0-21.6,9.693-21.6,21.601v507.6
+				c0,11.913,9.692,21.601,21.6,21.601h356.4c11.918,0,21.6-9.688,21.6-21.601V133.207
+				C475.2,132.796,475.137,132.398,475.095,131.997z M201.454,504.173c-14.123,11.733-35.598,17.297-61.836,17.297
+				c-15.715,0-26.844-0.997-34.404-1.983V387.861c11.14-1.793,25.65-2.784,40.964-2.784c25.449,0,41.958,4.577,54.875,14.312
+				c13.922,10.341,22.663,26.842,22.663,50.509C223.715,475.549,214.368,493.235,201.454,504.173z M329.084,509.341
+				c-9.745,7.757-26.046,12.129-52.689,12.129c-14.916,0-26.051-0.997-32.812-1.983V387.861c7.966-1.592,24.058-2.784,39.169-2.784
+				c18.493,0,29.826,1.793,39.571,7.552c9.345,4.973,16.105,14.111,16.105,26.24c0,11.929-6.961,23.066-22.075,28.635v0.401
+				c15.314,4.171,26.642,15.704,26.642,33.001C343.006,493.035,337.427,502.58,329.084,509.341z M445.596,410.928h-51.501v30.617
+				h48.115v24.648h-48.115v53.884h-30.417V386.068h81.918V410.928z M97.2,366.758V21.605h250.204v110.516
+				c0,5.962,4.83,10.8,10.8,10.8h95.396l0.011,223.837H97.2z"/>
+		</g>
+		<g>
+			<path d="M261.025,299.769c-2.22-0.912-4.008-2.389-5.292-4.192c-1.189,0.021-2.355,0.079-3.565,0.079
+				c-39.295,0-66.677-13.753-66.677-26.093v-20.239c12.263,11.232,37.491,18.404,66.677,18.404c1.777,0,3.489-0.113,5.234-0.164
+				c1.01-0.917,2.128-1.716,3.446-2.262c1.582-0.62,2.758-1.767,3.33-3.143s0.551-3.035-0.048-4.406
+				c-0.374-0.815-0.551-1.695-0.714-2.576c-3.633,0.253-7.359,0.422-11.243,0.422c-39.295,0-66.678-13.756-66.678-26.093v-20.24
+				c12.264,11.232,37.492,18.402,66.678,18.402c29.228,0,54.456-7.169,66.711-18.412v13.46h0.681c2.769,0,5.278,1.018,7.325,2.653
+				v-17.381c0-3.059-0.887-6.162-2.669-9.218l-0.231-0.391c1.846-3.006,2.9-6.188,2.9-9.502v-21.516
+				c0-2.909-0.812-5.875-2.416-8.807l-0.189-0.34c0.475-1.305,0.812-2.645,0.812-4.071c0-17.384-37.568-26.768-72.924-26.768
+				c-35.322,0-72.876,9.384-72.876,26.768c0,1.427,0.33,2.767,0.809,4.071l-0.189,0.34c-1.614,2.932-2.413,5.898-2.413,8.807v21.516
+				c0,3.309,1.042,6.484,2.887,9.492l-0.243,0.412c-1.759,3.056-2.645,6.148-2.645,9.207V229.5c0,3.207,0.981,6.275,2.731,9.208
+				l-0.087,0.134c-1.759,3.045-2.645,6.149-2.645,9.197v21.513c0,19.111,32.798,34.093,74.669,34.093
+				c4.131,0,8.139-0.189,12.07-0.464c-0.039-0.116-0.021-0.253-0.071-0.358C263.601,301.451,262.451,300.322,261.025,299.769z
+				 M185.496,170.968c11.981,10.159,39.77,15.42,66.678,15.42c26.947,0,54.72-5.266,66.711-15.42v17.909
+				c0,12.337-27.379,26.083-66.711,26.083c-39.295,0-66.678-13.745-66.678-26.083V170.968z"/>
+			<path d="M340.875,296.494c0.733-0.295,1.324-0.865,1.608-1.551c0.28-0.686,0.28-1.497-0.021-2.178
+				c-0.902-2.062-0.438-4.487,1.134-6.064l4.197-4.203c1.055-1.039,2.415-1.613,3.845-1.613h0.986l1.123,0.433
+				c0.875,0.385,1.619,0.348,2.283,0.073c0.707-0.295,1.276-0.875,1.54-1.571c0.802-1.993,2.615-3.265,4.714-3.417v-3.908
+				c0-0.406-0.264-0.789-0.648-0.952c-4.14-1.619-7.493-4.892-9.191-8.997c-1.703-4.113-1.661-8.812,0.132-12.877
+				c0.163-0.367,0.079-0.82-0.211-1.118l-8.864-8.862c-0.485-0.49-0.85-0.335-1.129-0.222c-4.039,1.788-8.754,1.849-12.867,0.138
+				c-4.103-1.69-7.383-5.055-9.007-9.197c-0.147-0.372-0.517-0.638-0.938-0.638h-12.53c-0.411,0-0.796,0.266-0.938,0.638
+				c-1.619,4.147-4.904,7.507-9.007,9.197c-4.087,1.706-8.807,1.661-12.878-0.138c-0.253-0.103-0.622-0.269-1.118,0.222
+				l-8.867,8.862c-0.285,0.298-0.372,0.751-0.203,1.123c1.78,4.071,1.83,8.759,0.135,12.873c-1.701,4.105-5.055,7.377-9.202,8.997
+				c-0.382,0.158-0.638,0.524-0.638,0.952v12.535c0,0.412,0.256,0.781,0.638,0.939c4.153,1.618,7.496,4.894,9.202,8.996
+				c1.695,4.113,1.646,8.807-0.143,12.883c-0.161,0.364-0.074,0.817,0.211,1.113l8.867,8.869c0.496,0.496,0.85,0.332,1.123,0.201
+				c2.083-0.907,4.298-1.382,6.587-1.382c2.173,0,4.282,0.422,6.286,1.255c4.103,1.698,7.388,5.052,9.007,9.197
+				c0.143,0.369,0.527,0.633,0.938,0.633h12.53c0.427,0,0.791-0.254,0.938-0.633c1.624-4.146,4.898-7.499,9.007-9.197
+				c4.003-1.656,8.543-1.635,12.524-0.006c0.058-0.532,0.189-1.055,0.411-1.561c0.322-0.712,0.322-1.534,0.037-2.226
+				c-0.279-0.68-0.87-1.249-1.566-1.523c-2.125-0.854-3.475-2.837-3.475-5.084v-5.938
+				C337.438,299.315,338.787,297.327,340.875,296.494z M313.295,303.123c-13.437,0-24.368-10.922-24.368-24.379
+				c0-13.421,10.921-24.372,24.368-24.372c13.442,0,24.385,10.945,24.385,24.372C337.669,292.201,326.737,303.123,313.295,303.123z"
+				/>
+			<path d="M389.148,301.108c-1.962-0.765-3.555-2.32-4.355-4.256c-0.802-1.951-0.781-4.177,0.062-6.102
+				c0.085-0.174,0.043-0.396-0.095-0.543l-4.197-4.192c-0.232-0.227-0.411-0.152-0.538-0.1c-1.909,0.844-4.155,0.875-6.091,0.062
+				c-1.951-0.801-3.507-2.394-4.271-4.355c-0.068-0.18-0.242-0.306-0.443-0.306h-5.933c-0.2,0-0.385,0.116-0.453,0.306
+				c-0.76,1.962-2.32,3.555-4.261,4.355c-1.936,0.812-4.172,0.791-6.097-0.062c-0.121-0.053-0.3-0.127-0.532,0.1l-4.203,4.192
+				c-0.132,0.147-0.179,0.358-0.1,0.543c0.859,1.925,0.87,4.15,0.068,6.102c-0.812,1.936-2.395,3.491-4.356,4.256
+				c-0.185,0.079-0.306,0.248-0.306,0.453v5.943c0,0.19,0.121,0.364,0.306,0.443c1.962,0.77,3.544,2.32,4.356,4.261
+				c0.802,1.94,0.785,4.171-0.068,6.106c-0.079,0.169-0.032,0.38,0.1,0.527l4.203,4.198c0.232,0.231,0.406,0.158,0.532,0.1
+				c0.986-0.433,2.036-0.659,3.122-0.659c1.023,0,2.025,0.2,2.964,0.596c1.951,0.808,3.512,2.39,4.271,4.356
+				c0.074,0.179,0.253,0.301,0.453,0.301h5.933c0.201,0,0.375-0.111,0.443-0.301c0.765-1.957,2.32-3.549,4.271-4.356
+				c1.946-0.807,4.171-0.774,6.096,0.063c0.122,0.059,0.301,0.121,0.533-0.1l4.192-4.198c0.132-0.137,0.179-0.348,0.1-0.527
+				c-0.854-1.93-0.864-4.166-0.068-6.106c0.807-1.94,2.389-3.491,4.361-4.261c0.18-0.079,0.301-0.253,0.301-0.443v-5.943
+				C389.454,301.367,389.328,301.188,389.148,301.108z M366.251,316.08c-6.37,0-11.538-5.179-11.538-11.549
+				c0-6.365,5.168-11.555,11.538-11.555c6.365,0,11.544,5.189,11.544,11.555C377.795,310.901,372.616,316.08,366.251,316.08z"/>
+		</g>
+	</g>
+</g>
+</svg>
--- a/plugins/dbgate-plugin-dbf/package.json
+++ b/plugins/dbgate-plugin-dbf/package.json
@@ -0,0 +1,43 @@
+{
+  "name": "dbgate-plugin-dbf",
+  "main": "dist/backend.js",
+  "version": "6.0.0-alpha.1",
+  "homepage": "https://dbgate.org",
+  "description": "DBF import/export plugin for DbGate",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate"
+  },
+  "author": "Jan Prochazka",
+  "license": "GPL-3.0",
+  "keywords": [
+    "dbf",
+    "import",
+    "export",
+    "dbgate",
+    "dbgatebuiltin"
+  ],
+  "files": [
+    "dist",
+    "icon.svg"
+  ],
+  "scripts": {
+    "build:frontend": "webpack --config webpack-frontend.config",
+    "build:frontend:watch": "webpack --watch --config webpack-frontend.config",
+    "build:backend": "webpack --config webpack-backend.config.js",
+    "build": "yarn build:frontend && yarn build:backend",
+    "plugin": "yarn build && yarn pack && dbgate-plugin dbgate-plugin-dbf",
+    "copydist": "yarn build && yarn pack && dbgate-copydist ../dist/dbgate-plugin-dbf",
+    "plugout": "dbgate-plugout dbgate-plugin-dbf",
+    "prepublishOnly": "yarn build"
+  },
+  "devDependencies": {
+    "dbgate-plugin-tools": "^1.0.7",
+    "webpack": "^5.91.0",
+    "webpack-cli": "^5.1.4"
+  },
+  "dependencies": {
+    "dbffile": "^1.12.0",
+    "lodash": "^4.17.21"
+  }
+}
--- a/plugins/dbgate-plugin-dbf/prettier.config.js
+++ b/plugins/dbgate-plugin-dbf/prettier.config.js
@@ -0,0 +1,8 @@
+module.exports = {
+  trailingComma: 'es5',
+  tabWidth: 2,
+  semi: true,
+  singleQuote: true,
+  arrowParen: 'avoid',
+  printWidth: 120,
+};
--- a/plugins/dbgate-plugin-dbf/src/backend/index.js
+++ b/plugins/dbgate-plugin-dbf/src/backend/index.js
@@ -0,0 +1,12 @@
+const reader = require('./reader');
+// const writer = require('./writer');
+
+module.exports = {
+  packageName: 'dbgate-plugin-dbf',
+  shellApi: {
+    reader,
+  },
+  initialize(dbgateEnv) {
+    reader.initialize(dbgateEnv);
+  },
+};
--- a/plugins/dbgate-plugin-dbf/src/backend/reader.js
+++ b/plugins/dbgate-plugin-dbf/src/backend/reader.js
@@ -0,0 +1,85 @@
+const _ = require('lodash');
+const csv = require('csv');
+const fs = require('fs');
+const stream = require('stream');
+const { DBFFile } = require('dbffile');
+
+let dbgateApi;
+
+function getFieldType(field) {
+  const { type, size } = field;
+  switch (type) {
+    case 'C':
+      return `varchar(${size})`;
+    case 'N':
+      return 'numeric';
+    case 'F':
+      return 'float';
+    case 'Y':
+      return 'money';
+    case 'I':
+      return 'int';
+    case 'L':
+      return 'boolean';
+    case 'D':
+      return 'date';
+    case 'T':
+      return 'datetime';
+    case 'B':
+      return 'duouble';
+    case 'M':
+      return 'varchar(max)';
+    default:
+      return 'string';
+  }
+}
+
+async function reader({ fileName, encoding = 'ISO-8859-1', includeDeletedRecords = false, limitRows = undefined }) {
+  console.log(`Reading file ${fileName}`);
+  const downloadedFile = await dbgateApi.download(fileName);
+
+  const pass = new stream.PassThrough({
+    objectMode: true,
+  });
+
+  (async () => {
+    try {
+      // Open the DBF file
+      const dbf = await DBFFile.open(downloadedFile, { encoding, includeDeletedRecords });
+
+      const columns = dbf.fields.map((field) => ({
+        columnName: field.name,
+        dataType: getFieldType(field),
+      }));
+
+      pass.write({
+        __isStreamHeader: true,
+        columns,
+      })
+
+      let readedRows = 0;
+      // Read each record and push it into the stream
+      for await (const record of dbf) {
+        // Emit the record as a chunk
+        pass.write(record);
+        readedRows++;
+        if (limitRows && readedRows >= limitRows) {
+          break;
+        }
+      }
+
+      pass.end();
+    } catch (error) {
+      // If any error occurs, destroy the stream with the error
+      pass.end();
+    }
+  })();
+
+  return pass;
+}
+
+reader.initialize = (dbgateEnv) => {
+  dbgateApi = dbgateEnv.dbgateApi;
+};
+
+module.exports = reader;
--- a/plugins/dbgate-plugin-dbf/src/frontend/index.js
+++ b/plugins/dbgate-plugin-dbf/src/frontend/index.js
@@ -0,0 +1,25 @@
+const fileFormat = {
+  packageName: 'dbgate-plugin-dbf',
+  // file format identifier
+  storageType: 'dbf',
+  // file extension without leading dot
+  extension: 'dbf',
+  // human readable file format name
+  name: 'DBF',
+  // function name from backend, which contains reader factory, postfixed by package name
+  readerFunc: 'reader@dbgate-plugin-dbf',
+
+  args: [
+    {
+      type: 'text',
+      name: 'encoding',
+      label: 'Encoding',
+      apiName: 'encoding',
+    },
+  ],
+
+};
+
+export default {
+  fileFormats: [fileFormat],
+};
--- a/plugins/dbgate-plugin-dbf/webpack-backend.config.js
+++ b/plugins/dbgate-plugin-dbf/webpack-backend.config.js
@@ -0,0 +1,29 @@
+var webpack = require('webpack');
+var path = require('path');
+
+const packageJson = require('./package.json');
+const buildPluginExternals = require('../../common/buildPluginExternals');
+const externals = buildPluginExternals(packageJson);
+
+var config = {
+  context: __dirname + '/src/backend',
+
+  entry: {
+    app: './index.js',
+  },
+  target: 'node',
+  output: {
+    path: path.resolve(__dirname, 'dist'),
+    filename: 'backend.js',
+    libraryTarget: 'commonjs2',
+  },
+
+  // uncomment for disable minimalization
+  //   optimization: {
+  //     minimize: false,
+  //   },
+
+  externals,
+};
+
+module.exports = config;
--- a/plugins/dbgate-plugin-dbf/webpack-frontend.config.js
+++ b/plugins/dbgate-plugin-dbf/webpack-frontend.config.js
@@ -0,0 +1,30 @@
+var webpack = require("webpack");
+var path = require("path");
+
+var config = {
+  context: __dirname + "/src/frontend",
+
+  entry: {
+    app: "./index.js",
+  },
+  target: "web",
+  output: {
+    path: path.resolve(__dirname, "dist"),
+    filename: "frontend.js",
+    libraryTarget: "var",
+    library: 'plugin',
+  },
+
+  plugins: [
+    new webpack.DefinePlugin({
+      'global.DBGATE_PACKAGES': 'window.DBGATE_PACKAGES',
+    }),
+  ],
+
+  // uncomment for disable minimalization
+  // optimization: {
+  //   minimize: false,
+  // },
+};
+
+module.exports = config;
--- a/yarn.lock
+++ b/yarn.lock
@@ -4221,6 +4221,13 @@ dateformat@^4.6.3:
  resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5"
  integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==

+dbffile@^1.12.0:
+  version "1.12.0"
+  resolved "https://registry.yarnpkg.com/dbffile/-/dbffile-1.12.0.tgz#626593b5248b273b3194bbb57e21bb8f1661802c"
+  integrity sha512-repLNtp1jyODRyAbqSaCiComsLi9/2w+ljYC760TXf69ExFVoJzdDr5STBx21gunM59RipYK5zHeDIuGf1LA8w==
+  dependencies:
+    iconv-lite "^0.4.24"
+
 dbgate-plugin-tools@^1.0.4, dbgate-plugin-tools@^1.0.7, dbgate-plugin-tools@^1.0.8:
  version "1.0.8"
  resolved "https://registry.yarnpkg.com/dbgate-plugin-tools/-/dbgate-plugin-tools-1.0.8.tgz#bb16f38ff7161d9b57c08f0918e152593e7ac695"