dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-18 21:55:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/duckdb-2

Browse Source
This commit is contained in:
SPRINX0\prochazka
2025-04-24 09:25:45 +02:00
parent 041397b137 e18254e4d8
commit 7752db3916
117 changed files with 5141 additions and 2829 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
packages/api/src/controllers/archive.js
View File

@@ -2,14 +2,20 @@ const fs = require('fs-extra');
const readline = require('readline');
const crypto = require('crypto');
const path = require('path');
const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('../utility/directories');
const { archivedir, clearArchiveLinksCache, resolveArchiveFolder, uploadsdir } = require('../utility/directories');
const socket = require('../utility/socket');
const loadFilesRecursive = require('../utility/loadFilesRecursive');
const getJslFileName = require('../utility/getJslFileName');
const { getLogger, extractErrorLogData } = require('dbgate-tools');
const { getLogger, extractErrorLogData, jsonLinesParse } = require('dbgate-tools');
const dbgateApi = require('../shell');
const jsldata = require('./jsldata');
const platformInfo = require('../utility/platformInfo');
const { isProApp } = require('../utility/checkLicense');
const listZipEntries = require('../utility/listZipEntries');
const unzipJsonLinesFile = require('../shell/unzipJsonLinesFile');
const { zip } = require('lodash');
const zipDirectory = require('../shell/zipDirectory');
const unzipDirectory = require('../shell/unzipDirectory');
const logger = getLogger('archive');
@@ -47,9 +53,31 @@ module.exports = {
return folder;
},
async getZipFiles({ file }) {
const entries = await listZipEntries(path.join(archivedir(), file));
const files = entries.map(entry => {
let name = entry.fileName;
if (isProApp() && entry.fileName.endsWith('.jsonl')) {
name = entry.fileName.slice(0, -6);
}
return {
name: name,
label: name,
type: isProApp() && entry.fileName.endsWith('.jsonl') ? 'jsonl' : 'other',
};
});
return files;
},
files_meta: true,
async files({ folder }) {
try {
if (folder.endsWith('.zip')) {
if (await fs.exists(path.join(archivedir(), folder))) {
return this.getZipFiles({ file: folder });
}
return [];
}
const dir = resolveArchiveFolder(folder);
if (!(await fs.exists(dir))) return [];
const files = await loadFilesRecursive(dir); // fs.readdir(dir);
@@ -91,6 +119,16 @@ module.exports = {
return true;
},
createFile_meta: true,
async createFile({ folder, file, fileType, tableInfo }) {
await fs.writeFile(
path.join(resolveArchiveFolder(folder), `${file}.${fileType}`),
tableInfo ? JSON.stringify({ __isStreamHeader: true, tableInfo }) : ''
);
socket.emitChanged(`archive-files-changed`, { folder });
return true;
},
deleteFile_meta: true,
async deleteFile({ folder, file, fileType }) {
await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
@@ -158,7 +196,7 @@ module.exports = {
deleteFolder_meta: true,
async deleteFolder({ folder }) {
if (!folder) throw new Error('Missing folder parameter');
if (folder.endsWith('.link')) {
if (folder.endsWith('.link') || folder.endsWith('.zip')) {
await fs.unlink(path.join(archivedir(), folder));
} else {
await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
@@ -204,9 +242,10 @@ module.exports = {
},
async getNewArchiveFolder({ database }) {
const isLink = database.endsWith(database);
const name = isLink ? database.slice(0, -5) : database;
const suffix = isLink ? '.link' : '';
const isLink = database.endsWith('.link');
const isZip = database.endsWith('.zip');
const name = isLink ? database.slice(0, -5) : isZip ? database.slice(0, -4) : database;
const suffix = isLink ? '.link' : isZip ? '.zip' : '';
if (!(await fs.exists(path.join(archivedir(), database)))) return database;
let index = 2;
while (await fs.exists(path.join(archivedir(), `${name}${index}${suffix}`))) {
@@ -214,4 +253,58 @@ module.exports = {
}
return `${name}${index}${suffix}`;
},
getArchiveData_meta: true,
async getArchiveData({ folder, file }) {
let rows;
if (folder.endsWith('.zip')) {
rows = await unzipJsonLinesFile(path.join(archivedir(), folder), `${file}.jsonl`);
} else {
rows = jsonLinesParse(await fs.readFile(path.join(archivedir(), folder, `${file}.jsonl`), { encoding: 'utf8' }));
}
return rows.filter(x => !x.__isStreamHeader);
},
saveUploadedZip_meta: true,
async saveUploadedZip({ filePath, fileName }) {
if (!fileName?.endsWith('.zip')) {
throw new Error(`${fileName} is not a ZIP file`);
}
const folder = await this.getNewArchiveFolder({ database: fileName });
await fs.copyFile(filePath, path.join(archivedir(), folder));
socket.emitChanged(`archive-folders-changed`);
return null;
},
zip_meta: true,
async zip({ folder }) {
const newFolder = await this.getNewArchiveFolder({ database: folder + '.zip' });
await zipDirectory(path.join(archivedir(), folder), path.join(archivedir(), newFolder));
socket.emitChanged(`archive-folders-changed`);
return null;
},
unzip_meta: true,
async unzip({ folder }) {
const newFolder = await this.getNewArchiveFolder({ database: folder.slice(0, -4) });
await unzipDirectory(path.join(archivedir(), folder), path.join(archivedir(), newFolder));
socket.emitChanged(`archive-folders-changed`);
return null;
},
getZippedPath_meta: true,
async getZippedPath({ folder }) {
if (folder.endsWith('.zip')) {
return { filePath: path.join(archivedir(), folder) };
}
const uploadName = crypto.randomUUID();
const filePath = path.join(uploadsdir(), uploadName);
await zipDirectory(path.join(archivedir(), folder), filePath);
return { filePath };
},
};

4
packages/api/src/controllers/auth.js
View File

@@ -12,6 +12,7 @@ const {
getAuthProviderById,
} = require('../auth/authProvider');
const storage = require('./storage');
const { decryptPasswordString } = require('../utility/crypting');
const logger = getLogger('auth');
@@ -44,6 +45,7 @@ function authMiddleware(req, res, next) {
'/connections/dblogin-auth',
'/connections/dblogin-auth-token',
'/health',
'/__health',
];
// console.log('********************* getAuthProvider()', getAuthProvider());
@@ -95,7 +97,7 @@ module.exports = {
let adminPassword = process.env.ADMIN_PASSWORD;
if (!adminPassword) {
const adminConfig = await storage.readConfig({ group: 'admin' });
adminPassword = adminConfig?.adminPassword;
adminPassword = decryptPasswordString(adminConfig?.adminPassword);
}
if (adminPassword && adminPassword == password) {
return {

155
packages/api/src/controllers/config.js
View File

@@ -19,6 +19,14 @@ const storage = require('./storage');
const { getAuthProxyUrl } = require('../utility/authProxy');
const { getPublicHardwareFingerprint } = require('../utility/hardwareFingerprint');
const { extractErrorMessage } = require('dbgate-tools');
const {
generateTransportEncryptionKey,
createTransportEncryptor,
recryptConnection,
getInternalEncryptor,
recryptUser,
recryptObjectPasswordFieldInPlace,
} = require('../utility/crypting');
const lock = new AsyncLock();
@@ -107,6 +115,7 @@ module.exports = {
datadir(),
processArgs.runE2eTests ? 'connections-e2etests.jsonl' : 'connections.jsonl'
),
supportCloudAutoUpgrade: !!process.env.CLOUD_UPGRADE_FILE,
...currentVersion,
};
@@ -144,7 +153,7 @@ module.exports = {
const res = {
...value,
};
if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
if (platformInfo.isElectron && value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
// res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
res['app.useNativeMenu'] = false;
}
@@ -161,14 +170,19 @@ module.exports = {
async loadSettings() {
try {
const settingsText = await fs.readFile(
path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
{ encoding: 'utf-8' }
);
return {
...this.fillMissingSettings(JSON.parse(settingsText)),
'other.licenseKey': platformInfo.isElectron ? await this.loadLicenseKey() : undefined,
};
if (process.env.STORAGE_DATABASE) {
const settings = await storage.readConfig({ group: 'settings' });
return this.fillMissingSettings(settings);
} else {
const settingsText = await fs.readFile(
path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
{ encoding: 'utf-8' }
);
return {
...this.fillMissingSettings(JSON.parse(settingsText)),
'other.licenseKey': platformInfo.isElectron ? await this.loadLicenseKey() : undefined,
};
}
} catch (err) {
return this.fillMissingSettings({});
}
@@ -246,19 +260,31 @@ module.exports = {
const res = await lock.acquire('settings', async () => {
const currentValue = await this.loadSettings();
try {
const updated = {
...currentValue,
..._.omit(values, ['other.licenseKey']),
};
await fs.writeFile(
path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
JSON.stringify(updated, undefined, 2)
);
// this.settingsValue = updated;
let updated = currentValue;
if (process.env.STORAGE_DATABASE) {
updated = {
...currentValue,
...values,
};
await storage.writeConfig({
group: 'settings',
config: updated,
});
} else {
updated = {
...currentValue,
..._.omit(values, ['other.licenseKey']),
};
await fs.writeFile(
path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
JSON.stringify(updated, undefined, 2)
);
// this.settingsValue = updated;
if (currentValue['other.licenseKey'] != values['other.licenseKey']) {
await this.saveLicenseKey({ licenseKey: values['other.licenseKey'] });
socket.emitChanged(`config-changed`);
if (currentValue['other.licenseKey'] != values['other.licenseKey']) {
await this.saveLicenseKey({ licenseKey: values['other.licenseKey'] });
socket.emitChanged(`config-changed`);
}
}
socket.emitChanged(`settings-changed`);
@@ -281,4 +307,91 @@ module.exports = {
const resp = await checkLicenseKey(licenseKey);
return resp;
},
recryptDatabaseForExport(db) {
const encryptionKey = generateTransportEncryptionKey();
const transportEncryptor = createTransportEncryptor(encryptionKey);
const config = _.cloneDeep([
...(db.config?.filter(c => !(c.group == 'admin' && c.key == 'encryptionKey')) || []),
{ group: 'admin', key: 'encryptionKey', value: encryptionKey },
]);
const adminPassword = config.find(c => c.group == 'admin' && c.key == 'adminPassword');
recryptObjectPasswordFieldInPlace(adminPassword, 'value', getInternalEncryptor(), transportEncryptor);
return {
...db,
connections: db.connections?.map(conn => recryptConnection(conn, getInternalEncryptor(), transportEncryptor)),
users: db.users?.map(conn => recryptUser(conn, getInternalEncryptor(), transportEncryptor)),
config,
};
},
recryptDatabaseFromImport(db) {
const encryptionKey = db.config?.find(c => c.group == 'admin' && c.key == 'encryptionKey')?.value;
if (!encryptionKey) {
throw new Error('Missing encryption key in the database');
}
const config = _.cloneDeep(db.config || []).filter(c => !(c.group == 'admin' && c.key == 'encryptionKey'));
const transportEncryptor = createTransportEncryptor(encryptionKey);
const adminPassword = config.find(c => c.group == 'admin' && c.key == 'adminPassword');
recryptObjectPasswordFieldInPlace(adminPassword, 'value', transportEncryptor, getInternalEncryptor());
return {
...db,
connections: db.connections?.map(conn => recryptConnection(conn, transportEncryptor, getInternalEncryptor())),
users: db.users?.map(conn => recryptUser(conn, transportEncryptor, getInternalEncryptor())),
config,
};
},
exportConnectionsAndSettings_meta: true,
async exportConnectionsAndSettings(_params, req) {
if (!hasPermission(`admin/config`, req)) {
throw new Error('Permission denied: admin/config');
}
if (connections.portalConnections) {
throw new Error('Not allowed');
}
if (process.env.STORAGE_DATABASE) {
const db = await storage.getExportedDatabase();
return this.recryptDatabaseForExport(db);
}
return this.recryptDatabaseForExport({
connections: (await connections.list(null, req)).map((conn, index) => ({
..._.omit(conn, ['_id']),
id: index + 1,
conid: conn._id,
})),
});
},
importConnectionsAndSettings_meta: true,
async importConnectionsAndSettings({ db }, req) {
if (!hasPermission(`admin/config`, req)) {
throw new Error('Permission denied: admin/config');
}
if (connections.portalConnections) {
throw new Error('Not allowed');
}
const recryptedDb = this.recryptDatabaseFromImport(db);
if (process.env.STORAGE_DATABASE) {
await storage.replicateImportedDatabase(recryptedDb);
} else {
await connections.importFromArray(
recryptedDb.connections.map(conn => ({
..._.omit(conn, ['conid', 'id']),
_id: conn.conid,
}))
);
}
return true;
},
};

16
packages/api/src/controllers/connections.js
View File

@@ -107,8 +107,8 @@ function getPortalCollections() {
trustServerCertificate: process.env[`SSL_TRUST_CERTIFICATE_${id}`],
}));
for(const conn of connections) {
for(const prop in process.env) {
for (const conn of connections) {
for (const prop in process.env) {
if (prop.startsWith(`CONNECTION_${conn._id}_`)) {
const name = prop.substring(`CONNECTION_${conn._id}_`.length);
conn[name] = process.env[prop];
@@ -321,6 +321,18 @@ module.exports = {
return res;
},
importFromArray(list) {
this.datastore.transformAll(connections => {
const mapped = connections.map(x => {
const found = list.find(y => y._id == x._id);
if (found) return found;
return x;
});
return [...mapped, ...list.filter(x => !connections.find(y => y._id == x._id))];
});
socket.emitChanged('connection-list-changed');
},
async checkUnsavedConnectionsLimit() {
if (!this.datastore) {
return;

26
packages/api/src/controllers/databaseConnections.js
View File

@@ -37,6 +37,8 @@ const loadModelTransform = require('../utility/loadModelTransform');
const exportDbModelSql = require('../utility/exportDbModelSql');
const axios = require('axios');
const { callTextToSqlApi, callCompleteOnCursorApi, callRefactorSqlQueryApi } = require('../utility/authProxy');
const { decryptConnection } = require('../utility/crypting');
const { getSshTunnel } = require('../utility/sshTunnel');
const logger = getLogger('databaseConnections');
@@ -140,6 +142,11 @@ module.exports = {
if (newOpened.disconnected) return;
this.close(conid, database, false);
});
subprocess.on('error', err => {
logger.error(extractErrorLogData(err), 'Error in database connection subprocess');
if (newOpened.disconnected) return;
this.close(conid, database, false);
});
subprocess.send({
msgtype: 'connect',
@@ -619,9 +626,26 @@ module.exports = {
command,
{ conid, database, outputFile, inputFile, options, selectedTables, skippedTables, argsFormat }
) {
const connection = await connections.getCore({ conid });
const sourceConnection = await connections.getCore({ conid });
const connection = {
...decryptConnection(sourceConnection),
};
const driver = requireEngineDriver(connection);
if (!connection.port && driver.defaultPort) {
connection.port = driver.defaultPort.toString();
}
if (connection.useSshTunnel) {
const tunnel = await getSshTunnel(connection);
if (tunnel.state == 'error') {
throw new Error(tunnel.message);
}
connection.server = tunnel.localHost;
connection.port = tunnel.localPort;
}
const settingsValue = await config.getSettings();
const externalTools = {};

59
packages/api/src/controllers/files.js
View File

@@ -9,6 +9,9 @@ const scheduler = require('./scheduler');
const getDiagramExport = require('../utility/getDiagramExport');
const apps = require('./apps');
const getMapExport = require('../utility/getMapExport');
const dbgateApi = require('../shell');
const { getLogger } = require('dbgate-tools');
const logger = getLogger('files');
function serialize(format, data) {
if (format == 'text') return data;
@@ -219,4 +222,60 @@ module.exports = {
return path.join(dir, file);
}
},
createZipFromJsons_meta: true,
async createZipFromJsons({ db, filePath }) {
logger.info(`Creating zip file from JSONS ${filePath}`);
await dbgateApi.zipJsonLinesData(db, filePath);
return true;
},
getJsonsFromZip_meta: true,
async getJsonsFromZip({ filePath }) {
const res = await dbgateApi.unzipJsonLinesData(filePath);
return res;
},
downloadText_meta: true,
async downloadText({ uri }, req) {
if (!uri) return null;
const filePath = await dbgateApi.download(uri);
const text = await fs.readFile(filePath, {
encoding: 'utf-8',
});
return text;
},
saveUploadedFile_meta: true,
async saveUploadedFile({ filePath, fileName }) {
const FOLDERS = ['sql', 'sqlite'];
for (const folder of FOLDERS) {
if (fileName.toLowerCase().endsWith('.' + folder)) {
logger.info(`Saving ${folder} file ${fileName}`);
await fs.copyFile(filePath, path.join(filesdir(), folder, fileName));
socket.emitChanged(`files-changed`, { folder: folder });
socket.emitChanged(`all-files-changed`);
return {
name: path.basename(filePath),
folder: folder,
};
}
}
throw new Error(`${fileName} doesn't have one of supported extensions: ${FOLDERS.join(', ')}`);
},
exportFile_meta: true,
async exportFile({ folder, file, filePath }, req) {
if (!hasPermission(`files/${folder}/read`, req)) return false;
await fs.copyFile(path.join(filesdir(), folder, file), filePath);
return true;
},
simpleCopy_meta: true,
async simpleCopy({ sourceFilePath, targetFilePath }, req) {
await fs.copyFile(sourceFilePath, targetFilePath);
return true;
},
};

9
packages/api/src/controllers/jsldata.js
View File

@@ -8,6 +8,8 @@ const getJslFileName = require('../utility/getJslFileName');
const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
const requirePluginFunction = require('../utility/requirePluginFunction');
const socket = require('../utility/socket');
const crypto = require('crypto');
const dbgateApi = require('../shell');
function readFirstLine(file) {
return new Promise((resolve, reject) => {
@@ -293,4 +295,11 @@ module.exports = {
})),
};
},
downloadJslData_meta: true,
async downloadJslData({ uri }) {
const jslid = crypto.randomUUID();
await dbgateApi.download(uri, { targetFile: getJslFileName(jslid) });
return { jslid };
},
};

24
packages/api/src/controllers/runners.js
View File

@@ -96,9 +96,9 @@ module.exports = {
handle_ping() {},
handle_freeData(runid, { freeData }) {
handle_dataResult(runid, { dataResult }) {
const { resolve } = this.requests[runid];
resolve(freeData);
resolve(dataResult);
delete this.requests[runid];
},
@@ -328,4 +328,24 @@ module.exports = {
});
return promise;
},
scriptResult_meta: true,
async scriptResult({ script }) {
if (script.type != 'json') {
return { errorMessage: 'Only JSON scripts are allowed' };
}
const promise = new Promise((resolve, reject) => {
const runid = crypto.randomUUID();
this.requests[runid] = { resolve, reject, exitOnStreamError: true };
const cloned = _.cloneDeepWith(script, node => {
if (node?.$replace == 'runid') {
return runid;
}
});
const js = jsonScriptToJavascript(cloned);
this.startCore(runid, scriptTemplate(js, false));
});
return promise;
},
};

5
packages/api/src/controllers/serverConnections.js
View File

@@ -98,6 +98,11 @@ module.exports = {
if (newOpened.disconnected) return;
this.close(conid, false);
});
subprocess.on('error', err => {
logger.error(extractErrorLogData(err), 'Error in server connection subprocess');
if (newOpened.disconnected) return;
this.close(conid, false);
});
subprocess.send({ msgtype: 'connect', ...connection, globalSettings: await config.getSettings() });
return newOpened;
});

4
packages/api/src/controllers/storage.js
View File

@@ -4,6 +4,10 @@ module.exports = {
return null;
},
async getExportedDatabase() {
return {};
},
getConnection_meta: true,
async getConnection({ conid }) {
return null;

46
packages/api/src/controllers/uploads.js
View File

@@ -39,52 +39,6 @@ module.exports = {
});
},
uploadDataFile_meta: {
method: 'post',
raw: true,
},
uploadDataFile(req, res) {
const { data } = req.files || {};
if (!data) {
res.json(null);
return;
}
if (data.name.toLowerCase().endsWith('.sql')) {
logger.info(`Uploading SQL file ${data.name}, size=${data.size}`);
data.mv(path.join(filesdir(), 'sql', data.name), () => {
res.json({
name: data.name,
folder: 'sql',
});
socket.emitChanged(`files-changed`, { folder: 'sql' });
socket.emitChanged(`all-files-changed`);
});
return;
}
res.json(null);
},
saveDataFile_meta: true,
async saveDataFile({ filePath }) {
if (filePath.toLowerCase().endsWith('.sql')) {
logger.info(`Saving SQL file ${filePath}`);
await fs.copyFile(filePath, path.join(filesdir(), 'sql', path.basename(filePath)));
socket.emitChanged(`files-changed`, { folder: 'sql' });
socket.emitChanged(`all-files-changed`);
return {
name: path.basename(filePath),
folder: 'sql',
};
}
return null;
},
get_meta: {
method: 'get',
raw: true,

8
packages/api/src/main.js
View File

@@ -38,7 +38,7 @@ const { getLogger } = require('dbgate-tools');
const { getDefaultAuthProvider } = require('./auth/authProvider');
const startCloudUpgradeTimer = require('./utility/cloudUpgrade');
const { isProApp } = require('./utility/checkLicense');
const getHealthStatus = require('./utility/healthStatus');
const { getHealthStatus, getHealthStatusSprinx } = require('./utility/healthStatus');
const logger = getLogger('main');
@@ -124,6 +124,12 @@ function start() {
res.end(JSON.stringify(health, null, 2));
});
app.get(getExpressPath('/__health'), async function (req, res) {
res.setHeader('Content-Type', 'application/json');
const health = await getHealthStatusSprinx();
res.end(JSON.stringify(health, null, 2));
});
app.use(bodyParser.json({ limit: '50mb' }));
app.use(

16
packages/api/src/proc/connectProcess.js
View File

@@ -4,6 +4,8 @@ const { connectUtility } = require('../utility/connectUtility');
const { handleProcessCommunication } = require('../utility/processComm');
const { pickSafeConnectionInfo } = require('../utility/crypting');
const _ = require('lodash');
const { getLogger, extractErrorLogData } = require('dbgate-tools');
const logger = getLogger('connectProcess');
const formatErrorDetail = (e, connection) => `${e.stack}
@@ -23,12 +25,22 @@ function start() {
try {
const driver = requireEngineDriver(connection);
const dbhan = await connectUtility(driver, connection, 'app');
const res = await driver.getVersion(dbhan);
let version = {
version: 'Unknown',
};
try {
version = await driver.getVersion(dbhan);
} catch (err) {
logger.error(extractErrorLogData(err), 'Error getting DB server version');
version = {
version: 'Unknown',
};
}
let databases = undefined;
if (requestDbList) {
databases = await driver.listDatabases(dbhan);
}
process.send({ msgtype: 'connected', ...res, databases });
process.send({ msgtype: 'connected', ...version, databases });
await driver.close(dbhan);
} catch (e) {
console.error(e);

13
packages/api/src/proc/databaseConnectionProcess.js
View File

@@ -120,10 +120,15 @@ function setStatusName(name) {
async function readVersion() {
const driver = requireEngineDriver(storedConnection);
const version = await driver.getVersion(dbhan);
logger.debug(`Got server version: ${version.version}`);
process.send({ msgtype: 'version', version });
serverVersion = version;
try {
const version = await driver.getVersion(dbhan);
logger.debug(`Got server version: ${version.version}`);
serverVersion = version;
} catch (err) {
logger.error(extractErrorLogData(err), 'Error getting DB server version');
serverVersion = { version: 'Unknown' };
}
process.send({ msgtype: 'version', version: serverVersion });
}
async function handleConnect({ connection, structure, globalSettings }) {

8
packages/api/src/proc/serverConnectionProcess.js
View File

@@ -46,7 +46,13 @@ async function handleRefresh() {
async function readVersion() {
const driver = requireEngineDriver(storedConnection);
const version = await driver.getVersion(dbhan);
let version;
try {
version = await driver.getVersion(dbhan);
} catch (err) {
logger.error(extractErrorLogData(err), 'Error getting DB server version');
version = { version: 'Unknown' };
}
process.send({ msgtype: 'version', version });
}

4
packages/api/src/shell/archiveReader.js
View File

@@ -3,7 +3,9 @@ const { archivedir, resolveArchiveFolder } = require('../utility/directories');
const jsonLinesReader = require('./jsonLinesReader');
function archiveReader({ folderName, fileName, ...other }) {
const jsonlFile = path.join(resolveArchiveFolder(folderName), `${fileName}.jsonl`);
const jsonlFile = folderName.endsWith('.zip')
? `zip://archive:${folderName}//${fileName}.jsonl`
: path.join(resolveArchiveFolder(folderName), `${fileName}.jsonl`);
const res = jsonLinesReader({ fileName: jsonlFile, ...other });
return res;
}

4
packages/api/src/shell/collectorWriter.js
View File

@@ -15,9 +15,9 @@ class CollectorWriterStream extends stream.Writable {
_final(callback) {
process.send({
msgtype: 'freeData',
msgtype: 'dataResult',
runid: this.runid,
freeData: { rows: this.rows, structure: this.structure },
dataResult: { rows: this.rows, structure: this.structure },
});
callback();
}

61
packages/api/src/shell/dataDuplicator.js
View File

@@ -1,61 +0,0 @@
const stream = require('stream');
const path = require('path');
const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools');
const requireEngineDriver = require('../utility/requireEngineDriver');
const { connectUtility } = require('../utility/connectUtility');
const logger = getLogger('dataDuplicator');
const { DataDuplicator } = require('dbgate-datalib');
const copyStream = require('./copyStream');
const jsonLinesReader = require('./jsonLinesReader');
const { resolveArchiveFolder } = require('../utility/directories');
async function dataDuplicator({
connection,
archive,
folder,
items,
options,
analysedStructure = null,
driver,
systemConnection,
}) {
if (!driver) driver = requireEngineDriver(connection);
const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
try {
if (!analysedStructure) {
analysedStructure = await driver.analyseFull(dbhan);
}
const sourceDir = archive
? resolveArchiveFolder(archive)
: folder?.startsWith('archive:')
? resolveArchiveFolder(folder.substring('archive:'.length))
: folder;
const dupl = new DataDuplicator(
dbhan,
driver,
analysedStructure,
items.map(item => ({
name: item.name,
operation: item.operation,
matchColumns: item.matchColumns,
openStream:
item.openStream || (() => jsonLinesReader({ fileName: path.join(sourceDir, `${item.name}.jsonl`) })),
})),
stream,
copyStream,
options
);
await dupl.run();
} finally {
if (!systemConnection) {
await driver.close(dbhan);
}
}
}
module.exports = dataDuplicator;

96
packages/api/src/shell/dataReplicator.js Normal file
View File

@@ -0,0 +1,96 @@
const stream = require('stream');
const path = require('path');
const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools');
const requireEngineDriver = require('../utility/requireEngineDriver');
const { connectUtility } = require('../utility/connectUtility');
const logger = getLogger('datareplicator');
const { DataReplicator } = require('dbgate-datalib');
const { compileCompoudEvalCondition } = require('dbgate-filterparser');
const copyStream = require('./copyStream');
const jsonLinesReader = require('./jsonLinesReader');
const { resolveArchiveFolder } = require('../utility/directories');
const { evaluateCondition } = require('dbgate-sqltree');
function compileOperationFunction(enabled, condition) {
if (!enabled) return _row => false;
const conditionCompiled = compileCompoudEvalCondition(condition);
if (condition) {
return row => evaluateCondition(conditionCompiled, row);
}
return _row => true;
}
async function dataReplicator({
connection,
archive,
folder,
items,
options,
analysedStructure = null,
driver,
systemConnection,
}) {
if (!driver) driver = requireEngineDriver(connection);
const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
try {
if (!analysedStructure) {
analysedStructure = await driver.analyseFull(dbhan);
}
let joinPath;
if (archive?.endsWith('.zip')) {
joinPath = file => `zip://archive:${archive}//${file}`;
} else {
const sourceDir = archive
? resolveArchiveFolder(archive)
: folder?.startsWith('archive:')
? resolveArchiveFolder(folder.substring('archive:'.length))
: folder;
joinPath = file => path.join(sourceDir, file);
}
const repl = new DataReplicator(
dbhan,
driver,
analysedStructure,
items.map(item => {
return {
name: item.name,
matchColumns: item.matchColumns,
findExisting: compileOperationFunction(item.findExisting, item.findCondition),
createNew: compileOperationFunction(item.createNew, item.createCondition),
updateExisting: compileOperationFunction(item.updateExisting, item.updateCondition),
deleteMissing: !!item.deleteMissing,
deleteRestrictionColumns: item.deleteRestrictionColumns ?? [],
openStream: item.openStream
? item.openStream
: item.jsonArray
? () => stream.Readable.from(item.jsonArray)
: () => jsonLinesReader({ fileName: joinPath(`${item.name}.jsonl`) }),
};
}),
stream,
copyStream,
options
);
await repl.run();
if (options?.runid) {
process.send({
msgtype: 'dataResult',
runid: options?.runid,
dataResult: repl.result,
});
}
return repl.result;
} finally {
if (!systemConnection) {
await driver.close(dbhan);
}
}
}
module.exports = dataReplicator;

28
packages/api/src/shell/download.js
View File

@@ -1,14 +1,30 @@
const crypto = require('crypto');
const path = require('path');
const { uploadsdir } = require('../utility/directories');
const { uploadsdir, archivedir } = require('../utility/directories');
const { downloadFile } = require('../utility/downloader');
const extractSingleFileFromZip = require('../utility/extractSingleFileFromZip');
async function download(url) {
if (url && url.match(/(^http:\/\/)|(^https:\/\/)/)) {
const tmpFile = path.join(uploadsdir(), crypto.randomUUID());
await downloadFile(url, tmpFile);
return tmpFile;
async function download(url, options = {}) {
const { targetFile } = options || {};
if (url) {
if (url.match(/(^http:\/\/)|(^https:\/\/)/)) {
const destFile = targetFile || path.join(uploadsdir(), crypto.randomUUID());
await downloadFile(url, destFile);
return destFile;
}
const zipMatch = url.match(/^zip\:\/\/(.*)\/\/(.*)$/);
if (zipMatch) {
const destFile = targetFile || path.join(uploadsdir(), crypto.randomUUID());
let zipFile = zipMatch[1];
if (zipFile.startsWith('archive:')) {
zipFile = path.join(archivedir(), zipFile.substring('archive:'.length));
}
await extractSingleFileFromZip(zipFile, zipMatch[2], destFile);
return destFile;
}
}
return url;
}

14
packages/api/src/shell/index.js
View File

@@ -25,7 +25,7 @@ const importDatabase = require('./importDatabase');
const loadDatabase = require('./loadDatabase');
const generateModelSql = require('./generateModelSql');
const modifyJsonLinesReader = require('./modifyJsonLinesReader');
const dataDuplicator = require('./dataDuplicator');
const dataReplicator = require('./dataReplicator');
const dbModelToJson = require('./dbModelToJson');
const jsonToDbModel = require('./jsonToDbModel');
const jsonReader = require('./jsonReader');
@@ -35,6 +35,11 @@ const autoIndexForeignKeysTransform = require('./autoIndexForeignKeysTransform')
const generateDeploySql = require('./generateDeploySql');
const dropAllDbObjects = require('./dropAllDbObjects');
const importDbFromFolder = require('./importDbFromFolder');
const zipDirectory = require('./zipDirectory');
const unzipDirectory = require('./unzipDirectory');
const zipJsonLinesData = require('./zipJsonLinesData');
const unzipJsonLinesData = require('./unzipJsonLinesData');
const unzipJsonLinesFile = require('./unzipJsonLinesFile');
const dbgateApi = {
queryReader,
@@ -64,7 +69,7 @@ const dbgateApi = {
loadDatabase,
generateModelSql,
modifyJsonLinesReader,
dataDuplicator,
dataReplicator,
dbModelToJson,
jsonToDbModel,
dataTypeMapperTransform,
@@ -73,6 +78,11 @@ const dbgateApi = {
generateDeploySql,
dropAllDbObjects,
importDbFromFolder,
zipDirectory,
unzipDirectory,
zipJsonLinesData,
unzipJsonLinesData,
unzipJsonLinesFile,
};
requirePlugin.initializeDbgateApi(dbgateApi);

7
packages/api/src/shell/jsonLinesWriter.js
View File

@@ -36,9 +36,10 @@ async function jsonLinesWriter({ fileName, encoding = 'utf-8', header = true })
logger.info(`Writing file ${fileName}`);
const stringify = new StringifyStream({ header });
const fileStream = fs.createWriteStream(fileName, encoding);
stringify.pipe(fileStream);
stringify['finisher'] = fileStream;
return stringify;
return [stringify, fileStream];
// stringify.pipe(fileStream);
// stringify['finisher'] = fileStream;
// return stringify;
}
module.exports = jsonLinesWriter;

91
packages/api/src/shell/unzipDirectory.js Normal file
View File

@@ -0,0 +1,91 @@
const yauzl = require('yauzl');
const fs = require('fs');
const path = require('path');
const { getLogger, extractErrorLogData } = require('dbgate-tools');
const logger = getLogger('unzipDirectory');
/**
* Extracts an entire ZIP file, preserving its internal directory layout.
*
* @param {string} zipPath Path to the ZIP file on disk.
* @param {string} outputDirectory Folder to create / overwrite with the contents.
* @returns {Promise<boolean>} Resolves `true` on success, rejects on error.
*/
function unzipDirectory(zipPath, outputDirectory) {
return new Promise((resolve, reject) => {
yauzl.open(zipPath, { lazyEntries: true }, (err, zipFile) => {
if (err) return reject(err);
/** Pending per-file extractions – we resolve the main promise after they’re all done */
const pending = [];
// kick things off
zipFile.readEntry();
zipFile.on('entry', entry => {
const destPath = path.join(outputDirectory, entry.fileName);
// Handle directories (their names always end with “/” in ZIPs)
if (/\/$/.test(entry.fileName)) {
// Ensure directory exists, then continue to next entry
fs.promises
.mkdir(destPath, { recursive: true })
.then(() => zipFile.readEntry())
.catch(reject);
return;
}
// Handle files
const filePromise = fs.promises
.mkdir(path.dirname(destPath), { recursive: true }) // make sure parent dirs exist
.then(
() =>
new Promise((res, rej) => {
zipFile.openReadStream(entry, (err, readStream) => {
if (err) return rej(err);
const writeStream = fs.createWriteStream(destPath);
readStream.pipe(writeStream);
// proceed to next entry once we’ve consumed *this* one
readStream.on('end', () => zipFile.readEntry());
writeStream.on('finish', () => {
logger.info(`Extracted "${entry.fileName}" → "${destPath}".`);
res();
});
writeStream.on('error', writeErr => {
logger.error(
extractErrorLogData(writeErr),
`Error extracting "${entry.fileName}" from "${zipPath}".`
);
rej(writeErr);
});
});
})
);
pending.push(filePromise);
});
// Entire archive enumerated; wait for all streams to finish
zipFile.on('end', () => {
Promise.all(pending)
.then(() => {
logger.info(`Archive "${zipPath}" fully extracted to "${outputDirectory}".`);
resolve(true);
})
.catch(reject);
});
zipFile.on('error', err => {
logger.error(extractErrorLogData(err), `ZIP file error in ${zipPath}.`);
reject(err);
});
});
});
}
module.exports = unzipDirectory;

60
packages/api/src/shell/unzipJsonLinesData.js Normal file
View File

@@ -0,0 +1,60 @@
const yauzl = require('yauzl');
const fs = require('fs');
const { jsonLinesParse } = require('dbgate-tools');
function unzipJsonLinesData(zipPath) {
return new Promise((resolve, reject) => {
// Open the zip file
yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
if (err) {
return reject(err);
}
const results = {};
// Start reading entries
zipfile.readEntry();
zipfile.on('entry', entry => {
// Only process .json files
if (/\.jsonl$/i.test(entry.fileName)) {
zipfile.openReadStream(entry, (err, readStream) => {
if (err) {
return reject(err);
}
const chunks = [];
readStream.on('data', chunk => chunks.push(chunk));
readStream.on('end', () => {
try {
const fileContent = Buffer.concat(chunks).toString('utf-8');
const parsedJson = jsonLinesParse(fileContent);
results[entry.fileName.replace(/\.jsonl$/, '')] = parsedJson;
} catch (parseError) {
return reject(parseError);
}
// Move to the next entry
zipfile.readEntry();
});
});
} else {
// Not a JSON file, skip
zipfile.readEntry();
}
});
// Resolve when no more entries
zipfile.on('end', () => {
resolve(results);
});
// Catch errors from zipfile
zipfile.on('error', zipErr => {
reject(zipErr);
});
});
});
}
module.exports = unzipJsonLinesData;

59
packages/api/src/shell/unzipJsonLinesFile.js Normal file
View File

@@ -0,0 +1,59 @@
const yauzl = require('yauzl');
const fs = require('fs');
const { jsonLinesParse } = require('dbgate-tools');
function unzipJsonLinesFile(zipPath, fileInZip) {
return new Promise((resolve, reject) => {
// Open the zip file
yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
if (err) {
return reject(err);
}
let result = null;
// Start reading entries
zipfile.readEntry();
zipfile.on('entry', entry => {
if (entry.fileName == fileInZip) {
zipfile.openReadStream(entry, (err, readStream) => {
if (err) {
return reject(err);
}
const chunks = [];
readStream.on('data', chunk => chunks.push(chunk));
readStream.on('end', () => {
try {
const fileContent = Buffer.concat(chunks).toString('utf-8');
const parsedJson = jsonLinesParse(fileContent);
result = parsedJson;
} catch (parseError) {
return reject(parseError);
}
// Move to the next entry
zipfile.readEntry();
});
});
} else {
// Not a JSON file, skip
zipfile.readEntry();
}
});
// Resolve when no more entries
zipfile.on('end', () => {
resolve(result);
});
// Catch errors from zipfile
zipfile.on('error', zipErr => {
reject(zipErr);
});
});
});
}
module.exports = unzipJsonLinesFile;

49
packages/api/src/shell/zipDirectory.js Normal file
View File

@@ -0,0 +1,49 @@
const fs = require('fs');
const path = require('path');
const archiver = require('archiver');
const { getLogger, extractErrorLogData } = require('dbgate-tools');
const { archivedir } = require('../utility/directories');
const logger = getLogger('compressDirectory');
function zipDirectory(inputDirectory, outputFile) {
if (outputFile.startsWith('archive:')) {
outputFile = path.join(archivedir(), outputFile.substring('archive:'.length));
}
return new Promise((resolve, reject) => {
const output = fs.createWriteStream(outputFile);
const archive = archiver('zip', { zlib: { level: 9 } }); // level: 9 => best compression
// Listen for all archive data to be written
output.on('close', () => {
logger.info(`ZIP file created (${archive.pointer()} total bytes)`);
resolve();
});
archive.on('warning', err => {
logger.warn(extractErrorLogData(err), `Warning while creating ZIP: ${err.message}`);
});
archive.on('error', err => {
logger.error(extractErrorLogData(err), `Error while creating ZIP: ${err.message}`);
reject(err);
});
// Pipe archive data to the file
archive.pipe(output);
// Append files from a folder
archive.directory(inputDirectory, false, entryData => {
if (entryData.name.endsWith('.zip')) {
return false; // returning false means "do not include"
}
// otherwise, include it
return entryData;
});
// Finalize the archive
archive.finalize();
});
}
module.exports = zipDirectory;

49
packages/api/src/shell/zipJsonLinesData.js Normal file
View File

@@ -0,0 +1,49 @@
const fs = require('fs');
const _ = require('lodash');
const path = require('path');
const archiver = require('archiver');
const { getLogger, extractErrorLogData, jsonLinesStringify } = require('dbgate-tools');
const { archivedir } = require('../utility/directories');
const logger = getLogger('compressDirectory');
function zipDirectory(jsonDb, outputFile) {
if (outputFile.startsWith('archive:')) {
outputFile = path.join(archivedir(), outputFile.substring('archive:'.length));
}
return new Promise((resolve, reject) => {
const output = fs.createWriteStream(outputFile);
const archive = archiver('zip', { zlib: { level: 9 } }); // level: 9 => best compression
// Listen for all archive data to be written
output.on('close', () => {
logger.info(`ZIP file created (${archive.pointer()} total bytes)`);
resolve();
});
archive.on('warning', err => {
logger.warn(extractErrorLogData(err), `Warning while creating ZIP: ${err.message}`);
});
archive.on('error', err => {
logger.error(extractErrorLogData(err), `Error while creating ZIP: ${err.message}`);
reject(err);
});
// Pipe archive data to the file
archive.pipe(output);
for (const key in jsonDb) {
const data = jsonDb[key];
if (_.isArray(data)) {
const jsonString = jsonLinesStringify(data);
archive.append(jsonString, { name: `${key}.jsonl` });
}
}
// Finalize the archive
archive.finalize();
});
}
module.exports = zipDirectory;

819
packages/api/src/storageModel.js Normal file
View File

@@ -0,0 +1,819 @@
module.exports = {
"tables": [
{
"pureName": "auth_methods",
"columns": [
{
"pureName": "auth_methods",
"columnName": "id",
"dataType": "int",
"notNull": true
},
{
"pureName": "auth_methods",
"columnName": "name",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "auth_methods",
"columnName": "type",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "auth_methods",
"columnName": "amoid",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "auth_methods",
"columnName": "is_disabled",
"dataType": "int",
"notNull": false
},
{
"pureName": "auth_methods",
"columnName": "is_default",
"dataType": "int",
"notNull": false
},
{
"pureName": "auth_methods",
"columnName": "is_collapsed",
"dataType": "int",
"notNull": false
}
],
"foreignKeys": [],
"primaryKey": {
"pureName": "auth_methods",
"constraintType": "primaryKey",
"columns": [
{
"columnName": "id"
}
]
},
"preloadedRows": [
{
"id": -1,
"amoid": "790ca4d2-7f01-4800-955b-d691b890cc50",
"name": "Anonymous",
"type": "none"
},
{
"id": -2,
"amoid": "53db1cbf-f488-44d9-8670-7162510eb09c",
"name": "Local",
"type": "local"
}
]
},
{
"pureName": "auth_methods_config",
"columns": [
{
"pureName": "auth_methods_config",
"columnName": "id",
"dataType": "int",
"autoIncrement": true,
"notNull": true
},
{
"pureName": "auth_methods_config",
"columnName": "auth_method_id",
"dataType": "int",
"notNull": true
},
{
"pureName": "auth_methods_config",
"columnName": "key",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "auth_methods_config",
"columnName": "value",
"dataType": "varchar(250)",
"notNull": false
}
],
"foreignKeys": [
{
"constraintType": "foreignKey",
"pureName": "auth_methods_config",
"refTableName": "auth_methods",
"deleteAction": "CASCADE",
"columns": [
{
"columnName": "auth_method_id",
"refColumnName": "id"
}
]
}
],
"primaryKey": {
"pureName": "auth_methods_config",
"constraintType": "primaryKey",
"columns": [
{
"columnName": "id"
}
]
}
},
{
"pureName": "config",
"columns": [
{
"pureName": "config",
"columnName": "id",
"dataType": "int",
"autoIncrement": true,
"notNull": true
},
{
"pureName": "config",
"columnName": "group",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "config",
"columnName": "key",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "config",
"columnName": "value",
"dataType": "varchar(1000)",
"notNull": false
}
],
"foreignKeys": [],
"primaryKey": {
"pureName": "config",
"constraintType": "primaryKey",
"columns": [
{
"columnName": "id"
}
]
}
},
{
"pureName": "connections",
"columns": [
{
"pureName": "connections",
"columnName": "id",
"dataType": "int",
"autoIncrement": true,
"notNull": true
},
{
"pureName": "connections",
"columnName": "conid",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "displayName",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "connectionColor",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "engine",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "server",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "databaseFile",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "useDatabaseUrl",
"dataType": "int",
"notNull": false
},
{
"pureName": "connections",
"columnName": "databaseUrl",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "authType",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "port",
"dataType": "varchar(20)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "serviceName",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "serviceNameType",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "socketPath",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "user",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "password",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "passwordMode",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "treeKeySeparator",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "windowsDomain",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "isReadOnly",
"dataType": "int",
"notNull": false
},
{
"pureName": "connections",
"columnName": "trustServerCertificate",
"dataType": "int",
"notNull": false
},
{
"pureName": "connections",
"columnName": "defaultDatabase",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "singleDatabase",
"dataType": "int",
"notNull": false
},
{
"pureName": "connections",
"columnName": "useSshTunnel",
"dataType": "int",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sshHost",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sshPort",
"dataType": "varchar(20)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sshMode",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sshKeyFile",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sshKeyfilePassword",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sshLogin",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sshPassword",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sshBastionHost",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "useSsl",
"dataType": "int",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sslCaFile",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sslCertFilePassword",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sslKeyFile",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "sslRejectUnauthorized",
"dataType": "int",
"notNull": false
},
{
"pureName": "connections",
"columnName": "clientLibraryPath",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "useRedirectDbLogin",
"dataType": "int",
"notNull": false
},
{
"pureName": "connections",
"columnName": "allowedDatabases",
"dataType": "varchar(500)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "allowedDatabasesRegex",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "endpoint",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "endpointKey",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "accessKeyId",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "secretAccessKey",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "connections",
"columnName": "awsRegion",
"dataType": "varchar(250)",
"notNull": false
}
],
"foreignKeys": [],
"primaryKey": {
"pureName": "connections",
"constraintType": "primaryKey",
"columns": [
{
"columnName": "id"
}
]
}
},
{
"pureName": "roles",
"columns": [
{
"pureName": "roles",
"columnName": "id",
"dataType": "int",
"autoIncrement": true,
"notNull": true
},
{
"pureName": "roles",
"columnName": "name",
"dataType": "varchar(250)",
"notNull": false
}
],
"foreignKeys": [],
"primaryKey": {
"pureName": "roles",
"constraintType": "primaryKey",
"columns": [
{
"columnName": "id"
}
]
},
"preloadedRows": [
{
"id": -1,
"name": "anonymous-user"
},
{
"id": -2,
"name": "logged-user"
},
{
"id": -3,
"name": "superadmin"
}
]
},
{
"pureName": "role_connections",
"columns": [
{
"pureName": "role_connections",
"columnName": "id",
"dataType": "int",
"autoIncrement": true,
"notNull": true
},
{
"pureName": "role_connections",
"columnName": "role_id",
"dataType": "int",
"notNull": true
},
{
"pureName": "role_connections",
"columnName": "connection_id",
"dataType": "int",
"notNull": true
}
],
"foreignKeys": [
{
"constraintType": "foreignKey",
"pureName": "role_connections",
"refTableName": "roles",
"deleteAction": "CASCADE",
"columns": [
{
"columnName": "role_id",
"refColumnName": "id"
}
]
},
{
"constraintType": "foreignKey",
"pureName": "role_connections",
"refTableName": "connections",
"deleteAction": "CASCADE",
"columns": [
{
"columnName": "connection_id",
"refColumnName": "id"
}
]
}
],
"primaryKey": {
"pureName": "role_connections",
"constraintType": "primaryKey",
"columns": [
{
"columnName": "id"
}
]
}
},
{
"pureName": "role_permissions",
"columns": [
{
"pureName": "role_permissions",
"columnName": "id",
"dataType": "int",
"autoIncrement": true,
"notNull": true
},
{
"pureName": "role_permissions",
"columnName": "role_id",
"dataType": "int",
"notNull": true
},
{
"pureName": "role_permissions",
"columnName": "permission",
"dataType": "varchar(250)",
"notNull": true
}
],
"foreignKeys": [
{
"constraintType": "foreignKey",
"pureName": "role_permissions",
"refTableName": "roles",
"deleteAction": "CASCADE",
"columns": [
{
"columnName": "role_id",
"refColumnName": "id"
}
]
}
],
"primaryKey": {
"pureName": "role_permissions",
"constraintType": "primaryKey",
"columns": [
{
"columnName": "id"
}
]
}
},
{
"pureName": "users",
"columns": [
{
"pureName": "users",
"columnName": "id",
"dataType": "int",
"autoIncrement": true,
"notNull": true
},
{
"pureName": "users",
"columnName": "login",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "users",
"columnName": "password",
"dataType": "varchar(250)",
"notNull": false
},
{
"pureName": "users",
"columnName": "email",
"dataType": "varchar(250)",
"notNull": false
}
],
"foreignKeys": [],
"primaryKey": {
"pureName": "users",
"constraintType": "primaryKey",
"columns": [
{
"columnName": "id"
}
]
}
},
{
"pureName": "user_connections",
"columns": [
{
"pureName": "user_connections",
"columnName": "id",
"dataType": "int",
"autoIncrement": true,
"notNull": true
},
{
"pureName": "user_connections",
"columnName": "user_id",
"dataType": "int",
"notNull": true
},
{
"pureName": "user_connections",
"columnName": "connection_id",
"dataType": "int",
"notNull": true
}
],
"foreignKeys": [
{
"constraintType": "foreignKey",
"pureName": "user_connections",
"refTableName": "users",
"deleteAction": "CASCADE",
"columns": [
{
"columnName": "user_id",
"refColumnName": "id"
}
]
},
{
"constraintType": "foreignKey",
"pureName": "user_connections",
"refTableName": "connections",
"deleteAction": "CASCADE",
"columns": [
{
"columnName": "connection_id",
"refColumnName": "id"
}
]
}
],
"primaryKey": {
"pureName": "user_connections",
"constraintType": "primaryKey",
"columns": [
{
"columnName": "id"
}
]
}
},
{
"pureName": "user_permissions",
"columns": [
{
"pureName": "user_permissions",
"columnName": "id",
"dataType": "int",
"autoIncrement": true,
"notNull": true
},
{
"pureName": "user_permissions",
"columnName": "user_id",
"dataType": "int",
"notNull": true
},
{
"pureName": "user_permissions",
"columnName": "permission",
"dataType": "varchar(250)",
"notNull": true
}
],
"foreignKeys": [
{
"constraintType": "foreignKey",
"pureName": "user_permissions",
"refTableName": "users",
"deleteAction": "CASCADE",
"columns": [
{
"columnName": "user_id",
"refColumnName": "id"
}
]
}
],
"primaryKey": {
"pureName": "user_permissions",
"constraintType": "primaryKey",
"columns": [
{
"columnName": "id"
}
]
}
},
{
"pureName": "user_roles",
"columns": [
{
"pureName": "user_roles",
"columnName": "id",
"dataType": "int",
"autoIncrement": true,
"notNull": true
},
{
"pureName": "user_roles",
"columnName": "user_id",
"dataType": "int",
"notNull": true
},
{
"pureName": "user_roles",
"columnName": "role_id",
"dataType": "int",
"notNull": true
}
],
"foreignKeys": [
{
"constraintType": "foreignKey",
"pureName": "user_roles",
"refTableName": "users",
"deleteAction": "CASCADE",
"columns": [
{
"columnName": "user_id",
"refColumnName": "id"
}
]
},
{
"constraintType": "foreignKey",
"pureName": "user_roles",
"refTableName": "roles",
"deleteAction": "CASCADE",
"columns": [
{
"columnName": "role_id",
"refColumnName": "id"
}
]
}
],
"primaryKey": {
"pureName": "user_roles",
"constraintType": "primaryKey",
"columns": [
{
"columnName": "id"
}
]
}
}
],
"collections": [],
"views": [],
"matviews": [],
"functions": [],
"procedures": [],
"triggers": []
};

4
packages/api/src/utility/DatastoreProxy.js
View File

@@ -60,6 +60,10 @@ class DatastoreProxy {
// if (this.disconnected) return;
this.subprocess = null;
});
this.subprocess.on('error', err => {
logger.error(extractErrorLogData(err), 'Error in data store subprocess');
this.subprocess = null;
});
this.subprocess.send({ msgtype: 'open', file: this.file });
}
return this.subprocess;

15
packages/api/src/utility/cloudUpgrade.js
View File

@@ -4,11 +4,20 @@ const fsp = require('fs/promises');
const semver = require('semver');
const currentVersion = require('../currentVersion');
const { getLogger, extractErrorLogData } = require('dbgate-tools');
const { storageReadConfig } = require('../controllers/storageDb');
const logger = getLogger('cloudUpgrade');
async function checkCloudUpgrade() {
try {
if (process.env.STORAGE_DATABASE) {
const settings = await storageReadConfig('settings');
if (settings['cloud.useAutoUpgrade'] != 1) {
// auto-upgrade not allowed
return;
}
}
const resp = await axios.default.get('https://api.github.com/repos/dbgate/dbgate/releases/latest');
const json = resp.data;
const version = json.name.substring(1);
@@ -43,7 +52,11 @@ async function checkCloudUpgrade() {
logger.info(`Downloaded new version from ${zipUrl}`);
} else {
logger.info(`Checked version ${version} is not newer than ${cloudDownloadedVersion ?? currentVersion.version}, upgrade skippped`);
logger.info(
`Checked version ${version} is not newer than ${
cloudDownloadedVersion ?? currentVersion.version
}, upgrade skippped`
);
}
} catch (err) {
logger.error(extractErrorLogData(err), 'Error checking cloud upgrade');

4
packages/api/src/utility/connectUtility.js
View File

@@ -96,7 +96,9 @@ async function connectUtility(driver, storedConnection, connectionMode, addition
...decryptConnection(connectionLoaded),
};
if (!connection.port && driver.defaultPort) connection.port = driver.defaultPort.toString();
if (!connection.port && driver.defaultPort) {
connection.port = driver.defaultPort.toString();
}
if (connection.useSshTunnel) {
const tunnel = await getSshTunnelProxy(connection);

84
packages/api/src/utility/crypting.js
View File

@@ -5,12 +5,16 @@ const path = require('path');
const _ = require('lodash');
const { datadir } = require('./directories');
const { encryptionKeyArg } = require('./processArgs');
const defaultEncryptionKey = 'mQAUaXhavRGJDxDTXSCg7Ej0xMmGCrx6OKA07DIMBiDcYYkvkaXjTAzPUEHEHEf9';
let _encryptionKey = null;
function loadEncryptionKey() {
if (encryptionKeyArg) {
return encryptionKeyArg;
}
if (_encryptionKey) {
return _encryptionKey;
}
@@ -55,7 +59,7 @@ async function loadEncryptionKeyFromExternal(storedValue, setStoredValue) {
let _encryptor = null;
function getEncryptor() {
function getInternalEncryptor() {
if (_encryptor) {
return _encryptor;
}
@@ -63,11 +67,25 @@ function getEncryptor() {
return _encryptor;
}
function encryptPasswordString(password) {
if (password && !password.startsWith('crypt:')) {
return 'crypt:' + getInternalEncryptor().encrypt(password);
}
return password;
}
function decryptPasswordString(password) {
if (password && password.startsWith('crypt:')) {
return getInternalEncryptor().decrypt(password.substring('crypt:'.length));
}
return password;
}
function encryptObjectPasswordField(obj, field) {
if (obj && obj[field] && !obj[field].startsWith('crypt:')) {
return {
...obj,
[field]: 'crypt:' + getEncryptor().encrypt(obj[field]),
[field]: 'crypt:' + getInternalEncryptor().encrypt(obj[field]),
};
}
return obj;
@@ -77,7 +95,7 @@ function decryptObjectPasswordField(obj, field) {
if (obj && obj[field] && obj[field].startsWith('crypt:')) {
return {
...obj,
[field]: getEncryptor().decrypt(obj[field].substring('crypt:'.length)),
[field]: getInternalEncryptor().decrypt(obj[field].substring('crypt:'.length)),
};
}
return obj;
@@ -131,6 +149,54 @@ function pickSafeConnectionInfo(connection) {
function setEncryptionKey(encryptionKey) {
_encryptionKey = encryptionKey;
_encryptor = null;
global.ENCRYPTION_KEY = encryptionKey;
}
function getEncryptionKey() {
return _encryptionKey;
}
function generateTransportEncryptionKey() {
const encryptor = simpleEncryptor.createEncryptor(defaultEncryptionKey);
const result = {
encryptionKey: crypto.randomBytes(32).toString('hex'),
};
return encryptor.encrypt(result);
}
function createTransportEncryptor(encryptionData) {
const encryptor = simpleEncryptor.createEncryptor(defaultEncryptionKey);
const data = encryptor.decrypt(encryptionData);
const res = simpleEncryptor.createEncryptor(data['encryptionKey']);
return res;
}
function recryptObjectPasswordField(obj, field, decryptEncryptor, encryptEncryptor) {
if (obj && obj[field] && obj[field].startsWith('crypt:')) {
return {
...obj,
[field]: 'crypt:' + encryptEncryptor.encrypt(decryptEncryptor.decrypt(obj[field].substring('crypt:'.length))),
};
}
return obj;
}
function recryptObjectPasswordFieldInPlace(obj, field, decryptEncryptor, encryptEncryptor) {
if (obj && obj[field] && obj[field].startsWith('crypt:')) {
obj[field] = 'crypt:' + encryptEncryptor.encrypt(decryptEncryptor.decrypt(obj[field].substring('crypt:'.length)));
}
}
function recryptConnection(connection, decryptEncryptor, encryptEncryptor) {
connection = recryptObjectPasswordField(connection, 'password', decryptEncryptor, encryptEncryptor);
connection = recryptObjectPasswordField(connection, 'sshPassword', decryptEncryptor, encryptEncryptor);
connection = recryptObjectPasswordField(connection, 'sshKeyfilePassword', decryptEncryptor, encryptEncryptor);
return connection;
}
function recryptUser(user, decryptEncryptor, encryptEncryptor) {
user = recryptObjectPasswordField(user, 'password', decryptEncryptor, encryptEncryptor);
return user;
}
module.exports = {
@@ -142,4 +208,16 @@ module.exports = {
maskConnection,
pickSafeConnectionInfo,
loadEncryptionKeyFromExternal,
getEncryptionKey,
setEncryptionKey,
encryptPasswordString,
decryptPasswordString,
getInternalEncryptor,
recryptConnection,
recryptUser,
generateTransportEncryptionKey,
createTransportEncryptor,
recryptObjectPasswordField,
recryptObjectPasswordFieldInPlace,
};

77
packages/api/src/utility/extractSingleFileFromZip.js Normal file
View File

@@ -0,0 +1,77 @@
const yauzl = require('yauzl');
const fs = require('fs');
const { getLogger, extractErrorLogData } = require('dbgate-tools');
const logger = getLogger('extractSingleFileFromZip');
/**
* Extracts a single file from a ZIP using yauzl.
* Stops reading the rest of the archive once the file is found.
*
* @param {string} zipPath - Path to the ZIP file on disk.
* @param {string} fileInZip - The file path *inside* the ZIP to extract.
* @param {string} outputPath - Where to write the extracted file on disk.
* @returns {Promise<boolean>} - Resolves with a success message or a "not found" message.
*/
function extractSingleFileFromZip(zipPath, fileInZip, outputPath) {
return new Promise((resolve, reject) => {
yauzl.open(zipPath, { lazyEntries: true }, (err, zipFile) => {
if (err) return reject(err);
let fileFound = false;
// Start reading the first entry
zipFile.readEntry();
zipFile.on('entry', entry => {
// Compare the entry name to the file we want
if (entry.fileName === fileInZip) {
fileFound = true;
// Open a read stream for this entry
zipFile.openReadStream(entry, (err, readStream) => {
if (err) return reject(err);
// Create a write stream to outputPath
const writeStream = fs.createWriteStream(outputPath);
readStream.pipe(writeStream);
// When the read stream ends, we can close the zipFile
readStream.on('end', () => {
// We won't read further entries
zipFile.close();
});
// When the file is finished writing, resolve
writeStream.on('finish', () => {
logger.info(`File "${fileInZip}" extracted to "${outputPath}".`);
resolve(true);
});
// Handle write errors
writeStream.on('error', writeErr => {
logger.error(extractErrorLogData(writeErr), `Error extracting "${fileInZip}" from "${zipPath}".`);
reject(writeErr);
});
});
} else {
// Not the file we want; skip to the next entry
zipFile.readEntry();
}
});
// If we reach the end without finding the file
zipFile.on('end', () => {
if (!fileFound) {
resolve(false);
}
});
// Handle general errors
zipFile.on('error', err => {
logger.error(extractErrorLogData(err), `ZIP file error in ${zipPath}.`);
reject(err);
});
});
});
}
module.exports = extractSingleFileFromZip;

2
packages/api/src/utility/getMapExport.js
View File

@@ -22,6 +22,8 @@ const getMapExport = (geoJson) => {
})
.addTo(map);
leaflet.control.scale().addTo(map);
const geoJsonObj = leaflet
.geoJSON(${JSON.stringify(geoJson)}, {
style: function () {

13
packages/api/src/utility/healthStatus.js
View File

@@ -24,4 +24,15 @@ async function getHealthStatus() {
};
}
module.exports = getHealthStatus;
async function getHealthStatusSprinx() {
return {
overallStatus: 'OK',
timeStamp: new Date().toISOString(),
timeStampUnix: Math.floor(Date.now() / 1000),
};
}
module.exports = {
getHealthStatus,
getHealthStatusSprinx,
};

41
packages/api/src/utility/listZipEntries.js Normal file
View File

@@ -0,0 +1,41 @@
const yauzl = require('yauzl');
const path = require('path');
/**
* Lists the files in a ZIP archive using yauzl,
* returning an array of { fileName, uncompressedSize } objects.
*
* @param {string} zipPath - The path to the ZIP file.
* @returns {Promise<Array<{fileName: string, uncompressedSize: number}>>}
*/
function listZipEntries(zipPath) {
return new Promise((resolve, reject) => {
yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
if (err) return reject(err);
const entries = [];
// Start reading entries
zipfile.readEntry();
// Handle each entry
zipfile.on('entry', entry => {
entries.push({
fileName: entry.fileName,
uncompressedSize: entry.uncompressedSize,
});
// Move on to the next entry (we’re only listing, not reading file data)
zipfile.readEntry();
});
// Finished reading all entries
zipfile.on('end', () => resolve(entries));
// Handle errors
zipfile.on('error', err => reject(err));
});
});
}
module.exports = listZipEntries;

5
packages/api/src/utility/processArgs.js
View File

@@ -17,6 +17,7 @@ const processDisplayName = getNamedArg('--process-display-name');
const listenApi = process.argv.includes('--listen-api');
const listenApiChild = process.argv.includes('--listen-api-child') || listenApi;
const runE2eTests = process.argv.includes('--run-e2e-tests');
const encryptionKeyArg = getNamedArg('--encryption-key');
function getPassArgs() {
const res = [];
@@ -31,6 +32,9 @@ function getPassArgs() {
if (runE2eTests) {
res.push('--run-e2e-tests');
}
if (global['ENCRYPTION_KEY']) {
res.push('--encryption-key', global['ENCRYPTION_KEY']);
}
return res;
}
@@ -45,4 +49,5 @@ module.exports = {
listenApiChild,
processDisplayName,
runE2eTests,
encryptionKeyArg,
};

15
packages/api/src/utility/sshTunnel.js
View File

@@ -57,10 +57,21 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
}
});
subprocess.on('exit', code => {
logger.info('SSH forward process exited');
logger.info(`SSH forward process exited with code ${code}`);
delete sshTunnelCache[tunnelCacheKey];
if (!promiseHandled) {
reject(new Error('SSH forward process exited, try to change "Local host address for SSH connections" in Settings/Connections'));
reject(
new Error(
'SSH forward process exited, try to change "Local host address for SSH connections" in Settings/Connections'
)
);
}
});
subprocess.on('error', error => {
logger.error(extractErrorLogData(error), 'SSH forward process error');
delete sshTunnelCache[tunnelCacheKey];
if (!promiseHandled) {
reject(error);
}
});
});