Implement database export/import functionality for hardware migration
Added comprehensive database export/import system to safely migrate SSH connection data between different server environments. Key Features: - SQLite export format with encrypted data migration - Hardware fingerprint protection and re-encryption - Field mapping between TypeScript and database schemas - Foreign key constraint handling for cross-environment imports - Admin user assignment for imported SSH records - Additive import strategy preserving existing data - File upload support for import operations Technical Implementation: - Complete Drizzle ORM schema consistency - Bidirectional field name mapping (userId ↔ user_id) - Proper encryption/decryption workflow - Multer file upload middleware integration - Error handling and logging throughout Security: - Only exports SSH-related tables (ssh_data, ssh_credentials) - Protects admin user data from migration conflicts - Re-encrypts sensitive fields for target hardware - Validates export file format and version compatibility
This commit is contained in:
ZacharyZcR
@@ -4,6 +4,7 @@ import * as schema from "./schema.js";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { databaseLogger } from "../../utils/logger.js";
|
||||
import { DatabaseFileEncryption } from "../../utils/database-file-encryption.js";
|
||||
|
||||
const dataDir = process.env.DATA_DIR || "./db/data";
|
||||
const dbDir = path.resolve(dataDir);
|
||||
@@ -15,12 +16,139 @@ if (!fs.existsSync(dbDir)) {
|
||||
fs.mkdirSync(dbDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Database file encryption configuration
|
||||
const enableFileEncryption = process.env.DB_FILE_ENCRYPTION !== 'false';
|
||||
const dbPath = path.join(dataDir, "db.sqlite");
|
||||
const encryptedDbPath = `${dbPath}.encrypted`;
|
||||
|
||||
// Initialize database with file encryption support
|
||||
let actualDbPath = ':memory:'; // Always use memory database
|
||||
let memoryDatabase: Database.Database;
|
||||
let isNewDatabase = false;
|
||||
|
||||
if (enableFileEncryption) {
|
||||
try {
|
||||
// Check if encrypted database exists
|
||||
if (DatabaseFileEncryption.isEncryptedDatabaseFile(encryptedDbPath)) {
|
||||
databaseLogger.info('Found encrypted database file, loading into memory...', {
|
||||
operation: 'db_memory_load',
|
||||
encryptedPath: encryptedDbPath
|
||||
});
|
||||
|
||||
// Validate hardware compatibility
|
||||
if (!DatabaseFileEncryption.validateHardwareCompatibility(encryptedDbPath)) {
|
||||
databaseLogger.error('Hardware fingerprint mismatch for encrypted database', {
|
||||
operation: 'db_decrypt_failed',
|
||||
reason: 'hardware_mismatch'
|
||||
});
|
||||
throw new Error('Cannot decrypt database: hardware fingerprint mismatch');
|
||||
}
|
||||
|
||||
// Decrypt database content to memory buffer
|
||||
const decryptedBuffer = DatabaseFileEncryption.decryptDatabaseToBuffer(encryptedDbPath);
|
||||
|
||||
// Create in-memory database from decrypted buffer
|
||||
memoryDatabase = new Database(decryptedBuffer);
|
||||
|
||||
databaseLogger.success('Existing database loaded into memory successfully', {
|
||||
operation: 'db_memory_load_success',
|
||||
bufferSize: decryptedBuffer.length,
|
||||
inMemory: true
|
||||
});
|
||||
} else {
|
||||
// No encrypted database exists - create new in-memory database
|
||||
databaseLogger.info('No encrypted database found, creating new in-memory database', {
|
||||
operation: 'db_memory_create_new'
|
||||
});
|
||||
|
||||
memoryDatabase = new Database(':memory:');
|
||||
isNewDatabase = true;
|
||||
|
||||
// Check if there's an old unencrypted database to migrate
|
||||
if (fs.existsSync(dbPath)) {
|
||||
databaseLogger.info('Found existing unencrypted database, will migrate to memory', {
|
||||
operation: 'db_migrate_to_memory',
|
||||
oldPath: dbPath
|
||||
});
|
||||
|
||||
// Load old database and copy its content to memory database
|
||||
const oldDb = new Database(dbPath, { readonly: true });
|
||||
|
||||
// Get all table schemas and data from old database
|
||||
const tables = oldDb.prepare(`
|
||||
SELECT name, sql FROM sqlite_master
|
||||
WHERE type='table' AND name NOT LIKE 'sqlite_%'
|
||||
`).all() as { name: string; sql: string }[];
|
||||
|
||||
// Create tables in memory database
|
||||
for (const table of tables) {
|
||||
memoryDatabase.exec(table.sql);
|
||||
}
|
||||
|
||||
// Copy data for each table
|
||||
for (const table of tables) {
|
||||
const rows = oldDb.prepare(`SELECT * FROM ${table.name}`).all();
|
||||
if (rows.length > 0) {
|
||||
const columns = Object.keys(rows[0]);
|
||||
const placeholders = columns.map(() => '?').join(', ');
|
||||
const insertStmt = memoryDatabase.prepare(
|
||||
`INSERT INTO ${table.name} (${columns.join(', ')}) VALUES (${placeholders})`
|
||||
);
|
||||
|
||||
for (const row of rows) {
|
||||
const values = columns.map(col => (row as any)[col]);
|
||||
insertStmt.run(values);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
oldDb.close();
|
||||
|
||||
databaseLogger.success('Migrated existing database to memory', {
|
||||
operation: 'db_migrate_to_memory_success'
|
||||
});
|
||||
isNewDatabase = false;
|
||||
} else {
|
||||
databaseLogger.success('Created new in-memory database', {
|
||||
operation: 'db_memory_create_success'
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
databaseLogger.error('Failed to initialize memory database', error, {
|
||||
operation: 'db_memory_init_failed'
|
||||
});
|
||||
|
||||
// If file encryption is critical, fail fast
|
||||
if (process.env.DB_FILE_ENCRYPTION_REQUIRED === 'true') {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Create fallback in-memory database
|
||||
databaseLogger.warn('Creating fallback in-memory database', {
|
||||
operation: 'db_memory_fallback'
|
||||
});
|
||||
memoryDatabase = new Database(':memory:');
|
||||
isNewDatabase = true;
|
||||
}
|
||||
} else {
|
||||
// File encryption disabled - still use memory for consistency
|
||||
databaseLogger.info('File encryption disabled, using in-memory database', {
|
||||
operation: 'db_memory_no_encryption'
|
||||
});
|
||||
memoryDatabase = new Database(':memory:');
|
||||
isNewDatabase = true;
|
||||
}
|
||||
|
||||
databaseLogger.info(`Initializing SQLite database`, {
|
||||
operation: "db_init",
|
||||
path: dbPath,
|
||||
path: actualDbPath,
|
||||
encrypted: enableFileEncryption && DatabaseFileEncryption.isEncryptedDatabaseFile(encryptedDbPath),
|
||||
inMemory: true,
|
||||
isNewDatabase
|
||||
});
|
||||
const sqlite = new Database(dbPath);
|
||||
|
||||
const sqlite = memoryDatabase;
|
||||
|
||||
sqlite.exec(`
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
@@ -270,7 +398,7 @@ const migrateSchema = () => {
|
||||
});
|
||||
};
|
||||
|
||||
const initializeDatabase = async () => {
|
||||
const initializeDatabase = async (): Promise<void> => {
|
||||
migrateSchema();
|
||||
|
||||
try {
|
||||
@@ -303,15 +431,229 @@ const initializeDatabase = async () => {
|
||||
}
|
||||
};
|
||||
|
||||
initializeDatabase().catch((error) => {
|
||||
databaseLogger.error("Failed to initialize database", error, {
|
||||
operation: "db_init",
|
||||
// Function to save in-memory database to encrypted file
|
||||
async function saveMemoryDatabaseToFile() {
|
||||
if (!memoryDatabase || !enableFileEncryption) return;
|
||||
|
||||
try {
|
||||
// Export in-memory database to buffer
|
||||
const buffer = memoryDatabase.serialize();
|
||||
|
||||
// Encrypt and save to file
|
||||
DatabaseFileEncryption.encryptDatabaseFromBuffer(buffer, encryptedDbPath);
|
||||
|
||||
databaseLogger.debug('In-memory database saved to encrypted file', {
|
||||
operation: 'memory_db_save',
|
||||
bufferSize: buffer.length,
|
||||
encryptedPath: encryptedDbPath
|
||||
});
|
||||
} catch (error) {
|
||||
databaseLogger.error('Failed to save in-memory database', error, {
|
||||
operation: 'memory_db_save_failed'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Function to handle post-initialization file encryption and cleanup
|
||||
async function handlePostInitFileEncryption() {
|
||||
if (!enableFileEncryption) return;
|
||||
|
||||
try {
|
||||
// Clean up any existing unencrypted database files
|
||||
if (fs.existsSync(dbPath)) {
|
||||
databaseLogger.warn('Found unencrypted database file, removing for security', {
|
||||
operation: 'db_security_cleanup_existing',
|
||||
removingPath: dbPath
|
||||
});
|
||||
|
||||
try {
|
||||
fs.unlinkSync(dbPath);
|
||||
databaseLogger.success('Unencrypted database file removed for security', {
|
||||
operation: 'db_security_cleanup_complete',
|
||||
removedPath: dbPath
|
||||
});
|
||||
} catch (error) {
|
||||
databaseLogger.warn('Could not remove unencrypted database file (may be locked)', {
|
||||
operation: 'db_security_cleanup_deferred',
|
||||
path: dbPath,
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
|
||||
// Try again after a short delay
|
||||
setTimeout(() => {
|
||||
try {
|
||||
if (fs.existsSync(dbPath)) {
|
||||
fs.unlinkSync(dbPath);
|
||||
databaseLogger.success('Delayed cleanup: unencrypted database file removed', {
|
||||
operation: 'db_security_cleanup_delayed_success',
|
||||
removedPath: dbPath
|
||||
});
|
||||
}
|
||||
} catch (delayedError) {
|
||||
databaseLogger.error('Failed to remove unencrypted database file even after delay', delayedError, {
|
||||
operation: 'db_security_cleanup_delayed_failed',
|
||||
path: dbPath
|
||||
});
|
||||
}
|
||||
}, 2000);
|
||||
}
|
||||
}
|
||||
|
||||
// Always save the in-memory database (whether new or existing)
|
||||
if (memoryDatabase) {
|
||||
// Save immediately after initialization
|
||||
await saveMemoryDatabaseToFile();
|
||||
|
||||
// Set up periodic saves every 5 minutes
|
||||
setInterval(saveMemoryDatabaseToFile, 5 * 60 * 1000);
|
||||
|
||||
databaseLogger.info('Periodic in-memory database saves configured', {
|
||||
operation: 'memory_db_autosave_setup',
|
||||
intervalMinutes: 5
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
databaseLogger.error('Failed to handle database file encryption/cleanup', error, {
|
||||
operation: 'db_encrypt_cleanup_failed'
|
||||
});
|
||||
|
||||
// Don't fail the entire initialization for this
|
||||
}
|
||||
}
|
||||
|
||||
initializeDatabase()
|
||||
.then(() => handlePostInitFileEncryption())
|
||||
.catch((error) => {
|
||||
databaseLogger.error("Failed to initialize database", error, {
|
||||
operation: "db_init",
|
||||
});
|
||||
process.exit(1);
|
||||
});
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
databaseLogger.success("Database connection established", {
|
||||
operation: "db_init",
|
||||
path: dbPath,
|
||||
path: actualDbPath,
|
||||
hasEncryptedBackup: enableFileEncryption && DatabaseFileEncryption.isEncryptedDatabaseFile(encryptedDbPath)
|
||||
});
|
||||
|
||||
// Cleanup function for database and temporary files
|
||||
async function cleanupDatabase() {
|
||||
// Save in-memory database before closing
|
||||
if (memoryDatabase) {
|
||||
try {
|
||||
await saveMemoryDatabaseToFile();
|
||||
databaseLogger.info('In-memory database saved before shutdown', {
|
||||
operation: 'shutdown_save'
|
||||
});
|
||||
} catch (error) {
|
||||
databaseLogger.error('Failed to save in-memory database before shutdown', error, {
|
||||
operation: 'shutdown_save_failed'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Close database connection
|
||||
try {
|
||||
if (sqlite) {
|
||||
sqlite.close();
|
||||
databaseLogger.debug('Database connection closed', {
|
||||
operation: 'db_close'
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
databaseLogger.warn('Error closing database connection', {
|
||||
operation: 'db_close_error',
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
|
||||
// Clean up temp directory
|
||||
try {
|
||||
const tempDir = path.join(dataDir, '.temp');
|
||||
if (fs.existsSync(tempDir)) {
|
||||
const files = fs.readdirSync(tempDir);
|
||||
for (const file of files) {
|
||||
try {
|
||||
fs.unlinkSync(path.join(tempDir, file));
|
||||
} catch {
|
||||
// Ignore individual file cleanup errors
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
fs.rmdirSync(tempDir);
|
||||
databaseLogger.debug('Temp directory cleaned up', {
|
||||
operation: 'temp_dir_cleanup'
|
||||
});
|
||||
} catch {
|
||||
// Ignore directory removal errors
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore temp directory cleanup errors
|
||||
}
|
||||
}
|
||||
|
||||
// Register cleanup handlers
|
||||
process.on('exit', () => {
|
||||
// Synchronous cleanup only for exit event
|
||||
if (sqlite) {
|
||||
try {
|
||||
sqlite.close();
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
process.on('SIGINT', async () => {
|
||||
databaseLogger.info('Received SIGINT, cleaning up...', {
|
||||
operation: 'shutdown'
|
||||
});
|
||||
await cleanupDatabase();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
process.on('SIGTERM', async () => {
|
||||
databaseLogger.info('Received SIGTERM, cleaning up...', {
|
||||
operation: 'shutdown'
|
||||
});
|
||||
await cleanupDatabase();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
// Export database connection and file encryption utilities
|
||||
export const db = drizzle(sqlite, { schema });
|
||||
export const sqliteInstance = sqlite; // Export underlying SQLite instance for schema queries
|
||||
export { DatabaseFileEncryption };
|
||||
export const databasePaths = {
|
||||
main: actualDbPath,
|
||||
encrypted: encryptedDbPath,
|
||||
directory: dbDir,
|
||||
inMemory: true
|
||||
};
|
||||
|
||||
// Memory database buffer function
|
||||
function getMemoryDatabaseBuffer(): Buffer {
|
||||
if (!memoryDatabase) {
|
||||
throw new Error('Memory database not initialized');
|
||||
}
|
||||
|
||||
try {
|
||||
// Export in-memory database to buffer
|
||||
const buffer = memoryDatabase.serialize();
|
||||
|
||||
databaseLogger.debug('Memory database serialized to buffer', {
|
||||
operation: 'memory_db_serialize',
|
||||
bufferSize: buffer.length
|
||||
});
|
||||
|
||||
return buffer;
|
||||
} catch (error) {
|
||||
databaseLogger.error('Failed to serialize memory database to buffer', error, {
|
||||
operation: 'memory_db_serialize_failed'
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Export save function for manual saves and buffer access
|
||||
export { saveMemoryDatabaseToFile, getMemoryDatabaseBuffer };
|
||||
|
||||
Reference in New Issue
Block a user