From 46f842afce5cca75a0dd47e377b8918aec528552 Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Wed, 24 Sep 2025 03:50:38 +0800 Subject: [PATCH] feat: Implement dual-stage database migration with lazy field encryption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1: Database file migration (startup) - Add DatabaseMigration class for safe unencrypted → encrypted DB migration - Disable foreign key constraints during migration to prevent constraint failures - Create timestamped backups and verification checks - Rename original files instead of deletion for safety Phase 2: Lazy field encryption (user login) - Add LazyFieldEncryption utility for plaintext field detection - Implement gradual migration of sensitive fields using user KEK - Update DataCrypto to handle mixed plaintext/encrypted data - Integrate lazy encryption into AuthManager login flow Key improvements: - Non-destructive migration with comprehensive backup strategy - Automatic detection and handling of plaintext vs encrypted fields - User-transparent migration during normal login process - Complete migration logging and admin API endpoints - Foreign key constraint handling during database structure migration Resolves data decryption errors during Docker updates by providing seamless transition from plaintext to encrypted storage. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- MIGRATION-TESTING.md | 323 +++++++++++++++ src/backend/database/database.ts | 150 +++++++ src/backend/database/db/index.ts | 187 +++++---- src/backend/utils/auth-manager.ts | 65 ++- src/backend/utils/data-crypto.ts | 181 +++++++- src/backend/utils/database-migration.ts | 457 +++++++++++++++++++++ src/backend/utils/lazy-field-encryption.ts | 295 +++++++++++++ 7 files changed, 1560 insertions(+), 98 deletions(-) create mode 100644 MIGRATION-TESTING.md create mode 100644 src/backend/utils/database-migration.ts create mode 100644 src/backend/utils/lazy-field-encryption.ts diff --git a/MIGRATION-TESTING.md b/MIGRATION-TESTING.md new file mode 100644 index 00000000..faebdf67 --- /dev/null +++ b/MIGRATION-TESTING.md @@ -0,0 +1,323 @@ +# Database Migration Testing Guide + +## Overview + +This document outlines the testing procedures for the automatic database migration system that migrates unencrypted SQLite databases to encrypted format during Docker deployment updates. + +## Migration System Features + +✅ **Automatic Detection**: Detects unencrypted databases on startup +✅ **Safe Backup**: Creates timestamped backups before migration +✅ **Integrity Verification**: Validates migration completeness +✅ **Non-destructive**: Original files are renamed, not deleted +✅ **Cleanup**: Removes old backup files (keeps latest 3) +✅ **Admin API**: Migration status and history endpoints +✅ **Detailed Logging**: Comprehensive migration logs + +## Test Scenarios + +### Scenario 1: Fresh Installation (No Migration Needed) +**Setup**: Clean Docker container with no existing database files +**Expected**: +- New encrypted database created +- No migration messages in logs +- Status API shows "Fresh installation detected" + +**Test Commands**: +```bash +# Clean start +docker run --rm termix:latest +# Check logs for "fresh installation" +# GET /database/migration/status should show needsMigration: false +``` + +### Scenario 2: Standard Migration (Unencrypted → Encrypted) +**Setup**: Existing unencrypted `db.sqlite` file with user data +**Expected**: +- Automatic migration on startup +- Backup file created (`.migration-backup-{timestamp}`) +- Original file renamed (`.migrated-{timestamp}`) +- Encrypted database created successfully +- All data preserved and accessible + +**Test Commands**: +```bash +# 1. Create test data in unencrypted format +docker run -v /host/data:/app/data termix:old-version +# Add some SSH hosts and credentials via UI + +# 2. Stop container and update to new version +docker stop container_id +docker run -v /host/data:/app/data termix:latest + +# 3. Check migration logs +docker logs container_id | grep -i migration + +# 4. Verify data integrity via API +curl -H "Authorization: Bearer $TOKEN" http://localhost:8081/database/migration/status +``` + +### Scenario 3: Already Encrypted (No Migration Needed) +**Setup**: Only encrypted database file exists +**Expected**: +- No migration performed +- Database loads normally +- Status API shows "Only encrypted database exists" + +**Test Commands**: +```bash +# Start with existing encrypted database +docker run -v /host/encrypted-data:/app/data termix:latest +# Verify no migration messages in logs +``` + +### Scenario 4: Both Files Exist (Safety Mode) +**Setup**: Both encrypted and unencrypted databases present +**Expected**: +- Migration skipped for safety +- Warning logged about manual intervention +- Both files preserved +- Uses encrypted database + +**Test Commands**: +```bash +# Manually create both files +touch /host/data/db.sqlite +touch /host/data/db.sqlite.encrypted +docker run -v /host/data:/app/data termix:latest +# Check for safety warning in logs +``` + +### Scenario 5: Migration Failure Recovery +**Setup**: Simulate migration failure (corrupted source file) +**Expected**: +- Migration fails safely +- Backup file preserved +- Original unencrypted file untouched +- Clear error message with recovery instructions + +**Test Commands**: +```bash +# Create corrupted database file +echo "corrupted" > /host/data/db.sqlite +docker run -v /host/data:/app/data termix:latest +# Verify error handling and backup preservation +``` + +### Scenario 6: Large Database Migration +**Setup**: Large unencrypted database (>100MB with many records) +**Expected**: +- Migration completes successfully +- Performance is acceptable (under 30 seconds) +- Memory usage stays reasonable +- All data integrity checks pass + +**Test Commands**: +```bash +# Create large dataset first +# Monitor migration duration and memory usage +docker stats container_id +``` + +## API Testing + +### Migration Status Endpoint +```bash +# Admin access required +curl -H "Authorization: Bearer $ADMIN_TOKEN" \ + http://localhost:8081/database/migration/status + +# Expected response: +{ + "migrationStatus": { + "needsMigration": false, + "hasUnencryptedDb": false, + "hasEncryptedDb": true, + "unencryptedDbSize": 0, + "reason": "Only encrypted database exists. No migration needed." + }, + "files": { + "unencryptedDbSize": 0, + "encryptedDbSize": 524288, + "backupFiles": 2, + "migratedFiles": 1 + }, + "recommendations": [ + "Database is properly encrypted", + "No action required" + ] +} +``` + +### Migration History Endpoint +```bash +curl -H "Authorization: Bearer $ADMIN_TOKEN" \ + http://localhost:8081/database/migration/history + +# Expected response: +{ + "files": [ + { + "name": "db.sqlite.migration-backup-2024-09-24T10-30-00-000Z", + "size": 262144, + "created": "2024-09-24T10:30:00.000Z", + "modified": "2024-09-24T10:30:00.000Z", + "type": "backup" + } + ], + "summary": { + "totalBackups": 1, + "totalMigrated": 1, + "oldestBackup": "2024-09-24T10:30:00.000Z", + "newestBackup": "2024-09-24T10:30:00.000Z" + } +} +``` + +## Log Analysis + +### Successful Migration Logs +Look for these log entries: +``` +[INFO] Migration status check completed - needsMigration: true +[INFO] Starting automatic database migration +[INFO] Creating migration backup +[SUCCESS] Migration backup created successfully +[INFO] Found tables to migrate - tableCount: 8 +[SUCCESS] Migration integrity verification completed +[INFO] Creating encrypted database file +[SUCCESS] Database migration completed successfully +``` + +### Migration Skipped (Safety) Logs +``` +[INFO] Migration status check completed - needsMigration: false +[INFO] Both encrypted and unencrypted databases exist. Skipping migration for safety +[WARN] Manual intervention may be required +``` + +### Migration Failure Logs +``` +[ERROR] Database migration failed +[ERROR] Backup available at: /app/data/db.sqlite.migration-backup-{timestamp} +[ERROR] Manual intervention required to recover data +``` + +## Manual Recovery Procedures + +### If Migration Fails: +1. **Locate backup file**: `db.sqlite.migration-backup-{timestamp}` +2. **Restore original**: `cp backup-file db.sqlite` +3. **Check logs**: Look for specific error details +4. **Fix issue**: Address the root cause (permissions, disk space, etc.) +5. **Retry**: Restart container to trigger migration again + +### If Both Databases Exist: +1. **Check dates**: Determine which file is newer +2. **Backup both**: Make copies before proceeding +3. **Remove older**: Delete the outdated database file +4. **Restart**: Container will detect single database + +### Emergency Data Recovery: +1. **Backup files are SQLite**: Can be opened with any SQLite client +2. **Manual export**: Use SQLite tools to export data +3. **Re-import**: Use Termix import functionality + +## Performance Expectations + +| Database Size | Expected Migration Time | Memory Usage | +|---------------|------------------------|--------------| +| < 10MB | < 5 seconds | < 50MB | +| 10-50MB | 5-15 seconds | < 100MB | +| 50-200MB | 15-45 seconds | < 200MB | +| 200MB+ | 45+ seconds | < 500MB | + +## Validation Checklist + +After migration, verify: +- [ ] All SSH hosts are accessible +- [ ] SSH credentials work correctly +- [ ] File manager recent/pinned items preserved +- [ ] User settings maintained +- [ ] OIDC configuration intact +- [ ] Admin users still have admin privileges +- [ ] Backup file exists and is valid SQLite +- [ ] Original file renamed (not deleted) +- [ ] Encrypted file is properly encrypted +- [ ] Migration APIs respond correctly + +## Monitoring Commands + +```bash +# Watch migration in real-time +docker logs -f container_id | grep -i migration + +# Check file sizes before/after +ls -la /host/data/db.sqlite* + +# Verify encrypted file +file /host/data/db.sqlite.encrypted + +# Monitor system resources during migration +docker stats container_id + +# Test database connectivity after migration +curl -H "Authorization: Bearer $TOKEN" \ + http://localhost:8081/hosts/list +``` + +## Common Issues & Solutions + +### Issue: "Permission denied" during backup creation +**Solution**: Check container file permissions and volume mounts + +### Issue: "Insufficient disk space" during migration +**Solution**: Free up space, migration requires 2x database size temporarily + +### Issue: "Database locked" error +**Solution**: Ensure no other processes are accessing the database file + +### Issue: Migration hangs indefinitely +**Solution**: Check for very large BLOB data, increase timeout or migrate manually + +### Issue: Encrypted file fails validation +**Solution**: Check DATABASE_KEY environment variable, ensure it's stable + +## Security Considerations + +- **Backup files contain unencrypted data**: Secure backup file access +- **Migration logs may contain sensitive info**: Review log retention policies +- **Temporary files during migration**: Ensure secure temp directory +- **Original files are preserved**: Plan for secure cleanup of old files +- **Admin API access**: Ensure proper authentication and authorization + +## Integration with CI/CD + +For automated testing in CI/CD pipelines: + +```bash +#!/bin/bash +# Migration integration test +set -e + +# Start with unencrypted test data +docker run -d --name test-migration \ + -v ./test-data:/app/data \ + termix:latest + +# Wait for startup +sleep 30 + +# Check migration status +RESPONSE=$(curl -s -H "Authorization: Bearer $TEST_TOKEN" \ + http://localhost:8081/database/migration/status) + +# Validate migration success +echo "$RESPONSE" | jq '.migrationStatus.needsMigration == false' + +# Cleanup +docker stop test-migration +docker rm test-migration +``` + +This comprehensive testing approach ensures the migration system handles all edge cases safely and provides administrators with full visibility into the migration process. \ No newline at end of file diff --git a/src/backend/database/database.ts b/src/backend/database/database.ts index 2f2dc20b..f93126e6 100644 --- a/src/backend/database/database.ts +++ b/src/backend/database/database.ts @@ -15,6 +15,7 @@ import { databaseLogger, apiLogger } from "../utils/logger.js"; import { AuthManager } from "../utils/auth-manager.js"; import { DataCrypto } from "../utils/data-crypto.js"; import { DatabaseFileEncryption } from "../utils/database-file-encryption.js"; +import { DatabaseMigration } from "../utils/database-migration.js"; import { UserDataExport } from "../utils/user-data-export.js"; import { UserDataImport } from "../utils/user-data-import.js"; import https from "https"; @@ -1311,6 +1312,155 @@ async function initializeSecurity() { } } +// Database migration status endpoint - for administrators to check migration status +app.get("/database/migration/status", authenticateJWT, requireAdmin, async (req, res) => { + try { + const dataDir = process.env.DATA_DIR || "./db/data"; + const migration = new DatabaseMigration(dataDir); + const status = migration.checkMigrationStatus(); + + apiLogger.info("Migration status requested", { + operation: "migration_status_api", + userId: (req as any).userId, + }); + + // Get migration-related files info + const dbPath = path.join(dataDir, "db.sqlite"); + const encryptedDbPath = `${dbPath}.encrypted`; + + const files = fs.readdirSync(dataDir); + const backupFiles = files.filter(f => f.includes('.migration-backup-')); + const migratedFiles = files.filter(f => f.includes('.migrated-')); + + // Get file sizes + let unencryptedSize = 0; + let encryptedSize = 0; + + if (status.hasUnencryptedDb) { + try { + unencryptedSize = fs.statSync(dbPath).size; + } catch (error) { + // File might be locked or deleted + } + } + + if (status.hasEncryptedDb) { + try { + encryptedSize = fs.statSync(encryptedDbPath).size; + } catch (error) { + // File might not exist + } + } + + res.json({ + migrationStatus: status, + files: { + unencryptedDbSize: unencryptedSize, + encryptedDbSize: encryptedSize, + backupFiles: backupFiles.length, + migratedFiles: migratedFiles.length, + }, + recommendations: getMigrationRecommendations(status), + }); + + } catch (error) { + apiLogger.error("Failed to get migration status", error, { + operation: "migration_status_api_failed", + }); + res.status(500).json({ + error: "Failed to get migration status", + details: error instanceof Error ? error.message : "Unknown error", + }); + } +}); + +// Database migration history endpoint - shows backup and migrated files +app.get("/database/migration/history", authenticateJWT, requireAdmin, async (req, res) => { + try { + const dataDir = process.env.DATA_DIR || "./db/data"; + + apiLogger.info("Migration history requested", { + operation: "migration_history_api", + userId: (req as any).userId, + }); + + const files = fs.readdirSync(dataDir); + + const backupFiles = files + .filter(f => f.includes('.migration-backup-')) + .map(f => { + const filePath = path.join(dataDir, f); + const stats = fs.statSync(filePath); + return { + name: f, + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + type: 'backup', + }; + }) + .sort((a, b) => b.modified.getTime() - a.modified.getTime()); + + const migratedFiles = files + .filter(f => f.includes('.migrated-')) + .map(f => { + const filePath = path.join(dataDir, f); + const stats = fs.statSync(filePath); + return { + name: f, + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + type: 'migrated', + }; + }) + .sort((a, b) => b.modified.getTime() - a.modified.getTime()); + + res.json({ + files: [...backupFiles, ...migratedFiles], + summary: { + totalBackups: backupFiles.length, + totalMigrated: migratedFiles.length, + oldestBackup: backupFiles.length > 0 ? backupFiles[backupFiles.length - 1].created : null, + newestBackup: backupFiles.length > 0 ? backupFiles[0].created : null, + }, + }); + + } catch (error) { + apiLogger.error("Failed to get migration history", error, { + operation: "migration_history_api_failed", + }); + res.status(500).json({ + error: "Failed to get migration history", + details: error instanceof Error ? error.message : "Unknown error", + }); + } +}); + +// Helper function to generate migration recommendations +function getMigrationRecommendations(status: any): string[] { + const recommendations: string[] = []; + + if (status.needsMigration) { + recommendations.push("Automatic migration will occur on next server restart"); + recommendations.push("Ensure DATABASE_KEY environment variable is properly set"); + recommendations.push("Consider manual backup before restart if desired"); + } else if (status.hasUnencryptedDb && status.hasEncryptedDb) { + recommendations.push("Both encrypted and unencrypted databases found"); + recommendations.push("This may indicate a previous migration was interrupted"); + recommendations.push("Manual intervention may be required"); + recommendations.push("Check logs for migration history"); + } else if (status.hasEncryptedDb && !status.hasUnencryptedDb) { + recommendations.push("Database is properly encrypted"); + recommendations.push("No action required"); + } else if (!status.hasEncryptedDb && !status.hasUnencryptedDb) { + recommendations.push("Fresh installation detected"); + recommendations.push("Database will be created encrypted on first use"); + } + + return recommendations; +} + app.listen(HTTP_PORT, async () => { // Ensure uploads directory exists const uploadsDir = path.join(process.cwd(), "uploads"); diff --git a/src/backend/database/db/index.ts b/src/backend/database/db/index.ts index 52949e72..68708397 100644 --- a/src/backend/database/db/index.ts +++ b/src/backend/database/db/index.ts @@ -6,6 +6,7 @@ import path from "path"; import { databaseLogger } from "../../utils/logger.js"; import { DatabaseFileEncryption } from "../../utils/database-file-encryption.js"; import { SystemCrypto } from "../../utils/system-crypto.js"; +import { DatabaseMigration } from "../../utils/database-migration.js"; const dataDir = process.env.DATA_DIR || "./db/data"; const dbDir = path.resolve(dataDir); @@ -83,49 +84,86 @@ async function initializeDatabaseAsync(): Promise { operation: "db_memory_create_success", }); } else { - memoryDatabase = new Database(":memory:"); - isNewDatabase = true; + // No encrypted database exists - check if we need to migrate + const migration = new DatabaseMigration(dataDir); + const migrationStatus = migration.checkMigrationStatus(); - // Check if there's an old unencrypted database to migrate - if (fs.existsSync(dbPath)) { - // Load old database and copy its content to memory database - const oldDb = new Database(dbPath, { readonly: true }); + databaseLogger.info("Migration status check completed", { + operation: "migration_status", + needsMigration: migrationStatus.needsMigration, + hasUnencryptedDb: migrationStatus.hasUnencryptedDb, + hasEncryptedDb: migrationStatus.hasEncryptedDb, + unencryptedDbSize: migrationStatus.unencryptedDbSize, + reason: migrationStatus.reason, + }); - // Get all table schemas and data from old database - const tables = oldDb - .prepare( - ` - SELECT name, sql FROM sqlite_master - WHERE type='table' AND name NOT LIKE 'sqlite_%' - `, - ) - .all() as { name: string; sql: string }[]; + if (migrationStatus.needsMigration) { + // Perform automatic migration + databaseLogger.info("Starting automatic database migration", { + operation: "auto_migration_start", + unencryptedDbSize: migrationStatus.unencryptedDbSize, + }); - // Create tables in memory database - for (const table of tables) { - memoryDatabase.exec(table.sql); - } + const migrationResult = await migration.migrateDatabase(); - // Copy data for each table - for (const table of tables) { - const rows = oldDb.prepare(`SELECT * FROM ${table.name}`).all(); - if (rows.length > 0) { - const columns = Object.keys(rows[0]); - const placeholders = columns.map(() => "?").join(", "); - const insertStmt = memoryDatabase.prepare( - `INSERT INTO ${table.name} (${columns.join(", ")}) VALUES (${placeholders})`, - ); + if (migrationResult.success) { + databaseLogger.success("Automatic database migration completed successfully", { + operation: "auto_migration_success", + migratedTables: migrationResult.migratedTables, + migratedRows: migrationResult.migratedRows, + duration: migrationResult.duration, + backupPath: migrationResult.backupPath, + }); - for (const row of rows) { - const values = columns.map((col) => (row as any)[col]); - insertStmt.run(values); - } + // Clean up old backup files + migration.cleanupOldBackups(); + + // Load the newly created encrypted database + if (DatabaseFileEncryption.isEncryptedDatabaseFile(encryptedDbPath)) { + databaseLogger.info("Loading migrated encrypted database into memory", { + operation: "load_migrated_db", + encryptedPath: encryptedDbPath, + }); + + const decryptedBuffer = await DatabaseFileEncryption.decryptDatabaseToBuffer(encryptedDbPath); + memoryDatabase = new Database(decryptedBuffer); + isNewDatabase = false; // We have migrated data + + databaseLogger.success("Migrated encrypted database loaded successfully", { + operation: "load_migrated_db_success", + decryptedSize: decryptedBuffer.length, + }); + } else { + throw new Error("Migration completed but encrypted database file not found"); } + } else { + // Migration failed - this is critical + databaseLogger.error("Automatic database migration failed", null, { + operation: "auto_migration_failed", + error: migrationResult.error, + migratedTables: migrationResult.migratedTables, + migratedRows: migrationResult.migratedRows, + duration: migrationResult.duration, + backupPath: migrationResult.backupPath, + }); + + // 🔥 CRITICAL: Migration failure with existing data + console.error("🚨 DATABASE MIGRATION FAILED - THIS IS CRITICAL!"); + console.error("Migration error:", migrationResult.error); + console.error("Backup available at:", migrationResult.backupPath); + console.error("Manual intervention required to recover data."); + + throw new Error(`Database migration failed: ${migrationResult.error}. Backup available at: ${migrationResult.backupPath}`); } + } else { + // No migration needed - create fresh database + memoryDatabase = new Database(":memory:"); + isNewDatabase = true; - oldDb.close(); - - isNewDatabase = false; + databaseLogger.info("Creating fresh in-memory database", { + operation: "fresh_db_create", + reason: migrationStatus.reason, + }); } } } catch (error) { @@ -479,65 +517,25 @@ async function saveMemoryDatabaseToFile() { } } -// Function to handle post-initialization file encryption and cleanup +// Function to handle post-initialization file encryption and periodic saves async function handlePostInitFileEncryption() { if (!enableFileEncryption) return; try { - // Clean up any existing unencrypted database files + // Check for any remaining unencrypted database files that may need attention if (fs.existsSync(dbPath)) { + // This could happen if migration was skipped or if there are multiple database files databaseLogger.warn( - "Found unencrypted database file, removing for security", + "Unencrypted database file still exists after initialization", { - operation: "db_security_cleanup_existing", - removingPath: dbPath, + operation: "db_security_check", + path: dbPath, + note: "This may be normal if migration was skipped for safety reasons", }, ); - try { - fs.unlinkSync(dbPath); - databaseLogger.success( - "Unencrypted database file removed for security", - { - operation: "db_security_cleanup_complete", - removedPath: dbPath, - }, - ); - } catch (error) { - databaseLogger.warn( - "Could not remove unencrypted database file (may be locked)", - { - operation: "db_security_cleanup_deferred", - path: dbPath, - error: error instanceof Error ? error.message : "Unknown error", - }, - ); - - // Try again after a short delay - setTimeout(() => { - try { - if (fs.existsSync(dbPath)) { - fs.unlinkSync(dbPath); - databaseLogger.success( - "Delayed cleanup: unencrypted database file removed", - { - operation: "db_security_cleanup_delayed_success", - removedPath: dbPath, - }, - ); - } - } catch (delayedError) { - databaseLogger.error( - "Failed to remove unencrypted database file even after delay", - delayedError, - { - operation: "db_security_cleanup_delayed_failed", - path: dbPath, - }, - ); - } - }, 2000); - } + // Don't automatically delete - let migration logic handle this + // This provides better safety and transparency } // Always save the in-memory database (whether new or existing) @@ -545,15 +543,32 @@ async function handlePostInitFileEncryption() { // Save immediately after initialization await saveMemoryDatabaseToFile(); + databaseLogger.info("Setting up periodic database saves", { + operation: "db_periodic_save_setup", + interval: "5 minutes", + }); + // Set up periodic saves every 5 minutes setInterval(saveMemoryDatabaseToFile, 5 * 60 * 1000); } + + // Perform migration cleanup on startup (remove old backup files) + try { + const migration = new DatabaseMigration(dataDir); + migration.cleanupOldBackups(); + } catch (cleanupError) { + databaseLogger.warn("Failed to cleanup old migration files", { + operation: "migration_cleanup_startup_failed", + error: cleanupError instanceof Error ? cleanupError.message : "Unknown error", + }); + } + } catch (error) { databaseLogger.error( - "Failed to handle database file encryption/cleanup", + "Failed to handle database file encryption setup", error, { - operation: "db_encrypt_cleanup_failed", + operation: "db_encrypt_setup_failed", }, ); diff --git a/src/backend/utils/auth-manager.ts b/src/backend/utils/auth-manager.ts index 43033ca5..e849fb2b 100644 --- a/src/backend/utils/auth-manager.ts +++ b/src/backend/utils/auth-manager.ts @@ -1,6 +1,7 @@ import jwt from "jsonwebtoken"; import { UserCrypto } from "./user-crypto.js"; import { SystemCrypto } from "./system-crypto.js"; +import { DataCrypto } from "./data-crypto.js"; import { databaseLogger } from "./logger.js"; import type { Request, Response, NextFunction } from "express"; @@ -67,10 +68,70 @@ class AuthManager { } /** - * User login - use UserCrypto + * User login with lazy encryption migration */ async authenticateUser(userId: string, password: string): Promise { - return await this.userCrypto.authenticateUser(userId, password); + const authenticated = await this.userCrypto.authenticateUser(userId, password); + + if (authenticated) { + // Trigger lazy encryption migration for user's sensitive fields + await this.performLazyEncryptionMigration(userId); + } + + return authenticated; + } + + /** + * Perform lazy encryption migration for user's sensitive data + * This runs asynchronously after successful login + */ + private async performLazyEncryptionMigration(userId: string): Promise { + try { + const userDataKey = this.getUserDataKey(userId); + if (!userDataKey) { + databaseLogger.warn("Cannot perform lazy encryption migration - user data key not available", { + operation: "lazy_encryption_migration_no_key", + userId, + }); + return; + } + + // Import database connection - need to access raw SQLite for migration + const { getDb } = await import("../database/db/index.js"); + const db = getDb(); + + // Get the underlying SQLite instance + const sqlite = (db as any)._.session.db; + + // Perform the migration + const migrationResult = await DataCrypto.migrateUserSensitiveFields( + userId, + userDataKey, + sqlite + ); + + if (migrationResult.migrated) { + databaseLogger.success("Lazy encryption migration completed for user", { + operation: "lazy_encryption_migration_success", + userId, + migratedTables: migrationResult.migratedTables, + migratedFieldsCount: migrationResult.migratedFieldsCount, + }); + } else { + databaseLogger.debug("No lazy encryption migration needed for user", { + operation: "lazy_encryption_migration_not_needed", + userId, + }); + } + + } catch (error) { + // Log error but don't fail the login process + databaseLogger.error("Lazy encryption migration failed", error, { + operation: "lazy_encryption_migration_error", + userId, + error: error instanceof Error ? error.message : "Unknown error", + }); + } } /** diff --git a/src/backend/utils/data-crypto.ts b/src/backend/utils/data-crypto.ts index fb9aa387..512a68af 100644 --- a/src/backend/utils/data-crypto.ts +++ b/src/backend/utils/data-crypto.ts @@ -1,4 +1,5 @@ import { FieldCrypto } from "./field-crypto.js"; +import { LazyFieldEncryption } from "./lazy-field-encryption.js"; import { UserCrypto } from "./user-crypto.js"; import { databaseLogger } from "./logger.js"; @@ -43,13 +44,8 @@ class DataCrypto { } /** - * Decrypt record - either succeeds or fails - * - * Removed all: - * - isEncrypted() checks - * - legacy data handling - * - "backward compatibility" logic - * - migration on access + * Decrypt record with lazy encryption support + * Handles both encrypted and plaintext fields (from migration) */ static decryptRecord(tableName: string, record: any, userId: string, userDataKey: Buffer): any { if (!record) return record; @@ -59,9 +55,8 @@ class DataCrypto { for (const [fieldName, value] of Object.entries(record)) { if (FieldCrypto.shouldEncryptField(tableName, fieldName) && value) { - // Simple rule: sensitive fields must be encrypted JSON format - // If not, it's data corruption, fail directly - decryptedRecord[fieldName] = FieldCrypto.decryptField( + // Use lazy encryption to handle both plaintext and encrypted data + decryptedRecord[fieldName] = LazyFieldEncryption.safeGetFieldValue( value as string, userDataKey, recordId, @@ -81,6 +76,172 @@ class DataCrypto { return records.map((record) => this.decryptRecord(tableName, record, userId, userDataKey)); } + /** + * Migrate user's plaintext sensitive fields to encrypted format + * Called during user login to gradually encrypt legacy data + */ + static async migrateUserSensitiveFields( + userId: string, + userDataKey: Buffer, + db: any + ): Promise<{ + migrated: boolean; + migratedTables: string[]; + migratedFieldsCount: number; + }> { + let migrated = false; + const migratedTables: string[] = []; + let migratedFieldsCount = 0; + + try { + databaseLogger.info("Starting user sensitive fields migration", { + operation: "user_sensitive_migration_start", + userId, + }); + + // Check if migration is needed + const { needsMigration, plaintextFields } = await LazyFieldEncryption.checkUserNeedsMigration( + userId, + userDataKey, + db + ); + + if (!needsMigration) { + databaseLogger.info("No migration needed for user", { + operation: "user_sensitive_migration_not_needed", + userId, + }); + return { migrated: false, migratedTables: [], migratedFieldsCount: 0 }; + } + + databaseLogger.info("User requires sensitive field migration", { + operation: "user_sensitive_migration_required", + userId, + plaintextFieldsCount: plaintextFields.length, + }); + + // Process ssh_data table + const sshDataRecords = db.prepare("SELECT * FROM ssh_data WHERE user_id = ?").all(userId); + for (const record of sshDataRecords) { + const sensitiveFields = LazyFieldEncryption.getSensitiveFieldsForTable('ssh_data'); + const { updatedRecord, migratedFields, needsUpdate } = LazyFieldEncryption.migrateRecordSensitiveFields( + record, + sensitiveFields, + userDataKey, + record.id.toString() + ); + + if (needsUpdate) { + // Update the record in database + const updateQuery = ` + UPDATE ssh_data + SET password = ?, key = ?, key_password = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + `; + db.prepare(updateQuery).run( + updatedRecord.password || null, + updatedRecord.key || null, + updatedRecord.key_password || null, + record.id + ); + + migratedFieldsCount += migratedFields.length; + if (!migratedTables.includes('ssh_data')) { + migratedTables.push('ssh_data'); + } + migrated = true; + } + } + + // Process ssh_credentials table + const sshCredentialsRecords = db.prepare("SELECT * FROM ssh_credentials WHERE user_id = ?").all(userId); + for (const record of sshCredentialsRecords) { + const sensitiveFields = LazyFieldEncryption.getSensitiveFieldsForTable('ssh_credentials'); + const { updatedRecord, migratedFields, needsUpdate } = LazyFieldEncryption.migrateRecordSensitiveFields( + record, + sensitiveFields, + userDataKey, + record.id.toString() + ); + + if (needsUpdate) { + // Update the record in database + const updateQuery = ` + UPDATE ssh_credentials + SET password = ?, key = ?, key_password = ?, private_key = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + `; + db.prepare(updateQuery).run( + updatedRecord.password || null, + updatedRecord.key || null, + updatedRecord.key_password || null, + updatedRecord.private_key || null, + record.id + ); + + migratedFieldsCount += migratedFields.length; + if (!migratedTables.includes('ssh_credentials')) { + migratedTables.push('ssh_credentials'); + } + migrated = true; + } + } + + // Process users table + const userRecord = db.prepare("SELECT * FROM users WHERE id = ?").get(userId); + if (userRecord) { + const sensitiveFields = LazyFieldEncryption.getSensitiveFieldsForTable('users'); + const { updatedRecord, migratedFields, needsUpdate } = LazyFieldEncryption.migrateRecordSensitiveFields( + userRecord, + sensitiveFields, + userDataKey, + userId + ); + + if (needsUpdate) { + // Update the record in database + const updateQuery = ` + UPDATE users + SET totp_secret = ?, totp_backup_codes = ? + WHERE id = ? + `; + db.prepare(updateQuery).run( + updatedRecord.totp_secret || null, + updatedRecord.totp_backup_codes || null, + userId + ); + + migratedFieldsCount += migratedFields.length; + if (!migratedTables.includes('users')) { + migratedTables.push('users'); + } + migrated = true; + } + } + + if (migrated) { + databaseLogger.success("User sensitive fields migration completed", { + operation: "user_sensitive_migration_success", + userId, + migratedTables, + migratedFieldsCount, + }); + } + + return { migrated, migratedTables, migratedFieldsCount }; + + } catch (error) { + databaseLogger.error("User sensitive fields migration failed", error, { + operation: "user_sensitive_migration_failed", + userId, + error: error instanceof Error ? error.message : "Unknown error", + }); + + // Don't throw error to avoid breaking user login + return { migrated: false, migratedTables: [], migratedFieldsCount: 0 }; + } + } + /** * Get user data key */ diff --git a/src/backend/utils/database-migration.ts b/src/backend/utils/database-migration.ts new file mode 100644 index 00000000..0cb772c9 --- /dev/null +++ b/src/backend/utils/database-migration.ts @@ -0,0 +1,457 @@ +import Database from "better-sqlite3"; +import fs from "fs"; +import path from "path"; +import { databaseLogger } from "./logger.js"; +import { DatabaseFileEncryption } from "./database-file-encryption.js"; + +export interface MigrationResult { + success: boolean; + error?: string; + migratedTables: number; + migratedRows: number; + backupPath?: string; + duration: number; +} + +export interface MigrationStatus { + needsMigration: boolean; + hasUnencryptedDb: boolean; + hasEncryptedDb: boolean; + unencryptedDbSize: number; + reason: string; +} + +export class DatabaseMigration { + private dataDir: string; + private unencryptedDbPath: string; + private encryptedDbPath: string; + + constructor(dataDir: string) { + this.dataDir = dataDir; + this.unencryptedDbPath = path.join(dataDir, "db.sqlite"); + this.encryptedDbPath = `${this.unencryptedDbPath}.encrypted`; + } + + /** + * 检查是否需要迁移以及迁移状态 + */ + checkMigrationStatus(): MigrationStatus { + const hasUnencryptedDb = fs.existsSync(this.unencryptedDbPath); + const hasEncryptedDb = DatabaseFileEncryption.isEncryptedDatabaseFile(this.encryptedDbPath); + + let unencryptedDbSize = 0; + if (hasUnencryptedDb) { + try { + unencryptedDbSize = fs.statSync(this.unencryptedDbPath).size; + } catch (error) { + databaseLogger.warn("Could not get unencrypted database file size", { + operation: "migration_status_check", + error: error instanceof Error ? error.message : "Unknown error", + }); + } + } + + // 确定迁移状态 + let needsMigration = false; + let reason = ""; + + if (hasEncryptedDb && hasUnencryptedDb) { + // 两个都存在：可能是之前迁移失败或中断 + needsMigration = false; + reason = "Both encrypted and unencrypted databases exist. Skipping migration for safety. Manual intervention may be required."; + } else if (hasEncryptedDb && !hasUnencryptedDb) { + // 只有加密数据库：无需迁移 + needsMigration = false; + reason = "Only encrypted database exists. No migration needed."; + } else if (!hasEncryptedDb && hasUnencryptedDb) { + // 只有未加密数据库：需要迁移 + needsMigration = true; + reason = "Unencrypted database found. Migration to encrypted format required."; + } else { + // 都不存在：全新安装 + needsMigration = false; + reason = "No existing database found. This is a fresh installation."; + } + + return { + needsMigration, + hasUnencryptedDb, + hasEncryptedDb, + unencryptedDbSize, + reason, + }; + } + + /** + * 创建未加密数据库的安全备份 + */ + private createBackup(): string { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const backupPath = `${this.unencryptedDbPath}.migration-backup-${timestamp}`; + + try { + databaseLogger.info("Creating migration backup", { + operation: "migration_backup_create", + source: this.unencryptedDbPath, + backup: backupPath, + }); + + fs.copyFileSync(this.unencryptedDbPath, backupPath); + + // 验证备份完整性 + const originalSize = fs.statSync(this.unencryptedDbPath).size; + const backupSize = fs.statSync(backupPath).size; + + if (originalSize !== backupSize) { + throw new Error(`Backup size mismatch: original=${originalSize}, backup=${backupSize}`); + } + + databaseLogger.success("Migration backup created successfully", { + operation: "migration_backup_created", + backupPath, + fileSize: backupSize, + }); + + return backupPath; + } catch (error) { + databaseLogger.error("Failed to create migration backup", error, { + operation: "migration_backup_failed", + source: this.unencryptedDbPath, + backup: backupPath, + }); + throw new Error(`Backup creation failed: ${error instanceof Error ? error.message : "Unknown error"}`); + } + } + + /** + * 验证数据库迁移的完整性 + */ + private async verifyMigration(originalDb: Database.Database, memoryDb: Database.Database): Promise { + try { + databaseLogger.info("Verifying migration integrity", { + operation: "migration_verify_start", + }); + + // 临时禁用外键约束以进行验证查询 + memoryDb.exec("PRAGMA foreign_keys = OFF"); + + // 获取原数据库的表列表 + const originalTables = originalDb + .prepare(` + SELECT name FROM sqlite_master + WHERE type='table' AND name NOT LIKE 'sqlite_%' + ORDER BY name + `) + .all() as { name: string }[]; + + // 获取内存数据库的表列表 + const memoryTables = memoryDb + .prepare(` + SELECT name FROM sqlite_master + WHERE type='table' AND name NOT LIKE 'sqlite_%' + ORDER BY name + `) + .all() as { name: string }[]; + + // 检查表数量是否一致 + if (originalTables.length !== memoryTables.length) { + databaseLogger.error("Table count mismatch during migration verification", null, { + operation: "migration_verify_failed", + originalCount: originalTables.length, + memoryCount: memoryTables.length, + }); + return false; + } + + let totalOriginalRows = 0; + let totalMemoryRows = 0; + + // 逐表验证行数 + for (const table of originalTables) { + const originalCount = originalDb.prepare(`SELECT COUNT(*) as count FROM ${table.name}`).get() as { count: number }; + const memoryCount = memoryDb.prepare(`SELECT COUNT(*) as count FROM ${table.name}`).get() as { count: number }; + + totalOriginalRows += originalCount.count; + totalMemoryRows += memoryCount.count; + + if (originalCount.count !== memoryCount.count) { + databaseLogger.error("Row count mismatch for table during migration verification", null, { + operation: "migration_verify_table_failed", + table: table.name, + originalRows: originalCount.count, + memoryRows: memoryCount.count, + }); + return false; + } + + databaseLogger.debug("Table verification passed", { + operation: "migration_verify_table_success", + table: table.name, + rows: originalCount.count, + }); + } + + databaseLogger.success("Migration integrity verification completed", { + operation: "migration_verify_success", + tables: originalTables.length, + totalRows: totalOriginalRows, + }); + + // 重新启用外键约束 + memoryDb.exec("PRAGMA foreign_keys = ON"); + + return true; + } catch (error) { + databaseLogger.error("Migration verification failed", error, { + operation: "migration_verify_error", + }); + return false; + } + } + + /** + * 执行数据库迁移 + */ + async migrateDatabase(): Promise { + const startTime = Date.now(); + let backupPath: string | undefined; + let migratedTables = 0; + let migratedRows = 0; + + try { + databaseLogger.info("Starting database migration from unencrypted to encrypted format", { + operation: "migration_start", + source: this.unencryptedDbPath, + target: this.encryptedDbPath, + }); + + // 1. 创建安全备份 + backupPath = this.createBackup(); + + // 2. 打开原数据库（只读） + const originalDb = new Database(this.unencryptedDbPath, { readonly: true }); + + // 3. 创建内存数据库 + const memoryDb = new Database(":memory:"); + + try { + // 4. 获取所有表结构 + const tables = originalDb + .prepare(` + SELECT name, sql FROM sqlite_master + WHERE type='table' AND name NOT LIKE 'sqlite_%' + `) + .all() as { name: string; sql: string }[]; + + databaseLogger.info("Found tables to migrate", { + operation: "migration_tables_found", + tableCount: tables.length, + tables: tables.map(t => t.name), + }); + + // 5. 在内存数据库中创建表结构 + for (const table of tables) { + memoryDb.exec(table.sql); + migratedTables++; + + databaseLogger.debug("Table structure created", { + operation: "migration_table_created", + table: table.name, + }); + } + + // 6. 禁用外键约束以避免插入顺序问题 + databaseLogger.info("Disabling foreign key constraints for migration", { + operation: "migration_disable_fk", + }); + memoryDb.exec("PRAGMA foreign_keys = OFF"); + + // 7. 复制每个表的数据 + for (const table of tables) { + const rows = originalDb.prepare(`SELECT * FROM ${table.name}`).all(); + + if (rows.length > 0) { + const columns = Object.keys(rows[0]); + const placeholders = columns.map(() => "?").join(", "); + const insertStmt = memoryDb.prepare( + `INSERT INTO ${table.name} (${columns.join(", ")}) VALUES (${placeholders})` + ); + + // 使用事务批量插入 + const insertTransaction = memoryDb.transaction((dataRows: any[]) => { + for (const row of dataRows) { + const values = columns.map((col) => row[col]); + insertStmt.run(values); + } + }); + + insertTransaction(rows); + migratedRows += rows.length; + + databaseLogger.debug("Table data migrated", { + operation: "migration_table_data", + table: table.name, + rows: rows.length, + }); + } + } + + // 8. 重新启用外键约束 + databaseLogger.info("Re-enabling foreign key constraints after migration", { + operation: "migration_enable_fk", + }); + memoryDb.exec("PRAGMA foreign_keys = ON"); + + // 验证外键约束现在是否正常 + const fkCheckResult = memoryDb.prepare("PRAGMA foreign_key_check").all(); + if (fkCheckResult.length > 0) { + databaseLogger.error("Foreign key constraints violations detected after migration", null, { + operation: "migration_fk_check_failed", + violations: fkCheckResult, + }); + throw new Error(`Foreign key violations detected: ${JSON.stringify(fkCheckResult)}`); + } + + databaseLogger.success("Foreign key constraints verification passed", { + operation: "migration_fk_check_success", + }); + + // 9. 验证迁移完整性 + const verificationPassed = await this.verifyMigration(originalDb, memoryDb); + if (!verificationPassed) { + throw new Error("Migration integrity verification failed"); + } + + // 10. 导出内存数据库到缓冲区 + const buffer = memoryDb.serialize(); + + // 11. 创建加密数据库文件 + databaseLogger.info("Creating encrypted database file", { + operation: "migration_encrypt_start", + bufferSize: buffer.length, + }); + + await DatabaseFileEncryption.encryptDatabaseFromBuffer(buffer, this.encryptedDbPath); + + // 12. 验证加密文件 + if (!DatabaseFileEncryption.isEncryptedDatabaseFile(this.encryptedDbPath)) { + throw new Error("Encrypted database file verification failed"); + } + + // 13. 清理：重命名原文件而不是删除 + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const migratedPath = `${this.unencryptedDbPath}.migrated-${timestamp}`; + + fs.renameSync(this.unencryptedDbPath, migratedPath); + + databaseLogger.success("Database migration completed successfully", { + operation: "migration_complete", + migratedTables, + migratedRows, + duration: Date.now() - startTime, + backupPath, + migratedPath, + encryptedDbPath: this.encryptedDbPath, + }); + + return { + success: true, + migratedTables, + migratedRows, + backupPath, + duration: Date.now() - startTime, + }; + + } finally { + // 确保数据库连接关闭 + originalDb.close(); + memoryDb.close(); + } + + } catch (error) { + const errorMessage = error instanceof Error ? error.message : "Unknown error"; + + databaseLogger.error("Database migration failed", error, { + operation: "migration_failed", + migratedTables, + migratedRows, + duration: Date.now() - startTime, + backupPath, + }); + + return { + success: false, + error: errorMessage, + migratedTables, + migratedRows, + backupPath, + duration: Date.now() - startTime, + }; + } + } + + /** + * 清理旧的备份文件（保留最近3个） + */ + cleanupOldBackups(): void { + try { + const backupPattern = /\.migration-backup-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z$/; + const migratedPattern = /\.migrated-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z$/; + + const files = fs.readdirSync(this.dataDir); + + // 查找备份文件和已迁移文件 + const backupFiles = files.filter(f => backupPattern.test(f)) + .map(f => ({ + name: f, + path: path.join(this.dataDir, f), + mtime: fs.statSync(path.join(this.dataDir, f)).mtime, + })) + .sort((a, b) => b.mtime.getTime() - a.mtime.getTime()); + + const migratedFiles = files.filter(f => migratedPattern.test(f)) + .map(f => ({ + name: f, + path: path.join(this.dataDir, f), + mtime: fs.statSync(path.join(this.dataDir, f)).mtime, + })) + .sort((a, b) => b.mtime.getTime() - a.mtime.getTime()); + + // 保留最近3个备份文件 + const backupsToDelete = backupFiles.slice(3); + const migratedToDelete = migratedFiles.slice(3); + + for (const file of [...backupsToDelete, ...migratedToDelete]) { + try { + fs.unlinkSync(file.path); + databaseLogger.debug("Cleaned up old migration file", { + operation: "migration_cleanup", + file: file.name, + }); + } catch (error) { + databaseLogger.warn("Failed to cleanup old migration file", { + operation: "migration_cleanup_failed", + file: file.name, + error: error instanceof Error ? error.message : "Unknown error", + }); + } + } + + if (backupsToDelete.length > 0 || migratedToDelete.length > 0) { + databaseLogger.info("Migration cleanup completed", { + operation: "migration_cleanup_complete", + deletedBackups: backupsToDelete.length, + deletedMigrated: migratedToDelete.length, + remainingBackups: Math.min(backupFiles.length, 3), + remainingMigrated: Math.min(migratedFiles.length, 3), + }); + } + + } catch (error) { + databaseLogger.warn("Migration cleanup failed", { + operation: "migration_cleanup_error", + error: error instanceof Error ? error.message : "Unknown error", + }); + } + } +} \ No newline at end of file diff --git a/src/backend/utils/lazy-field-encryption.ts b/src/backend/utils/lazy-field-encryption.ts new file mode 100644 index 00000000..c46637ab --- /dev/null +++ b/src/backend/utils/lazy-field-encryption.ts @@ -0,0 +1,295 @@ +import { FieldCrypto } from "./field-crypto.js"; +import { databaseLogger } from "./logger.js"; + +/** + * 延迟字段加密 - 处理从明文到加密的平滑迁移 + * 用于在用户登录时将明文敏感数据逐步加密 + */ +export class LazyFieldEncryption { + /** + * 检测字段是否为明文（未加密） + */ + static isPlaintextField(value: string): boolean { + if (!value) return false; + + try { + const parsed = JSON.parse(value); + // 如果能解析为JSON且包含加密数据结构，则认为已加密 + if (parsed && typeof parsed === 'object' && + parsed.data && parsed.iv && parsed.tag && parsed.salt && parsed.recordId) { + return false; // 已加密 + } + // JSON格式但不是加密结构，视为明文 + return true; + } catch (jsonError) { + // 无法解析为JSON，视为明文 + return true; + } + } + + /** + * 安全获取字段值 - 自动处理明文和加密数据 + * 如果是明文，直接返回；如果已加密，则解密 + */ + static safeGetFieldValue( + fieldValue: string, + userKEK: Buffer, + recordId: string, + fieldName: string + ): string { + if (!fieldValue) return ""; + + if (this.isPlaintextField(fieldValue)) { + // 明文数据，直接返回 + databaseLogger.debug("Field detected as plaintext, returning as-is", { + operation: "lazy_encryption_plaintext_detected", + recordId, + fieldName, + valuePreview: fieldValue.substring(0, 10) + "...", + }); + return fieldValue; + } else { + // 加密数据，需要解密 + try { + const decrypted = FieldCrypto.decryptField(fieldValue, userKEK, recordId, fieldName); + databaseLogger.debug("Field decrypted successfully", { + operation: "lazy_encryption_decrypt_success", + recordId, + fieldName, + }); + return decrypted; + } catch (error) { + databaseLogger.error("Failed to decrypt field", error, { + operation: "lazy_encryption_decrypt_failed", + recordId, + fieldName, + error: error instanceof Error ? error.message : "Unknown error", + }); + throw error; + } + } + } + + /** + * 迁移明文字段到加密状态 + * 返回加密后的值，如果已经加密则返回原值 + */ + static migrateFieldToEncrypted( + fieldValue: string, + userKEK: Buffer, + recordId: string, + fieldName: string + ): { encrypted: string; wasPlaintext: boolean } { + if (!fieldValue) { + return { encrypted: "", wasPlaintext: false }; + } + + if (this.isPlaintextField(fieldValue)) { + // 明文数据，需要加密 + try { + const encrypted = FieldCrypto.encryptField(fieldValue, userKEK, recordId, fieldName); + + databaseLogger.info("Field migrated from plaintext to encrypted", { + operation: "lazy_encryption_migrate_success", + recordId, + fieldName, + plaintextLength: fieldValue.length, + }); + + return { encrypted, wasPlaintext: true }; + } catch (error) { + databaseLogger.error("Failed to encrypt plaintext field", error, { + operation: "lazy_encryption_migrate_failed", + recordId, + fieldName, + error: error instanceof Error ? error.message : "Unknown error", + }); + throw error; + } + } else { + // 已经加密，无需处理 + databaseLogger.debug("Field already encrypted, no migration needed", { + operation: "lazy_encryption_already_encrypted", + recordId, + fieldName, + }); + return { encrypted: fieldValue, wasPlaintext: false }; + } + } + + /** + * 批量迁移记录中的敏感字段 + */ + static migrateRecordSensitiveFields( + record: any, + sensitiveFields: string[], + userKEK: Buffer, + recordId: string + ): { + updatedRecord: any; + migratedFields: string[]; + needsUpdate: boolean + } { + const updatedRecord = { ...record }; + const migratedFields: string[] = []; + let needsUpdate = false; + + for (const fieldName of sensitiveFields) { + const fieldValue = record[fieldName]; + + if (fieldValue && this.isPlaintextField(fieldValue)) { + try { + const { encrypted } = this.migrateFieldToEncrypted( + fieldValue, + userKEK, + recordId, + fieldName + ); + + updatedRecord[fieldName] = encrypted; + migratedFields.push(fieldName); + needsUpdate = true; + + databaseLogger.debug("Record field migrated to encrypted", { + operation: "lazy_encryption_record_field_migrated", + recordId, + fieldName, + }); + } catch (error) { + databaseLogger.error("Failed to migrate record field", error, { + operation: "lazy_encryption_record_field_failed", + recordId, + fieldName, + }); + // 不抛出错误，继续处理其他字段 + } + } + } + + if (needsUpdate) { + databaseLogger.info("Record requires sensitive field migration", { + operation: "lazy_encryption_record_migration_needed", + recordId, + migratedFields, + totalMigratedFields: migratedFields.length, + }); + } + + return { updatedRecord, migratedFields, needsUpdate }; + } + + /** + * 获取敏感字段列表 - 定义哪些字段需要延迟加密 + */ + static getSensitiveFieldsForTable(tableName: string): string[] { + const sensitiveFieldsMap: Record = { + 'ssh_data': ['password', 'key', 'key_password'], + 'ssh_credentials': ['password', 'key', 'key_password', 'private_key'], + 'users': ['totp_secret', 'totp_backup_codes'], + }; + + return sensitiveFieldsMap[tableName] || []; + } + + /** + * 检查用户是否有需要迁移的明文数据 + */ + static async checkUserNeedsMigration( + userId: string, + userKEK: Buffer, + db: any + ): Promise<{ + needsMigration: boolean; + plaintextFields: Array<{ table: string; recordId: string; fields: string[] }>; + }> { + const plaintextFields: Array<{ table: string; recordId: string; fields: string[] }> = []; + let needsMigration = false; + + try { + // 检查 ssh_data 表 + const sshHosts = db.prepare("SELECT * FROM ssh_data WHERE user_id = ?").all(userId); + for (const host of sshHosts) { + const sensitiveFields = this.getSensitiveFieldsForTable('ssh_data'); + const hostPlaintextFields: string[] = []; + + for (const field of sensitiveFields) { + if (host[field] && this.isPlaintextField(host[field])) { + hostPlaintextFields.push(field); + needsMigration = true; + } + } + + if (hostPlaintextFields.length > 0) { + plaintextFields.push({ + table: 'ssh_data', + recordId: host.id.toString(), + fields: hostPlaintextFields, + }); + } + } + + // 检查 ssh_credentials 表 + const sshCredentials = db.prepare("SELECT * FROM ssh_credentials WHERE user_id = ?").all(userId); + for (const credential of sshCredentials) { + const sensitiveFields = this.getSensitiveFieldsForTable('ssh_credentials'); + const credentialPlaintextFields: string[] = []; + + for (const field of sensitiveFields) { + if (credential[field] && this.isPlaintextField(credential[field])) { + credentialPlaintextFields.push(field); + needsMigration = true; + } + } + + if (credentialPlaintextFields.length > 0) { + plaintextFields.push({ + table: 'ssh_credentials', + recordId: credential.id.toString(), + fields: credentialPlaintextFields, + }); + } + } + + // 检查 users 表中的敏感字段 + const user = db.prepare("SELECT * FROM users WHERE id = ?").get(userId); + if (user) { + const sensitiveFields = this.getSensitiveFieldsForTable('users'); + const userPlaintextFields: string[] = []; + + for (const field of sensitiveFields) { + if (user[field] && this.isPlaintextField(user[field])) { + userPlaintextFields.push(field); + needsMigration = true; + } + } + + if (userPlaintextFields.length > 0) { + plaintextFields.push({ + table: 'users', + recordId: userId, + fields: userPlaintextFields, + }); + } + } + + databaseLogger.info("User migration check completed", { + operation: "lazy_encryption_user_check", + userId, + needsMigration, + plaintextFieldsCount: plaintextFields.length, + totalPlaintextFields: plaintextFields.reduce((sum, item) => sum + item.fields.length, 0), + }); + + return { needsMigration, plaintextFields }; + + } catch (error) { + databaseLogger.error("Failed to check user migration needs", error, { + operation: "lazy_encryption_user_check_failed", + userId, + error: error instanceof Error ? error.message : "Unknown error", + }); + + return { needsMigration: false, plaintextFields: [] }; + } + } +} \ No newline at end of file