dev-1.7.0 #294

Merged
ZacharyZcR merged 73 commits from main into dev-1.7.0 2025-09-25 04:56:32 +00:00
4 changed files with 320 additions and 172 deletions
Showing only changes of commit ed11b309f4 - Show all commits

View File

@@ -679,7 +679,7 @@ app.post("/database/backup", async (req, res) => {
const backupPath = path.join(backupDir, backupFileName);
// Create encrypted backup directly from memory buffer
DatabaseFileEncryption.encryptDatabaseFromBuffer(dbBuffer, backupPath);
await DatabaseFileEncryption.encryptDatabaseFromBuffer(dbBuffer, backupPath);
res.json({
success: true,
@@ -718,7 +718,7 @@ app.post("/database/restore", async (req, res) => {
// Hardware compatibility check removed - no longer required
const restoredPath = DatabaseFileEncryption.restoreFromEncryptedBackup(
const restoredPath = await DatabaseFileEncryption.restoreFromEncryptedBackup(
backupPath,
targetPath,
);

View File

@@ -5,6 +5,7 @@ import fs from "fs";
import path from "path";
import { databaseLogger } from "../../utils/logger.js";
import { DatabaseFileEncryption } from "../../utils/database-file-encryption.js";
import { SystemCrypto } from "../../utils/system-crypto.js";
const dataDir = process.env.DATA_DIR || "./db/data";
const dbDir = path.resolve(dataDir);
@@ -25,105 +26,116 @@ const encryptedDbPath = `${dbPath}.encrypted`;
let actualDbPath = ":memory:"; // Always use memory database
let memoryDatabase: Database.Database;
let isNewDatabase = false;
let sqlite: Database.Database; // Module-level sqlite instance
if (enableFileEncryption) {
try {
// Check if encrypted database exists
if (DatabaseFileEncryption.isEncryptedDatabaseFile(encryptedDbPath)) {
databaseLogger.info(
"Found encrypted database file, loading into memory...",
{
operation: "db_memory_load",
encryptedPath: encryptedDbPath,
},
);
// Async initialization function to handle SystemCrypto and DatabaseFileEncryption
async function initializeDatabaseAsync(): Promise<void> {
// Initialize SystemCrypto database key first
const systemCrypto = SystemCrypto.getInstance();
await systemCrypto.initializeDatabaseKey();
// Hardware compatibility check removed - using fixed seed encryption
if (enableFileEncryption) {
try {
// Check if encrypted database exists
if (DatabaseFileEncryption.isEncryptedDatabaseFile(encryptedDbPath)) {
databaseLogger.info(
"Found encrypted database file, loading into memory...",
{
operation: "db_memory_load",
encryptedPath: encryptedDbPath,
},
);
// Decrypt database content to memory buffer
const decryptedBuffer =
DatabaseFileEncryption.decryptDatabaseToBuffer(encryptedDbPath);
// Decrypt database content to memory buffer (now async)
const decryptedBuffer =
await DatabaseFileEncryption.decryptDatabaseToBuffer(encryptedDbPath);
// Create in-memory database from decrypted buffer
memoryDatabase = new Database(decryptedBuffer);
} else {
memoryDatabase = new Database(":memory:");
isNewDatabase = true;
// Create in-memory database from decrypted buffer
memoryDatabase = new Database(decryptedBuffer);
} else {
memoryDatabase = new Database(":memory:");
isNewDatabase = true;
// Check if there's an old unencrypted database to migrate
if (fs.existsSync(dbPath)) {
// Load old database and copy its content to memory database
const oldDb = new Database(dbPath, { readonly: true });
// Check if there's an old unencrypted database to migrate
if (fs.existsSync(dbPath)) {
// Load old database and copy its content to memory database
const oldDb = new Database(dbPath, { readonly: true });
// Get all table schemas and data from old database
const tables = oldDb
.prepare(
`
SELECT name, sql FROM sqlite_master
WHERE type='table' AND name NOT LIKE 'sqlite_%'
`,
)
.all() as { name: string; sql: string }[];
// Get all table schemas and data from old database
const tables = oldDb
.prepare(
`
SELECT name, sql FROM sqlite_master
WHERE type='table' AND name NOT LIKE 'sqlite_%'
`,
)
.all() as { name: string; sql: string }[];
// Create tables in memory database
for (const table of tables) {
memoryDatabase.exec(table.sql);
}
// Create tables in memory database
for (const table of tables) {
memoryDatabase.exec(table.sql);
}
// Copy data for each table
for (const table of tables) {
const rows = oldDb.prepare(`SELECT * FROM ${table.name}`).all();
if (rows.length > 0) {
const columns = Object.keys(rows[0]);
const placeholders = columns.map(() => "?").join(", ");
const insertStmt = memoryDatabase.prepare(
`INSERT INTO ${table.name} (${columns.join(", ")}) VALUES (${placeholders})`,
);
// Copy data for each table
for (const table of tables) {
const rows = oldDb.prepare(`SELECT * FROM ${table.name}`).all();
if (rows.length > 0) {
const columns = Object.keys(rows[0]);
const placeholders = columns.map(() => "?").join(", ");
const insertStmt = memoryDatabase.prepare(
`INSERT INTO ${table.name} (${columns.join(", ")}) VALUES (${placeholders})`,
);
for (const row of rows) {
const values = columns.map((col) => (row as any)[col]);
insertStmt.run(values);
for (const row of rows) {
const values = columns.map((col) => (row as any)[col]);
insertStmt.run(values);
}
}
}
oldDb.close();
isNewDatabase = false;
}
oldDb.close();
isNewDatabase = false;
} else {
}
}
} catch (error) {
databaseLogger.error("Failed to initialize memory database", error, {
operation: "db_memory_init_failed",
});
} catch (error) {
databaseLogger.error("Failed to initialize memory database", error, {
operation: "db_memory_init_failed",
});
// If file encryption is critical, fail fast
if (process.env.DB_FILE_ENCRYPTION_REQUIRED === "true") {
throw error;
}
// If file encryption is critical, fail fast
if (process.env.DB_FILE_ENCRYPTION_REQUIRED === "true") {
throw error;
}
memoryDatabase = new Database(":memory:");
isNewDatabase = true;
}
} else {
memoryDatabase = new Database(":memory:");
isNewDatabase = true;
}
} else {
memoryDatabase = new Database(":memory:");
isNewDatabase = true;
}
databaseLogger.info(`Initializing SQLite database`, {
operation: "db_init",
path: actualDbPath,
encrypted:
enableFileEncryption &&
DatabaseFileEncryption.isEncryptedDatabaseFile(encryptedDbPath),
inMemory: true,
isNewDatabase,
});
// Main async initialization function that combines database setup with schema creation
async function initializeCompleteDatabase(): Promise<void> {
// First initialize the database and SystemCrypto
await initializeDatabaseAsync();
const sqlite = memoryDatabase;
databaseLogger.info(`Initializing SQLite database`, {
operation: "db_init",
path: actualDbPath,
encrypted:
enableFileEncryption &&
DatabaseFileEncryption.isEncryptedDatabaseFile(encryptedDbPath),
inMemory: true,
isNewDatabase,
});
sqlite.exec(`
// Create module-level sqlite instance after database is initialized
sqlite = memoryDatabase;
sqlite.exec(`
CREATE TABLE IF NOT EXISTS users (
id TEXT PRIMARY KEY,
username TEXT NOT NULL,
@@ -244,6 +256,33 @@ sqlite.exec(`
);
`);
// Run schema migrations
migrateSchema();
// Initialize default settings
try {
const row = sqlite
.prepare("SELECT value FROM settings WHERE key = 'allow_registration'")
.get();
if (!row) {
databaseLogger.info("Initializing default settings", {
operation: "db_init",
setting: "allow_registration",
});
sqlite
.prepare(
"INSERT INTO settings (key, value) VALUES ('allow_registration', 'true')",
gemini-code-assist[bot] commented 2025-09-23 19:31:37 +00:00 (Migrated from github.com)
Review

medium

This is an excellent and critical piece of error handling. Failing fast when database decryption fails is the correct approach to prevent data corruption or loss. The detailed logging and explicit error message will be invaluable for debugging if such a critical failure occurs. This significantly improves the robustness of the database initialization process.

![medium](https://www.gstatic.com/codereviewagent/medium-priority.svg) This is an excellent and critical piece of error handling. Failing fast when database decryption fails is the correct approach to prevent data corruption or loss. The detailed logging and explicit error message will be invaluable for debugging if such a critical failure occurs. This significantly improves the robustness of the database initialization process.
)
.run();
}
} catch (e) {
databaseLogger.warn("Could not initialize default settings", {
operation: "db_init",
error: e,
});
}
}
const addColumnIfNotExists = (
table: string,
column: string,
@@ -366,33 +405,6 @@ const migrateSchema = () => {
});
};
const initializeDatabase = async (): Promise<void> => {
migrateSchema();
try {
const row = sqlite
.prepare("SELECT value FROM settings WHERE key = 'allow_registration'")
.get();
if (!row) {
databaseLogger.info("Initializing default settings", {
operation: "db_init",
setting: "allow_registration",
});
sqlite
.prepare(
"INSERT INTO settings (key, value) VALUES ('allow_registration', 'true')",
)
.run();
} else {
}
} catch (e) {
databaseLogger.warn("Could not initialize default settings", {
operation: "db_init",
error: e,
});
}
};
// Function to save in-memory database to encrypted file
async function saveMemoryDatabaseToFile() {
if (!memoryDatabase || !enableFileEncryption) return;
@@ -401,8 +413,8 @@ async function saveMemoryDatabaseToFile() {
// Export in-memory database to buffer
const buffer = memoryDatabase.serialize();
// Encrypt and save to file
DatabaseFileEncryption.encryptDatabaseFromBuffer(buffer, encryptedDbPath);
// Encrypt and save to file (now async)
await DatabaseFileEncryption.encryptDatabaseFromBuffer(buffer, encryptedDbPath);
databaseLogger.debug("In-memory database saved to encrypted file", {
operation: "memory_db_save",
@@ -498,7 +510,7 @@ async function handlePostInitFileEncryption() {
}
}
initializeDatabase()
initializeCompleteDatabase()
.then(() => handlePostInitFileEncryption())
.catch((error) => {
databaseLogger.error("Failed to initialize database", error, {

View File

@@ -2,54 +2,44 @@ import crypto from "crypto";
import fs from "fs";
import path from "path";
import { databaseLogger } from "./logger.js";
import { SystemCrypto } from "./system-crypto.js";
interface EncryptedFileMetadata {
iv: string;
tag: string;
version: string;
fingerprint: string;
salt: string;
algorithm: string;
keySource?: string; // Track where the key comes from (SystemCrypto) - v2 only
salt?: string; // Legacy v1 format only
}
/**
* Database file encryption - encrypts the entire SQLite database file at rest
* This provides an additional security layer on top of field-level encryption
* Uses SystemCrypto for key management - no more fixed seed garbage!
*
* Linus principles applied:
* - Remove hardcoded keys security disaster
* - Use SystemCrypto instance keys for proper per-instance security
* - Simple and direct, no complex key derivation
*/
class DatabaseFileEncryption {
private static readonly VERSION = "v1";
private static readonly VERSION = "v2";
private static readonly ALGORITHM = "aes-256-gcm";
private static readonly KEY_ITERATIONS = 100000;
private static readonly ENCRYPTED_FILE_SUFFIX = ".encrypted";
private static readonly METADATA_FILE_SUFFIX = ".meta";
/**
* Generate file encryption key from fixed seed (no hardware dependency)
*/
private static generateFileEncryptionKey(salt: Buffer): Buffer {
// Use fixed seed for file encryption - simpler and more reliable than hardware fingerprint
const fixedSeed = process.env.DB_FILE_KEY || "termix-database-file-encryption-seed-v1";
const key = crypto.pbkdf2Sync(
fixedSeed,
salt,
this.KEY_ITERATIONS,
32, // 256 bits for AES-256
"sha256",
);
return key;
}
private static systemCrypto = SystemCrypto.getInstance();
/**
* Encrypt database from buffer (for in-memory databases)
*/
static encryptDatabaseFromBuffer(buffer: Buffer, targetPath: string): string {
static async encryptDatabaseFromBuffer(buffer: Buffer, targetPath: string): Promise<string> {
try {
// Get database key from SystemCrypto (no more fixed seed garbage!)
const key = await this.systemCrypto.getDatabaseKey();
// Generate encryption components
const salt = crypto.randomBytes(32);
const iv = crypto.randomBytes(16);
const key = this.generateFileEncryptionKey(salt);
// Encrypt the buffer
const cipher = crypto.createCipheriv(this.ALGORITHM, key, iv) as any;
@@ -61,9 +51,9 @@ class DatabaseFileEncryption {
iv: iv.toString("hex"),
tag: tag.toString("hex"),
version: this.VERSION,
fingerprint: "termix-v1-file", // Fixed identifier instead of hardware fingerprint
salt: salt.toString("hex"),
fingerprint: "termix-v2-systemcrypto", // SystemCrypto managed key
algorithm: this.ALGORITHM,
keySource: "SystemCrypto",
};
// Write encrypted file and metadata
@@ -86,7 +76,7 @@ class DatabaseFileEncryption {
/**
* Encrypt database file
*/
static encryptDatabaseFile(sourcePath: string, targetPath?: string): string {
static async encryptDatabaseFile(sourcePath: string, targetPath?: string): Promise<string> {
if (!fs.existsSync(sourcePath)) {
throw new Error(`Source database file does not exist: ${sourcePath}`);
}
@@ -99,10 +89,11 @@ class DatabaseFileEncryption {
// Read source file
const sourceData = fs.readFileSync(sourcePath);
// Get database key from SystemCrypto (no more fixed seed garbage!)
const key = await this.systemCrypto.getDatabaseKey();
// Generate encryption components
const salt = crypto.randomBytes(32);
const iv = crypto.randomBytes(16);
const key = this.generateFileEncryptionKey(salt);
// Encrypt the file
const cipher = crypto.createCipheriv(this.ALGORITHM, key, iv) as any;
@@ -117,9 +108,9 @@ class DatabaseFileEncryption {
iv: iv.toString("hex"),
tag: tag.toString("hex"),
version: this.VERSION,
fingerprint: "termix-v1-file", // Fixed identifier instead of hardware fingerprint
salt: salt.toString("hex"),
fingerprint: "termix-v2-systemcrypto", // SystemCrypto managed key
algorithm: this.ALGORITHM,
keySource: "SystemCrypto",
};
// Write encrypted file and metadata
@@ -151,7 +142,7 @@ class DatabaseFileEncryption {
/**
* Decrypt database file to buffer (for in-memory usage)
*/
static decryptDatabaseToBuffer(encryptedPath: string): Buffer {
static async decryptDatabaseToBuffer(encryptedPath: string): Promise<Buffer> {
if (!fs.existsSync(encryptedPath)) {
throw new Error(
`Encrypted database file does not exist: ${encryptedPath}`,
@@ -168,19 +159,29 @@ class DatabaseFileEncryption {
const metadataContent = fs.readFileSync(metadataPath, "utf8");
const metadata: EncryptedFileMetadata = JSON.parse(metadataContent);
// Validate metadata version
if (metadata.version !== this.VERSION) {
throw new Error(`Unsupported encryption version: ${metadata.version}`);
}
// Hardware fingerprint validation removed - no longer required
// Read encrypted data
const encryptedData = fs.readFileSync(encryptedPath);
// Generate decryption key
const salt = Buffer.from(metadata.salt, "hex");
const key = this.generateFileEncryptionKey(salt);
// Get decryption key based on version
let key: Buffer;
if (metadata.version === "v2") {
// New v2 format: use SystemCrypto key
key = await this.systemCrypto.getDatabaseKey();
} else if (metadata.version === "v1") {
// Legacy v1 format: use deprecated salt-based key derivation
databaseLogger.warn("Decrypting legacy v1 encrypted database - consider upgrading", {
operation: "decrypt_legacy_v1",
path: encryptedPath
});
if (!metadata.salt) {
throw new Error("v1 encrypted file missing required salt field");
}
const salt = Buffer.from(metadata.salt, "hex");
const fixedSeed = process.env.DB_FILE_KEY || "termix-database-file-encryption-seed-v1";
key = crypto.pbkdf2Sync(fixedSeed, salt, 100000, 32, "sha256");
} else {
throw new Error(`Unsupported encryption version: ${metadata.version}`);
}
// Decrypt to buffer
const decipher = crypto.createDecipheriv(
@@ -210,10 +211,10 @@ class DatabaseFileEncryption {
/**
* Decrypt database file
*/
static decryptDatabaseFile(
static async decryptDatabaseFile(
encryptedPath: string,
targetPath?: string,
): string {
): Promise<string> {
if (!fs.existsSync(encryptedPath)) {
throw new Error(
`Encrypted database file does not exist: ${encryptedPath}`,
@@ -233,19 +234,29 @@ class DatabaseFileEncryption {
const metadataContent = fs.readFileSync(metadataPath, "utf8");
const metadata: EncryptedFileMetadata = JSON.parse(metadataContent);
// Validate metadata version
if (metadata.version !== this.VERSION) {
throw new Error(`Unsupported encryption version: ${metadata.version}`);
}
// Hardware fingerprint validation removed - no longer required
// Read encrypted data
const encryptedData = fs.readFileSync(encryptedPath);
// Generate decryption key
const salt = Buffer.from(metadata.salt, "hex");
const key = this.generateFileEncryptionKey(salt);
// Get decryption key based on version
let key: Buffer;
if (metadata.version === "v2") {
// New v2 format: use SystemCrypto key
key = await this.systemCrypto.getDatabaseKey();
} else if (metadata.version === "v1") {
// Legacy v1 format: use deprecated salt-based key derivation
databaseLogger.warn("Decrypting legacy v1 encrypted database - consider upgrading", {
operation: "decrypt_legacy_v1",
path: encryptedPath
});
if (!metadata.salt) {
throw new Error("v1 encrypted file missing required salt field");
}
const salt = Buffer.from(metadata.salt, "hex");
const fixedSeed = process.env.DB_FILE_KEY || "termix-database-file-encryption-seed-v1";
key = crypto.pbkdf2Sync(fixedSeed, salt, 100000, 32, "sha256");
} else {
throw new Error(`Unsupported encryption version: ${metadata.version}`);
}
// Decrypt the file
const decipher = crypto.createDecipheriv(
@@ -344,10 +355,10 @@ class DatabaseFileEncryption {
/**
* Securely backup database by creating encrypted copy
*/
static createEncryptedBackup(
static async createEncryptedBackup(
databasePath: string,
backupDir: string,
): string {
): Promise<string> {
if (!fs.existsSync(databasePath)) {
throw new Error(`Database file does not exist: ${databasePath}`);
}
@@ -363,7 +374,7 @@ class DatabaseFileEncryption {
const backupPath = path.join(backupDir, backupFileName);
try {
const encryptedPath = this.encryptDatabaseFile(databasePath, backupPath);
const encryptedPath = await this.encryptDatabaseFile(databasePath, backupPath);
databaseLogger.info("Encrypted database backup created", {
operation: "database_backup",
@@ -386,16 +397,16 @@ class DatabaseFileEncryption {
/**
* Restore database from encrypted backup
*/
static restoreFromEncryptedBackup(
static async restoreFromEncryptedBackup(
backupPath: string,
targetPath: string,
): string {
): Promise<string> {
if (!this.isEncryptedDatabaseFile(backupPath)) {
throw new Error("Invalid encrypted backup file");
}
try {
const restoredPath = this.decryptDatabaseFile(backupPath, targetPath);
const restoredPath = await this.decryptDatabaseFile(backupPath, targetPath);
databaseLogger.info("Database restored from encrypted backup", {
operation: "database_restore",

View File

@@ -7,7 +7,7 @@ import { eq } from "drizzle-orm";
import { databaseLogger } from "./logger.js";
/**
* SystemCrypto - Open source friendly JWT key management
* SystemCrypto - Open source friendly system key management
*
* Linus principles:
* - Remove complex "system master key" layer - doesn't solve real threats
@@ -18,10 +18,13 @@ import { databaseLogger } from "./logger.js";
class SystemCrypto {
private static instance: SystemCrypto;
private jwtSecret: string | null = null;
private databaseKey: Buffer | null = null;
// Storage path configuration
private static readonly JWT_SECRET_FILE = path.join(process.cwd(), '.termix', 'jwt.key');
private static readonly JWT_SECRET_DB_KEY = 'system_jwt_secret';
private static readonly DATABASE_KEY_FILE = path.join(process.cwd(), '.termix', 'db.key');
private static readonly DATABASE_KEY_DB_KEY = 'system_database_key';
private constructor() {}
@@ -95,6 +98,58 @@ class SystemCrypto {
return this.jwtSecret!;
}
/**
* Initialize database encryption key - same pattern as JWT but for database file encryption
*/
async initializeDatabaseKey(): Promise<void> {
try {
databaseLogger.info("Initializing database encryption key", {
operation: "db_key_init",
});
// 1. Environment variable priority (production best practice)
const envKey = process.env.DATABASE_KEY;
if (envKey && envKey.length >= 64) {
this.databaseKey = Buffer.from(envKey, 'hex');
databaseLogger.info("✅ Using database key from environment variable", {
operation: "db_key_env_loaded",
source: "environment"
});
return;
}
// 2. Check filesystem storage
const fileKey = await this.loadDatabaseKeyFromFile();
if (fileKey) {
this.databaseKey = fileKey;
databaseLogger.info("✅ Loaded database key from file", {
operation: "db_key_file_loaded",
source: "file"
});
return;
}
// 3. Generate new key and persist (NO database storage to avoid circular dependency)
await this.generateAndStoreDatabaseKey();
} catch (error) {
databaseLogger.error("Failed to initialize database key", error, {
operation: "db_key_init_failed",
});
throw new Error("Database key initialization failed");
}
}
/**
* Get database encryption key
*/
async getDatabaseKey(): Promise<Buffer> {
if (!this.databaseKey) {
await this.initializeDatabaseKey();
}
return this.databaseKey!;
}
/**
* Generate new key and persist storage
*/
@@ -168,7 +223,77 @@ class SystemCrypto {
return null;
}
// ===== Database storage methods =====
// ===== Database key generation and storage methods =====
/**
* Generate new database key and persist to file storage only
* (avoid circular dependency with database)
*/
private async generateAndStoreDatabaseKey(): Promise<void> {
const newKey = crypto.randomBytes(32); // 256-bit key for AES-256
const instanceId = crypto.randomBytes(8).toString('hex');
databaseLogger.info("🔑 Generating new database encryption key for this Termix instance", {
operation: "db_key_generate",
instanceId
});
// Only try file storage (no database storage to avoid circular dependency)
try {
await this.saveDatabaseKeyToFile(newKey);
databaseLogger.info("✅ Database key saved to file", {
operation: "db_key_file_saved",
path: SystemCrypto.DATABASE_KEY_FILE
});
} catch (fileError) {
databaseLogger.error("❌ Failed to save database key to file", {
operation: "db_key_file_save_failed",
error: fileError instanceof Error ? fileError.message : "Unknown error",
note: "Database encryption cannot work without persistent key storage"
});
throw new Error("Database key file storage is required for database encryption");
}
this.databaseKey = newKey;
databaseLogger.success("🔐 This Termix instance now has a unique database encryption key", {
operation: "db_key_generated_success",
instanceId,
note: "Database file is now encrypted at rest"
});
}
/**
* Save database key to file (binary format)
*/
private async saveDatabaseKeyToFile(key: Buffer): Promise<void> {
const dir = path.dirname(SystemCrypto.DATABASE_KEY_FILE);
await fs.mkdir(dir, { recursive: true });
await fs.writeFile(SystemCrypto.DATABASE_KEY_FILE, key.toString('hex'), {
mode: 0o600 // Only owner can read/write
});
}
/**
* Load database key from file
*/
private async loadDatabaseKeyFromFile(): Promise<Buffer | null> {
try {
const keyHex = await fs.readFile(SystemCrypto.DATABASE_KEY_FILE, 'utf8');
if (keyHex.trim().length >= 64) { // 32 bytes = 64 hex chars
return Buffer.from(keyHex.trim(), 'hex');
}
databaseLogger.warn("Database key file exists but too short", {
operation: "db_key_file_invalid",
length: keyHex.length
});
} catch (error) {
// File doesn't exist or can't be read, this is normal
}
return null;
}
// ===== JWT Database storage methods =====
/**
* Save key to database (plaintext storage, don't pretend encryption helps)