import express from "express"; import net from "net"; import cors from "cors"; import cookieParser from "cookie-parser"; import { Client, type ConnectConfig } from "ssh2"; import { getDb } from "../database/db/index.js"; import { sshData, sshCredentials } from "../database/db/schema.js"; import { eq, and } from "drizzle-orm"; import { statsLogger, sshLogger } from "../utils/logger.js"; import { SimpleDBOps } from "../utils/simple-db-ops.js"; import { AuthManager } from "../utils/auth-manager.js"; import type { AuthenticatedRequest, ProxyNode } from "../../types/index.js"; import { collectCpuMetrics } from "./widgets/cpu-collector.js"; import { collectMemoryMetrics } from "./widgets/memory-collector.js"; import { collectDiskMetrics } from "./widgets/disk-collector.js"; import { collectNetworkMetrics } from "./widgets/network-collector.js"; import { collectUptimeMetrics } from "./widgets/uptime-collector.js"; import { collectProcessesMetrics } from "./widgets/processes-collector.js"; import { collectSystemMetrics } from "./widgets/system-collector.js"; import { collectLoginStats } from "./widgets/login-stats-collector.js"; import { createSocks5Connection } from "../utils/socks5-helper.js"; async function resolveJumpHost( hostId: number, userId: string, ): Promise { try { const hosts = await SimpleDBOps.select( getDb() .select() .from(sshData) .where(and(eq(sshData.id, hostId), eq(sshData.userId, userId))), "ssh_data", userId, ); if (hosts.length === 0) { return null; } const host = hosts[0]; if (host.credentialId) { const credentials = await SimpleDBOps.select( getDb() .select() .from(sshCredentials) .where( and( eq(sshCredentials.id, host.credentialId as number), eq(sshCredentials.userId, userId), ), ), "ssh_credentials", userId, ); if (credentials.length > 0) { const credential = credentials[0]; return { ...host, password: credential.password, key: credential.private_key || credential.privateKey || credential.key, keyPassword: credential.key_password || credential.keyPassword, keyType: credential.key_type || credential.keyType, authType: credential.auth_type || credential.authType, }; } } return host; } catch (error) { statsLogger.error("Failed to resolve jump host", error, { operation: "resolve_jump_host", hostId, userId, }); return null; } } async function createJumpHostChain( jumpHosts: Array<{ hostId: number }>, userId: string, ): Promise { if (!jumpHosts || jumpHosts.length === 0) { return null; } let currentClient: Client | null = null; const clients: Client[] = []; try { for (let i = 0; i < jumpHosts.length; i++) { const jumpHostConfig = await resolveJumpHost(jumpHosts[i].hostId, userId); if (!jumpHostConfig) { statsLogger.error(`Jump host ${i + 1} not found`, undefined, { operation: "jump_host_chain", hostId: jumpHosts[i].hostId, }); clients.forEach((c) => c.end()); return null; } const jumpClient = new Client(); clients.push(jumpClient); const connected = await new Promise((resolve) => { const timeout = setTimeout(() => { resolve(false); }, 30000); jumpClient.on("ready", () => { clearTimeout(timeout); resolve(true); }); jumpClient.on("error", (err) => { clearTimeout(timeout); statsLogger.error(`Jump host ${i + 1} connection failed`, err, { operation: "jump_host_connect", hostId: jumpHostConfig.id, ip: jumpHostConfig.ip, }); resolve(false); }); const connectConfig: any = { host: jumpHostConfig.ip, port: jumpHostConfig.port || 22, username: jumpHostConfig.username, tryKeyboard: true, readyTimeout: 30000, }; if (jumpHostConfig.authType === "password" && jumpHostConfig.password) { connectConfig.password = jumpHostConfig.password; } else if (jumpHostConfig.authType === "key" && jumpHostConfig.key) { const cleanKey = jumpHostConfig.key .trim() .replace(/\r\n/g, "\n") .replace(/\r/g, "\n"); connectConfig.privateKey = Buffer.from(cleanKey, "utf8"); if (jumpHostConfig.keyPassword) { connectConfig.passphrase = jumpHostConfig.keyPassword; } } if (currentClient) { currentClient.forwardOut( "127.0.0.1", 0, jumpHostConfig.ip, jumpHostConfig.port || 22, (err, stream) => { if (err) { clearTimeout(timeout); resolve(false); return; } connectConfig.sock = stream; jumpClient.connect(connectConfig); }, ); } else { jumpClient.connect(connectConfig); } }); if (!connected) { clients.forEach((c) => c.end()); return null; } currentClient = jumpClient; } return currentClient; } catch (error) { statsLogger.error("Failed to create jump host chain", error, { operation: "jump_host_chain", }); clients.forEach((c) => c.end()); return null; } } interface PooledConnection { client: Client; lastUsed: number; inUse: boolean; hostKey: string; } interface MetricsSession { client: Client; isConnected: boolean; lastActive: number; timeout?: NodeJS.Timeout; activeOperations: number; hostId: number; userId: string; } interface PendingTOTPSession { client: Client; finish: (responses: string[]) => void; config: ConnectConfig; createdAt: number; sessionId: string; hostId: number; userId: string; prompts?: Array<{ prompt: string; echo: boolean }>; totpPromptIndex?: number; resolvedPassword?: string; totpAttempts: number; } const metricsSessions: Record = {}; const pendingTOTPSessions: Record = {}; function cleanupMetricsSession(sessionId: string) { const session = metricsSessions[sessionId]; if (session) { if (session.activeOperations > 0) { statsLogger.warn( `Deferring metrics session cleanup - ${session.activeOperations} active operations`, { operation: "cleanup_deferred", sessionId, activeOperations: session.activeOperations, }, ); scheduleMetricsSessionCleanup(sessionId); return; } try { session.client.end(); } catch (error) {} clearTimeout(session.timeout); delete metricsSessions[sessionId]; } } function scheduleMetricsSessionCleanup(sessionId: string) { const session = metricsSessions[sessionId]; if (session) { if (session.timeout) clearTimeout(session.timeout); session.timeout = setTimeout( () => { cleanupMetricsSession(sessionId); }, 30 * 60 * 1000, ); } } function getSessionKey(hostId: number, userId: string): string { return `${userId}:${hostId}`; } class SSHConnectionPool { private connections = new Map(); private maxConnectionsPerHost = 3; private connectionTimeout = 30000; private cleanupInterval: NodeJS.Timeout; constructor() { this.cleanupInterval = setInterval( () => { this.cleanup(); }, 2 * 60 * 1000, ); } private getHostKey(host: SSHHostWithCredentials): string { const socks5Key = host.useSocks5 ? `:socks5:${host.socks5Host}:${host.socks5Port}:${JSON.stringify(host.socks5ProxyChain || [])}` : ""; return `${host.ip}:${host.port}:${host.username}${socks5Key}`; } private isConnectionHealthy(client: Client): boolean { try { const sock = (client as any)._sock; if (sock && (sock.destroyed || !sock.writable)) { return false; } return true; } catch (error) { return false; } } async getConnection(host: SSHHostWithCredentials): Promise { const hostKey = this.getHostKey(host); let connections = this.connections.get(hostKey) || []; const available = connections.find((conn) => !conn.inUse); if (available) { if (!this.isConnectionHealthy(available.client)) { statsLogger.warn("Removing unhealthy connection from pool", { operation: "remove_dead_connection", hostKey, }); try { available.client.end(); } catch (error) { // Ignore cleanup errors } connections = connections.filter((c) => c !== available); this.connections.set(hostKey, connections); } else { available.inUse = true; available.lastUsed = Date.now(); return available.client; } } if (connections.length < this.maxConnectionsPerHost) { const client = await this.createConnection(host); const pooled: PooledConnection = { client, lastUsed: Date.now(), inUse: true, hostKey, }; connections.push(pooled); this.connections.set(hostKey, connections); return client; } return new Promise((resolve) => { const checkAvailable = () => { const available = connections.find((conn) => !conn.inUse); if (available) { available.inUse = true; available.lastUsed = Date.now(); resolve(available.client); } else { setTimeout(checkAvailable, 100); } }; checkAvailable(); }); } private async createConnection( host: SSHHostWithCredentials, ): Promise { return new Promise(async (resolve, reject) => { const config = buildSshConfig(host); const client = new Client(); const timeout = setTimeout(() => { client.end(); reject(new Error("SSH connection timeout")); }, this.connectionTimeout); client.on("ready", () => { clearTimeout(timeout); resolve(client); }); client.on("error", (err) => { clearTimeout(timeout); reject(err); }); client.on( "keyboard-interactive", ( name: string, instructions: string, instructionsLang: string, prompts: Array<{ prompt: string; echo: boolean }>, finish: (responses: string[]) => void, ) => { const totpPromptIndex = prompts.findIndex((p) => /verification code|verification_code|token|otp|2fa|authenticator|google.*auth/i.test( p.prompt, ), ); if (totpPromptIndex !== -1) { const sessionId = `totp-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; pendingTOTPSessions[sessionId] = { client, finish, config, createdAt: Date.now(), sessionId, hostId: host.id, userId: host.userId!, prompts: prompts.map((p) => ({ prompt: p.prompt, echo: p.echo ?? false, })), totpPromptIndex, resolvedPassword: host.password, totpAttempts: 0, }; return; } else if (host.password) { const responses = prompts.map((p) => { if (/password/i.test(p.prompt)) { return host.password || ""; } return ""; }); finish(responses); } else { finish(prompts.map(() => "")); } }, ); try { if ( host.useSocks5 && (host.socks5Host || (host.socks5ProxyChain && host.socks5ProxyChain.length > 0)) ) { try { const socks5Socket = await createSocks5Connection( host.ip, host.port, { useSocks5: host.useSocks5, socks5Host: host.socks5Host, socks5Port: host.socks5Port, socks5Username: host.socks5Username, socks5Password: host.socks5Password, socks5ProxyChain: host.socks5ProxyChain, }, ); if (socks5Socket) { config.sock = socks5Socket; client.connect(config); return; } else { statsLogger.error("SOCKS5 socket is null", undefined, { operation: "socks5_socket_null", hostIp: host.ip, }); } } catch (socks5Error) { clearTimeout(timeout); statsLogger.error("SOCKS5 connection error", socks5Error, { operation: "socks5_connection_error", hostIp: host.ip, errorMessage: socks5Error instanceof Error ? socks5Error.message : "Unknown", }); reject( new Error( "SOCKS5 proxy connection failed: " + (socks5Error instanceof Error ? socks5Error.message : "Unknown error"), ), ); return; } } if (host.jumpHosts && host.jumpHosts.length > 0 && host.userId) { const jumpClient = await createJumpHostChain( host.jumpHosts, host.userId, ); if (!jumpClient) { clearTimeout(timeout); reject(new Error("Failed to establish jump host chain")); return; } jumpClient.forwardOut( "127.0.0.1", 0, host.ip, host.port, (err, stream) => { if (err) { clearTimeout(timeout); jumpClient.end(); reject( new Error( "Failed to forward through jump host: " + err.message, ), ); return; } config.sock = stream; client.connect(config); }, ); } else { client.connect(config); } } catch (err) { clearTimeout(timeout); reject(err); } }); } releaseConnection(host: SSHHostWithCredentials, client: Client): void { const hostKey = this.getHostKey(host); const connections = this.connections.get(hostKey) || []; const pooled = connections.find((conn) => conn.client === client); if (pooled) { pooled.inUse = false; pooled.lastUsed = Date.now(); } } clearHostConnections(host: SSHHostWithCredentials): void { const hostKey = this.getHostKey(host); const connections = this.connections.get(hostKey) || []; for (const conn of connections) { try { conn.client.end(); } catch (error) { statsLogger.error("Error closing connection during cleanup", error, { operation: "clear_connection_error", }); } } this.connections.delete(hostKey); } private cleanup(): void { const now = Date.now(); const maxAge = 10 * 60 * 1000; let totalCleaned = 0; let totalUnhealthy = 0; for (const [hostKey, connections] of this.connections.entries()) { const activeConnections = connections.filter((conn) => { if (!conn.inUse && now - conn.lastUsed > maxAge) { try { conn.client.end(); } catch (error) {} totalCleaned++; return false; } if (!this.isConnectionHealthy(conn.client)) { statsLogger.warn("Removing unhealthy connection during cleanup", { operation: "cleanup_unhealthy", hostKey, inUse: conn.inUse, }); try { conn.client.end(); } catch (error) {} totalUnhealthy++; return false; } return true; }); if (activeConnections.length === 0) { this.connections.delete(hostKey); } else { this.connections.set(hostKey, activeConnections); } } } clearAllConnections(): void { for (const [hostKey, connections] of this.connections.entries()) { for (const conn of connections) { try { conn.client.end(); } catch (error) { statsLogger.error( "Error closing connection during full cleanup", error, { operation: "clear_all_error", hostKey, }, ); } } } this.connections.clear(); } destroy(): void { clearInterval(this.cleanupInterval); for (const connections of this.connections.values()) { for (const conn of connections) { try { conn.client.end(); } catch (error) {} } } this.connections.clear(); } } class RequestQueue { private queues = new Map Promise>>(); private processing = new Set(); private requestTimeout = 60000; async queueRequest(hostId: number, request: () => Promise): Promise { return new Promise((resolve, reject) => { const wrappedRequest = async () => { try { const result = await Promise.race([ request(), new Promise((_, rej) => setTimeout( () => rej( new Error( `Request timeout after ${this.requestTimeout}ms for host ${hostId}`, ), ), this.requestTimeout, ), ), ]); resolve(result); } catch (error) { reject(error); } }; const queue = this.queues.get(hostId) || []; queue.push(wrappedRequest); this.queues.set(hostId, queue); this.processQueue(hostId); }); } private async processQueue(hostId: number): Promise { if (this.processing.has(hostId)) return; this.processing.add(hostId); const queue = this.queues.get(hostId) || []; while (queue.length > 0) { const request = queue.shift(); if (request) { try { await request(); } catch (error) {} } } this.processing.delete(hostId); const currentQueue = this.queues.get(hostId); if (currentQueue && currentQueue.length > 0) { this.processQueue(hostId); } } } interface CachedMetrics { data: unknown; timestamp: number; hostId: number; } class MetricsCache { private cache = new Map(); private ttl = 30000; get(hostId: number): unknown | null { const cached = this.cache.get(hostId); if (cached && Date.now() - cached.timestamp < this.ttl) { return cached.data; } return null; } set(hostId: number, data: unknown): void { this.cache.set(hostId, { data, timestamp: Date.now(), hostId, }); } clear(hostId?: number): void { if (hostId) { this.cache.delete(hostId); } else { this.cache.clear(); } } } interface AuthFailureRecord { count: number; lastFailure: number; reason: "TOTP" | "AUTH" | "TIMEOUT"; permanent: boolean; } class AuthFailureTracker { private failures = new Map(); private maxRetries = 3; private backoffBase = 5000; recordFailure( hostId: number, reason: "TOTP" | "AUTH" | "TIMEOUT", permanent = false, ): void { const existing = this.failures.get(hostId); if (existing) { existing.count++; existing.lastFailure = Date.now(); existing.reason = reason; if (permanent) existing.permanent = true; } else { this.failures.set(hostId, { count: 1, lastFailure: Date.now(), reason, permanent, }); } } shouldSkip(hostId: number): boolean { const record = this.failures.get(hostId); if (!record) return false; if (record.reason === "TOTP" || record.permanent) { return true; } if (record.count >= this.maxRetries) { return true; } const backoffTime = this.backoffBase * Math.pow(2, record.count - 1); const timeSinceFailure = Date.now() - record.lastFailure; return timeSinceFailure < backoffTime; } getSkipReason(hostId: number): string | null { const record = this.failures.get(hostId); if (!record) return null; if (record.reason === "TOTP") { return "TOTP authentication required (metrics unavailable)"; } if (record.permanent) { return "Authentication permanently failed"; } if (record.count >= this.maxRetries) { return `Too many authentication failures (${record.count} attempts)`; } const backoffTime = this.backoffBase * Math.pow(2, record.count - 1); const timeSinceFailure = Date.now() - record.lastFailure; const remainingTime = Math.ceil((backoffTime - timeSinceFailure) / 1000); if (timeSinceFailure < backoffTime) { return `Retry in ${remainingTime}s (attempt ${record.count}/${this.maxRetries})`; } return null; } reset(hostId: number): void { this.failures.delete(hostId); } cleanup(): void { const maxAge = 60 * 60 * 1000; const now = Date.now(); for (const [hostId, record] of this.failures.entries()) { if (!record.permanent && now - record.lastFailure > maxAge) { this.failures.delete(hostId); } } } } class PollingBackoff { private failures = new Map(); private baseDelay = 30000; private maxDelay = 600000; private maxRetries = 5; recordFailure(hostId: number): void { const existing = this.failures.get(hostId) || { count: 0, nextRetry: 0 }; const delay = Math.min( this.baseDelay * Math.pow(2, existing.count), this.maxDelay, ); this.failures.set(hostId, { count: existing.count + 1, nextRetry: Date.now() + delay, }); } shouldSkip(hostId: number): boolean { const backoff = this.failures.get(hostId); if (!backoff) return false; if (backoff.count >= this.maxRetries) { return true; } return Date.now() < backoff.nextRetry; } getBackoffInfo(hostId: number): string | null { const backoff = this.failures.get(hostId); if (!backoff) return null; if (backoff.count >= this.maxRetries) { return `Max retries exceeded (${backoff.count} failures) - polling suspended`; } const remainingMs = backoff.nextRetry - Date.now(); if (remainingMs > 0) { const remainingSec = Math.ceil(remainingMs / 1000); return `Retry in ${remainingSec}s (attempt ${backoff.count}/${this.maxRetries})`; } return null; } reset(hostId: number): void { this.failures.delete(hostId); } cleanup(): void { const maxAge = 60 * 60 * 1000; const now = Date.now(); for (const [hostId, backoff] of this.failures.entries()) { if (backoff.count < this.maxRetries && now - backoff.nextRetry > maxAge) { this.failures.delete(hostId); } } } } const connectionPool = new SSHConnectionPool(); const requestQueue = new RequestQueue(); const metricsCache = new MetricsCache(); const authFailureTracker = new AuthFailureTracker(); const pollingBackoff = new PollingBackoff(); const authManager = AuthManager.getInstance(); type HostStatus = "online" | "offline"; interface SSHHostWithCredentials { id: number; name: string; ip: string; port: number; username: string; folder: string; tags: string[]; pin: boolean; authType: string; password?: string; key?: string; keyPassword?: string; keyType?: string; credentialId?: number; enableTerminal: boolean; enableTunnel: boolean; enableFileManager: boolean; defaultPath: string; tunnelConnections: unknown[]; jumpHosts?: Array<{ hostId: number }>; statsConfig?: string | StatsConfig; createdAt: string; updatedAt: string; userId: string; useSocks5?: boolean; socks5Host?: string; socks5Port?: number; socks5Username?: string; socks5Password?: string; socks5ProxyChain?: ProxyNode[]; } type StatusEntry = { status: HostStatus; lastChecked: string; }; interface StatsConfig { enabledWidgets: string[]; statusCheckEnabled: boolean; statusCheckInterval: number; metricsEnabled: boolean; metricsInterval: number; } const DEFAULT_STATS_CONFIG: StatsConfig = { enabledWidgets: ["cpu", "memory", "disk", "network", "uptime", "system"], statusCheckEnabled: true, statusCheckInterval: 30, metricsEnabled: true, metricsInterval: 30, }; interface HostPollingConfig { host: SSHHostWithCredentials; statsConfig: StatsConfig; statusTimer?: NodeJS.Timeout; metricsTimer?: NodeJS.Timeout; } class PollingManager { private pollingConfigs = new Map(); private statusStore = new Map(); private metricsStore = new Map< number, { data: Awaited>; timestamp: number; } >(); parseStatsConfig(statsConfigStr?: string | StatsConfig): StatsConfig { if (!statsConfigStr) { return DEFAULT_STATS_CONFIG; } let parsed: StatsConfig; if (typeof statsConfigStr === "object") { parsed = statsConfigStr; } else { try { let temp: any = JSON.parse(statsConfigStr); if (typeof temp === "string") { temp = JSON.parse(temp); } parsed = temp; } catch (error) { statsLogger.warn( `Failed to parse statsConfig: ${error instanceof Error ? error.message : "Unknown error"}`, { operation: "parse_stats_config_error", statsConfigStr, }, ); return DEFAULT_STATS_CONFIG; } } const result = { ...DEFAULT_STATS_CONFIG, ...parsed }; return result; } async startPollingForHost( host: SSHHostWithCredentials, options?: { statusOnly?: boolean }, ): Promise { const statsConfig = this.parseStatsConfig(host.statsConfig); const statusOnly = options?.statusOnly ?? false; const existingConfig = this.pollingConfigs.get(host.id); if (existingConfig) { if (existingConfig.statusTimer) { clearInterval(existingConfig.statusTimer); existingConfig.statusTimer = undefined; } if (existingConfig.metricsTimer) { clearInterval(existingConfig.metricsTimer); existingConfig.metricsTimer = undefined; } } if (!statsConfig.statusCheckEnabled && !statsConfig.metricsEnabled) { this.pollingConfigs.delete(host.id); this.statusStore.delete(host.id); this.metricsStore.delete(host.id); return; } const config: HostPollingConfig = { host, statsConfig, }; if (statsConfig.statusCheckEnabled) { const intervalMs = statsConfig.statusCheckInterval * 1000; this.pollHostStatus(host); config.statusTimer = setInterval(() => { const latestConfig = this.pollingConfigs.get(host.id); if (latestConfig && latestConfig.statsConfig.statusCheckEnabled) { this.pollHostStatus(latestConfig.host); } }, intervalMs); } else { this.statusStore.delete(host.id); } if (!statusOnly && statsConfig.metricsEnabled) { const intervalMs = statsConfig.metricsInterval * 1000; await this.pollHostMetrics(host); config.metricsTimer = setInterval(() => { const latestConfig = this.pollingConfigs.get(host.id); if (latestConfig && latestConfig.statsConfig.metricsEnabled) { this.pollHostMetrics(latestConfig.host); } }, intervalMs); } else { this.metricsStore.delete(host.id); } this.pollingConfigs.set(host.id, config); } private async pollHostStatus(host: SSHHostWithCredentials): Promise { const refreshedHost = await fetchHostById(host.id, host.userId); if (!refreshedHost) { statsLogger.warn("Host not found during status polling", { operation: "poll_host_status", hostId: host.id, }); return; } try { const isOnline = await tcpPing( refreshedHost.ip, refreshedHost.port, 5000, ); const statusEntry: StatusEntry = { status: isOnline ? "online" : "offline", lastChecked: new Date().toISOString(), }; this.statusStore.set(refreshedHost.id, statusEntry); } catch (error) { const statusEntry: StatusEntry = { status: "offline", lastChecked: new Date().toISOString(), }; this.statusStore.set(refreshedHost.id, statusEntry); } } private async pollHostMetrics(host: SSHHostWithCredentials): Promise { const refreshedHost = await fetchHostById(host.id, host.userId); if (!refreshedHost) { statsLogger.warn("Host not found during metrics polling", { operation: "poll_host_metrics", hostId: host.id, }); return; } const config = this.pollingConfigs.get(refreshedHost.id); if (!config || !config.statsConfig.metricsEnabled) { return; } const hasExistingMetrics = this.metricsStore.has(refreshedHost.id); if (hasExistingMetrics && pollingBackoff.shouldSkip(host.id)) { const backoffInfo = pollingBackoff.getBackoffInfo(host.id); return; } try { const metrics = await collectMetrics(refreshedHost); this.metricsStore.set(refreshedHost.id, { data: metrics, timestamp: Date.now(), }); pollingBackoff.reset(refreshedHost.id); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); pollingBackoff.recordFailure(refreshedHost.id); const latestConfig = this.pollingConfigs.get(refreshedHost.id); if (latestConfig && latestConfig.statsConfig.metricsEnabled) { const backoffInfo = pollingBackoff.getBackoffInfo(refreshedHost.id); statsLogger.error("Failed to collect metrics for host", { operation: "metrics_poll_failed", hostId: refreshedHost.id, hostName: refreshedHost.name, error: errorMessage, backoff: backoffInfo, }); } } } stopPollingForHost(hostId: number, clearData = true): void { const config = this.pollingConfigs.get(hostId); if (config) { if (config.statusTimer) { clearInterval(config.statusTimer); config.statusTimer = undefined; } if (config.metricsTimer) { clearInterval(config.metricsTimer); config.metricsTimer = undefined; } this.pollingConfigs.delete(hostId); if (clearData) { this.statusStore.delete(hostId); this.metricsStore.delete(hostId); } } } stopMetricsOnly(hostId: number): void { const config = this.pollingConfigs.get(hostId); if (config?.metricsTimer) { clearInterval(config.metricsTimer); config.metricsTimer = undefined; } } getStatus(hostId: number): StatusEntry | undefined { return this.statusStore.get(hostId); } getAllStatuses(): Map { return this.statusStore; } getMetrics( hostId: number, ): | { data: Awaited>; timestamp: number } | undefined { return this.metricsStore.get(hostId); } async initializePolling(userId: string): Promise { const hosts = await fetchAllHosts(userId); for (const host of hosts) { await this.startPollingForHost(host, { statusOnly: true }); } } async refreshHostPolling(userId: string): Promise { const hosts = await fetchAllHosts(userId); const currentHostIds = new Set(hosts.map((h) => h.id)); for (const hostId of this.pollingConfigs.keys()) { this.stopPollingForHost(hostId, false); } for (const hostId of this.statusStore.keys()) { if (!currentHostIds.has(hostId)) { this.statusStore.delete(hostId); this.metricsStore.delete(hostId); } } for (const host of hosts) { await this.startPollingForHost(host, { statusOnly: true }); } } destroy(): void { for (const hostId of this.pollingConfigs.keys()) { this.stopPollingForHost(hostId); } } } const pollingManager = new PollingManager(); function validateHostId( req: express.Request, res: express.Response, next: express.NextFunction, ) { const id = Number(req.params.id); if (!id || !Number.isInteger(id) || id <= 0) { return res.status(400).json({ error: "Invalid host ID" }); } next(); } const app = express(); app.use( cors({ origin: (origin, callback) => { if (!origin) return callback(null, true); const allowedOrigins = [ "http://localhost:5173", "http://localhost:3000", "http://127.0.0.1:5173", "http://127.0.0.1:3000", ]; if (allowedOrigins.includes(origin)) { return callback(null, true); } if (origin.startsWith("https://")) { return callback(null, true); } if (origin.startsWith("http://")) { return callback(null, true); } callback(new Error("Not allowed by CORS")); }, credentials: true, methods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], allowedHeaders: [ "Content-Type", "Authorization", "User-Agent", "X-Electron-App", ], }), ); app.use(cookieParser()); app.use(express.json({ limit: "1mb" })); app.use(authManager.createAuthMiddleware()); async function fetchAllHosts( userId: string, ): Promise { try { const hosts = await SimpleDBOps.select( getDb().select().from(sshData).where(eq(sshData.userId, userId)), "ssh_data", userId, ); const hostsWithCredentials: SSHHostWithCredentials[] = []; for (const host of hosts) { try { const hostWithCreds = await resolveHostCredentials(host, userId); if (hostWithCreds) { hostsWithCredentials.push(hostWithCreds); } } catch (err) { statsLogger.warn( `Failed to resolve credentials for host ${host.id}: ${err instanceof Error ? err.message : "Unknown error"}`, ); } } return hostsWithCredentials.filter((h) => !!h.id && !!h.ip && !!h.port); } catch (err) { statsLogger.error("Failed to fetch hosts from database", err); return []; } } async function fetchHostById( id: number, userId: string, ): Promise { try { if (!SimpleDBOps.isUserDataUnlocked(userId)) { return undefined; } const hosts = await SimpleDBOps.select( getDb() .select() .from(sshData) .where(and(eq(sshData.id, id), eq(sshData.userId, userId))), "ssh_data", userId, ); if (hosts.length === 0) { return undefined; } const host = hosts[0]; return await resolveHostCredentials(host, userId); } catch (err) { statsLogger.error(`Failed to fetch host ${id}`, err); return undefined; } } async function resolveHostCredentials( host: Record, userId: string, ): Promise { try { const baseHost: Record = { id: host.id, name: host.name, ip: host.ip, port: host.port, username: host.username, folder: host.folder || "", tags: typeof host.tags === "string" ? host.tags ? host.tags.split(",").filter(Boolean) : [] : [], pin: !!host.pin, authType: host.authType, enableTerminal: !!host.enableTerminal, enableTunnel: !!host.enableTunnel, enableFileManager: !!host.enableFileManager, defaultPath: host.defaultPath || "/", tunnelConnections: host.tunnelConnections ? JSON.parse(host.tunnelConnections as string) : [], jumpHosts: host.jumpHosts ? JSON.parse(host.jumpHosts as string) : [], statsConfig: host.statsConfig || undefined, createdAt: host.createdAt, updatedAt: host.updatedAt, userId: host.userId, useSocks5: !!host.useSocks5, socks5Host: host.socks5Host || undefined, socks5Port: host.socks5Port || undefined, socks5Username: host.socks5Username || undefined, socks5Password: host.socks5Password || undefined, socks5ProxyChain: host.socks5ProxyChain ? JSON.parse(host.socks5ProxyChain as string) : undefined, }; if (host.credentialId) { try { const credentials = await SimpleDBOps.select( getDb() .select() .from(sshCredentials) .where( and( eq(sshCredentials.id, host.credentialId as number), eq(sshCredentials.userId, userId), ), ), "ssh_credentials", userId, ); if (credentials.length > 0) { const credential = credentials[0]; baseHost.credentialId = credential.id; baseHost.username = credential.username; baseHost.authType = credential.auth_type || credential.authType; if (credential.password) { baseHost.password = credential.password; } if (credential.key) { baseHost.key = credential.key; } if (credential.key_password || credential.keyPassword) { baseHost.keyPassword = credential.key_password || credential.keyPassword; } if (credential.key_type || credential.keyType) { baseHost.keyType = credential.key_type || credential.keyType; } } else { addLegacyCredentials(baseHost, host); } } catch (error) { statsLogger.warn( `Failed to resolve credential ${host.credentialId} for host ${host.id}: ${error instanceof Error ? error.message : "Unknown error"}`, ); addLegacyCredentials(baseHost, host); } } else { addLegacyCredentials(baseHost, host); } return baseHost as unknown as SSHHostWithCredentials; } catch (error) { statsLogger.error( `Failed to resolve host credentials for host ${host.id}: ${error instanceof Error ? error.message : "Unknown error"}`, ); return undefined; } } function addLegacyCredentials( baseHost: Record, host: Record, ): void { baseHost.password = host.password || null; baseHost.key = host.key || null; baseHost.keyPassword = host.key_password || host.keyPassword || null; baseHost.keyType = host.keyType; } function buildSshConfig(host: SSHHostWithCredentials): ConnectConfig { const base: ConnectConfig = { host: host.ip, port: host.port, username: host.username, tryKeyboard: true, keepaliveInterval: 30000, keepaliveCountMax: 3, readyTimeout: 60000, tcpKeepAlive: true, tcpKeepAliveInitialDelay: 30000, env: { TERM: "xterm-256color", LANG: "en_US.UTF-8", LC_ALL: "en_US.UTF-8", LC_CTYPE: "en_US.UTF-8", LC_MESSAGES: "en_US.UTF-8", LC_MONETARY: "en_US.UTF-8", LC_NUMERIC: "en_US.UTF-8", LC_TIME: "en_US.UTF-8", LC_COLLATE: "en_US.UTF-8", COLORTERM: "truecolor", }, algorithms: { kex: [ "curve25519-sha256", "curve25519-sha256@libssh.org", "ecdh-sha2-nistp521", "ecdh-sha2-nistp384", "ecdh-sha2-nistp256", "diffie-hellman-group-exchange-sha256", "diffie-hellman-group14-sha256", "diffie-hellman-group14-sha1", "diffie-hellman-group-exchange-sha1", "diffie-hellman-group1-sha1", ], serverHostKey: [ "ssh-ed25519", "ecdsa-sha2-nistp521", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp256", "rsa-sha2-512", "rsa-sha2-256", "ssh-rsa", "ssh-dss", ], cipher: [ "chacha20-poly1305@openssh.com", "aes256-gcm@openssh.com", "aes128-gcm@openssh.com", "aes256-ctr", "aes192-ctr", "aes128-ctr", "aes256-cbc", "aes192-cbc", "aes128-cbc", "3des-cbc", ], hmac: [ "hmac-sha2-512-etm@openssh.com", "hmac-sha2-256-etm@openssh.com", "hmac-sha2-512", "hmac-sha2-256", "hmac-sha1", "hmac-md5", ], compress: ["none", "zlib@openssh.com", "zlib"], }, } as ConnectConfig; if (host.authType === "password") { if (!host.password) { throw new Error(`No password available for host ${host.ip}`); } base.password = host.password; } else if (host.authType === "key") { if (!host.key) { throw new Error(`No SSH key available for host ${host.ip}`); } try { if (!host.key.includes("-----BEGIN") || !host.key.includes("-----END")) { throw new Error("Invalid private key format"); } const cleanKey = host.key .trim() .replace(/\r\n/g, "\n") .replace(/\r/g, "\n"); (base as Record).privateKey = Buffer.from( cleanKey, "utf8", ); if (host.keyPassword) { (base as Record).passphrase = host.keyPassword; } } catch (keyError) { statsLogger.error( `SSH key format error for host ${host.ip}: ${keyError instanceof Error ? keyError.message : "Unknown error"}`, ); throw new Error(`Invalid SSH key format for host ${host.ip}`); } } else if (host.authType === "none") { } else { throw new Error( `Unsupported authentication type '${host.authType}' for host ${host.ip}`, ); } return base; } async function withSshConnection( host: SSHHostWithCredentials, fn: (client: Client) => Promise, ): Promise { const client = await connectionPool.getConnection(host); try { const result = await fn(client); return result; } finally { connectionPool.releaseConnection(host, client); } } async function collectMetrics(host: SSHHostWithCredentials): Promise<{ cpu: { percent: number | null; cores: number | null; load: [number, number, number] | null; }; memory: { percent: number | null; usedGiB: number | null; totalGiB: number | null; }; disk: { percent: number | null; usedHuman: string | null; totalHuman: string | null; availableHuman: string | null; }; network: { interfaces: Array<{ name: string; ip: string; state: string; rxBytes: string | null; txBytes: string | null; }>; }; uptime: { seconds: number | null; formatted: string | null; }; processes: { total: number | null; running: number | null; top: Array<{ pid: string; user: string; cpu: string; mem: string; command: string; }>; }; system: { hostname: string | null; kernel: string | null; os: string | null; }; }> { if (authFailureTracker.shouldSkip(host.id)) { const reason = authFailureTracker.getSkipReason(host.id); throw new Error(reason || "Authentication failed"); } const cached = metricsCache.get(host.id); if (cached) { return cached as ReturnType extends Promise ? T : never; } return requestQueue.queueRequest(host.id, async () => { const sessionKey = getSessionKey(host.id, host.userId!); const existingSession = metricsSessions[sessionKey]; try { const collectFn = async (client: Client) => { const cpu = await collectCpuMetrics(client); const memory = await collectMemoryMetrics(client); const disk = await collectDiskMetrics(client); const network = await collectNetworkMetrics(client); const uptime = await collectUptimeMetrics(client); const processes = await collectProcessesMetrics(client); const system = await collectSystemMetrics(client); let login_stats = { recentLogins: [], failedLogins: [], totalLogins: 0, uniqueIPs: 0, }; try { login_stats = await collectLoginStats(client); } catch (e) {} const result = { cpu, memory, disk, network, uptime, processes, system, login_stats, }; metricsCache.set(host.id, result); return result; }; if (existingSession && existingSession.isConnected) { existingSession.activeOperations++; try { const result = await collectFn(existingSession.client); existingSession.lastActive = Date.now(); return result; } finally { existingSession.activeOperations--; } } else { return await withSshConnection(host, collectFn); } } catch (error) { if (error instanceof Error) { if (error.message.includes("TOTP authentication required")) { throw error; } else if ( error.message.includes("No password available") || error.message.includes("Unsupported authentication type") || error.message.includes("No SSH key available") ) { authFailureTracker.recordFailure(host.id, "AUTH", true); } else if ( error.message.includes("authentication") || error.message.includes("Permission denied") || error.message.includes("All configured authentication methods failed") ) { authFailureTracker.recordFailure(host.id, "AUTH"); } else if ( error.message.includes("timeout") || error.message.includes("ETIMEDOUT") ) { authFailureTracker.recordFailure(host.id, "TIMEOUT"); } } throw error; } }); } function tcpPing( host: string, port: number, timeoutMs = 5000, ): Promise { return new Promise((resolve) => { const socket = new net.Socket(); let settled = false; const onDone = (result: boolean) => { if (settled) return; settled = true; try { socket.destroy(); } catch (error) {} resolve(result); }; socket.setTimeout(timeoutMs); socket.once("connect", () => onDone(true)); socket.once("timeout", () => onDone(false)); socket.once("error", () => onDone(false)); socket.connect(port, host); }); } app.get("/status", async (req, res) => { const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } const statuses = pollingManager.getAllStatuses(); if (statuses.size === 0) { await pollingManager.initializePolling(userId); } const result: Record = {}; for (const [id, entry] of pollingManager.getAllStatuses().entries()) { result[id] = entry; } res.json(result); }); app.get("/status/:id", validateHostId, async (req, res) => { const id = Number(req.params.id); const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } const statuses = pollingManager.getAllStatuses(); if (statuses.size === 0) { await pollingManager.initializePolling(userId); } const statusEntry = pollingManager.getStatus(id); if (!statusEntry) { return res.status(404).json({ error: "Status not available" }); } res.json(statusEntry); }); app.post("/clear-connections", async (req, res) => { const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } connectionPool.clearAllConnections(); res.json({ message: "All SSH connections cleared" }); }); app.post("/refresh", async (req, res) => { const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } connectionPool.clearAllConnections(); await pollingManager.refreshHostPolling(userId); res.json({ message: "Polling refreshed" }); }); app.post("/host-updated", async (req, res) => { const userId = (req as AuthenticatedRequest).userId; const { hostId } = req.body; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } if (!hostId || typeof hostId !== "number") { return res.status(400).json({ error: "Invalid hostId" }); } try { const host = await fetchHostById(hostId, userId); if (host) { connectionPool.clearHostConnections(host); await pollingManager.startPollingForHost(host); res.json({ message: "Host polling started" }); } else { res.status(404).json({ error: "Host not found" }); } } catch (error) { statsLogger.error("Failed to start polling for host", error, { operation: "host_updated", hostId, userId, }); res.status(500).json({ error: "Failed to start polling" }); } }); app.post("/host-deleted", async (req, res) => { const userId = (req as AuthenticatedRequest).userId; const { hostId } = req.body; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } if (!hostId || typeof hostId !== "number") { return res.status(400).json({ error: "Invalid hostId" }); } try { pollingManager.stopPollingForHost(hostId, true); res.json({ message: "Host polling stopped" }); } catch (error) { statsLogger.error("Failed to stop polling for host", error, { operation: "host_deleted", hostId, userId, }); res.status(500).json({ error: "Failed to stop polling" }); } }); app.get("/metrics/:id", validateHostId, async (req, res) => { const id = Number(req.params.id); const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } const metricsData = pollingManager.getMetrics(id); if (!metricsData) { return res.status(404).json({ error: "Metrics not available", cpu: { percent: null, cores: null, load: null }, memory: { percent: null, usedGiB: null, totalGiB: null }, disk: { percent: null, usedHuman: null, totalHuman: null, availableHuman: null, }, network: { interfaces: [] }, uptime: { seconds: null, formatted: null }, processes: { total: null, running: null, top: [] }, system: { hostname: null, kernel: null, os: null }, lastChecked: new Date().toISOString(), }); } res.json({ ...metricsData.data, lastChecked: new Date(metricsData.timestamp).toISOString(), }); }); app.post("/metrics/start/:id", validateHostId, async (req, res) => { const id = Number(req.params.id); const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } try { const host = await fetchHostById(id, userId); if (!host) { return res.status(404).json({ error: "Host not found" }); } const sessionKey = getSessionKey(host.id, userId); const existingSession = metricsSessions[sessionKey]; if (existingSession && existingSession.isConnected) { return res.json({ success: true }); } const config = buildSshConfig(host); const client = new Client(); const connectionPromise = new Promise<{ success: boolean; requires_totp?: boolean; sessionId?: string; prompt?: string; }>((resolve, reject) => { let isResolved = false; const timeout = setTimeout(() => { if (!isResolved) { isResolved = true; client.end(); reject(new Error("Connection timeout")); } }, 60000); client.on( "keyboard-interactive", (name, instructions, instructionsLang, prompts, finish) => { const totpPromptIndex = prompts.findIndex((p) => /verification code|verification_code|token|otp|2fa|authenticator|google.*auth/i.test( p.prompt, ), ); if (totpPromptIndex !== -1) { const sessionId = `totp-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; pendingTOTPSessions[sessionId] = { client, finish, config, createdAt: Date.now(), sessionId, hostId: host.id, userId: host.userId!, prompts: prompts.map((p) => ({ prompt: p.prompt, echo: p.echo ?? false, })), totpPromptIndex, resolvedPassword: host.password, totpAttempts: 0, }; clearTimeout(timeout); if (!isResolved) { isResolved = true; resolve({ success: false, requires_totp: true, sessionId, prompt: prompts[totpPromptIndex].prompt, }); } return; } else { const responses = prompts.map((p) => { if (/password/i.test(p.prompt) && host.password) { return host.password; } return ""; }); finish(responses); } }, ); client.on("ready", () => { clearTimeout(timeout); if (!isResolved) { isResolved = true; metricsSessions[sessionKey] = { client, isConnected: true, lastActive: Date.now(), activeOperations: 0, hostId: host.id, userId, }; scheduleMetricsSessionCleanup(sessionKey); pollingManager.startPollingForHost(host).catch((error) => { statsLogger.error("Failed to start polling after connection", { operation: "start_polling_error", hostId: host.id, error: error instanceof Error ? error.message : String(error), }); }); resolve({ success: true }); } }); client.on("error", (error) => { clearTimeout(timeout); if (!isResolved) { isResolved = true; statsLogger.error("SSH connection error in metrics/start", { operation: "metrics_start_ssh_error", hostId: host.id, error: error instanceof Error ? error.message : String(error), }); reject(error); } }); if ( host.useSocks5 && (host.socks5Host || (host.socks5ProxyChain && host.socks5ProxyChain.length > 0)) ) { createSocks5Connection(host.ip, host.port, { useSocks5: host.useSocks5, socks5Host: host.socks5Host, socks5Port: host.socks5Port, socks5Username: host.socks5Username, socks5Password: host.socks5Password, socks5ProxyChain: host.socks5ProxyChain, }) .then((socks5Socket) => { if (socks5Socket) { config.sock = socks5Socket; } client.connect(config); }) .catch((error) => { if (!isResolved) { isResolved = true; clearTimeout(timeout); reject(error); } }); } else { client.connect(config); } }); const result = await connectionPromise; res.json(result); } catch (error) { statsLogger.error("Failed to start metrics collection", { operation: "metrics_start_error", hostId: id, error: error instanceof Error ? error.message : String(error), }); res.status(500).json({ error: error instanceof Error ? error.message : "Failed to start metrics collection", }); } }); app.post("/metrics/stop/:id", validateHostId, async (req, res) => { const id = Number(req.params.id); const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } try { const sessionKey = getSessionKey(id, userId); const session = metricsSessions[sessionKey]; if (session) { cleanupMetricsSession(sessionKey); } pollingManager.stopMetricsOnly(id); res.json({ success: true }); } catch (error) { statsLogger.error("Failed to stop metrics collection", { operation: "metrics_stop_error", hostId: id, error: error instanceof Error ? error.message : String(error), }); res.status(500).json({ error: error instanceof Error ? error.message : "Failed to stop metrics collection", }); } }); app.post("/metrics/connect-totp", async (req, res) => { const { sessionId, totpCode } = req.body; const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } if (!sessionId || !totpCode) { return res.status(400).json({ error: "Missing sessionId or totpCode" }); } const session = pendingTOTPSessions[sessionId]; if (!session) { return res.status(404).json({ error: "TOTP session not found or expired" }); } if (Date.now() - session.createdAt > 180000) { delete pendingTOTPSessions[sessionId]; try { session.client.end(); } catch {} return res.status(408).json({ error: "TOTP session timeout" }); } if (session.userId !== userId) { return res.status(403).json({ error: "Unauthorized" }); } session.totpAttempts++; if (session.totpAttempts > 3) { delete pendingTOTPSessions[sessionId]; try { session.client.end(); } catch {} return res.status(429).json({ error: "Too many TOTP attempts" }); } try { const responses = (session.prompts || []).map((p, idx) => { if (idx === session.totpPromptIndex) { return totpCode.trim(); } else if (/password/i.test(p.prompt) && session.resolvedPassword) { return session.resolvedPassword; } return ""; }); const connectionPromise = new Promise((resolve, reject) => { const timeout = setTimeout(() => { reject(new Error("TOTP verification timeout")); }, 30000); session.client.once( "keyboard-interactive", (name, instructions, instructionsLang, prompts, finish) => { statsLogger.warn("Second keyboard-interactive received after TOTP", { operation: "totp_second_keyboard_interactive", hostId: session.hostId, sessionId, prompts: prompts.map((p) => p.prompt), }); const secondResponses = prompts.map((p) => { if (/password/i.test(p.prompt) && session.resolvedPassword) { return session.resolvedPassword; } return ""; }); finish(secondResponses); }, ); session.client.once("ready", () => { clearTimeout(timeout); resolve(); }); session.client.once("error", (error) => { clearTimeout(timeout); statsLogger.error("SSH client error after TOTP", { operation: "totp_client_error", hostId: session.hostId, sessionId, error: error instanceof Error ? error.message : String(error), }); reject(error); }); }); session.finish(responses); await connectionPromise; const sessionKey = getSessionKey(session.hostId, userId); metricsSessions[sessionKey] = { client: session.client, isConnected: true, lastActive: Date.now(), activeOperations: 0, hostId: session.hostId, userId, }; scheduleMetricsSessionCleanup(sessionKey); delete pendingTOTPSessions[sessionId]; const host = await fetchHostById(session.hostId, userId); if (host) { pollingManager.startPollingForHost(host).catch((error) => { statsLogger.error("Failed to start polling after TOTP", { operation: "totp_polling_start_error", hostId: session.hostId, error: error instanceof Error ? error.message : String(error), }); }); } res.json({ success: true }); } catch (error) { statsLogger.error("TOTP verification failed", { operation: "totp_verification_failed", hostId: session.hostId, sessionId, error: error instanceof Error ? error.message : String(error), }); if (session.totpAttempts >= 3) { delete pendingTOTPSessions[sessionId]; try { session.client.end(); } catch {} } res.status(401).json({ error: "TOTP verification failed", attemptsRemaining: Math.max(0, 3 - session.totpAttempts), }); } }); process.on("SIGINT", () => { pollingManager.destroy(); connectionPool.destroy(); process.exit(0); }); process.on("SIGTERM", () => { pollingManager.destroy(); connectionPool.destroy(); process.exit(0); }); const PORT = 30005; app.listen(PORT, async () => { try { await authManager.initialize(); } catch (err) { statsLogger.error("Failed to initialize AuthManager", err, { operation: "auth_init_error", }); } setInterval( () => { authFailureTracker.cleanup(); pollingBackoff.cleanup(); }, 10 * 60 * 1000, ); });