feat: Add many terminal customizations
This commit is contained in:
@@ -331,20 +331,12 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
|
||||
}
|
||||
} else if (resolvedCredentials.authType === "password") {
|
||||
if (!resolvedCredentials.password || !resolvedCredentials.password.trim()) {
|
||||
fileLogger.warn(
|
||||
"Password authentication requested but no password provided",
|
||||
{
|
||||
operation: "file_connect",
|
||||
sessionId,
|
||||
hostId,
|
||||
},
|
||||
);
|
||||
return res
|
||||
.status(400)
|
||||
.json({ error: "Password required for password authentication" });
|
||||
}
|
||||
// Set password to offer both password and keyboard-interactive methods
|
||||
config.password = resolvedCredentials.password;
|
||||
} else if (resolvedCredentials.authType === "none") {
|
||||
// Don't set password in config - rely on keyboard-interactive
|
||||
} else {
|
||||
fileLogger.warn(
|
||||
"No valid authentication method provided for file manager",
|
||||
@@ -458,15 +450,6 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
|
||||
finish: (responses: string[]) => void,
|
||||
) => {
|
||||
const promptTexts = prompts.map((p) => p.prompt);
|
||||
fileLogger.info("Keyboard-interactive authentication requested", {
|
||||
operation: "file_keyboard_interactive",
|
||||
hostId,
|
||||
sessionId,
|
||||
promptsCount: prompts.length,
|
||||
prompts: promptTexts,
|
||||
alreadyResponded: keyboardInteractiveResponded,
|
||||
});
|
||||
|
||||
const totpPromptIndex = prompts.findIndex((p) =>
|
||||
/verification code|verification_code|token|otp|2fa|authenticator|google.*auth/i.test(
|
||||
p.prompt,
|
||||
@@ -474,26 +457,26 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
|
||||
);
|
||||
|
||||
if (totpPromptIndex !== -1) {
|
||||
// TOTP prompt detected - need user input
|
||||
if (responseSent) {
|
||||
fileLogger.warn("Response already sent, ignoring TOTP prompt", {
|
||||
operation: "file_keyboard_interactive",
|
||||
hostId,
|
||||
sessionId,
|
||||
const responses = prompts.map((p) => {
|
||||
if (/password/i.test(p.prompt) && resolvedCredentials.password) {
|
||||
return resolvedCredentials.password;
|
||||
}
|
||||
return "";
|
||||
});
|
||||
finish(responses);
|
||||
return;
|
||||
}
|
||||
responseSent = true;
|
||||
|
||||
if (pendingTOTPSessions[sessionId]) {
|
||||
fileLogger.warn(
|
||||
"TOTP session already exists, ignoring duplicate keyboard-interactive",
|
||||
{
|
||||
operation: "file_keyboard_interactive",
|
||||
hostId,
|
||||
sessionId,
|
||||
},
|
||||
);
|
||||
const responses = prompts.map((p) => {
|
||||
if (/password/i.test(p.prompt) && resolvedCredentials.password) {
|
||||
return resolvedCredentials.password;
|
||||
}
|
||||
return "";
|
||||
});
|
||||
finish(responses);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -515,36 +498,75 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
|
||||
resolvedPassword: resolvedCredentials.password,
|
||||
};
|
||||
|
||||
fileLogger.info("Created TOTP session", {
|
||||
operation: "file_keyboard_interactive_totp",
|
||||
hostId,
|
||||
sessionId,
|
||||
prompt: prompts[totpPromptIndex].prompt,
|
||||
promptsCount: prompts.length,
|
||||
});
|
||||
|
||||
res.json({
|
||||
requires_totp: true,
|
||||
sessionId,
|
||||
prompt: prompts[totpPromptIndex].prompt,
|
||||
});
|
||||
} else {
|
||||
// Non-TOTP prompts (password, etc.) - respond automatically
|
||||
if (keyboardInteractiveResponded) {
|
||||
fileLogger.warn(
|
||||
"Already responded to keyboard-interactive, ignoring subsequent prompt",
|
||||
{
|
||||
operation: "file_keyboard_interactive",
|
||||
hostId,
|
||||
sessionId,
|
||||
prompts: promptTexts,
|
||||
},
|
||||
);
|
||||
// Non-TOTP prompts (password, etc.)
|
||||
const hasStoredPassword =
|
||||
resolvedCredentials.password &&
|
||||
resolvedCredentials.authType !== "none";
|
||||
|
||||
// Check if this is a password prompt
|
||||
const passwordPromptIndex = prompts.findIndex((p) =>
|
||||
/password/i.test(p.prompt),
|
||||
);
|
||||
|
||||
// If no stored password (including authType "none"), prompt the user
|
||||
if (!hasStoredPassword && passwordPromptIndex !== -1) {
|
||||
if (responseSent) {
|
||||
const responses = prompts.map((p) => {
|
||||
if (/password/i.test(p.prompt) && resolvedCredentials.password) {
|
||||
return resolvedCredentials.password;
|
||||
}
|
||||
return "";
|
||||
});
|
||||
finish(responses);
|
||||
return;
|
||||
}
|
||||
responseSent = true;
|
||||
|
||||
if (pendingTOTPSessions[sessionId]) {
|
||||
const responses = prompts.map((p) => {
|
||||
if (/password/i.test(p.prompt) && resolvedCredentials.password) {
|
||||
return resolvedCredentials.password;
|
||||
}
|
||||
return "";
|
||||
});
|
||||
finish(responses);
|
||||
return;
|
||||
}
|
||||
|
||||
keyboardInteractiveResponded = true;
|
||||
|
||||
pendingTOTPSessions[sessionId] = {
|
||||
client,
|
||||
finish,
|
||||
config,
|
||||
createdAt: Date.now(),
|
||||
sessionId,
|
||||
hostId,
|
||||
ip,
|
||||
port,
|
||||
username,
|
||||
userId,
|
||||
prompts,
|
||||
totpPromptIndex: passwordPromptIndex,
|
||||
resolvedPassword: resolvedCredentials.password,
|
||||
};
|
||||
|
||||
res.json({
|
||||
requires_totp: true,
|
||||
sessionId,
|
||||
prompt: prompts[passwordPromptIndex].prompt,
|
||||
isPassword: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
keyboardInteractiveResponded = true;
|
||||
|
||||
// Auto-respond with stored credentials if available
|
||||
const responses = prompts.map((p) => {
|
||||
if (/password/i.test(p.prompt) && resolvedCredentials.password) {
|
||||
return resolvedCredentials.password;
|
||||
@@ -552,16 +574,7 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
|
||||
return "";
|
||||
});
|
||||
|
||||
fileLogger.info("Auto-responding to keyboard-interactive prompts", {
|
||||
operation: "file_keyboard_interactive_response",
|
||||
hostId,
|
||||
sessionId,
|
||||
hasPassword: !!resolvedCredentials.password,
|
||||
responsesProvided: responses.filter((r) => r !== "").length,
|
||||
totalPrompts: prompts.length,
|
||||
prompts: promptTexts,
|
||||
});
|
||||
|
||||
keyboardInteractiveResponded = true;
|
||||
finish(responses);
|
||||
}
|
||||
},
|
||||
@@ -619,14 +632,6 @@ app.post("/ssh/file_manager/ssh/connect-totp", async (req, res) => {
|
||||
.json({ error: "TOTP session timeout. Please reconnect." });
|
||||
}
|
||||
|
||||
fileLogger.info("Submitting TOTP code to SSH server", {
|
||||
operation: "file_totp_verify",
|
||||
sessionId,
|
||||
userId,
|
||||
codeLength: totpCode.length,
|
||||
promptsCount: session.prompts?.length || 0,
|
||||
});
|
||||
|
||||
// Build responses for ALL prompts, just like in terminal.ts
|
||||
const responses = (session.prompts || []).map((p, index) => {
|
||||
if (index === session.totpPromptIndex) {
|
||||
@@ -649,9 +654,9 @@ app.post("/ssh/file_manager/ssh/connect-totp", async (req, res) => {
|
||||
let responseSent = false;
|
||||
let responseTimeout: NodeJS.Timeout;
|
||||
|
||||
// Remove old event listeners from /connect endpoint to avoid conflicts
|
||||
session.client.removeAllListeners("ready");
|
||||
session.client.removeAllListeners("error");
|
||||
// Don't remove event listeners - just add our own 'once' handlers
|
||||
// The ssh2 library manages multiple listeners correctly
|
||||
// Removing them can cause the connection to become unstable
|
||||
|
||||
// CRITICAL: Attach event listeners BEFORE calling finish() to avoid race condition
|
||||
session.client.once("ready", () => {
|
||||
@@ -661,78 +666,69 @@ app.post("/ssh/file_manager/ssh/connect-totp", async (req, res) => {
|
||||
|
||||
delete pendingTOTPSessions[sessionId];
|
||||
|
||||
sshSessions[sessionId] = {
|
||||
client: session.client,
|
||||
isConnected: true,
|
||||
lastActive: Date.now(),
|
||||
};
|
||||
scheduleSessionCleanup(sessionId);
|
||||
// Add a small delay to let SSH2 stabilize the connection after keyboard-interactive
|
||||
// This prevents "Not connected" errors when immediately trying to exec commands
|
||||
setTimeout(() => {
|
||||
sshSessions[sessionId] = {
|
||||
client: session.client,
|
||||
isConnected: true,
|
||||
lastActive: Date.now(),
|
||||
};
|
||||
scheduleSessionCleanup(sessionId);
|
||||
|
||||
fileLogger.success("TOTP verification successful", {
|
||||
operation: "file_totp_verify",
|
||||
sessionId,
|
||||
userId,
|
||||
});
|
||||
res.json({
|
||||
status: "success",
|
||||
message: "TOTP verified, SSH connection established",
|
||||
});
|
||||
|
||||
res.json({
|
||||
status: "success",
|
||||
message: "TOTP verified, SSH connection established",
|
||||
});
|
||||
|
||||
// Log activity to dashboard API
|
||||
if (session.hostId && session.userId) {
|
||||
(async () => {
|
||||
try {
|
||||
const hosts = await SimpleDBOps.select(
|
||||
getDb()
|
||||
.select()
|
||||
.from(sshData)
|
||||
.where(
|
||||
and(
|
||||
eq(sshData.id, session.hostId!),
|
||||
eq(sshData.userId, session.userId!),
|
||||
// Log activity to dashboard API after connection is stable
|
||||
if (session.hostId && session.userId) {
|
||||
(async () => {
|
||||
try {
|
||||
const hosts = await SimpleDBOps.select(
|
||||
getDb()
|
||||
.select()
|
||||
.from(sshData)
|
||||
.where(
|
||||
and(
|
||||
eq(sshData.id, session.hostId!),
|
||||
eq(sshData.userId, session.userId!),
|
||||
),
|
||||
),
|
||||
),
|
||||
"ssh_data",
|
||||
session.userId!,
|
||||
);
|
||||
"ssh_data",
|
||||
session.userId!,
|
||||
);
|
||||
|
||||
const hostName =
|
||||
hosts.length > 0 && hosts[0].name
|
||||
? hosts[0].name
|
||||
: `${session.username}@${session.ip}:${session.port}`;
|
||||
const hostName =
|
||||
hosts.length > 0 && hosts[0].name
|
||||
? hosts[0].name
|
||||
: `${session.username}@${session.ip}:${session.port}`;
|
||||
|
||||
const authManager = AuthManager.getInstance();
|
||||
await axios.post(
|
||||
"http://localhost:30006/activity/log",
|
||||
{
|
||||
type: "file_manager",
|
||||
hostId: session.hostId,
|
||||
hostName,
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${await authManager.generateJWTToken(session.userId!)}`,
|
||||
const authManager = AuthManager.getInstance();
|
||||
await axios.post(
|
||||
"http://localhost:30006/activity/log",
|
||||
{
|
||||
type: "file_manager",
|
||||
hostId: session.hostId,
|
||||
hostName,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
fileLogger.info("File manager activity logged (TOTP)", {
|
||||
operation: "activity_log",
|
||||
userId: session.userId,
|
||||
hostId: session.hostId,
|
||||
hostName,
|
||||
});
|
||||
} catch (error) {
|
||||
fileLogger.warn("Failed to log file manager activity (TOTP)", {
|
||||
operation: "activity_log_error",
|
||||
userId: session.userId,
|
||||
hostId: session.hostId,
|
||||
error: error instanceof Error ? error.message : "Unknown error",
|
||||
});
|
||||
}
|
||||
})();
|
||||
}
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${await authManager.generateJWTToken(session.userId!)}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
fileLogger.warn("Failed to log file manager activity (TOTP)", {
|
||||
operation: "activity_log_error",
|
||||
userId: session.userId,
|
||||
hostId: session.hostId,
|
||||
error: error instanceof Error ? error.message : "Unknown error",
|
||||
});
|
||||
}
|
||||
})();
|
||||
}
|
||||
}, 200); // Give SSH2 connection 200ms to fully stabilize after keyboard-interactive
|
||||
});
|
||||
|
||||
session.client.once("error", (err) => {
|
||||
|
||||
@@ -112,13 +112,9 @@ class SSHConnectionPool {
|
||||
);
|
||||
|
||||
if (totpPrompt) {
|
||||
statsLogger.warn(
|
||||
`Server Stats cannot handle TOTP for host ${host.ip}. Connection will fail.`,
|
||||
{
|
||||
operation: "server_stats_totp_detected",
|
||||
hostId: host.id,
|
||||
},
|
||||
);
|
||||
// Record TOTP failure as permanent - never retry
|
||||
// The recordFailure method will log this once
|
||||
authFailureTracker.recordFailure(host.id, "TOTP", true);
|
||||
client.end();
|
||||
reject(
|
||||
new Error(
|
||||
@@ -272,9 +268,109 @@ class MetricsCache {
|
||||
}
|
||||
}
|
||||
|
||||
interface AuthFailureRecord {
|
||||
count: number;
|
||||
lastFailure: number;
|
||||
reason: "TOTP" | "AUTH" | "TIMEOUT";
|
||||
permanent: boolean; // If true, don't retry at all
|
||||
}
|
||||
|
||||
class AuthFailureTracker {
|
||||
private failures = new Map<number, AuthFailureRecord>();
|
||||
private maxRetries = 3;
|
||||
private backoffBase = 60000; // 1 minute base backoff
|
||||
|
||||
recordFailure(
|
||||
hostId: number,
|
||||
reason: "TOTP" | "AUTH" | "TIMEOUT",
|
||||
permanent = false,
|
||||
): void {
|
||||
const existing = this.failures.get(hostId);
|
||||
if (existing) {
|
||||
existing.count++;
|
||||
existing.lastFailure = Date.now();
|
||||
existing.reason = reason;
|
||||
if (permanent) existing.permanent = true;
|
||||
} else {
|
||||
this.failures.set(hostId, {
|
||||
count: 1,
|
||||
lastFailure: Date.now(),
|
||||
reason,
|
||||
permanent,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
shouldSkip(hostId: number): boolean {
|
||||
const record = this.failures.get(hostId);
|
||||
if (!record) return false;
|
||||
|
||||
// Always skip TOTP hosts
|
||||
if (record.reason === "TOTP" || record.permanent) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Skip if we've exceeded max retries
|
||||
if (record.count >= this.maxRetries) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Calculate exponential backoff
|
||||
const backoffTime = this.backoffBase * Math.pow(2, record.count - 1);
|
||||
const timeSinceFailure = Date.now() - record.lastFailure;
|
||||
|
||||
return timeSinceFailure < backoffTime;
|
||||
}
|
||||
|
||||
getSkipReason(hostId: number): string | null {
|
||||
const record = this.failures.get(hostId);
|
||||
if (!record) return null;
|
||||
|
||||
if (record.reason === "TOTP") {
|
||||
return "TOTP authentication required (metrics unavailable)";
|
||||
}
|
||||
|
||||
if (record.permanent) {
|
||||
return "Authentication permanently failed";
|
||||
}
|
||||
|
||||
if (record.count >= this.maxRetries) {
|
||||
return `Too many authentication failures (${record.count} attempts)`;
|
||||
}
|
||||
|
||||
const backoffTime = this.backoffBase * Math.pow(2, record.count - 1);
|
||||
const timeSinceFailure = Date.now() - record.lastFailure;
|
||||
const remainingTime = Math.ceil((backoffTime - timeSinceFailure) / 1000);
|
||||
|
||||
if (timeSinceFailure < backoffTime) {
|
||||
return `Retry in ${remainingTime}s (attempt ${record.count}/${this.maxRetries})`;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
reset(hostId: number): void {
|
||||
this.failures.delete(hostId);
|
||||
// Don't log reset - it's not important
|
||||
}
|
||||
|
||||
cleanup(): void {
|
||||
// Clean up old failures (older than 1 hour)
|
||||
const maxAge = 60 * 60 * 1000;
|
||||
const now = Date.now();
|
||||
|
||||
for (const [hostId, record] of this.failures.entries()) {
|
||||
if (!record.permanent && now - record.lastFailure > maxAge) {
|
||||
this.failures.delete(hostId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const connectionPool = new SSHConnectionPool();
|
||||
const requestQueue = new RequestQueue();
|
||||
const metricsCache = new MetricsCache();
|
||||
const authFailureTracker = new AuthFailureTracker();
|
||||
const authManager = AuthManager.getInstance();
|
||||
|
||||
type HostStatus = "online" | "offline";
|
||||
@@ -729,6 +825,13 @@ async function collectMetrics(host: SSHHostWithCredentials): Promise<{
|
||||
os: string | null;
|
||||
};
|
||||
}> {
|
||||
// Check if we should skip this host due to auth failures
|
||||
if (authFailureTracker.shouldSkip(host.id)) {
|
||||
const reason = authFailureTracker.getSkipReason(host.id);
|
||||
// Don't log - just skip silently to avoid spam
|
||||
throw new Error(reason || "Authentication failed");
|
||||
}
|
||||
|
||||
const cached = metricsCache.get(host.id);
|
||||
if (cached) {
|
||||
return cached as ReturnType<typeof collectMetrics> extends Promise<infer T>
|
||||
@@ -1070,11 +1173,32 @@ async function collectMetrics(host: SSHHostWithCredentials): Promise<{
|
||||
return result;
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes("TOTP authentication required")
|
||||
) {
|
||||
throw error;
|
||||
// Record authentication failures for backoff
|
||||
if (error instanceof Error) {
|
||||
if (error.message.includes("TOTP authentication required")) {
|
||||
// TOTP failures are already recorded in keyboard-interactive handler
|
||||
throw error;
|
||||
} else if (
|
||||
error.message.includes("No password available") ||
|
||||
error.message.includes("Unsupported authentication type") ||
|
||||
error.message.includes("No SSH key available")
|
||||
) {
|
||||
// Configuration errors - permanent failures, don't retry
|
||||
// recordFailure will log once when first detected
|
||||
authFailureTracker.recordFailure(host.id, "AUTH", true);
|
||||
} else if (
|
||||
error.message.includes("authentication") ||
|
||||
error.message.includes("Permission denied") ||
|
||||
error.message.includes("All configured authentication methods failed")
|
||||
) {
|
||||
// recordFailure will log once when first detected
|
||||
authFailureTracker.recordFailure(host.id, "AUTH");
|
||||
} else if (
|
||||
error.message.includes("timeout") ||
|
||||
error.message.includes("ETIMEDOUT")
|
||||
) {
|
||||
authFailureTracker.recordFailure(host.id, "TIMEOUT");
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
@@ -1257,13 +1381,17 @@ app.get("/metrics/:id", validateHostId, async (req, res) => {
|
||||
const metrics = await collectMetrics(host);
|
||||
res.json({ ...metrics, lastChecked: new Date().toISOString() });
|
||||
} catch (err) {
|
||||
const errorMessage = err instanceof Error ? err.message : "Unknown error";
|
||||
|
||||
// Check if this is a skip due to auth failure tracking
|
||||
if (
|
||||
err instanceof Error &&
|
||||
err.message.includes("TOTP authentication required")
|
||||
errorMessage.includes("TOTP authentication required") ||
|
||||
errorMessage.includes("metrics unavailable")
|
||||
) {
|
||||
// Don't log as error - this is expected for TOTP hosts
|
||||
return res.status(403).json({
|
||||
error: "TOTP_REQUIRED",
|
||||
message: "Server Stats unavailable for TOTP-enabled servers",
|
||||
message: errorMessage,
|
||||
cpu: { percent: null, cores: null, load: null },
|
||||
memory: { percent: null, usedGiB: null, totalGiB: null },
|
||||
disk: {
|
||||
@@ -1280,7 +1408,43 @@ app.get("/metrics/:id", validateHostId, async (req, res) => {
|
||||
});
|
||||
}
|
||||
|
||||
statsLogger.error("Failed to collect metrics", err);
|
||||
// Check if this is a skip due to too many failures or config issues
|
||||
if (
|
||||
errorMessage.includes("Too many authentication failures") ||
|
||||
errorMessage.includes("Retry in") ||
|
||||
errorMessage.includes("Invalid configuration") ||
|
||||
errorMessage.includes("Authentication failed")
|
||||
) {
|
||||
// Don't log - return error silently to avoid spam
|
||||
return res.status(429).json({
|
||||
error: "UNAVAILABLE",
|
||||
message: errorMessage,
|
||||
cpu: { percent: null, cores: null, load: null },
|
||||
memory: { percent: null, usedGiB: null, totalGiB: null },
|
||||
disk: {
|
||||
percent: null,
|
||||
usedHuman: null,
|
||||
totalHuman: null,
|
||||
availableHuman: null,
|
||||
},
|
||||
network: { interfaces: [] },
|
||||
uptime: { seconds: null, formatted: null },
|
||||
processes: { total: null, running: null, top: [] },
|
||||
system: { hostname: null, kernel: null, os: null },
|
||||
lastChecked: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// Only log unexpected errors
|
||||
if (
|
||||
!errorMessage.includes("timeout") &&
|
||||
!errorMessage.includes("offline") &&
|
||||
!errorMessage.includes("permanently") &&
|
||||
!errorMessage.includes("none") &&
|
||||
!errorMessage.includes("No password")
|
||||
) {
|
||||
statsLogger.error("Failed to collect metrics", err);
|
||||
}
|
||||
|
||||
if (err instanceof Error && err.message.includes("timeout")) {
|
||||
return res.status(504).json({
|
||||
@@ -1339,4 +1503,12 @@ app.listen(PORT, async () => {
|
||||
operation: "auth_init_error",
|
||||
});
|
||||
}
|
||||
|
||||
// Cleanup old auth failures every 10 minutes
|
||||
setInterval(
|
||||
() => {
|
||||
authFailureTracker.cleanup();
|
||||
},
|
||||
10 * 60 * 1000,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -313,12 +313,6 @@ wss.on("connection", async (ws: WebSocket, req) => {
|
||||
const totpData = data as TOTPResponseData;
|
||||
if (keyboardInteractiveFinish && totpData?.code) {
|
||||
const totpCode = totpData.code;
|
||||
sshLogger.info("TOTP code received from user", {
|
||||
operation: "totp_response",
|
||||
userId,
|
||||
codeLength: totpCode.length,
|
||||
});
|
||||
|
||||
keyboardInteractiveFinish([totpCode]);
|
||||
keyboardInteractiveFinish = null;
|
||||
} else {
|
||||
@@ -512,177 +506,167 @@ wss.on("connection", async (ws: WebSocket, req) => {
|
||||
sshConn.on("ready", () => {
|
||||
clearTimeout(connectionTimeout);
|
||||
|
||||
// Small delay to let connection stabilize after keyboard-interactive auth
|
||||
// This helps prevent "No response from server" errors with TOTP
|
||||
setTimeout(() => {
|
||||
// Check if connection still exists (might have been cleaned up)
|
||||
if (!sshConn) {
|
||||
sshLogger.warn(
|
||||
"SSH connection was cleaned up before shell could be created",
|
||||
{
|
||||
// Immediately try to create shell - don't delay as it can cause connection to be cleaned up
|
||||
// The connection is already ready at this point
|
||||
if (!sshConn) {
|
||||
sshLogger.warn(
|
||||
"SSH connection was cleaned up before shell could be created",
|
||||
{
|
||||
operation: "ssh_shell",
|
||||
hostId: id,
|
||||
ip,
|
||||
port,
|
||||
username,
|
||||
},
|
||||
);
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
message:
|
||||
"SSH connection was closed before terminal could be created",
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
sshConn.shell(
|
||||
{
|
||||
rows: data.rows,
|
||||
cols: data.cols,
|
||||
term: "xterm-256color",
|
||||
} as PseudoTtyOptions,
|
||||
(err, stream) => {
|
||||
if (err) {
|
||||
sshLogger.error("Shell error", err, {
|
||||
operation: "ssh_shell",
|
||||
hostId: id,
|
||||
ip,
|
||||
port,
|
||||
username,
|
||||
},
|
||||
);
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
message:
|
||||
"SSH connection was closed before terminal could be created",
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
sshConn.shell(
|
||||
{
|
||||
rows: data.rows,
|
||||
cols: data.cols,
|
||||
term: "xterm-256color",
|
||||
} as PseudoTtyOptions,
|
||||
(err, stream) => {
|
||||
if (err) {
|
||||
sshLogger.error("Shell error", err, {
|
||||
operation: "ssh_shell",
|
||||
hostId: id,
|
||||
ip,
|
||||
port,
|
||||
username,
|
||||
});
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
message: "Shell error: " + err.message,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
sshStream = stream;
|
||||
|
||||
stream.on("data", (data: Buffer) => {
|
||||
try {
|
||||
const utf8String = data.toString("utf-8");
|
||||
ws.send(JSON.stringify({ type: "data", data: utf8String }));
|
||||
} catch (error) {
|
||||
sshLogger.error("Error encoding terminal data", error, {
|
||||
operation: "terminal_data_encoding",
|
||||
hostId: id,
|
||||
dataLength: data.length,
|
||||
});
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "data",
|
||||
data: data.toString("latin1"),
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
stream.on("close", () => {
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "disconnected",
|
||||
message: "Connection lost",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
stream.on("error", (err: Error) => {
|
||||
sshLogger.error("SSH stream error", err, {
|
||||
operation: "ssh_stream",
|
||||
hostId: id,
|
||||
ip,
|
||||
port,
|
||||
username,
|
||||
});
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
message: "SSH stream error: " + err.message,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
setupPingInterval();
|
||||
|
||||
if (initialPath && initialPath.trim() !== "") {
|
||||
const cdCommand = `cd "${initialPath.replace(/"/g, '\\"')}" && pwd\n`;
|
||||
stream.write(cdCommand);
|
||||
}
|
||||
|
||||
if (executeCommand && executeCommand.trim() !== "") {
|
||||
setTimeout(() => {
|
||||
const command = `${executeCommand}\n`;
|
||||
stream.write(command);
|
||||
}, 500);
|
||||
}
|
||||
|
||||
ws.send(
|
||||
JSON.stringify({ type: "connected", message: "SSH connected" }),
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
message: "Shell error: " + err.message,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Log activity to dashboard API
|
||||
if (id && hostConfig.userId) {
|
||||
(async () => {
|
||||
try {
|
||||
// Fetch host name from database
|
||||
const hosts = await SimpleDBOps.select(
|
||||
getDb()
|
||||
.select()
|
||||
.from(sshData)
|
||||
.where(
|
||||
and(
|
||||
eq(sshData.id, id),
|
||||
eq(sshData.userId, hostConfig.userId!),
|
||||
),
|
||||
sshStream = stream;
|
||||
|
||||
stream.on("data", (data: Buffer) => {
|
||||
try {
|
||||
const utf8String = data.toString("utf-8");
|
||||
ws.send(JSON.stringify({ type: "data", data: utf8String }));
|
||||
} catch (error) {
|
||||
sshLogger.error("Error encoding terminal data", error, {
|
||||
operation: "terminal_data_encoding",
|
||||
hostId: id,
|
||||
dataLength: data.length,
|
||||
});
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "data",
|
||||
data: data.toString("latin1"),
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
stream.on("close", () => {
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "disconnected",
|
||||
message: "Connection lost",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
stream.on("error", (err: Error) => {
|
||||
sshLogger.error("SSH stream error", err, {
|
||||
operation: "ssh_stream",
|
||||
hostId: id,
|
||||
ip,
|
||||
port,
|
||||
username,
|
||||
});
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
message: "SSH stream error: " + err.message,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
setupPingInterval();
|
||||
|
||||
if (initialPath && initialPath.trim() !== "") {
|
||||
const cdCommand = `cd "${initialPath.replace(/"/g, '\\"')}" && pwd\n`;
|
||||
stream.write(cdCommand);
|
||||
}
|
||||
|
||||
if (executeCommand && executeCommand.trim() !== "") {
|
||||
setTimeout(() => {
|
||||
const command = `${executeCommand}\n`;
|
||||
stream.write(command);
|
||||
}, 500);
|
||||
}
|
||||
|
||||
ws.send(
|
||||
JSON.stringify({ type: "connected", message: "SSH connected" }),
|
||||
);
|
||||
|
||||
// Log activity to dashboard API
|
||||
if (id && hostConfig.userId) {
|
||||
(async () => {
|
||||
try {
|
||||
// Fetch host name from database
|
||||
const hosts = await SimpleDBOps.select(
|
||||
getDb()
|
||||
.select()
|
||||
.from(sshData)
|
||||
.where(
|
||||
and(
|
||||
eq(sshData.id, id),
|
||||
eq(sshData.userId, hostConfig.userId!),
|
||||
),
|
||||
"ssh_data",
|
||||
hostConfig.userId!,
|
||||
);
|
||||
),
|
||||
"ssh_data",
|
||||
hostConfig.userId!,
|
||||
);
|
||||
|
||||
const hostName =
|
||||
hosts.length > 0 && hosts[0].name
|
||||
? hosts[0].name
|
||||
: `${username}@${ip}:${port}`;
|
||||
const hostName =
|
||||
hosts.length > 0 && hosts[0].name
|
||||
? hosts[0].name
|
||||
: `${username}@${ip}:${port}`;
|
||||
|
||||
await axios.post(
|
||||
"http://localhost:30006/activity/log",
|
||||
{
|
||||
type: "terminal",
|
||||
hostId: id,
|
||||
hostName,
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${await authManager.generateJWTToken(hostConfig.userId!)}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
sshLogger.info("Terminal activity logged", {
|
||||
operation: "activity_log",
|
||||
userId: hostConfig.userId,
|
||||
await axios.post(
|
||||
"http://localhost:30006/activity/log",
|
||||
{
|
||||
type: "terminal",
|
||||
hostId: id,
|
||||
hostName,
|
||||
});
|
||||
} catch (error) {
|
||||
sshLogger.warn("Failed to log terminal activity", {
|
||||
operation: "activity_log_error",
|
||||
userId: hostConfig.userId,
|
||||
hostId: id,
|
||||
error:
|
||||
error instanceof Error ? error.message : "Unknown error",
|
||||
});
|
||||
}
|
||||
})();
|
||||
}
|
||||
},
|
||||
);
|
||||
}, 100); // Small delay to stabilize connection after keyboard-interactive auth
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${await authManager.generateJWTToken(hostConfig.userId!)}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
sshLogger.warn("Failed to log terminal activity", {
|
||||
operation: "activity_log_error",
|
||||
userId: hostConfig.userId,
|
||||
hostId: id,
|
||||
error:
|
||||
error instanceof Error ? error.message : "Unknown error",
|
||||
});
|
||||
}
|
||||
})();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
sshConn.on("error", (err: Error) => {
|
||||
@@ -738,6 +722,13 @@ wss.on("connection", async (ws: WebSocket, req) => {
|
||||
|
||||
sshConn.on("close", () => {
|
||||
clearTimeout(connectionTimeout);
|
||||
sshLogger.warn("SSH connection closed by server", {
|
||||
operation: "ssh_close",
|
||||
hostId: id,
|
||||
ip,
|
||||
port,
|
||||
hadStream: !!sshStream,
|
||||
});
|
||||
cleanupSSH(connectionTimeout);
|
||||
});
|
||||
|
||||
@@ -751,17 +742,6 @@ wss.on("connection", async (ws: WebSocket, req) => {
|
||||
finish: (responses: string[]) => void,
|
||||
) => {
|
||||
const promptTexts = prompts.map((p) => p.prompt);
|
||||
sshLogger.info("Keyboard-interactive authentication requested", {
|
||||
operation: "ssh_keyboard_interactive",
|
||||
hostId: id,
|
||||
promptsCount: prompts.length,
|
||||
instructions: instructions || "none",
|
||||
});
|
||||
console.log(
|
||||
`[SSH Keyboard-Interactive] Host ${id}: ${prompts.length} prompts:`,
|
||||
promptTexts,
|
||||
);
|
||||
|
||||
const totpPromptIndex = prompts.findIndex((p) =>
|
||||
/verification code|verification_code|token|otp|2fa|authenticator|google.*auth/i.test(
|
||||
p.prompt,
|
||||
@@ -769,11 +749,11 @@ wss.on("connection", async (ws: WebSocket, req) => {
|
||||
);
|
||||
|
||||
if (totpPromptIndex !== -1) {
|
||||
// TOTP prompt detected - need user input
|
||||
if (totpPromptSent) {
|
||||
sshLogger.warn("TOTP prompt already sent, ignoring duplicate", {
|
||||
operation: "ssh_keyboard_interactive",
|
||||
sshLogger.warn("TOTP prompt asked again - ignoring duplicate", {
|
||||
operation: "ssh_keyboard_interactive_totp_duplicate",
|
||||
hostId: id,
|
||||
prompts: promptTexts,
|
||||
});
|
||||
return;
|
||||
}
|
||||
@@ -783,7 +763,6 @@ wss.on("connection", async (ws: WebSocket, req) => {
|
||||
keyboardInteractiveFinish = (totpResponses: string[]) => {
|
||||
const totpCode = (totpResponses[0] || "").trim();
|
||||
|
||||
// Respond to ALL prompts, not just TOTP
|
||||
const responses = prompts.map((p, index) => {
|
||||
if (index === totpPromptIndex) {
|
||||
return totpCode;
|
||||
@@ -794,14 +773,6 @@ wss.on("connection", async (ws: WebSocket, req) => {
|
||||
return "";
|
||||
});
|
||||
|
||||
sshLogger.info("TOTP response being sent to SSH server", {
|
||||
operation: "totp_verification",
|
||||
hostId: id,
|
||||
totpCodeLength: totpCode.length,
|
||||
totalPrompts: prompts.length,
|
||||
responsesProvided: responses.filter((r) => r !== "").length,
|
||||
});
|
||||
|
||||
finish(responses);
|
||||
};
|
||||
ws.send(
|
||||
@@ -811,68 +782,57 @@ wss.on("connection", async (ws: WebSocket, req) => {
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
// Non-TOTP prompts (password, etc.)
|
||||
if (keyboardInteractiveResponded) {
|
||||
sshLogger.warn(
|
||||
"Already responded to keyboard-interactive, ignoring subsequent prompt",
|
||||
{
|
||||
operation: "ssh_keyboard_interactive",
|
||||
hostId: id,
|
||||
prompts: promptTexts,
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
keyboardInteractiveResponded = true;
|
||||
|
||||
// Check if we have stored credentials for auto-response
|
||||
const hasStoredPassword =
|
||||
resolvedCredentials.password &&
|
||||
resolvedCredentials.authType !== "none";
|
||||
|
||||
if (!hasStoredPassword && resolvedCredentials.authType === "none") {
|
||||
// For "none" auth type, prompt user for all keyboard-interactive inputs
|
||||
const passwordPromptIndex = prompts.findIndex((p) =>
|
||||
/password/i.test(p.prompt),
|
||||
);
|
||||
// Check if this is a password prompt
|
||||
const passwordPromptIndex = prompts.findIndex((p) =>
|
||||
/password/i.test(p.prompt),
|
||||
);
|
||||
|
||||
if (passwordPromptIndex !== -1) {
|
||||
// Ask user for password
|
||||
keyboardInteractiveFinish = (userResponses: string[]) => {
|
||||
const userInput = (userResponses[0] || "").trim();
|
||||
|
||||
// Build responses for all prompts
|
||||
const responses = prompts.map((p, index) => {
|
||||
if (index === passwordPromptIndex) {
|
||||
return userInput;
|
||||
}
|
||||
return "";
|
||||
});
|
||||
|
||||
sshLogger.info(
|
||||
"User-provided password being sent to SSH server",
|
||||
{
|
||||
operation: "interactive_password_verification",
|
||||
hostId: id,
|
||||
passwordLength: userInput.length,
|
||||
totalPrompts: prompts.length,
|
||||
},
|
||||
);
|
||||
|
||||
finish(responses);
|
||||
};
|
||||
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "password_required",
|
||||
prompt: prompts[passwordPromptIndex].prompt,
|
||||
}),
|
||||
);
|
||||
// If no stored password (including authType "none"), prompt the user
|
||||
if (!hasStoredPassword && passwordPromptIndex !== -1) {
|
||||
if (keyboardInteractiveResponded) {
|
||||
return;
|
||||
}
|
||||
keyboardInteractiveResponded = true;
|
||||
|
||||
keyboardInteractiveFinish = (userResponses: string[]) => {
|
||||
const userInput = (userResponses[0] || "").trim();
|
||||
|
||||
// Build responses for all prompts
|
||||
const responses = prompts.map((p, index) => {
|
||||
if (index === passwordPromptIndex) {
|
||||
return userInput;
|
||||
}
|
||||
return "";
|
||||
});
|
||||
|
||||
sshLogger.info(
|
||||
"User-provided password being sent to SSH server",
|
||||
{
|
||||
operation: "interactive_password_verification",
|
||||
hostId: id,
|
||||
passwordLength: userInput.length,
|
||||
totalPrompts: prompts.length,
|
||||
},
|
||||
);
|
||||
|
||||
finish(responses);
|
||||
};
|
||||
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "password_required",
|
||||
prompt: prompts[passwordPromptIndex].prompt,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Auto-respond with stored credentials
|
||||
// Auto-respond with stored credentials if available
|
||||
// Allow multiple responses - the server might ask multiple times during auth flow
|
||||
const responses = prompts.map((p) => {
|
||||
if (/password/i.test(p.prompt) && resolvedCredentials.password) {
|
||||
return resolvedCredentials.password;
|
||||
@@ -880,18 +840,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
|
||||
return "";
|
||||
});
|
||||
|
||||
sshLogger.info("Responding to keyboard-interactive prompts", {
|
||||
operation: "ssh_keyboard_interactive_response",
|
||||
hostId: id,
|
||||
hasPassword: !!resolvedCredentials.password,
|
||||
responsesProvided: responses.filter((r) => r !== "").length,
|
||||
totalPrompts: prompts.length,
|
||||
prompts: promptTexts,
|
||||
});
|
||||
|
||||
console.log(
|
||||
`[SSH Auto Response] Host ${id}: Sending ${responses.length} responses, ${responses.filter((r) => r !== "").length} non-empty`,
|
||||
);
|
||||
keyboardInteractiveResponded = true;
|
||||
finish(responses);
|
||||
}
|
||||
},
|
||||
@@ -963,18 +912,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
|
||||
};
|
||||
|
||||
if (resolvedCredentials.authType === "none") {
|
||||
// No credentials provided - rely entirely on keyboard-interactive authentication
|
||||
// This mimics the behavior of the ssh command-line client where it prompts for password/TOTP
|
||||
sshLogger.info(
|
||||
"Using interactive authentication (no stored credentials)",
|
||||
{
|
||||
operation: "ssh_auth_none",
|
||||
hostId: id,
|
||||
ip,
|
||||
username,
|
||||
},
|
||||
);
|
||||
// Don't set password or privateKey - let keyboard-interactive handle everything
|
||||
// Don't set password in config - rely on keyboard-interactive
|
||||
} else if (
|
||||
resolvedCredentials.authType === "key" &&
|
||||
resolvedCredentials.key
|
||||
@@ -1030,8 +968,6 @@ wss.on("connection", async (ws: WebSocket, req) => {
|
||||
);
|
||||
return;
|
||||
}
|
||||
// Set password to offer both password and keyboard-interactive methods
|
||||
connectConfig.password = resolvedCredentials.password;
|
||||
} else {
|
||||
sshLogger.error("No valid authentication method provided");
|
||||
ws.send(
|
||||
|
||||
Reference in New Issue
Block a user