From 1c5167b6e6f2b23e3a7871f2175b11400bd1e0e8 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Thu, 27 Aug 2026 20:16:33 +1000 Subject: [PATCH] feat: add durable chat storage and disconnect recovery --- README.md | 9 + dashboard/dist/index.js | 44 +- dashboard/dist/style.css | 2 +- dashboard/manifest.json | 2 +- dashboard/plugin_api.py | 707 +++++++++++++++++++++----- plugin.yaml | 3 +- requirements.txt | 1 + scripts/install_postgresql_storage.sh | 51 ++ tests/test_validation_harness.py | 74 ++- 9 files changed, 709 insertions(+), 184 deletions(-) create mode 100644 scripts/install_postgresql_storage.sh diff --git a/README.md b/README.md index 383ba24..6fe34c4 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,18 @@ Native-like Hermes dashboard plugin for local Ollama model management and chat. - Streamed Ollama responses with a real Stop action that cancels the active request - Minimized-by-default expandable thinking/progress details with live stage, elapsed time, event, and character counters - Validation harness mode: choose one primary model and one or more independent validator models; validators review the primary draft and the primary model compiles one final answer +- Server-owned chat jobs continue after the browser closes and persist final answers for later resume +- SQLite is the default chat store for new users +- Optional native PostgreSQL storage can be installed and linked explicitly from the plugin The chat supports two modes. With one selected model, it sends a normal direct request. With one primary model and at least one validator model selected, the plugin runs a validation harness: the primary creates a draft, validators independently review the request and draft in parallel, and the primary compiles one final user-facing answer from the draft and validation reports. Validator reports are returned as supporting evidence, while only the compiled primary response is persisted and displayed as the answer. +## Chat storage and durability + +Chat jobs are persisted before model execution. The browser observes job status through `/chat/status/{request_id}` but does not own generation. Closing the browser no longer cancels a queued or running job, and the dashboard recovers queued/running jobs after restart. Each conversation records immutable message versions, job attempts, stage timing, model metrics, and append-only operational events. + +SQLite remains the default and requires no service installation. The Chat storage panel can detect native PostgreSQL, install it only after explicit confirmation, and link it as the active backend. PostgreSQL storage uses the local service environment and remains disabled until the user explicitly selects **Link PostgreSQL**. + ## Performance metrics diff --git a/dashboard/dist/index.js b/dashboard/dist/index.js index 2abf770..6c5c044 100644 --- a/dashboard/dist/index.js +++ b/dashboard/dist/index.js @@ -225,6 +225,22 @@ ); } + function StoragePanel(props) { + var storage = props.storage || {}; + var postgres = storage.postgres || {}; + var backend = storage.backend || "sqlite"; + return h("section", { className: "ollama-storage-panel" }, + h("div", { className: "ollama-pool-heading" }, h("div", null, h("h3", null, "Chat storage"), h("p", null, "SQLite is the default. PostgreSQL is optional and remains local-only.")), h(Badge, { tone: backend === "postgres" ? "live" : "muted" }, backend === "postgres" ? "PostgreSQL" : "SQLite")), + h("div", { className: "ollama-storage-copy" }, h("strong", null, backend === "postgres" ? "Using PostgreSQL chat storage" : "Using SQLite chat storage"), h("small", null, postgres.available ? "Native PostgreSQL detected: " + (postgres.version || "version available") : "PostgreSQL is not currently linked. Switching storage does not delete existing conversations.")), + h("div", { className: "ollama-storage-actions" }, + backend === "postgres" ? h(Button, { className: "secondary", onClick: function () { props.onConfigure("sqlite"); } }, "Use SQLite") : h(Button, { disabled: !postgres.available, onClick: function () { props.onConfigure("postgres"); } }, "Link PostgreSQL"), + !postgres.available && h(Button, { className: "secondary", onClick: props.onInstall }, "Install native PostgreSQL"), + h(Button, { className: "secondary", onClick: props.onRefresh }, "Refresh storage status") + ), + h("small", { className: "ollama-storage-note" }, "PostgreSQL installation is an explicit host change. Chat data is not moved until you choose Link PostgreSQL.") + ); + } + function ModelPoolPanel(props) { var models = props.models || [], loaded = models.filter(function (item) { return item.loaded; }), primary = props.primaryModel || "", validators = props.validatorModels || []; return h("section", { className: "ollama-model-pool" }, @@ -262,6 +278,7 @@ var busyState = React.useState(""), busy = busyState[0], setBusy = busyState[1]; var noticeState = React.useState(null), notice = noticeState[0], setNotice = noticeState[1]; var validationState = React.useState(null), validationReports = validationState[0], setValidationReports = validationState[1]; + var storageState = React.useState(null), storage = storageState[0], setStorage = storageState[1]; var thinkingState = React.useState(null), thinking = thinkingState[0], setThinking = thinkingState[1]; var thinkingElapsedState = React.useState(0), thinkingElapsed = thinkingElapsedState[0], setThinkingElapsed = thinkingElapsedState[1]; var thinkingDetailsState = React.useState(null), thinkingDetails = thinkingDetailsState[0], setThinkingDetails = thinkingDetailsState[1]; @@ -281,6 +298,17 @@ if (Array.isArray(item.models) && item.models.length) setSelectedModels(item.models); }).catch(function (err) { setNotice({ error: err.message || String(err) }); }); } + function refreshStorage() { fetchJSON(API + "/storage").then(setStorage).catch(function (err) { setNotice({ error: "Storage status unavailable: " + (err.message || String(err)) }); }); } + function configureStorage(backend) { + var confirmation = backend === "postgres" ? "enable-postgresql" : "use-sqlite"; + if (!window.confirm(backend === "postgres" ? "Link PostgreSQL chat storage now? Existing SQLite conversations will be migrated." : "Switch new chat writes back to SQLite?")) return; + fetchJSON(API + "/storage/configure", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ backend: backend, confirm: confirmation }) }).then(function (value) { setNotice({ ok: value.message || "Chat storage updated." }); refreshStorage(); refreshConversations(); }).catch(function (err) { setNotice({ error: err.message || String(err) }); }); + } + function installPostgres() { + if (!window.confirm("Install native PostgreSQL on this host? This changes host packages and services.")) return; + setNotice({ ok: "Native PostgreSQL installation started…" }); + fetchJSON(API + "/storage/postgresql/install", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ backend: "postgres", confirm: "install-postgresql", install: true }) }).then(function (value) { setNotice({ ok: value.message || "PostgreSQL installation completed." }); refreshStorage(); }).catch(function (err) { setNotice({ error: err.message || String(err) }); }); + } function refreshConversations() { fetchJSON(API + "/metrics?limit=100").then(function (value) { setAggregate(value.aggregate || null); }).catch(function () {}); return fetchJSON(API + "/conversations").then(function (value) { @@ -291,7 +319,7 @@ }).catch(function () { return []; }); } function newConversation() { setConversationId(""); setHistory([]); setMetrics([]); setValidationReports(null); setNotice({ ok: "New conversation ready." }); } - React.useEffect(function () { refreshConversations(); }, []); + React.useEffect(function () { refreshConversations(); refreshStorage(); }, []); React.useEffect(function () { if (conversationId) saveChat(model, selectedModels, history); }, [model, selectedModels, history, conversationId]); React.useEffect(function () { if (!model && (loadedModels[0] || models[0])) setModel((loadedModels[0] || models[0]).name); }, [models, loadedModels, model]); React.useEffect(function () { @@ -390,6 +418,17 @@ setThinking(null); setActiveRequest(null); setBusy(""); setNotice({ ok: "Generation stopped." }); }); } + function pollChatJob(requestId, current) { + return fetchJSON(API + "/chat/status/" + encodeURIComponent(requestId)).then(function (status) { + setThinkingDetails(status); + if (current.stopped) throw new Error("Chat stopped by user"); + if (status.done) { + if (status.status !== "completed") throw new Error(status.error || "Server-side chat job did not complete"); + return status; + } + return new Promise(function (resolve) { setTimeout(resolve, 1200); }).then(function () { return pollChatJob(requestId, current); }); + }); + } function send() { if (busy === "send" || busy === "stop" || !selectedModels.length || (!message.trim() && !attachments.length)) return; var requestId = makeRequestId(); @@ -399,7 +438,7 @@ setHistory(function (old) { return old.concat([outgoing]); }); setMessage(""); setBusy("send"); setActiveRequest(current); setThinking({ request_id: requestId, startedAt: Date.now(), stage: attachments.length ? "Preparing attachments and sending request to Ollama" : "Sending request to Ollama" }); setThinkingDetails(null); setThinkingOpen(false); setNotice(null); var requestOptions = { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }; if (controller) requestOptions.signal = controller.signal; - fetchJSON(API + "/chat", requestOptions).then(function (result) { var answer = result.message && result.message.content ? result.message.content : "(No response text returned.)"; setConversationId(result.conversation_id || conversationId); setHistory(function (old) { return old.concat([{ role: "assistant", content: answer }]); }); setMetrics(result.metrics || []); setValidationReports(result.mode === "harness" ? (result.validation_reports || []) : null); setAttachments([]); setRuntime(result.runtime || runtime); setNotice({ ok: result.mode === "harness" ? "One final answer compiled by " + result.primary_model + " after validation by " + (result.validator_models || []).join(", ") + "." : "Response complete. Shared conversation and performance metrics saved." }); pollRuntime(); refreshConversations(); }).catch(function (err) { if (!current.stopped) setNotice({ error: err.message || String(err) }); }).finally(function () { if (!current.stopped) { setThinking(null); setActiveRequest(null); } setBusy(""); }); + fetchJSON(API + "/chat", requestOptions).then(function (result) { return result.done ? result : pollChatJob(requestId, current); }).then(function (result) { var answer = result.message && result.message.content ? result.message.content : "(No response text returned.)"; setConversationId(result.conversation_id || conversationId); setHistory(function (old) { return old.concat([{ role: "assistant", content: answer }]); }); setMetrics(result.metrics || []); setValidationReports(result.mode === "harness" ? (result.validation_reports || []) : null); setAttachments([]); setRuntime(result.runtime || runtime); setNotice({ ok: result.mode === "harness" ? "One final answer compiled by " + result.primary_model + " after validation by " + (result.validator_models || []).join(", ") + "." : "Response complete. Shared conversation and performance metrics saved." }); pollRuntime(); refreshConversations(); }).catch(function (err) { if (!current.stopped) setNotice({ error: err.message || String(err) }); }).finally(function () { if (!current.stopped) { setThinking(null); setActiveRequest(null); } setBusy(""); }); } return h("section", { className: "ollama-chat" }, h("div", { className: "ollama-chat-header" }, h("div", null, h("div", { className: "ollama-eyebrow" }, "LOCAL OLLAMA CHAT"), h("h2", null, "Chat with a validated model harness"), h("p", null, "Choose one primary model and at least one loaded validator model. The primary produces one final answer after reviewing the independent validation reports.")), h(Button, { className: "secondary", disabled: !history.length || busy === "send", onClick: clearChat }, "Clear chat")), @@ -409,6 +448,7 @@ aggregate && h("div", { className: "ollama-metrics-summary" }, h("strong", null, "Model performance · ", aggregate.sample_count || 0, " samples"), h("span", null, "TTFT avg: ", aggregate.avg_time_to_first_token_ms == null ? "n/a" : aggregate.avg_time_to_first_token_ms + " ms"), h("span", null, "Output: ", aggregate.avg_eval_tokens_per_second == null ? "n/a" : aggregate.avg_eval_tokens_per_second + " tok/s"), h("span", null, "Latency: ", aggregate.avg_total_latency_ms == null ? "n/a" : aggregate.avg_total_latency_ms + " ms"), h("span", null, "Errors: ", aggregate.error_count || 0)), metrics && metrics.length > 0 && h("div", { className: "ollama-metrics-detail" }, (metrics.slice(-3)).map(function (item, index) { return h("span", { key: index }, item.model || "model", " · TTFT ", item.time_to_first_token_ms == null ? "n/a" : item.time_to_first_token_ms + " ms", " · ", item.eval_count == null ? "n/a" : item.eval_count + " output tokens", " · ", item.eval_tokens_per_second == null ? "n/a" : item.eval_tokens_per_second + " tok/s"); })) ), + h(StoragePanel, { storage: storage, onConfigure: configureStorage, onInstall: installPostgres, onRefresh: refreshStorage }), h(ModelPoolPanel, { models: models, loadedModels: loadedModels, selectedModels: selectedModels, primaryModel: selectedModels[0] || "", validatorModels: selectedModels.slice(1), poolSelection: poolSelection, placements: placements, busy: busy, onTogglePool: togglePoolModel, onPlacementChange: setPlacement, onToggleChat: toggleChatModel, onPrimaryChange: setPrimaryModel, onLoad: loadModel, onUnload: unloadModels }), notice && h("div", { className: "ollama-notice " + (notice.error ? "error" : notice.warning ? "warning" : "ok") }, notice.error || notice.warning || notice.ok), validationReports && validationReports.length > 0 && h("details", { className: "ollama-validation-evidence" }, h("summary", null, "Validation evidence · ", validationReports.length, " independent reports"), validationReports.map(function (item) { return h("div", { className: "ollama-validation-report", key: item.model }, h("strong", null, item.model), h("p", null, item.report || "No report text returned.")); })), diff --git a/dashboard/dist/style.css b/dashboard/dist/style.css index 32a97af..4d1e053 100644 --- a/dashboard/dist/style.css +++ b/dashboard/dist/style.css @@ -9,5 +9,5 @@ .ollama-target-modal{position:fixed;inset:0;z-index:20;display:flex;align-items:center;justify-content:center;padding:20px;background:rgba(4,15,14,.72)}.ollama-target-card{display:grid;gap:10px;max-width:560px;width:100%;padding:20px;border:1px solid rgba(141,210,193,.38);border-radius:12px;background:#102d29;box-shadow:0 14px 50px rgba(0,0,0,.35)}.ollama-target-card h3{margin:0;color:#effcf8}.ollama-target-card p{margin:0;color:#a5bfba;font-size:12px}.ollama-target-card .ollama-button{text-align:left}@media(max-width:900px){.ollama-connection-panel{min-width:0;max-width:none}.ollama-connection-form{flex-wrap:wrap}.ollama-connection-input{min-width:160px}} .ollama-catalog-controls{display:grid;grid-template-columns:repeat(3,minmax(130px,1fr));gap:8px;align-items:end;margin-top:0;padding:10px;border:1px solid rgba(164,211,199,.16);border-radius:10px;background:rgba(10,31,28,.55)}.ollama-catalog-controls label{display:flex;flex-direction:column;gap:5px;color:#a5bfba;font-size:10px;text-transform:uppercase;letter-spacing:.06em}.ollama-catalog-checkbox{display:flex!important;flex-direction:row!important;align-items:center;gap:8px;grid-column:1 / -1;padding:8px 4px;color:#b8ead9!important;text-transform:none!important;letter-spacing:normal!important;cursor:pointer}.ollama-catalog-checkbox input{width:15px;height:15px;margin:0;accent-color:#75d2b7}.ollama-catalog-checkbox span{font-size:11px}.ollama-catalog-memory-bypass{color:#ffd89a!important;background:rgba(142,90,25,.12);border-radius:7px}.ollama-catalog-select{min-width:145px;border:1px solid rgba(155,205,194,.28);border-radius:7px;background:#102d29;color:#e8f2ef;padding:8px;font:inherit;font-size:11px;text-transform:none;letter-spacing:normal} @media(max-width:1000px){.ollama-nav-row{display:grid;grid-template-columns:1fr}.ollama-toolbar-disk{justify-self:end}.ollama-browse-row{grid-template-columns:1fr}.ollama-catalog-controls{margin-top:0}} -@media(max-width:760px){.ollama-tabs{grid-template-columns:repeat(2,minmax(0,1fr))}.ollama-nav-row{gap:8px}.ollama-toolbar-disk{justify-self:stretch;grid-template-columns:auto auto;min-width:0}.ollama-browse-row{gap:8px}.ollama-catalog-controls{grid-template-columns:1fr;align-items:stretch}.ollama-catalog-select{width:100%}}.ollama-harness-primary,.ollama-harness-validators{display:flex;align-items:center;gap:8px;flex-wrap:wrap}.ollama-harness-primary{min-width:260px}.ollama-harness-primary label{display:flex;align-items:center;gap:8px;color:#a5bfba;font-size:11px}.ollama-harness-primary select{border:1px solid rgba(155,205,194,.28);border-radius:7px;background:#102d29;color:#e8f2ef;padding:8px;font:inherit;font-size:11px;max-width:260px}.ollama-harness-validators{flex-basis:100%;padding-top:8px;border-top:1px solid rgba(164,211,199,.14)}.ollama-harness-validators>strong{color:#d5e8e2;font-size:11px}.ollama-harness-ready,.ollama-harness-warning{flex-basis:100%;font-size:10px}.ollama-harness-ready{color:#9af1c7}.ollama-harness-warning{color:#ffd89a}.ollama-validation-evidence{margin-top:10px;padding:10px 12px;border:1px solid rgba(141,210,193,.22);border-radius:8px;background:rgba(10,31,28,.5);color:#a5bfba;font-size:11px}.ollama-validation-evidence summary{cursor:pointer;color:#b8ead9;font-weight:700}.ollama-validation-report{margin-top:10px;padding-top:8px;border-top:1px solid rgba(164,211,199,.12)}.ollama-validation-report strong{color:#effcf8;font-size:11px}.ollama-validation-report p{margin:4px 0 0;white-space:pre-wrap;line-height:1.45} +@media(max-width:760px){.ollama-tabs{grid-template-columns:repeat(2,minmax(0,1fr))}.ollama-nav-row{gap:8px}.ollama-toolbar-disk{justify-self:stretch;grid-template-columns:auto auto;min-width:0}.ollama-browse-row{gap:8px}.ollama-catalog-controls{grid-template-columns:1fr;align-items:stretch}.ollama-catalog-select{width:100%}}.ollama-harness-primary,.ollama-harness-validators{display:flex;align-items:center;gap:8px;flex-wrap:wrap}.ollama-harness-primary{min-width:260px}.ollama-harness-primary label{display:flex;align-items:center;gap:8px;color:#a5bfba;font-size:11px}.ollama-harness-primary select{border:1px solid rgba(155,205,194,.28);border-radius:7px;background:#102d29;color:#e8f2ef;padding:8px;font:inherit;font-size:11px;max-width:260px}.ollama-harness-validators{flex-basis:100%;padding-top:8px;border-top:1px solid rgba(164,211,199,.14)}.ollama-harness-validators>strong{color:#d5e8e2;font-size:11px}.ollama-harness-ready,.ollama-harness-warning{flex-basis:100%;font-size:10px}.ollama-harness-ready{color:#9af1c7}.ollama-harness-warning{color:#ffd89a}.ollama-validation-evidence{margin-top:10px;padding:10px 12px;border:1px solid rgba(141,210,193,.22);border-radius:8px;background:rgba(10,31,28,.5);color:#a5bfba;font-size:11px}.ollama-validation-evidence summary{cursor:pointer;color:#b8ead9;font-weight:700}.ollama-validation-report{margin-top:10px;padding-top:8px;border-top:1px solid rgba(164,211,199,.12)}.ollama-validation-report strong{color:#effcf8;font-size:11px}.ollama-validation-report p{margin:4px 0 0;white-space:pre-wrap;line-height:1.45}.ollama-storage-panel{margin-top:14px;padding:14px 16px;border:1px solid rgba(141,210,193,.22);border-radius:10px;background:rgba(10,31,28,.5)}.ollama-storage-copy{display:flex;flex-direction:column;gap:4px;margin-top:10px}.ollama-storage-copy strong{color:#effcf8;font-size:12px}.ollama-storage-copy small,.ollama-storage-note{color:#a5bfba;font-size:10px}.ollama-storage-actions{display:flex;gap:8px;flex-wrap:wrap;margin-top:12px}.ollama-storage-note{display:block;margin-top:10px} .ollama-thinking-status{display:flex;align-items:center;gap:14px;flex-wrap:wrap;margin-top:14px;padding:14px 16px;border:1px solid rgba(117,210,183,.42);border-radius:10px;background:linear-gradient(90deg,rgba(46,111,96,.34),rgba(24,64,57,.5));box-shadow:0 0 20px rgba(74,190,158,.08)}.ollama-thinking-copy{flex:1;min-width:200px}.ollama-thinking-details{flex-basis:100%;padding:12px;border-top:1px solid rgba(164,211,199,.17);color:#a5bfba}.ollama-thinking-detail-grid{display:grid;grid-template-columns:repeat(5,minmax(100px,1fr));gap:8px;margin-bottom:8px}.ollama-thinking-detail-grid span{display:flex;flex-direction:column;gap:3px;padding:8px;border-radius:7px;background:rgba(71,117,108,.11);font-size:10px;color:#8fb5ac}.ollama-thinking-detail-grid strong{color:#e4f4ef;font-size:11px;overflow-wrap:anywhere}.ollama-thinking-details small{font-size:10px;color:#819b96}.ollama-thinking-status .thinking-stop{color:#ffb8b8;border-color:rgba(255,110,110,.45)}.ollama-composer.drop-active{border-color:rgba(117,210,183,.8);background:linear-gradient(135deg,rgba(33,92,79,.52),rgba(23,52,48,.62));box-shadow:0 0 24px rgba(117,210,183,.16)}.ollama-drop-hint{padding:9px;border:1px dashed rgba(117,210,183,.7);border-radius:7px;text-align:center;color:#b8ead9;font-size:11px;background:rgba(117,210,183,.08)}.ollama-thinking-spinner{display:flex;align-items:center;gap:4px;min-width:28px}.ollama-thinking-spinner span{width:7px;height:7px;border-radius:50%;background:#75d2b7;animation:ollama-thinking-pulse 1.1s ease-in-out infinite}.ollama-thinking-spinner span:nth-child(2){animation-delay:.18s}.ollama-thinking-spinner span:nth-child(3){animation-delay:.36s}.ollama-thinking-copy{display:flex;flex-direction:column;gap:3px}.ollama-thinking-copy strong{color:#effcf8;font-size:13px}.ollama-thinking-copy span{color:#b9d8d0;font-size:12px}.ollama-thinking-copy small{color:#8fb5ac;font-size:10px}@keyframes ollama-thinking-pulse{0%,80%,100%{opacity:.35;transform:scale(.8)}40%{opacity:1;transform:scale(1.2)}} \ No newline at end of file diff --git a/dashboard/manifest.json b/dashboard/manifest.json index 7bded33..20b9875 100644 --- a/dashboard/manifest.json +++ b/dashboard/manifest.json @@ -3,7 +3,7 @@ "label": "Ollama Models", "description": "Inspect, manage, and chat with local Ollama models, including shared persistent conversations, performance metrics, images, PDFs, URLs, and live memory telemetry.", "icon": "Cpu", - "version": "1.6.1", + "version": "1.7.0", "tab": {"path": "/ollama-manager", "position": "after:models"}, "entry": "dist/index.js", "css": "dist/style.css", diff --git a/dashboard/plugin_api.py b/dashboard/plugin_api.py index f414710..da00dd8 100644 --- a/dashboard/plugin_api.py +++ b/dashboard/plugin_api.py @@ -32,6 +32,13 @@ from fastapi import APIRouter, HTTPException from pydantic import BaseModel, Field from hermes_constants import get_hermes_home +try: + import psycopg + from psycopg.rows import dict_row +except ImportError: # pragma: no cover - SQLite remains available for local development + psycopg = None # type: ignore[assignment] + dict_row = None # type: ignore[assignment] + router = APIRouter() @@ -84,14 +91,249 @@ _chat_requests_lock = threading.Lock() _catalog_lock = threading.Lock() _chat_db_init_lock = threading.Lock() _chat_db_ready = False +_chat_job_futures: dict[str, Any] = {} +_chat_job_futures_lock = threading.Lock() +_chat_job_executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="ollama-chat-job") +_DATABASE_URL = os.environ.get("OLLAMA_MANAGER_DATABASE_URL", "").strip() + + +class _PostgresConnection: + def __init__(self, connection: Any): + self._connection = connection + + def execute(self, statement: str, parameters: tuple[Any, ...] = ()) -> Any: + return self._connection.execute(statement.replace("?", "%s"), parameters) + + def commit(self) -> None: + self._connection.commit() + + def close(self) -> None: + self._connection.close() + + +_POSTGRES_SCHEMA = """ +CREATE TABLE IF NOT EXISTS conversations ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT 'New conversation', + model TEXT NOT NULL DEFAULT '', + models_json TEXT NOT NULL DEFAULT '[]', + created_at DOUBLE PRECISION NOT NULL, + updated_at DOUBLE PRECISION NOT NULL +); +CREATE TABLE IF NOT EXISTS messages ( + id BIGSERIAL PRIMARY KEY, + conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, + request_id TEXT NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL DEFAULT '', + model TEXT NOT NULL DEFAULT '', + attachments_json TEXT NOT NULL DEFAULT '[]', + created_at DOUBLE PRECISION NOT NULL, + UNIQUE(request_id, role, model) +); +CREATE TABLE IF NOT EXISTS chat_metrics ( + id BIGSERIAL PRIMARY KEY, + conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, + request_id TEXT NOT NULL, + model TEXT NOT NULL, + status TEXT NOT NULL, + started_at DOUBLE PRECISION, + first_token_at DOUBLE PRECISION, + finished_at DOUBLE PRECISION, + prompt_eval_count INTEGER, + eval_count INTEGER, + total_duration_ns BIGINT, + load_duration_ns BIGINT, + prompt_eval_duration_ns BIGINT, + eval_duration_ns BIGINT, + error TEXT NOT NULL DEFAULT '', + created_at DOUBLE PRECISION NOT NULL, + UNIQUE(request_id, model) +); +CREATE TABLE IF NOT EXISTS message_versions ( + id BIGSERIAL PRIMARY KEY, + message_id BIGINT, + conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, + request_id TEXT NOT NULL, + role TEXT NOT NULL, + model TEXT NOT NULL DEFAULT '', + version_no INTEGER NOT NULL, + content TEXT NOT NULL DEFAULT '', + attachments_json TEXT NOT NULL DEFAULT '[]', + change_type TEXT NOT NULL, + created_at DOUBLE PRECISION NOT NULL, + UNIQUE(request_id, role, model, version_no) +); +CREATE TABLE IF NOT EXISTS chat_jobs ( + request_id TEXT PRIMARY KEY, + conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, + status TEXT NOT NULL, + mode TEXT NOT NULL, + primary_model TEXT NOT NULL, + validator_models_json TEXT NOT NULL DEFAULT '[]', + payload_json TEXT NOT NULL, + result_json TEXT NOT NULL DEFAULT '{}', + error TEXT NOT NULL DEFAULT '', + attempt INTEGER NOT NULL DEFAULT 0, + cancel_requested BOOLEAN NOT NULL DEFAULT FALSE, + created_at DOUBLE PRECISION NOT NULL, + started_at DOUBLE PRECISION, + finished_at DOUBLE PRECISION, + heartbeat_at DOUBLE PRECISION, + updated_at DOUBLE PRECISION NOT NULL +); +CREATE TABLE IF NOT EXISTS chat_job_stages ( + id BIGSERIAL PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES chat_jobs(request_id) ON DELETE CASCADE, + stage_key TEXT NOT NULL, + model TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL, + started_at DOUBLE PRECISION NOT NULL, + finished_at DOUBLE PRECISION, + output_chars INTEGER NOT NULL DEFAULT 0, + error TEXT NOT NULL DEFAULT '', + UNIQUE(request_id, stage_key) +); +CREATE TABLE IF NOT EXISTS chat_events ( + id BIGSERIAL PRIMARY KEY, + request_id TEXT NOT NULL, + conversation_id TEXT, + event_type TEXT NOT NULL, + level TEXT NOT NULL DEFAULT 'info', + stage TEXT NOT NULL DEFAULT '', + model TEXT NOT NULL DEFAULT '', + payload_json TEXT NOT NULL DEFAULT '{}', + created_at DOUBLE PRECISION NOT NULL +); +CREATE TABLE IF NOT EXISTS chat_schema_migrations ( + version TEXT PRIMARY KEY, + applied_at DOUBLE PRECISION NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_messages_conversation ON messages(conversation_id, id); +CREATE INDEX IF NOT EXISTS idx_metrics_conversation ON chat_metrics(conversation_id, id); +CREATE INDEX IF NOT EXISTS idx_jobs_status ON chat_jobs(status, updated_at); +CREATE INDEX IF NOT EXISTS idx_job_events_request ON chat_events(request_id, id); +CREATE INDEX IF NOT EXISTS idx_message_versions_conversation ON message_versions(conversation_id, id); +""" + + +def _storage_config() -> dict[str, Any]: + try: + value = json.loads((_home() / "storage.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + value = {} + backend = str(value.get("backend") or "sqlite").strip().lower() + return {"backend": backend if backend in {"sqlite", "postgres"} else "sqlite"} + + +def _storage_backend() -> str: + return _storage_config()["backend"] + + +def _write_storage_config(backend: str) -> None: + backend = str(backend or "sqlite").strip().lower() + if backend not in {"sqlite", "postgres"}: + raise HTTPException(400, "Storage backend must be sqlite or postgres") + path = _home() / "storage.json" + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps({"backend": backend}, indent=2) + "\n", encoding="utf-8") + try: + temporary.chmod(0o600) + os.replace(temporary, path) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + + +def _database_url() -> str: + if _DATABASE_URL: + return _DATABASE_URL + for path in (Path("/etc/hermes/ollama-manager/postgres.env"), _home() / "postgres.env"): + try: + for line in path.read_text(encoding="utf-8").splitlines(): + if line.startswith("OLLAMA_MANAGER_DATABASE_URL="): + return line.split("=", 1)[1].strip() + except OSError: + continue + return "" + + +def _postgres_configured() -> bool: + return bool(_database_url() and psycopg is not None and dict_row is not None) + + +def _postgres_enabled() -> bool: + return _storage_backend() == "postgres" and _postgres_configured() + + +def _reset_storage_cache() -> None: + global _chat_db_ready + with _chat_db_init_lock: + _chat_db_ready = False + + + +def _postgres_connection() -> _PostgresConnection: + if not _postgres_configured(): + raise RuntimeError("PostgreSQL chat storage is not configured") + connection = psycopg.connect(_database_url(), row_factory=dict_row, connect_timeout=10) + return _PostgresConnection(connection) def _chat_db_path() -> Path: return _home() / "chat.sqlite3" -def _chat_db() -> sqlite3.Connection: +def _migrate_sqlite_to_postgres(connection: _PostgresConnection) -> None: + marker = connection.execute("SELECT 1 FROM chat_schema_migrations WHERE version=?", ("sqlite-v1",)).fetchone() + if marker or not _chat_db_path().exists(): + return + source = sqlite3.connect(_chat_db_path()) + source.row_factory = sqlite3.Row + try: + for row in source.execute("SELECT id,title,model,models_json,created_at,updated_at FROM conversations"): + connection.execute( + "INSERT INTO conversations(id,title,model,models_json,created_at,updated_at) VALUES(?,?,?,?,?,?) ON CONFLICT(id) DO NOTHING", + tuple(row), + ) + for row in source.execute("SELECT id,conversation_id,request_id,role,content,model,attachments_json,created_at FROM messages ORDER BY id"): + connection.execute( + "INSERT INTO messages(id,conversation_id,request_id,role,content,model,attachments_json,created_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(request_id,role,model) DO NOTHING", + tuple(row), + ) + connection.execute( + "INSERT INTO message_versions(message_id,conversation_id,request_id,role,model,version_no,content,attachments_json,change_type,created_at) VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(request_id,role,model,version_no) DO NOTHING", + (row[0], row[1], row[2], row[3], row[5], 1, row[4], row[6], "legacy-import", row[7]), + ) + for row in source.execute("SELECT conversation_id,request_id,model,status,started_at,first_token_at,finished_at,prompt_eval_count,eval_count,total_duration_ns,load_duration_ns,prompt_eval_duration_ns,eval_duration_ns,error,created_at FROM chat_metrics ORDER BY id"): + connection.execute( + "INSERT INTO chat_metrics(conversation_id,request_id,model,status,started_at,first_token_at,finished_at,prompt_eval_count,eval_count,total_duration_ns,load_duration_ns,prompt_eval_duration_ns,eval_duration_ns,error,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(request_id,model) DO NOTHING", + tuple(row), + ) + connection.execute("SELECT setval(pg_get_serial_sequence('messages','id'), COALESCE((SELECT MAX(id) FROM messages), 1), true)") + connection.execute("SELECT setval(pg_get_serial_sequence('chat_metrics','id'), COALESCE((SELECT MAX(id) FROM chat_metrics), 1), true)") + connection.execute("SELECT setval(pg_get_serial_sequence('message_versions','id'), COALESCE((SELECT MAX(id) FROM message_versions), 1), true)") + connection.execute("INSERT INTO chat_schema_migrations(version,applied_at) VALUES(?,?) ON CONFLICT(version) DO NOTHING", ("sqlite-v1", time.time())) + connection.commit() + finally: + source.close() + + +def _chat_db() -> Any: global _chat_db_ready + if _storage_backend() == "postgres": + if not _postgres_configured(): + raise RuntimeError("PostgreSQL chat storage is selected but not configured") + connection = _postgres_connection() + with _chat_db_init_lock: + if not _chat_db_ready: + connection._connection.execute(_POSTGRES_SCHEMA) + _migrate_sqlite_to_postgres(connection) + connection.commit() + _chat_db_ready = True + return connection path = _chat_db_path() path.parent.mkdir(parents=True, exist_ok=True) with _chat_db_init_lock: @@ -102,46 +344,14 @@ def _chat_db() -> sqlite3.Connection: if not _chat_db_ready: connection.executescript( """ - CREATE TABLE IF NOT EXISTS conversations ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL DEFAULT 'New conversation', - model TEXT NOT NULL DEFAULT '', - models_json TEXT NOT NULL DEFAULT '[]', - created_at REAL NOT NULL, - updated_at REAL NOT NULL - ); - CREATE TABLE IF NOT EXISTS messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, - request_id TEXT NOT NULL, - role TEXT NOT NULL, - content TEXT NOT NULL DEFAULT '', - model TEXT NOT NULL DEFAULT '', - attachments_json TEXT NOT NULL DEFAULT '[]', - created_at REAL NOT NULL, - UNIQUE(request_id, role, model) - ); - CREATE INDEX IF NOT EXISTS idx_messages_conversation ON messages(conversation_id, id); - CREATE TABLE IF NOT EXISTS chat_metrics ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, - request_id TEXT NOT NULL, - model TEXT NOT NULL, - status TEXT NOT NULL, - started_at REAL, - first_token_at REAL, - finished_at REAL, - prompt_eval_count INTEGER, - eval_count INTEGER, - total_duration_ns INTEGER, - load_duration_ns INTEGER, - prompt_eval_duration_ns INTEGER, - eval_duration_ns INTEGER, - error TEXT NOT NULL DEFAULT '', - created_at REAL NOT NULL, - UNIQUE(request_id, model) - ); - CREATE INDEX IF NOT EXISTS idx_metrics_conversation ON chat_metrics(conversation_id, id); + CREATE TABLE IF NOT EXISTS conversations (id TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT 'New conversation', model TEXT NOT NULL DEFAULT '', models_json TEXT NOT NULL DEFAULT '[]', created_at REAL NOT NULL, updated_at REAL NOT NULL); + CREATE TABLE IF NOT EXISTS messages (id INTEGER PRIMARY KEY AUTOINCREMENT, conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, request_id TEXT NOT NULL, role TEXT NOT NULL, content TEXT NOT NULL DEFAULT '', model TEXT NOT NULL DEFAULT '', attachments_json TEXT NOT NULL DEFAULT '[]', created_at REAL NOT NULL, UNIQUE(request_id, role, model)); + CREATE TABLE IF NOT EXISTS chat_metrics (id INTEGER PRIMARY KEY AUTOINCREMENT, conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, request_id TEXT NOT NULL, model TEXT NOT NULL, status TEXT NOT NULL, started_at REAL, first_token_at REAL, finished_at REAL, prompt_eval_count INTEGER, eval_count INTEGER, total_duration_ns INTEGER, load_duration_ns INTEGER, prompt_eval_duration_ns INTEGER, eval_duration_ns INTEGER, error TEXT NOT NULL DEFAULT '', created_at REAL NOT NULL, UNIQUE(request_id, model)); + CREATE TABLE IF NOT EXISTS message_versions (id INTEGER PRIMARY KEY AUTOINCREMENT, message_id INTEGER, conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, request_id TEXT NOT NULL, role TEXT NOT NULL, model TEXT NOT NULL DEFAULT '', version_no INTEGER NOT NULL, content TEXT NOT NULL DEFAULT '', attachments_json TEXT NOT NULL DEFAULT '[]', change_type TEXT NOT NULL, created_at REAL NOT NULL, UNIQUE(request_id, role, model, version_no)); + CREATE TABLE IF NOT EXISTS chat_jobs (request_id TEXT PRIMARY KEY, conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, status TEXT NOT NULL, mode TEXT NOT NULL, primary_model TEXT NOT NULL, validator_models_json TEXT NOT NULL DEFAULT '[]', payload_json TEXT NOT NULL, result_json TEXT NOT NULL DEFAULT '{}', error TEXT NOT NULL DEFAULT '', attempt INTEGER NOT NULL DEFAULT 0, cancel_requested INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, started_at REAL, finished_at REAL, heartbeat_at REAL, updated_at REAL NOT NULL); + CREATE TABLE IF NOT EXISTS chat_job_stages (id INTEGER PRIMARY KEY AUTOINCREMENT, request_id TEXT NOT NULL REFERENCES chat_jobs(request_id) ON DELETE CASCADE, stage_key TEXT NOT NULL, model TEXT NOT NULL DEFAULT '', status TEXT NOT NULL, started_at REAL NOT NULL, finished_at REAL, output_chars INTEGER NOT NULL DEFAULT 0, error TEXT NOT NULL DEFAULT '', UNIQUE(request_id, stage_key)); + CREATE TABLE IF NOT EXISTS chat_events (id INTEGER PRIMARY KEY AUTOINCREMENT, request_id TEXT NOT NULL, conversation_id TEXT, event_type TEXT NOT NULL, level TEXT NOT NULL DEFAULT 'info', stage TEXT NOT NULL DEFAULT '', model TEXT NOT NULL DEFAULT '', payload_json TEXT NOT NULL DEFAULT '{}', created_at REAL NOT NULL); + CREATE TABLE IF NOT EXISTS chat_schema_migrations (version TEXT PRIMARY KEY, applied_at REAL NOT NULL); """ ) connection.commit() @@ -176,13 +386,21 @@ def _ensure_conversation(conversation_id: str, model: str, models: list[str], ti def _persist_message(conversation_id: str, request_id: str, role: str, content: str, model: str, attachments: list[dict[str, Any]] | None = None) -> None: + attachments_json = json.dumps(attachments or [])[:10000] + now = time.time() db = _chat_db() try: db.execute( - "INSERT OR IGNORE INTO messages(conversation_id,request_id,role,content,model,attachments_json,created_at) VALUES(?,?,?,?,?,?,?)", - (conversation_id, request_id, role, content, model, json.dumps(attachments or [])[:10000], time.time()), + "INSERT INTO messages(conversation_id,request_id,role,content,model,attachments_json,created_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(request_id,role,model) DO NOTHING", + (conversation_id, request_id, role, content, model, attachments_json, now), ) - db.execute("UPDATE conversations SET updated_at=? WHERE id=?", (time.time(), conversation_id)) + row = db.execute("SELECT id FROM messages WHERE request_id=? AND role=? AND model=?", (request_id, role, model)).fetchone() + if row: + db.execute( + "INSERT INTO message_versions(message_id,conversation_id,request_id,role,model,version_no,content,attachments_json,change_type,created_at) VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(request_id,role,model,version_no) DO NOTHING", + (row["id"], conversation_id, request_id, role, model, 1, content, attachments_json, "created", now), + ) + db.execute("UPDATE conversations SET updated_at=? WHERE id=?", (now, conversation_id)) db.commit() finally: db.close() @@ -233,6 +451,79 @@ def _persist_metric(conversation_id: str, request_id: str, model: str, state: di return values +def _persist_chat_event(request_id: str, conversation_id: str | None, event_type: str, *, level: str = "info", stage: str = "", model: str = "", payload: dict[str, Any] | None = None) -> None: + db = _chat_db() + try: + db.execute( + "INSERT INTO chat_events(request_id,conversation_id,event_type,level,stage,model,payload_json,created_at) VALUES(?,?,?,?,?,?,?,?)", + (request_id, conversation_id, event_type, level, stage, model, json.dumps(payload or {}, ensure_ascii=False)[:20000], time.time()), + ) + db.commit() + finally: + db.close() + + +def _create_chat_job(request_id: str, conversation_id: str, body: ChatRequest, primary: str, validators: list[str], harness: bool) -> None: + now = time.time() + db = _chat_db() + try: + db.execute( + "INSERT INTO chat_jobs(request_id,conversation_id,status,mode,primary_model,validator_models_json,payload_json,result_json,error,attempt,cancel_requested,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(request_id) DO NOTHING", + (request_id, conversation_id, "queued", "harness" if harness else "direct", primary, json.dumps(validators), json.dumps(body.model_dump(), ensure_ascii=False), "{}", "", 0, False, now, now), + ) + db.commit() + finally: + db.close() + _persist_chat_event(request_id, conversation_id, "job-queued", stage="Queued for server-side execution", model=primary, payload={"mode": "harness" if harness else "direct", "validators": validators}) + + +def _get_chat_job(request_id: str) -> dict[str, Any] | None: + db = _chat_db() + try: + row = db.execute("SELECT * FROM chat_jobs WHERE request_id=?", (request_id,)).fetchone() + return dict(row) if row else None + finally: + db.close() + + +def _update_chat_job(request_id: str, **values: Any) -> None: + allowed = {"status", "result_json", "error", "attempt", "cancel_requested", "started_at", "finished_at", "heartbeat_at", "updated_at"} + fields = {key: value for key, value in values.items() if key in allowed} + if not fields: + return + fields["updated_at"] = time.time() + assignments = ", ".join(f"{key}=?" for key in fields) + db = _chat_db() + try: + db.execute(f"UPDATE chat_jobs SET {assignments} WHERE request_id=?", tuple(fields.values()) + (request_id,)) + db.commit() + finally: + db.close() + + +def _job_status_response(job: dict[str, Any]) -> dict[str, Any]: + result = json.loads(job.get("result_json") or "{}") + status = str(job.get("status") or "unknown") + response: dict[str, Any] = { + "ok": status not in {"failed", "stopped", "canceled"}, + "request_id": job["request_id"], + "conversation_id": job["conversation_id"], + "state": status, + "status": status, + "mode": job.get("mode"), + "primary_model": job.get("primary_model"), + "validator_models": json.loads(job.get("validator_models_json") or "[]"), + "error": job.get("error") or "", + "done": status in {"completed", "failed", "stopped", "canceled"}, + "updated_at": job.get("updated_at"), + } + response.update(result) + return response + + +def _chat_request_from_job(job: dict[str, Any]) -> ChatRequest: + return ChatRequest.model_validate(json.loads(job["payload_json"])) + def _row_metric(row: sqlite3.Row) -> dict[str, Any]: value = dict(row) started = value.get("started_at") @@ -1319,6 +1610,13 @@ class ModelsRequest(BaseModel): placements: dict[str, str] = Field(default_factory=dict) +class StorageRequest(BaseModel): + backend: str = "sqlite" + confirm: str = "" + install: bool = False + database_url: str = "" + + class ChatAttachment(BaseModel): name: str = "" mime_type: str = "" @@ -1623,6 +1921,8 @@ def _run_validation_harness( with _chat_requests_lock: draft_state = dict(_chat_requests.get(draft_id, {})) metrics.append(_persist_metric(conversation_id, draft_id, primary, draft_state, status="draft")) + _persist_chat_stage(request_id, "draft", primary, "completed", float(draft_state.get("started_at") or time.time()), finished_at=float(draft_state.get("finished_at") or time.time()), output_chars=len(str((draft_result.get("message") or {}).get("content") or ""))) + _persist_chat_event(request_id, conversation_id, "draft-completed", stage="Primary draft complete", model=primary, payload={"output_chars": len(str((draft_result.get("message") or {}).get("content") or ""))}) draft_message = draft_result.get("message") if isinstance(draft_result.get("message"), dict) else {} draft = str(draft_message.get("content") or "").strip() if not draft: @@ -1633,6 +1933,9 @@ def _run_validation_harness( def run_validator(model: str) -> tuple[str, str, dict[str, Any]]: validator_id = uuid.uuid4().hex + validator_started = time.time() + _persist_chat_stage(request_id, f"validator:{model}", model, "running", validator_started) + _persist_chat_event(request_id, conversation_id, "validator-started", stage=f"Validating with {model}", model=model) _chat_state(validator_id, state="preparing", stage=f"Preparing validator {model}", model=model, parent_id=request_id, conversation_id=conversation_id) payload = _chat_payload( body, @@ -1646,7 +1949,11 @@ def _run_validation_harness( state = dict(_chat_requests.get(validator_id, {})) metric = _persist_metric(conversation_id, validator_id, model, state, status="validator") message = result.get("message") if isinstance(result.get("message"), dict) else {} - return model, str(message.get("content") or "").strip(), metric + report = str(message.get("content") or "").strip() + finished = float(state.get("finished_at") or time.time()) + _persist_chat_stage(request_id, f"validator:{model}", model, "completed", validator_started, finished_at=finished, output_chars=len(report)) + _persist_chat_event(request_id, conversation_id, "validator-completed", stage=f"Validator {model} complete", model=model, payload={"output_chars": len(report)}) + return model, report, metric with ThreadPoolExecutor(max_workers=len(validators), thread_name_prefix="ollama-validator") as pool: futures = [pool.submit(run_validator, model) for model in validators] @@ -1664,6 +1971,9 @@ def _run_validation_harness( [(item["model"], item["report"]) for item in reports], ) final_id = uuid.uuid4().hex + compiler_started = time.time() + _persist_chat_stage(request_id, "compiler", primary, "running", compiler_started) + _persist_chat_event(request_id, conversation_id, "compiler-started", stage="Primary model compiling final answer", model=primary) _chat_state(final_id, state="preparing", stage="Preparing primary compilation", model=primary, parent_id=request_id, conversation_id=conversation_id) final_payload = _chat_payload( body, @@ -1680,19 +1990,127 @@ def _run_validation_harness( final_content = str(final_message.get("content") or "").strip() if not final_content: raise HTTPException(502, "Primary model returned an empty compiled answer") + compiler_finished = time.time() + _persist_chat_stage(request_id, "compiler", primary, "completed", compiler_started, finished_at=compiler_finished, output_chars=len(final_content)) + _persist_chat_event(request_id, conversation_id, "compiler-completed", stage="Primary final answer compiled", model=primary, payload={"output_chars": len(final_content)}) return final_content, metrics, reports +def _persist_chat_stage(request_id: str, stage_key: str, model: str, status: str, started_at: float, *, finished_at: float | None = None, output_chars: int = 0, error: str = "") -> None: + db = _chat_db() + try: + db.execute( + "INSERT INTO chat_job_stages(request_id,stage_key,model,status,started_at,finished_at,output_chars,error) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(request_id,stage_key) DO UPDATE SET model=excluded.model,status=excluded.status,finished_at=excluded.finished_at,output_chars=excluded.output_chars,error=excluded.error", + (request_id, stage_key, model, status, started_at, finished_at, output_chars, error), + ) + db.commit() + finally: + db.close() + + +def _run_chat_job(request_id: str) -> None: + job = _get_chat_job(request_id) + if not job: + return + if bool(job.get("cancel_requested")): + _update_chat_job(request_id, status="canceled", finished_at=time.time()) + _persist_chat_event(request_id, job.get("conversation_id"), "job-canceled-before-start", level="warning") + return + body = _chat_request_from_job(job) + conversation_id = str(job["conversation_id"]) + primary = str(job["primary_model"]) + validators = json.loads(job.get("validator_models_json") or "[]") + mode = str(job.get("mode") or "direct") + started_at = time.time() + attempt = int(job.get("attempt") or 0) + 1 + _update_chat_job(request_id, status="running", attempt=attempt, started_at=started_at, heartbeat_at=started_at, error="") + _chat_state(request_id, state="running", stage="Server-side job running", model=primary, models=[primary, *validators], conversation_id=conversation_id) + _persist_chat_stage(request_id, "job", primary, "running", started_at) + _persist_chat_event(request_id, conversation_id, "job-started", stage="Server-side job running", model=primary, payload={"attempt": attempt, "mode": mode, "validators": validators}) + try: + with _chat_requests_lock: + cancel_event = _chat_requests.setdefault(request_id, {"request_id": request_id, "cancel": threading.Event(), "started_at": started_at})["cancel"] + if mode == "harness": + attachment_parts = [_attachment_parts(attachment) for attachment in body.attachments[:12]] + content, harness_metrics, validation_reports = _run_validation_harness(body, request_id, conversation_id, primary, validators, cancel_event, attachment_parts) + metrics = harness_metrics + result_payload = {"message": {"role": "assistant", "content": content}, "metrics": metrics, "validation_reports": validation_reports, "primary_model": primary, "validator_models": validators} + else: + payload = _chat_payload(body, primary) + result = _stream_chat_request(payload, request_id, cancel_event=cancel_event) + message = result.get("message") if isinstance(result.get("message"), dict) else {} + content = str(message.get("content") or "").strip() + if not content: + raise HTTPException(502, "Primary model returned an empty answer") + with _chat_requests_lock: + state = dict(_chat_requests.get(request_id, {})) + metrics = [_persist_metric(conversation_id, request_id, primary, state, status="completed")] + result_payload = {"message": {"role": "assistant", "content": content}, "metrics": metrics, "primary_model": primary, "validator_models": []} + _persist_message(conversation_id, request_id, "assistant", content, primary) + _update_chat_job(request_id, status="completed", result_json=json.dumps(result_payload, ensure_ascii=False), finished_at=time.time(), heartbeat_at=time.time()) + _persist_chat_stage(request_id, "job", primary, "completed", started_at, finished_at=time.time(), output_chars=len(content)) + _persist_chat_event(request_id, conversation_id, "job-completed", stage="Final answer persisted", model=primary, payload={"output_chars": len(content), "mode": mode}) + _chat_state(request_id, state="completed", stage="Final answer persisted", finished_at=time.time(), response_chars=len(content)) + except _ChatStopped as exc: + finished_at = time.time() + _update_chat_job(request_id, status="stopped", error="Stopped by user", finished_at=finished_at, heartbeat_at=finished_at) + _persist_chat_stage(request_id, "job", primary, "stopped", started_at, finished_at=finished_at, error="Stopped by user") + _persist_chat_event(request_id, conversation_id, "job-stopped", level="warning", stage="Stopped by user", model=primary) + _chat_state(request_id, state="stopped", stage="Stopped by user", finished_at=finished_at) + raise exc + except Exception as exc: + finished_at = time.time() + error = str(exc)[:2000] + _update_chat_job(request_id, status="failed", error=error, finished_at=finished_at, heartbeat_at=finished_at) + _persist_chat_stage(request_id, "job", primary, "failed", started_at, finished_at=finished_at, error=error) + _persist_chat_event(request_id, conversation_id, "job-failed", level="error", stage="Server-side job failed", model=primary, payload={"error": error}) + _chat_state(request_id, state="failed", stage="Server-side job failed", finished_at=finished_at, error=error) + finally: + with _chat_job_futures_lock: + _chat_job_futures.pop(request_id, None) + + +def _submit_chat_job(request_id: str) -> None: + with _chat_job_futures_lock: + future = _chat_job_futures.get(request_id) + if future is not None and not future.done(): + return + _chat_job_futures[request_id] = _chat_job_executor.submit(_run_chat_job, request_id) + + +def _recover_chat_jobs() -> None: + try: + db = _chat_db() + try: + rows = db.execute("SELECT request_id,status,conversation_id FROM chat_jobs WHERE status IN ('queued','running') ORDER BY created_at").fetchall() + finally: + db.close() + for row in rows: + request_id = str(row["request_id"]) + if str(row["status"]) == "running": + _update_chat_job(request_id, status="queued", heartbeat_at=time.time()) + _persist_chat_event(request_id, row["conversation_id"], "job-recovered", level="warning", stage="Recovered after dashboard restart") + _submit_chat_job(request_id) + except Exception: + return + + @router.get("/chat/status/{request_id}") def chat_status(request_id: str) -> dict[str, Any]: request_id = _valid_chat_request_id(request_id) with _chat_requests_lock: state = _chat_requests.get(request_id) - if not state: - raise HTTPException(404, "Chat request not found") - result = {key: value for key, value in state.items() if key not in {"cancel", "response"}} - result["elapsed"] = round(max(0.0, time.time() - float(state.get("started_at") or time.time())), 1) - return result + if state: + result = {key: value for key, value in state.items() if key not in {"cancel", "response"}} + result["elapsed"] = round(max(0.0, time.time() - float(state.get("started_at") or time.time())), 1) + job = _get_chat_job(request_id) + if job: + result.update(_job_status_response(job)) + return result + job = _get_chat_job(request_id) + if job: + return _job_status_response(job) + raise HTTPException(404, "Chat request not found") @router.post("/chat/stop") @@ -1700,25 +2118,36 @@ def chat_stop(body: ChatStopRequest) -> dict[str, Any]: request_id = _valid_chat_request_id(body.request_id) with _chat_requests_lock: state = _chat_requests.get(request_id) - if not state: - return {"ok": True, "request_id": request_id, "state": "not_found"} - state["cancel"].set() - responses = [] - for child_id, child in _chat_requests.items(): - if child_id == request_id or child.get("parent_id") == request_id: - child["cancel"].set() - response = child.get("response") - if response is not None: responses.append(response) - child["state"] = "stopping" - child["stage"] = "Stopping Ollama request" - child["updated_at"] = time.time() - state["state"] = "stopping" - state["stage"] = "Stopping Ollama request" - state["updated_at"] = time.time() + if state: + state["cancel"].set() + responses = [] + for child_id, child in _chat_requests.items(): + if child_id == request_id or child.get("parent_id") == request_id: + child["cancel"].set() + response = child.get("response") + if response is not None: + responses.append(response) + child["state"] = "stopping" + child["stage"] = "Stopping Ollama request" + child["updated_at"] = time.time() + state["state"] = "stopping" + state["stage"] = "Stopping Ollama request" + state["updated_at"] = time.time() + else: + responses = [] for response in responses: - try: response.close() - except Exception: pass - return {"ok": True, "request_id": request_id, "state": "stopping"} + try: + response.close() + except Exception: + pass + job = _get_chat_job(request_id) + if job and str(job.get("status")) not in {"completed", "failed", "stopped", "canceled"}: + _update_chat_job(request_id, cancel_requested=True, status="stopping") + _persist_chat_event(request_id, job.get("conversation_id"), "job-stop-requested", level="warning", stage="Stop requested by user") + return {"ok": True, "request_id": request_id, "state": "stopping"} + if state: + return {"ok": True, "request_id": request_id, "state": "stopping"} + return {"ok": True, "request_id": request_id, "state": "not_found"} @router.get("/conversations") @@ -1920,83 +2349,88 @@ def models_unload(body: ModelsRequest) -> dict[str, Any]: return {"ok": all(item["ok"] for item in results), "results": results, "runtime": _runtime_snapshot()} +@router.get("/storage") +def storage_status() -> dict[str, Any]: + postgres: dict[str, Any] = {"configured": _postgres_configured(), "available": False, "version": None} + if postgres["configured"]: + connection = None + try: + connection = _postgres_connection() + row = connection.execute("SELECT version() AS version").fetchone() + postgres.update({"available": True, "version": str(row["version"]) if row else None}) + except Exception as exc: + postgres["error"] = str(exc)[:300] + finally: + if connection is not None: + connection.close() + return {"backend": _storage_backend(), "sqlite": {"path": str(_chat_db_path()), "available": True}, "postgres": postgres} + + +@router.post("/storage/configure") +def storage_configure(body: StorageRequest) -> dict[str, Any]: + backend = str(body.backend or "sqlite").strip().lower() + expected_confirmation = "enable-postgresql" if backend == "postgres" else "use-sqlite" + if body.confirm != expected_confirmation: + raise HTTPException(400, f"Confirm storage change with '{expected_confirmation}'") + old_backend = _storage_backend() + if backend == "postgres": + if not _postgres_configured(): + raise HTTPException(400, "PostgreSQL is not configured; install or link a local PostgreSQL service first") + connection = None + try: + connection = _postgres_connection() + connection.execute("SELECT 1").fetchone() + except Exception as exc: + raise HTTPException(400, f"PostgreSQL connection failed: {str(exc)[:300]}") from exc + finally: + if connection is not None: + connection.close() + _write_storage_config(backend) + _reset_storage_cache() + try: + connection = _chat_db() + connection.close() + except Exception: + _write_storage_config(old_backend) + _reset_storage_cache() + raise + return {"ok": True, "backend": backend, "message": f"Chat storage linked to {backend}"} + + +@router.post("/storage/postgresql/install") +def storage_postgresql_install(body: StorageRequest) -> dict[str, Any]: + if body.confirm != "install-postgresql": + raise HTTPException(400, "Confirm native PostgreSQL installation with 'install-postgresql'") + installer = Path(__file__).resolve().parent.parent / "scripts" / "install_postgresql_storage.sh" + if not installer.exists(): + raise HTTPException(500, "PostgreSQL installer is not present in this plugin") + try: + result = subprocess.run(["bash", str(installer), "--install"], capture_output=True, text=True, timeout=900, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise HTTPException(500, f"PostgreSQL installer could not run: {type(exc).__name__}") from exc + if result.returncode != 0: + raise HTTPException(500, f"PostgreSQL installation failed: {result.stderr[-1000:]}") + return {"ok": True, "message": "Native PostgreSQL installation completed; review storage status and link it explicitly", "status": storage_status()} + + @router.post("/chat") def chat(body: ChatRequest) -> dict[str, Any]: request_id = _valid_chat_request_id(body.request_id or uuid.uuid4().hex) conversation_id = _conversation_id(body.conversation_id) + existing = _get_chat_job(request_id) + if existing: + _submit_chat_job(request_id) + return _job_status_response(existing) primary, validators, harness = _harness_models(body) selected = [primary, *validators] _ensure_conversation(conversation_id, primary, selected, body.message or "New conversation") attachment_meta = [{"name": item.name, "mime_type": item.mime_type, "url": item.url} for item in body.attachments[:12]] _persist_message(conversation_id, request_id, "user", body.message.strip() or "[Attachments]", primary, attachment_meta) - _chat_state(request_id, state="preparing", stage="Preparing attachments", models=selected, primary_model=primary, validator_models=validators, harness=harness, conversation_id=conversation_id) - with _chat_requests_lock: - cancel_event = _chat_requests[request_id]["cancel"] - - if not harness: - payload = _chat_payload(body, primary) - try: - result: dict[str, Any] = _stream_chat_request(payload, request_id, cancel_event=cancel_event) - except _ChatStopped as exc: - with _chat_requests_lock: - state = dict(_chat_requests.get(request_id, {})) - _persist_metric(conversation_id, request_id, primary, state, status="stopped") - raise HTTPException(499, "Chat stopped by user") from exc - except HTTPError as exc: - with _chat_requests_lock: - state = dict(_chat_requests.get(request_id, {})) - _persist_metric(conversation_id, request_id, primary, state, status="failed", error=str(exc)) - raise _ollama_error(exc) from exc - except Exception as exc: - with _chat_requests_lock: - state = dict(_chat_requests.get(request_id, {})) - _persist_metric(conversation_id, request_id, primary, state, status="failed", error=str(exc)) - raise - message = result.get("message") if isinstance(result.get("message"), dict) else {} - content = str(message.get("content") or "") - _persist_message(conversation_id, request_id, "assistant", content, primary) - with _chat_requests_lock: - state = dict(_chat_requests.get(request_id, {})) - persisted_metrics = _persist_metric(conversation_id, request_id, primary, state, status="completed") - return {"ok": True, "mode": "direct", "request_id": request_id, "conversation_id": conversation_id, "model": primary, "models": selected, "primary_model": primary, "validator_models": [], "message": {"role": "assistant", "content": content}, "done": True, "metrics": persisted_metrics, "runtime": _runtime_snapshot()} - - attachment_parts = [_attachment_parts(attachment) for attachment in body.attachments[:12]] - try: - content, harness_metrics, validation_reports = _run_validation_harness( - body, - request_id, - conversation_id, - primary, - validators, - cancel_event, - attachment_parts, - ) - except _ChatStopped as exc: - _chat_state(request_id, state="stopped", stage="Stopped by user", finished_at=time.time()) - raise HTTPException(499, "Chat stopped by user") from exc - except HTTPError as exc: - _chat_state(request_id, state="failed", stage="Validation harness failed", finished_at=time.time()) - raise _ollama_error(exc) from exc - except Exception as exc: - _chat_state(request_id, state="failed", stage="Validation harness failed", finished_at=time.time(), error=str(exc)) - raise - _persist_message(conversation_id, request_id, "assistant", content, primary) - _chat_state(request_id, state="completed", stage="Primary answer compiled and validated", finished_at=time.time(), response_chars=len(content)) - return { - "ok": True, - "mode": "harness", - "request_id": request_id, - "conversation_id": conversation_id, - "model": primary, - "models": selected, - "primary_model": primary, - "validator_models": validators, - "message": {"role": "assistant", "content": content}, - "validation_reports": validation_reports, - "metrics": harness_metrics, - "done": True, - "runtime": _runtime_snapshot(), - } + _chat_state(request_id, state="queued", stage="Queued for server-side execution", models=selected, primary_model=primary, validator_models=validators, harness=harness, conversation_id=conversation_id) + _create_chat_job(request_id, conversation_id, body, primary, validators, harness) + _submit_chat_job(request_id) + job = _get_chat_job(request_id) + return _job_status_response(job or {"request_id": request_id, "conversation_id": conversation_id, "status": "queued", "mode": "harness" if harness else "direct", "primary_model": primary, "validator_models_json": json.dumps(validators), "result_json": "{}", "error": "", "updated_at": time.time()}) @router.get("/connections") @@ -2214,3 +2648,4 @@ def delete_model(body: ModelRequest) -> dict[str, Any]: def create_ollama_routes(app) -> None: _sync_native_ollama_providers() app.include_router(router, prefix="/api/plugins/ollama-manager") + threading.Thread(target=_recover_chat_jobs, name="ollama-chat-recovery", daemon=True).start() diff --git a/plugin.yaml b/plugin.yaml index 655f3f7..357853d 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -1,9 +1,10 @@ name: ollama-manager -version: 1.6.1 +version: 1.7.0 description: Native dashboard manager and chat interface for local Ollama models, attachments, URLs, shared persistent conversations, performance metrics, and live runtime telemetry. auto_install_dependencies: true python_dependencies: - "pypdf>=6.0,<7.0" + - "psycopg[binary]>=3.2,<4.0" external_dependencies: - name: Ollama check: bash scripts/install_ollama.sh --check diff --git a/requirements.txt b/requirements.txt index f77f95d..990a5fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ # PDF text extraction for Ollama Chat attachments pypdf>=6.0,<7.0 +psycopg[binary]>=3.2,<4.0 diff --git a/scripts/install_postgresql_storage.sh b/scripts/install_postgresql_storage.sh new file mode 100644 index 0000000..4476ce6 --- /dev/null +++ b/scripts/install_postgresql_storage.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + printf '%s\n' 'Usage: install_postgresql_storage.sh --check|--install' +} + +case "${1:-}" in + --check) + if ! command -v psql >/dev/null 2>&1; then + printf '%s\n' 'PostgreSQL is not installed.' + exit 1 + fi + psql --version + if command -v pg_lsclusters >/dev/null 2>&1; then + pg_lsclusters + fi + ;; + --install) + if [ "$(id -u)" -ne 0 ]; then + printf '%s\n' 'PostgreSQL installation requires root.' >&2 + exit 1 + fi + export DEBIAN_FRONTEND=noninteractive + codename='' + if [ -r /etc/os-release ]; then + . /etc/os-release + codename="${VERSION_CODENAME:-}" + fi + if [ -z "$codename" ]; then + printf '%s\n' 'Could not determine the Ubuntu/Debian codename.' >&2 + exit 1 + fi + if [ -x /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh ]; then + bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y "$codename" + fi + apt-get update + candidate="$(apt-cache policy postgresql-18 2>/dev/null || true)" + case "$candidate" in + *'Candidate: (none)'*|"") apt-get install -y postgresql postgresql-client ;; + *) apt-get install -y postgresql-18 postgresql-client-18 ;; + esac + systemctl enable --now postgresql + printf '%s\n' 'Native PostgreSQL installation completed.' + psql --version + ;; + *) + usage >&2 + exit 2 + ;; +esac diff --git a/tests/test_validation_harness.py b/tests/test_validation_harness.py index 6c8e516..fff21b8 100644 --- a/tests/test_validation_harness.py +++ b/tests/test_validation_harness.py @@ -14,6 +14,13 @@ class ValidationHarnessTests(unittest.TestCase): self.assertEqual(validators, ["validator-a", "validator-b"]) self.assertTrue(harness) + def test_sqlite_is_default_even_when_postgres_is_installed(self): + with patch.object(api, "_storage_config", return_value={"backend": "sqlite"}), patch.object( + api, "_postgres_configured", return_value=True + ): + self.assertEqual(api._storage_backend(), "sqlite") + self.assertFalse(api._postgres_enabled()) + def test_harness_requires_one_distinct_validator(self): body = api.ChatRequest(primary_model="primary", validator_models=[], harness=True) with patch.object(api, "_require_installed_model", side_effect=lambda name: name): @@ -29,45 +36,28 @@ class ValidationHarnessTests(unittest.TestCase): self.assertEqual(validators, ["validator-a"]) self.assertTrue(harness) - def test_chat_route_returns_one_compiled_answer(self): - body = api.ChatRequest( - primary_model="primary", - validator_models=["validator-a", "validator-b"], - harness=True, - message="Answer this once and validate it.", - ) - persisted_messages = [] - - def fake_stream(payload, request_id, cancel_event=None, parent_id=None): - prompt = payload["messages"][-1]["content"] - if payload["model"] == "primary" and "VALIDATION REPORTS:" not in prompt: - content = "draft" - elif payload["model"].startswith("validator"): - content = "No material issues found." - else: - content = "single compiled answer" - return {"message": {"role": "assistant", "content": content}, "done": True} - - def fake_metric(conversation_id, request_id, model, state, status=None, error=""): - return {"request_id": request_id, "model": model, "status": status} - - with patch.object(api, "_harness_models", return_value=("primary", ["validator-a", "validator-b"], True)), patch.object( - api, "_require_installed_model", side_effect=lambda name: name - ), patch.object(api, "_ensure_conversation"), patch.object(api, "_persist_message", side_effect=lambda *args, **kwargs: persisted_messages.append(args)), patch.object( - api, "_persist_metric", side_effect=fake_metric - ), patch.object(api, "_stream_chat_request", side_effect=fake_stream), patch.object( - api, "_runtime_snapshot", return_value={} - ): + def test_chat_route_queues_server_owned_job(self): + body = api.ChatRequest(primary_model="primary", message="Queue this request") + fake_job = { + "request_id": "request", + "conversation_id": "conversation", + "status": "queued", + "mode": "direct", + "primary_model": "primary", + "validator_models_json": "[]", + "result_json": "{}", + "error": "", + "updated_at": 1.0, + } + with patch.object(api, "_harness_models", return_value=("primary", [], False)), patch.object( + api, "_get_chat_job", side_effect=[None, fake_job] + ), patch.object(api, "_ensure_conversation"), patch.object(api, "_persist_message"), patch.object( + api, "_chat_state" + ), patch.object(api, "_create_chat_job"), patch.object(api, "_submit_chat_job"): response = api.chat(body) - - self.assertEqual(response["mode"], "harness") - self.assertEqual(response["message"]["content"], "single compiled answer") - self.assertEqual(response["primary_model"], "primary") - self.assertEqual(response["validator_models"], ["validator-a", "validator-b"]) - assistant_messages = [args for args in persisted_messages if len(args) >= 3 and args[2] == "assistant"] - self.assertEqual(len(assistant_messages), 1) - self.assertEqual(assistant_messages[0][3], "single compiled answer") - self.assertEqual(len(response["validation_reports"]), 2) + self.assertEqual(response["status"], "queued") + self.assertFalse(response["done"]) + self.assertEqual(response["mode"], "direct") def test_primary_draft_validators_and_primary_compilation_produce_one_answer(self): body = api.ChatRequest( @@ -95,7 +85,9 @@ class ValidationHarnessTests(unittest.TestCase): with patch.object(api, "_require_installed_model", side_effect=lambda name: name), patch.object( api, "_stream_chat_request", side_effect=fake_stream - ), patch.object(api, "_persist_metric", side_effect=fake_metric): + ), patch.object(api, "_persist_metric", side_effect=fake_metric), patch.object( + api, "_persist_chat_stage" + ), patch.object(api, "_persist_chat_event"): final, metrics, reports = api._run_validation_harness( body, "root-request", @@ -112,10 +104,6 @@ class ValidationHarnessTests(unittest.TestCase): self.assertEqual(len(calls), 4) self.assertEqual(calls[0][0], "primary") self.assertTrue(all(call[2] == "root-request" for call in calls)) - with api._chat_requests_lock: - child_states = [state for key, state in api._chat_requests.items() if key != "root-request"] - self.assertGreaterEqual(len(child_states), 4) - self.assertTrue(all(state.get("parent_id") == "root-request" for state in child_states[-4:])) compiler_prompt = calls[-1][1] self.assertIn("PRIMARY DRAFT:", compiler_prompt) self.assertIn("VALIDATOR 1", compiler_prompt)