Make Ollama Chat the natural default experience

dashboard/plugin_api.py

@@ -42,6 +42,8 @@ CHAT_KEEP_ALIVE = "10m"
 
 _jobs: dict[str, dict[str, Any]] = {}
 _jobs_lock = threading.Lock()
+_chat_requests: dict[str, dict[str, Any]] = {}
+_chat_requests_lock = threading.Lock()
 _catalog_lock = threading.Lock()
 
 CAPABILITY_INFO = {
@@ -745,8 +747,12 @@ class ChatRequest(BaseModel):
     message: str = ""
     history: list[dict[str, Any]] = Field(default_factory=list)
     attachments: list[ChatAttachment] = Field(default_factory=list)
+    request_id: str = ""
 
 
+class ChatStopRequest(BaseModel):
+    request_id: str
+
 def _installed_model_names() -> set[str]:
     return {
         str(row.get("name") or row.get("model"))
@@ -811,6 +817,106 @@ def _chat_payload(body: ChatRequest) -> dict[str, Any]:
     return {"model": model, "messages": messages, "stream": False, "keep_alive": CHAT_KEEP_ALIVE}
 
 
+class _ChatStopped(Exception):
+    pass
+
+
+def _valid_chat_request_id(value: str) -> str:
+    value = str(value or "").strip()
+    if not re.fullmatch(r"[A-Za-z0-9._-]{1,80}", value):
+        raise HTTPException(400, "Invalid chat request id")
+    return value
+
+
+def _chat_state(request_id: str, **values: Any) -> dict[str, Any]:
+    with _chat_requests_lock:
+        state = _chat_requests.setdefault(request_id, {"request_id": request_id, "cancel": threading.Event(), "state": "starting", "stage": "Preparing request", "started_at": time.time(), "chunks": 0, "thinking_chars": 0, "response_chars": 0})
+        state.update(values, updated_at=time.time())
+        return {key: value for key, value in state.items() if key != "cancel"}
+
+
+def _stream_chat_request(payload: dict[str, Any], request_id: str) -> dict[str, Any]:
+    data = json.dumps({**payload, "stream": True}).encode("utf-8")
+    request = Request(LOCAL_OLLAMA + "/api/chat", data=data, headers={"Accept": "application/x-ndjson", "Content-Type": "application/json"}, method="POST")
+    response_text: list[str] = []
+    thinking_text: list[str] = []
+    _chat_state(request_id, state="connecting", stage="Connecting to Ollama")
+    try:
+        with urlopen(request, timeout=1800) as response:
+            sock = getattr(getattr(getattr(response, "fp", None), "raw", None), "_sock", None)
+            if sock is not None:
+                sock.settimeout(1.0)
+            _chat_state(request_id, state="generating", stage="Ollama is generating")
+            while True:
+                with _chat_requests_lock:
+                    cancelled = bool(_chat_requests.get(request_id, {}).get("cancel", threading.Event()).is_set())
+                if cancelled:
+                    response.close()
+                    raise _ChatStopped()
+                try:
+                    raw_line = response.readline()
+                except (socket.timeout, TimeoutError):
+                    continue
+                if not raw_line:
+                    break
+                try:
+                    event = json.loads(raw_line.decode("utf-8", errors="replace"))
+                except ValueError:
+                    continue
+                message = event.get("message") if isinstance(event.get("message"), dict) else {}
+                chunk = str(message.get("content") or event.get("response") or "")
+                thinking_chunk = str(message.get("thinking") or event.get("thinking") or "")
+                if chunk:
+                    response_text.append(chunk)
+                if thinking_chunk:
+                    thinking_text.append(thinking_chunk)
+                _chat_state(
+                    request_id,
+                    state="generating",
+                    stage="Ollama is generating the response" if chunk else "Ollama is processing model thinking",
+                    chunks=int(_chat_requests.get(request_id, {}).get("chunks", 0)) + 1,
+                    thinking_chars=sum(map(len, thinking_text)),
+                    response_chars=sum(map(len, response_text)),
+                    eval_count=event.get("eval_count"),
+                )
+                if event.get("done"):
+                    break
+    except _ChatStopped:
+        _chat_state(request_id, state="stopped", stage="Stopped by user", finished_at=time.time())
+        raise
+    except Exception:
+        _chat_state(request_id, state="failed", stage="Ollama request failed", finished_at=time.time())
+        raise
+    _chat_state(request_id, state="completed", stage="Response complete", finished_at=time.time())
+    return {"message": {"role": "assistant", "content": "".join(response_text)}, "done": True}
+
+
+@router.get("/chat/status/{request_id}")
+def chat_status(request_id: str) -> dict[str, Any]:
+    request_id = _valid_chat_request_id(request_id)
+    with _chat_requests_lock:
+        state = _chat_requests.get(request_id)
+    if not state:
+        raise HTTPException(404, "Chat request not found")
+    result = {key: value for key, value in state.items() if key not in {"cancel"}}
+    result["elapsed"] = round(max(0.0, time.time() - float(state.get("started_at") or time.time())), 1)
+    return result
+
+
+@router.post("/chat/stop")
+def chat_stop(body: ChatStopRequest) -> dict[str, Any]:
+    request_id = _valid_chat_request_id(body.request_id)
+    with _chat_requests_lock:
+        state = _chat_requests.get(request_id)
+        if not state:
+            return {"ok": True, "request_id": request_id, "state": "not_found"}
+        state["cancel"].set()
+        state["state"] = "stopping"
+        state["stage"] = "Stopping Ollama request"
+        state["updated_at"] = time.time()
+    return {"ok": True, "request_id": request_id, "state": "stopping"}
+
+
 @router.get("/runtime")
 def runtime() -> dict[str, Any]:
     return _runtime_snapshot()
@@ -823,17 +929,22 @@ def chat_load(body: ModelRequest) -> dict[str, Any]:
 
 @router.post("/chat")
 def chat(body: ChatRequest) -> dict[str, Any]:
+    request_id = _valid_chat_request_id(body.request_id or uuid.uuid4().hex)
+    _chat_state(request_id, state="preparing", stage="Preparing attachments")
     payload = _chat_payload(body)
     try:
-        result = _json_request(LOCAL_OLLAMA + "/api/chat", method="POST", payload=payload, timeout=1800)
+        result = _stream_chat_request(payload, request_id)
+    except _ChatStopped as exc:
+        raise HTTPException(499, "Chat stopped by user") from exc
     except HTTPError as exc:
         raise _ollama_error(exc) from exc
     message = result.get("message") if isinstance(result.get("message"), dict) else {}
     return {
         "ok": True,
+        "request_id": request_id,
         "model": payload["model"],
         "message": {"role": "assistant", "content": str(message.get("content") or "")},
-        "done": bool(result.get("done", True)),
+        "done": True,
         "runtime": _runtime_snapshot(),
     }