feat: show live Ollama model loading progress

dashboard/plugin_api.py

@@ -71,6 +71,8 @@ CHAT_KEEP_ALIVE = -1
 
 _jobs: dict[str, dict[str, Any]] = {}
 _jobs_lock = threading.Lock()
+_model_loads: dict[str, dict[str, Any]] = {}
+_model_loads_lock = threading.Lock()
 _chat_requests: dict[str, dict[str, Any]] = {}
 _chat_requests_lock = threading.Lock()
 _catalog_lock = threading.Lock()
@@ -445,6 +447,21 @@ def _local_ps() -> list[dict[str, Any]]:
         return []
 
 
+def _model_load_update(name: str, **values: Any) -> None:
+    with _model_loads_lock:
+        current = _model_loads.setdefault(name, {"name": name, "state": "queued", "stage": "Queued for Ollama", "started_at": time.time()})
+        current.update(values, updated_at=time.time())
+
+
+def _model_load_snapshot() -> list[dict[str, Any]]:
+    now = time.time()
+    with _model_loads_lock:
+        rows = [dict(value) for value in _model_loads.values()]
+    for row in rows:
+        row["elapsed"] = round(max(0.0, now - float(row.get("started_at") or now)), 1)
+    return sorted(rows, key=lambda row: row.get("started_at") or 0)
+
+
 def _read_meminfo() -> dict[str, int]:
     values: dict[str, int] = {}
     try:
@@ -507,6 +524,8 @@ def _runtime_snapshot() -> dict[str, Any]:
     swap_free = mem.get("SwapFree", 0)
     ps_rows = _local_ps()
     tag_rows = {str(row.get("name") or row.get("model")): row for row in _local_tags()}
+    load_rows = _model_load_snapshot()
+    active_loads = [row for row in load_rows if row.get("active")]
     model_memory = []
     for row in ps_rows:
         name = str(row.get("name") or row.get("model") or "")
@@ -528,6 +547,18 @@ def _runtime_snapshot() -> dict[str, Any]:
             "quantization": capability_view["quantization"],
             "permanent": True,
         })
+    gpu = _gpu_snapshot()
+    ollama_model_bytes = sum(int(row.get("total_bytes") or 0) for row in model_memory)
+    ollama_model_vram_bytes = sum(int(row.get("gpu_bytes") or 0) for row in model_memory)
+    model_loading = []
+    for row in active_loads:
+        tag = tag_rows.get(str(row.get("name")), {})
+        model_loading.append({
+            **row,
+            "estimated_bytes": int(tag.get("size") or 0),
+            "estimated_vram_bytes": 0,
+        })
+    ollama_target_model_bytes = ollama_model_bytes + sum(int(row.get("estimated_bytes") or 0) for row in model_loading)
     return {
         "captured_at": time.time(),
         "memory_total_bytes": total,
@@ -536,7 +567,12 @@ def _runtime_snapshot() -> dict[str, Any]:
         "swap_total_bytes": swap_total,
         "swap_used_bytes": max(0, swap_total - swap_free),
         "model_memory": model_memory,
-        "gpu": _gpu_snapshot(),
+        "model_loading": model_loading,
+        "model_loads": load_rows[-12:],
+        "ollama_model_bytes": ollama_model_bytes,
+        "ollama_model_vram_bytes": ollama_model_vram_bytes,
+        "ollama_target_model_bytes": ollama_target_model_bytes,
+        "gpu": gpu,
     }
 
 
@@ -1394,15 +1430,25 @@ def models_load(body: ModelsRequest) -> dict[str, Any]:
         raise HTTPException(400, "Select at least one model to load")
     results = []
     for name in names:
+        _model_load_update(name, active=True, state="queued", stage="Waiting for Ollama", started_at=time.time(), error="")
+    for name in names:
+        _model_load_update(name, state="loading", stage="Loading model into Ollama memory")
         try:
             results.append({"name": name, "ok": True, "result": _load_model(name)})
+            _model_load_update(name, state="checking", stage="Checking Ollama resident state")
         except Exception as exc:
             results.append({"name": name, "ok": False, "error": str(exc)})
+            _model_load_update(name, active=False, state="failed", stage="Ollama load failed", finished_at=time.time(), error=str(exc))
     resident_rows = _local_ps()
     resident_names = {str(row.get("name") or row.get("model")) for row in resident_rows}
     for item in results:
         item["resident"] = item["name"] in resident_names
     not_resident = [name for name in names if name not in resident_names]
+    for item in results:
+        if item["name"] in resident_names:
+            _model_load_update(item["name"], active=False, state="resident", stage="Model is resident in Ollama", finished_at=time.time())
+        elif item["ok"]:
+            _model_load_update(item["name"], active=False, state="evicted", stage="Ollama did not retain this model", finished_at=time.time())
     runtime = _runtime_snapshot()
     return {
         "ok": bool(results) and not not_resident and all(item["ok"] for item in results),