feat: retry evicted models in one load action

dashboard/plugin_api.py

@@ -1446,21 +1446,29 @@ def models_load(body: ModelsRequest) -> dict[str, Any]:
     if not names:
         raise HTTPException(400, "Select at least one model to load")
     placements = {name: _model_placement(body.placements.get(name)) for name in names}
-    results = []
-    for name in names:
-        _model_load_update(name, active=True, state="queued", stage="Waiting for Ollama", started_at=time.time(), error="", placement=placements[name])
-    for name in names:
-        placement = placements[name]
-        stage = "Loading into GPU + RAM" if placement == "gpu_ram" else "Loading into system RAM only"
-        _model_load_update(name, state="loading", stage=stage)
-        try:
-            results.append({"name": name, "placement": placement, "ok": True, "result": _load_model(name, placement)})
-            _model_load_update(name, state="checking", stage="Checking Ollama resident state")
-        except Exception as exc:
-            results.append({"name": name, "placement": placement, "ok": False, "error": str(exc)})
-            _model_load_update(name, active=False, state="failed", stage="Ollama load failed", finished_at=time.time(), error=str(exc))
+    # Load RAM-only models first, then GPU+RAM models. This avoids asking the
+    # GPU scheduler to rearrange an already GPU-resident runner unnecessarily.
+    ordered_names = sorted(names, key=lambda name: 0 if placements[name] == "ram_only" else 1)
+    results_by_name: dict[str, dict[str, Any]] = {}
+    for pass_index in range(2):
+        resident_now = {str(row.get("name") or row.get("model")) for row in _local_ps()}
+        missing_now = [name for name in ordered_names if name not in resident_now]
+        if not missing_now:
+            break
+        for name in missing_now:
+            placement = placements[name]
+            stage = "Loading into GPU + RAM" if placement == "gpu_ram" else "Loading into system RAM only"
+            _model_load_update(name, active=True, state="loading", stage=stage + " · pass " + str(pass_index + 1), attempt=pass_index + 1)
+            try:
+                result = _load_model(name, placement)
+                results_by_name[name] = {"name": name, "placement": placement, "ok": True, "result": result, "attempts": pass_index + 1}
+                _model_load_update(name, state="checking", stage="Checking Ollama resident state", attempt=pass_index + 1)
+            except Exception as exc:
+                results_by_name[name] = {"name": name, "placement": placement, "ok": False, "error": str(exc), "attempts": pass_index + 1}
+                _model_load_update(name, active=False, state="failed", stage="Ollama load failed", finished_at=time.time(), error=str(exc), attempt=pass_index + 1)
     resident_rows = _local_ps()
     resident_names = {str(row.get("name") or row.get("model")) for row in resident_rows}
+    results = [results_by_name.get(name, {"name": name, "placement": placements[name], "ok": name in resident_names, "attempts": 0}) for name in names]
     for item in results:
         item["resident"] = item["name"] in resident_names
     not_resident = [name for name in names if name not in resident_names]
@@ -1470,16 +1478,20 @@ def models_load(body: ModelsRequest) -> dict[str, Any]:
         elif item["ok"]:
             _model_load_update(item["name"], active=False, state="evicted", stage="Ollama did not retain this model", finished_at=time.time())
     runtime = _runtime_snapshot()
+    retry_count = sum(max(0, int(item.get("attempts") or 0) - 1) for item in results)
     return {
         "ok": bool(results) and not not_resident and all(item["ok"] for item in results),
         "requested": names,
         "resident": sorted(resident_names),
         "not_resident": not_resident,
         "results": results,
+        "retries": retry_count,
         "runtime": runtime,
         "keep_alive": "permanent",
         "message": (
-            "All selected models are resident."
+            "All selected models are resident. Automatic Ollama eviction recovery completed."
+            if not not_resident and retry_count
+            else "All selected models are resident."
             if not not_resident
             else "Ollama did not keep every selected model resident; see not_resident."
         ),