dennii/Hermes-Ollama_Models
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add RAM bypass and load safety guard

Browse Source
This commit is contained in:
Hermes Agent
2026-08-26 01:27:11 +10:00
parent b961b3986e
commit 6348b23881
6 changed files with 110 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dashboard/plugin_api.py
+91 -11
View File
@@ -1369,6 +1369,41 @@ def _model_placement(value: str | None) -> str:
return placement
PERMANENT_LOAD_RAM_LIMIT_PERCENT = 95.0
def _memory_usage_percent() -> float | None:
mem = _read_meminfo()
total = int(mem.get("MemTotal") or 0)
available = int(mem.get("MemAvailable") or mem.get("MemFree") or 0)
if not total:
return None
return round(max(0.0, min(100.0, (total - available) * 100 / total)), 1)
def _memory_safety_check() -> dict[str, Any]:
usage_percent = _memory_usage_percent()
return {
"threshold_percent": PERMANENT_LOAD_RAM_LIMIT_PERCENT,
"usage_percent": usage_percent,
"triggered": usage_percent is not None and usage_percent >= PERMANENT_LOAD_RAM_LIMIT_PERCENT,
}
def _unload_model(name: str) -> dict[str, Any]:
name = _require_installed_model(name)
try:
_json_request(
LOCAL_OLLAMA + "/api/generate",
method="POST",
payload={"model": name, "prompt": "", "stream": False, "keep_alive": 0},
timeout=120,
)
return {"name": name, "ok": True}
except Exception as exc:
return {"name": name, "ok": False, "error": str(exc)}
def _load_model(name: str, placement: str = "gpu_ram") -> dict[str, Any]:
name = _require_installed_model(name)
placement = _model_placement(placement)
@@ -1633,50 +1668,91 @@ def models_load(body: ModelsRequest) -> dict[str, Any]:
if not names:
raise HTTPException(400, "Select at least one model to load")
placements = {name: _model_placement(body.placements.get(name)) for name in names}
# Load RAM-only models first, then GPU+RAM models. This avoids asking the
# GPU scheduler to rearrange an already GPU-resident runner unnecessarily.
ordered_names = sorted(names, key=lambda name: 0 if placements[name] == "ram_only" else 1)
initial_resident = {str(row.get("name") or row.get("model")) for row in _local_ps()}
started_by_action: list[str] = []
unloaded_by_safety: list[dict[str, Any]] = []
results_by_name: dict[str, dict[str, Any]] = {}
safety = _memory_safety_check()
safety_triggered = bool(safety["triggered"])
for pass_index in range(2):
if safety_triggered:
break
resident_now = {str(row.get("name") or row.get("model")) for row in _local_ps()}
missing_now = [name for name in ordered_names if name not in resident_now]
if not missing_now:
break
for name in missing_now:
safety = _memory_safety_check()
if safety["triggered"]:
safety_triggered = True
break
placement = placements[name]
stage = "Loading into GPU + RAM" if placement == "gpu_ram" else "Loading into system RAM only"
_model_load_update(name, active=True, state="loading", stage=stage + " · pass " + str(pass_index + 1), attempt=pass_index + 1)
try:
result = _load_model(name, placement)
results_by_name[name] = {"name": name, "placement": placement, "ok": True, "result": result, "attempts": pass_index + 1}
if name not in initial_resident and name not in started_by_action:
started_by_action.append(name)
_model_load_update(name, state="checking", stage="Checking Ollama resident state", attempt=pass_index + 1)
safety = _memory_safety_check()
if safety["triggered"]:
safety_triggered = True
break
except Exception as exc:
results_by_name[name] = {"name": name, "placement": placement, "ok": False, "error": str(exc), "attempts": pass_index + 1}
_model_load_update(name, active=False, state="failed", stage="Ollama load failed", finished_at=time.time(), error=str(exc), attempt=pass_index + 1)
if safety_triggered:
for name in reversed(started_by_action):
unloaded = _unload_model(name)
unloaded_by_safety.append(unloaded)
_model_load_update(name, active=False, state="safety_rollback", stage="Unloaded after 95% RAM safety stop", finished_at=time.time())
resident_rows = _local_ps()
resident_names = {str(row.get("name") or row.get("model")) for row in resident_rows}
results = [results_by_name.get(name, {"name": name, "placement": placements[name], "ok": name in resident_names, "attempts": 0}) for name in names]
for item in results:
item["resident"] = item["name"] in resident_names
results = []
for name in names:
item = results_by_name.get(name, {"name": name, "placement": placements[name], "ok": name in resident_names, "attempts": 0})
item["resident"] = name in resident_names
if safety_triggered and name not in results_by_name:
item["ok"] = False
item["error"] = "Loading stopped by 95% RAM safety limit"
results.append(item)
not_resident = [name for name in names if name not in resident_names]
for item in results:
if item["name"] in resident_names:
_model_load_update(item["name"], active=False, state="resident", stage="Model is resident in Ollama", finished_at=time.time())
elif item["ok"]:
elif item.get("ok"):
_model_load_update(item["name"], active=False, state="evicted", stage="Ollama did not retain this model", finished_at=time.time())
runtime = _runtime_snapshot()
retry_count = sum(max(0, int(item.get("attempts") or 0) - 1) for item in results)
memory_safety = {
**safety,
"triggered": safety_triggered,
"unloaded_by_safety": unloaded_by_safety,
"message": (
"Permanent loading stopped because host RAM reached the 95% safety limit. Models started by this action were unloaded; models resident before this action were preserved."
if safety_triggered
else "RAM remained below the 95% permanent-load safety limit."
),
}
return {
"ok": bool(results) and not not_resident and all(item["ok"] for item in results),
"ok": bool(results) and not not_resident and all(item["ok"] for item in results) and not safety_triggered,
"requested": names,
"resident": sorted(resident_names),
"not_resident": not_resident,
"results": results,
"retries": retry_count,
"runtime": runtime,
"memory_safety": memory_safety,
"keep_alive": "permanent",
"message": (
"All selected models are resident. Automatic Ollama eviction recovery completed."
memory_safety["message"]
if safety_triggered
else "All selected models are resident. Automatic Ollama eviction recovery completed."
if not not_resident and retry_count
else "All selected models are resident."
if not not_resident
@@ -1876,18 +1952,21 @@ def status() -> dict[str, Any]:
if str(variant.get("name") or variant.get("model") or "") not in catalog_names
]
downloadable = []
all_downloadable = []
seen_downloads: set[str] = set()
for rank, row in enumerate(candidate_rows, 1):
name = str(row.get("name") or row.get("model") or "")
if _is_mlx(row) or not name or name in installed_names or name in seen_downloads:
continue
view = catalog_view(row, "catalog")
if not _known_ram_fit(view):
continue
view["memory_fit"] = _known_ram_fit(view)
view["memory_warning"] = "Estimated runtime RAM exceeds detected host RAM" if view["memory_fit"] is False else ""
if name in catalog_names:
view["popularity_rank"] = rank
seen_downloads.add(name)
downloadable.append(view)
all_downloadable.append(view)
if view["memory_fit"]:
downloadable.append(dict(view))
popular = _popular_fit_models(
[row for row in catalog.get("models", []) if not _is_mlx(row)],
@@ -1936,6 +2015,7 @@ def status() -> dict[str, Any]:
"smaller_fit_variants_substituted": True,
},
"catalog": downloadable,
"catalog_all": all_downloadable,
"catalog_filter": {
"max_expected_ram_gib": _host_ram_gib(),
"basis": "detected MemTotal from the running host",