feat: add per-model GPU and RAM placement

README.md

@@ -47,7 +47,15 @@ The Available downloads controls support:
 
 Popularity and date ordering use upstream metadata only; the plugin does not invent popularity, dates, RAM requirements, or token metrics.
 
-## Ollama capacity warnings
+## Per-model placement
+
+Each installed model in the Model pool has a persistent placement selector:
+
+- **GPU + RAM (automatic offload)**: Ollama uses GPU layers where it can and keeps the remainder in system RAM. This is the default.
+- **RAM only (CPU)**: the plugin sends Ollama `num_gpu: 0`, preventing GPU layer offload for that model.
+
+Placement is sent both when loading models and when chatting, so a RAM-only model is not silently reloaded with GPU offload. RAM-only models will not increase GPU VRAM usage; GPU+RAM models can still be evicted by Ollama if the GPU/device-memory scheduler cannot fit the runner.
+
 
 When Ollama accepts a load request but evicts one model while starting another, the dashboard displays an amber capacity warning rather than a red plugin error. On the verified host, Ollama logged that a 27.9 GiB runner would exceed available device memory with approximately 2.5 GiB GPU memory free. The T600 has 4 GiB VRAM, so two large multimodal models cannot be guaranteed resident simultaneously by the plugin. The actual resident set remains authoritative through `/api/ps`.
 

dashboard/dist/index.js

@@ -8,7 +8,18 @@
   var fetchJSON = SDK.fetchJSON;
   var API = "/api/plugins/ollama-manager";
   var CHAT_STORAGE_KEY = "hermes.ollama-manager.chat.v1";
+  var PLACEMENT_STORAGE_KEY = "hermes.ollama-manager.placement.v1";
 
+  function readSavedPlacements() {
+    try {
+      var raw = window.localStorage.getItem(PLACEMENT_STORAGE_KEY);
+      var value = raw ? JSON.parse(raw) : {};
+      return value && typeof value === "object" && !Array.isArray(value) ? value : {};
+    } catch (_) { return {}; }
+  }
+  function savePlacements(value) {
+    try { window.localStorage.setItem(PLACEMENT_STORAGE_KEY, JSON.stringify(value || {})); } catch (_) {}
+  }
   function readSavedChat() {
     try {
       var raw = window.localStorage.getItem(CHAT_STORAGE_KEY);
@@ -210,7 +221,7 @@
     var models = props.models || [], loaded = models.filter(function (item) { return item.loaded; });
     return h("section", { className: "ollama-model-pool" },
       h("div", { className: "ollama-pool-heading" }, h("div", null, h("h3", null, "Model pool"), h("p", null, "Choose installed models to keep permanently loaded. Loaded models remain available to Hermes Agent through Local Ollama.")), h(Badge, { tone: loaded.length ? "live" : "muted" }, loaded.length + " loaded")),
-      h("div", { className: "ollama-pool-grid" }, models.map(function (item) { var loaded = !!item.loaded, selected = props.poolSelection.indexOf(item.name) >= 0; return h("label", { className: "ollama-pool-item" + (loaded ? " loaded" : "") + (selected ? " selected" : ""), key: item.name, title: loaded ? "Loaded and resident in Ollama" : "Installed but not resident" }, h("input", { type: "checkbox", checked: selected, onChange: function () { props.onTogglePool(item.name); } }), h("span", null, h("strong", null, item.name), h("small", null, loaded ? "Loaded and resident · keep-alive active" : "Installed · not loaded", " · ", (item.capabilities || []).join(", ") || "capabilities unknown"))); })),
+      h("div", { className: "ollama-pool-grid" }, models.map(function (item) { var loaded = !!item.loaded, selected = props.poolSelection.indexOf(item.name) >= 0, placement = props.placements[item.name] || "gpu_ram"; return h("label", { className: "ollama-pool-item" + (loaded ? " loaded" : "") + (selected ? " selected" : ""), key: item.name, title: loaded ? "Loaded and resident in Ollama" : "Installed but not resident" }, h("input", { type: "checkbox", checked: selected, onChange: function () { props.onTogglePool(item.name); } }), h("span", null, h("strong", null, item.name), h("small", null, loaded ? "Loaded and resident · keep-alive active" : "Installed · not loaded", " · ", (item.capabilities || []).join(", ") || "capabilities unknown"), h("span", { className: "ollama-placement-control" }, h("small", null, "Placement"), h("select", { value: placement, onClick: function (event) { event.stopPropagation(); }, onChange: function (event) { event.stopPropagation(); props.onPlacementChange(item.name, event.target.value); } }, h("option", { value: "gpu_ram" }, "GPU + RAM (automatic offload)"), h("option", { value: "ram_only" }, "RAM only (CPU)"))))); })),
       h("div", { className: "ollama-pool-actions" }, h(Button, { disabled: !props.poolSelection.length || !!props.busy, onClick: props.onLoad }, props.busy === "/models/load" ? "Loading " + props.poolSelection.length + " model" + (props.poolSelection.length === 1 ? "" : "s") + "…" : "Load selected permanently"), h(Button, { className: "secondary", disabled: !props.poolSelection.length || !!props.busy, onClick: props.onUnload }, props.busy === "/models/unload" ? "Unloading…" : "Unload selected")),
       h("div", { className: "ollama-chat-model-selection" }, h("strong", null, "Models for this answer"), h("small", null, "Select one or more loaded models for parallel perspectives."), loaded.length ? loaded.map(function (item) { return h("label", { className: "loaded", key: item.name }, h("input", { type: "checkbox", checked: props.selectedModels.indexOf(item.name) >= 0, onChange: function () { props.onToggleChat(item.name); } }), item.name, " · ", (item.capabilities || []).join(", ")); }) : h("span", null, "Load one or more models above first."))
     );
@@ -225,6 +236,7 @@
     var poolState = React.useState(loadedModels.map(function (item) { return item.name; })), poolSelection = poolState[0], setPoolSelection = poolState[1];
     var poolLoadedSignature = React.useRef("");
     var chatLoadedSignature = React.useRef("");
+    var placementState = React.useState(readSavedPlacements()), placements = placementState[0], setPlacements = placementState[1];
     var messageState = React.useState(""), message = messageState[0], setMessage = messageState[1];
     var urlState = React.useState(""), url = urlState[0], setUrl = urlState[1];
     var attachState = React.useState([]), attachments = attachState[0], setAttachments = attachState[1];
@@ -312,12 +324,15 @@
     function pollRuntime() { fetchJSON(API + "/runtime").then(function (value) { setRuntime(value); setSamples(function (old) { return old.concat([{ used: Number(value.memory_used_bytes || 0), total: Number(value.memory_total_bytes || 0) }]).slice(-60); }); }).catch(function () {}); }
     React.useEffect(function () { pollRuntime(); var timer = setInterval(pollRuntime, 1000); return function () { clearInterval(timer); }; }, []);
 
+    React.useEffect(function () { savePlacements(placements); }, [placements]);
     function toggleIn(setter, name) { setter(function (old) { return old.indexOf(name) >= 0 ? old.filter(function (item) { return item !== name; }) : old.concat([name]); }); }
+    function setPlacement(name, value) { setPlacements(function (old) { var next = Object.assign({}, old); next[name] = value === "ram_only" ? "ram_only" : "gpu_ram"; return next; }); }
     function manageModels(endpoint, label) {
       if (!poolSelection.length) { setNotice({ error: "Select one or more installed models first." }); return; }
       var requested = poolSelection.slice();
+      var requestedPlacements = requested.reduce(function (result, name) { result[name] = placements[name] || "gpu_ram"; return result; }, {});
       setBusy(endpoint); setNotice({ ok: label + " in progress for " + requested.length + " model" + (requested.length === 1 ? "" : "s") + "…" });
-      fetchJSON(API + endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ names: requested }) }).then(function (result) {
+      fetchJSON(API + endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ names: requested, placements: requestedPlacements }) }).then(function (result) {
         if (endpoint === "/models/load") {
           var resident = Array.isArray(result.resident) ? result.resident : ((result.runtime && result.runtime.model_memory) || []).map(function (item) { return item.name; });
           var missing = Array.isArray(result.not_resident) ? result.not_resident : requested.filter(function (name) { return resident.indexOf(name) < 0; });
@@ -362,7 +377,7 @@
       var requestId = makeRequestId();
       var controller = typeof AbortController === "function" ? new AbortController() : null;
       var current = { id: requestId, controller: controller, stopped: false };
-      var outgoing = { role: "user", content: message.trim() || "[Attachments]" }, body = { model: selectedModels[0], models: selectedModels, message: message, history: history, attachments: attachments, request_id: requestId, conversation_id: conversationId };
+      var outgoing = { role: "user", content: message.trim() || "[Attachments]" }, body = { model: selectedModels[0], models: selectedModels, placements: placements, message: message, history: history, attachments: attachments, request_id: requestId, conversation_id: conversationId };
       setHistory(function (old) { return old.concat([outgoing]); }); setMessage(""); setBusy("send"); setActiveRequest(current); setThinking({ request_id: requestId, startedAt: Date.now(), stage: attachments.length ? "Preparing attachments and sending request to Ollama" : "Sending request to Ollama" }); setThinkingDetails(null); setThinkingOpen(false); setNotice(null);
       var requestOptions = { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) };
       if (controller) requestOptions.signal = controller.signal;
@@ -376,7 +391,7 @@
         aggregate && h("div", { className: "ollama-metrics-summary" }, h("strong", null, "Model performance · ", aggregate.sample_count || 0, " samples"), h("span", null, "TTFT avg: ", aggregate.avg_time_to_first_token_ms == null ? "n/a" : aggregate.avg_time_to_first_token_ms + " ms"), h("span", null, "Output: ", aggregate.avg_eval_tokens_per_second == null ? "n/a" : aggregate.avg_eval_tokens_per_second + " tok/s"), h("span", null, "Latency: ", aggregate.avg_total_latency_ms == null ? "n/a" : aggregate.avg_total_latency_ms + " ms"), h("span", null, "Errors: ", aggregate.error_count || 0)),
         metrics && metrics.length > 0 && h("div", { className: "ollama-metrics-detail" }, (metrics.slice(-3)).map(function (item, index) { return h("span", { key: index }, item.model || "model", " · TTFT ", item.time_to_first_token_ms == null ? "n/a" : item.time_to_first_token_ms + " ms", " · ", item.eval_count == null ? "n/a" : item.eval_count + " output tokens", " · ", item.eval_tokens_per_second == null ? "n/a" : item.eval_tokens_per_second + " tok/s"); }))
       ),
-      h(ModelPoolPanel, { models: models, loadedModels: loadedModels, selectedModels: selectedModels, poolSelection: poolSelection, busy: busy, onTogglePool: togglePoolModel, onToggleChat: toggleChatModel, onLoad: loadModel, onUnload: unloadModels }),
+      h(ModelPoolPanel, { models: models, loadedModels: loadedModels, selectedModels: selectedModels, poolSelection: poolSelection, placements: placements, busy: busy, onTogglePool: togglePoolModel, onPlacementChange: setPlacement, onToggleChat: toggleChatModel, onLoad: loadModel, onUnload: unloadModels }),
       notice && h("div", { className: "ollama-notice " + (notice.error ? "error" : notice.warning ? "warning" : "ok") }, notice.error || notice.warning || notice.ok),
       thinking && h(ThinkingStatus, { stage: thinkingDetails && thinkingDetails.stage ? thinkingDetails.stage : (thinkingElapsed < 1 ? thinking.stage : "Ollama is generating the response"), elapsed: thinkingDetails && thinkingDetails.elapsed != null ? thinkingDetails.elapsed : thinkingElapsed, details: thinkingDetails, expanded: thinkingOpen, onToggle: function () { setThinkingOpen(!thinkingOpen); }, onStop: stop }),
       h(RuntimePanel, { runtime: runtime, samples: samples }),

dashboard/manifest.json

@@ -3,7 +3,7 @@
   "label": "Ollama Models",
   "description": "Inspect, manage, and chat with local Ollama models, including shared persistent conversations, performance metrics, images, PDFs, URLs, and live memory telemetry.",
   "icon": "Cpu",
-  "version": "1.5.14",
+  "version": "1.5.15",
   "tab": {"path": "/ollama-manager", "position": "after:models"},
   "entry": "dist/index.js",
   "css": "dist/style.css",

dashboard/plugin_api.py

@@ -1116,6 +1116,7 @@ def _new_job(name: str, action: str, target: str = "local") -> str:
 class ModelRequest(BaseModel):
     name: str
     target: str = "local"
+    placement: str = "gpu_ram"
 
 
 class ConnectionRequest(BaseModel):
@@ -1125,6 +1126,7 @@ class ConnectionRequest(BaseModel):
 
 class ModelsRequest(BaseModel):
     names: list[str] = Field(default_factory=list)
+    placements: dict[str, str] = Field(default_factory=dict)
 
 
 class ChatAttachment(BaseModel):
@@ -1140,6 +1142,7 @@ class ChatRequest(BaseModel):
     message: str = ""
     history: list[dict[str, Any]] = Field(default_factory=list)
     attachments: list[ChatAttachment] = Field(default_factory=list)
+    placements: dict[str, str] = Field(default_factory=dict)
     request_id: str = ""
     conversation_id: str = ""
 
@@ -1172,18 +1175,29 @@ def _ollama_error(exc: HTTPError) -> HTTPException:
     return HTTPException(502, f"Ollama request failed: {detail}")
 
 
-def _load_model(name: str) -> dict[str, Any]:
+def _model_placement(value: str | None) -> str:
+    placement = str(value or "gpu_ram").strip().lower()
+    if placement not in {"gpu_ram", "ram_only"}:
+        raise HTTPException(400, "Model placement must be gpu_ram or ram_only")
+    return placement
+
+
+def _load_model(name: str, placement: str = "gpu_ram") -> dict[str, Any]:
     name = _require_installed_model(name)
+    placement = _model_placement(placement)
+    options: dict[str, Any] = {"num_predict": 1}
+    if placement == "ram_only":
+        options["num_gpu"] = 0
     try:
         result = _json_request(
             LOCAL_OLLAMA + "/api/generate",
             method="POST",
-            payload={"model": name, "prompt": "", "stream": False, "keep_alive": CHAT_KEEP_ALIVE, "options": {"num_predict": 1}},
+            payload={"model": name, "prompt": "", "stream": False, "keep_alive": CHAT_KEEP_ALIVE, "options": options},
             timeout=900,
         )
     except HTTPError as exc:
         raise _ollama_error(exc) from exc
-    return {"ok": True, "model": name, "response": result.get("response", ""), "runtime": _runtime_snapshot()}
+    return {"ok": True, "model": name, "placement": placement, "response": result.get("response", ""), "runtime": _runtime_snapshot()}
 
 
 def _chat_payload(body: ChatRequest, model_name: str | None = None) -> dict[str, Any]:
@@ -1208,7 +1222,10 @@ def _chat_payload(body: ChatRequest, model_name: str | None = None) -> dict[str,
     if images:
         user_message["images"] = images
     messages.append(user_message)
-    return {"model": model, "messages": messages, "stream": False, "keep_alive": CHAT_KEEP_ALIVE}
+    payload: dict[str, Any] = {"model": model, "messages": messages, "stream": False, "keep_alive": CHAT_KEEP_ALIVE}
+    if _model_placement(body.placements.get(model)) == "ram_only":
+        payload["options"] = {"num_gpu": 0}
+    return payload
 
 
 class _ChatStopped(Exception):
@@ -1420,7 +1437,7 @@ def runtime() -> dict[str, Any]:
 
 @router.post("/chat/load")
 def chat_load(body: ModelRequest) -> dict[str, Any]:
-    return _load_model(body.name)
+    return _load_model(body.name, body.placement)
 
 
 @router.post("/models/load")
@@ -1428,16 +1445,19 @@ def models_load(body: ModelsRequest) -> dict[str, Any]:
     names = list(dict.fromkeys(_valid_name(name) for name in body.names if str(name).strip()))[:12]
     if not names:
         raise HTTPException(400, "Select at least one model to load")
+    placements = {name: _model_placement(body.placements.get(name)) for name in names}
     results = []
     for name in names:
-        _model_load_update(name, active=True, state="queued", stage="Waiting for Ollama", started_at=time.time(), error="")
+        _model_load_update(name, active=True, state="queued", stage="Waiting for Ollama", started_at=time.time(), error="", placement=placements[name])
     for name in names:
-        _model_load_update(name, state="loading", stage="Loading model into Ollama memory")
+        placement = placements[name]
+        stage = "Loading into GPU + RAM" if placement == "gpu_ram" else "Loading into system RAM only"
+        _model_load_update(name, state="loading", stage=stage)
         try:
-            results.append({"name": name, "ok": True, "result": _load_model(name)})
+            results.append({"name": name, "placement": placement, "ok": True, "result": _load_model(name, placement)})
             _model_load_update(name, state="checking", stage="Checking Ollama resident state")
         except Exception as exc:
-            results.append({"name": name, "ok": False, "error": str(exc)})
+            results.append({"name": name, "placement": placement, "ok": False, "error": str(exc)})
             _model_load_update(name, active=False, state="failed", stage="Ollama load failed", finished_at=time.time(), error=str(exc))
     resident_rows = _local_ps()
     resident_names = {str(row.get("name") or row.get("model")) for row in resident_rows}

plugin.yaml

@@ -1,5 +1,5 @@
 name: ollama-manager
-version: 1.5.14
+version: 1.5.15
 description: Native dashboard manager and chat interface for local Ollama models, attachments, URLs, shared persistent conversations, performance metrics, and live runtime telemetry.
 auto_install_dependencies: true
 python_dependencies: