diff --git a/README.md b/README.md index 3541072..8a34be3 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,15 @@ The Available downloads controls support: Popularity and date ordering use upstream metadata only; the plugin does not invent popularity, dates, RAM requirements, or token metrics. -## Ollama capacity warnings +## Per-model placement + +Each installed model in the Model pool has a persistent placement selector: + +- **GPU + RAM (automatic offload)**: Ollama uses GPU layers where it can and keeps the remainder in system RAM. This is the default. +- **RAM only (CPU)**: the plugin sends Ollama `num_gpu: 0`, preventing GPU layer offload for that model. + +Placement is sent both when loading models and when chatting, so a RAM-only model is not silently reloaded with GPU offload. RAM-only models will not increase GPU VRAM usage; GPU+RAM models can still be evicted by Ollama if the GPU/device-memory scheduler cannot fit the runner. + When Ollama accepts a load request but evicts one model while starting another, the dashboard displays an amber capacity warning rather than a red plugin error. On the verified host, Ollama logged that a 27.9 GiB runner would exceed available device memory with approximately 2.5 GiB GPU memory free. The T600 has 4 GiB VRAM, so two large multimodal models cannot be guaranteed resident simultaneously by the plugin. The actual resident set remains authoritative through `/api/ps`. diff --git a/dashboard/dist/index.js b/dashboard/dist/index.js index 19f995a..17574b4 100644 --- a/dashboard/dist/index.js +++ b/dashboard/dist/index.js @@ -8,7 +8,18 @@ var fetchJSON = SDK.fetchJSON; var API = "/api/plugins/ollama-manager"; var CHAT_STORAGE_KEY = "hermes.ollama-manager.chat.v1"; + var PLACEMENT_STORAGE_KEY = "hermes.ollama-manager.placement.v1"; + function readSavedPlacements() { + try { + var raw = window.localStorage.getItem(PLACEMENT_STORAGE_KEY); + var value = raw ? JSON.parse(raw) : {}; + return value && typeof value === "object" && !Array.isArray(value) ? value : {}; + } catch (_) { return {}; } + } + function savePlacements(value) { + try { window.localStorage.setItem(PLACEMENT_STORAGE_KEY, JSON.stringify(value || {})); } catch (_) {} + } function readSavedChat() { try { var raw = window.localStorage.getItem(CHAT_STORAGE_KEY); @@ -210,7 +221,7 @@ var models = props.models || [], loaded = models.filter(function (item) { return item.loaded; }); return h("section", { className: "ollama-model-pool" }, h("div", { className: "ollama-pool-heading" }, h("div", null, h("h3", null, "Model pool"), h("p", null, "Choose installed models to keep permanently loaded. Loaded models remain available to Hermes Agent through Local Ollama.")), h(Badge, { tone: loaded.length ? "live" : "muted" }, loaded.length + " loaded")), - h("div", { className: "ollama-pool-grid" }, models.map(function (item) { var loaded = !!item.loaded, selected = props.poolSelection.indexOf(item.name) >= 0; return h("label", { className: "ollama-pool-item" + (loaded ? " loaded" : "") + (selected ? " selected" : ""), key: item.name, title: loaded ? "Loaded and resident in Ollama" : "Installed but not resident" }, h("input", { type: "checkbox", checked: selected, onChange: function () { props.onTogglePool(item.name); } }), h("span", null, h("strong", null, item.name), h("small", null, loaded ? "Loaded and resident · keep-alive active" : "Installed · not loaded", " · ", (item.capabilities || []).join(", ") || "capabilities unknown"))); })), + h("div", { className: "ollama-pool-grid" }, models.map(function (item) { var loaded = !!item.loaded, selected = props.poolSelection.indexOf(item.name) >= 0, placement = props.placements[item.name] || "gpu_ram"; return h("label", { className: "ollama-pool-item" + (loaded ? " loaded" : "") + (selected ? " selected" : ""), key: item.name, title: loaded ? "Loaded and resident in Ollama" : "Installed but not resident" }, h("input", { type: "checkbox", checked: selected, onChange: function () { props.onTogglePool(item.name); } }), h("span", null, h("strong", null, item.name), h("small", null, loaded ? "Loaded and resident · keep-alive active" : "Installed · not loaded", " · ", (item.capabilities || []).join(", ") || "capabilities unknown"), h("span", { className: "ollama-placement-control" }, h("small", null, "Placement"), h("select", { value: placement, onClick: function (event) { event.stopPropagation(); }, onChange: function (event) { event.stopPropagation(); props.onPlacementChange(item.name, event.target.value); } }, h("option", { value: "gpu_ram" }, "GPU + RAM (automatic offload)"), h("option", { value: "ram_only" }, "RAM only (CPU)"))))); })), h("div", { className: "ollama-pool-actions" }, h(Button, { disabled: !props.poolSelection.length || !!props.busy, onClick: props.onLoad }, props.busy === "/models/load" ? "Loading " + props.poolSelection.length + " model" + (props.poolSelection.length === 1 ? "" : "s") + "…" : "Load selected permanently"), h(Button, { className: "secondary", disabled: !props.poolSelection.length || !!props.busy, onClick: props.onUnload }, props.busy === "/models/unload" ? "Unloading…" : "Unload selected")), h("div", { className: "ollama-chat-model-selection" }, h("strong", null, "Models for this answer"), h("small", null, "Select one or more loaded models for parallel perspectives."), loaded.length ? loaded.map(function (item) { return h("label", { className: "loaded", key: item.name }, h("input", { type: "checkbox", checked: props.selectedModels.indexOf(item.name) >= 0, onChange: function () { props.onToggleChat(item.name); } }), item.name, " · ", (item.capabilities || []).join(", ")); }) : h("span", null, "Load one or more models above first.")) ); @@ -225,6 +236,7 @@ var poolState = React.useState(loadedModels.map(function (item) { return item.name; })), poolSelection = poolState[0], setPoolSelection = poolState[1]; var poolLoadedSignature = React.useRef(""); var chatLoadedSignature = React.useRef(""); + var placementState = React.useState(readSavedPlacements()), placements = placementState[0], setPlacements = placementState[1]; var messageState = React.useState(""), message = messageState[0], setMessage = messageState[1]; var urlState = React.useState(""), url = urlState[0], setUrl = urlState[1]; var attachState = React.useState([]), attachments = attachState[0], setAttachments = attachState[1]; @@ -312,12 +324,15 @@ function pollRuntime() { fetchJSON(API + "/runtime").then(function (value) { setRuntime(value); setSamples(function (old) { return old.concat([{ used: Number(value.memory_used_bytes || 0), total: Number(value.memory_total_bytes || 0) }]).slice(-60); }); }).catch(function () {}); } React.useEffect(function () { pollRuntime(); var timer = setInterval(pollRuntime, 1000); return function () { clearInterval(timer); }; }, []); + React.useEffect(function () { savePlacements(placements); }, [placements]); function toggleIn(setter, name) { setter(function (old) { return old.indexOf(name) >= 0 ? old.filter(function (item) { return item !== name; }) : old.concat([name]); }); } + function setPlacement(name, value) { setPlacements(function (old) { var next = Object.assign({}, old); next[name] = value === "ram_only" ? "ram_only" : "gpu_ram"; return next; }); } function manageModels(endpoint, label) { if (!poolSelection.length) { setNotice({ error: "Select one or more installed models first." }); return; } var requested = poolSelection.slice(); + var requestedPlacements = requested.reduce(function (result, name) { result[name] = placements[name] || "gpu_ram"; return result; }, {}); setBusy(endpoint); setNotice({ ok: label + " in progress for " + requested.length + " model" + (requested.length === 1 ? "" : "s") + "…" }); - fetchJSON(API + endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ names: requested }) }).then(function (result) { + fetchJSON(API + endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ names: requested, placements: requestedPlacements }) }).then(function (result) { if (endpoint === "/models/load") { var resident = Array.isArray(result.resident) ? result.resident : ((result.runtime && result.runtime.model_memory) || []).map(function (item) { return item.name; }); var missing = Array.isArray(result.not_resident) ? result.not_resident : requested.filter(function (name) { return resident.indexOf(name) < 0; }); @@ -362,7 +377,7 @@ var requestId = makeRequestId(); var controller = typeof AbortController === "function" ? new AbortController() : null; var current = { id: requestId, controller: controller, stopped: false }; - var outgoing = { role: "user", content: message.trim() || "[Attachments]" }, body = { model: selectedModels[0], models: selectedModels, message: message, history: history, attachments: attachments, request_id: requestId, conversation_id: conversationId }; + var outgoing = { role: "user", content: message.trim() || "[Attachments]" }, body = { model: selectedModels[0], models: selectedModels, placements: placements, message: message, history: history, attachments: attachments, request_id: requestId, conversation_id: conversationId }; setHistory(function (old) { return old.concat([outgoing]); }); setMessage(""); setBusy("send"); setActiveRequest(current); setThinking({ request_id: requestId, startedAt: Date.now(), stage: attachments.length ? "Preparing attachments and sending request to Ollama" : "Sending request to Ollama" }); setThinkingDetails(null); setThinkingOpen(false); setNotice(null); var requestOptions = { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }; if (controller) requestOptions.signal = controller.signal; @@ -376,7 +391,7 @@ aggregate && h("div", { className: "ollama-metrics-summary" }, h("strong", null, "Model performance · ", aggregate.sample_count || 0, " samples"), h("span", null, "TTFT avg: ", aggregate.avg_time_to_first_token_ms == null ? "n/a" : aggregate.avg_time_to_first_token_ms + " ms"), h("span", null, "Output: ", aggregate.avg_eval_tokens_per_second == null ? "n/a" : aggregate.avg_eval_tokens_per_second + " tok/s"), h("span", null, "Latency: ", aggregate.avg_total_latency_ms == null ? "n/a" : aggregate.avg_total_latency_ms + " ms"), h("span", null, "Errors: ", aggregate.error_count || 0)), metrics && metrics.length > 0 && h("div", { className: "ollama-metrics-detail" }, (metrics.slice(-3)).map(function (item, index) { return h("span", { key: index }, item.model || "model", " · TTFT ", item.time_to_first_token_ms == null ? "n/a" : item.time_to_first_token_ms + " ms", " · ", item.eval_count == null ? "n/a" : item.eval_count + " output tokens", " · ", item.eval_tokens_per_second == null ? "n/a" : item.eval_tokens_per_second + " tok/s"); })) ), - h(ModelPoolPanel, { models: models, loadedModels: loadedModels, selectedModels: selectedModels, poolSelection: poolSelection, busy: busy, onTogglePool: togglePoolModel, onToggleChat: toggleChatModel, onLoad: loadModel, onUnload: unloadModels }), + h(ModelPoolPanel, { models: models, loadedModels: loadedModels, selectedModels: selectedModels, poolSelection: poolSelection, placements: placements, busy: busy, onTogglePool: togglePoolModel, onPlacementChange: setPlacement, onToggleChat: toggleChatModel, onLoad: loadModel, onUnload: unloadModels }), notice && h("div", { className: "ollama-notice " + (notice.error ? "error" : notice.warning ? "warning" : "ok") }, notice.error || notice.warning || notice.ok), thinking && h(ThinkingStatus, { stage: thinkingDetails && thinkingDetails.stage ? thinkingDetails.stage : (thinkingElapsed < 1 ? thinking.stage : "Ollama is generating the response"), elapsed: thinkingDetails && thinkingDetails.elapsed != null ? thinkingDetails.elapsed : thinkingElapsed, details: thinkingDetails, expanded: thinkingOpen, onToggle: function () { setThinkingOpen(!thinkingOpen); }, onStop: stop }), h(RuntimePanel, { runtime: runtime, samples: samples }), diff --git a/dashboard/dist/style.css b/dashboard/dist/style.css index f428f76..0345508 100644 --- a/dashboard/dist/style.css +++ b/dashboard/dist/style.css @@ -1,7 +1,7 @@ .ollama-page{min-height:100%;padding:28px 32px 56px;color:#e8f2ef;background:linear-gradient(135deg,rgba(17,42,39,.92),rgba(10,24,23,.98));font-family:inherit}.ollama-hero{display:flex;justify-content:space-between;gap:24px;align-items:flex-start;border-bottom:1px solid rgba(164,211,199,.16);padding-bottom:22px}.ollama-eyebrow{font-size:10px;letter-spacing:.18em;color:#8ec8bb;font-weight:700}.ollama-hero h1{margin:7px 0 6px;font-size:30px;letter-spacing:.02em}.ollama-hero p{margin:0;color:#a5bfba;max-width:680px}.ollama-health{display:flex;align-items:center;gap:10px;flex-wrap:wrap;justify-content:flex-end;color:#9bb6b0;font-size:12px}.ollama-badge-row{display:flex;gap:6px;flex-wrap:wrap;margin-top:7px}.ollama-badge{display:inline-flex;align-items:center;border:1px solid rgba(152,204,192,.24);border-radius:999px;padding:4px 8px;color:#b4d1ca;font-size:10px;letter-spacing:.06em;text-transform:uppercase;background:rgba(80,125,115,.12)}.ollama-badge.live{color:#9af1c7;border-color:rgba(72,218,147,.4);background:rgba(35,137,91,.2)}.ollama-badge.installed{color:#bce1ff;border-color:rgba(89,168,231,.35);background:rgba(30,96,145,.2)}.ollama-badge.download{color:#f6d18f;border-color:rgba(244,182,80,.35);background:rgba(142,90,25,.18)}.ollama-badge.moe{color:#e2b4ff;border-color:rgba(197,107,255,.35);background:rgba(105,44,139,.2)}.ollama-badge.danger{color:#ffb1b1;border-color:rgba(255,100,100,.45);background:rgba(160,40,40,.18)}.ollama-button{border:1px solid rgba(155,205,194,.24);border-radius:7px;background:rgba(70,117,108,.18);color:#dbebe7;padding:8px 11px;cursor:pointer;font:inherit;font-size:11px;transition:background .15s,border-color .15s}.ollama-button:hover:not(:disabled){background:rgba(91,161,144,.3);border-color:rgba(155,230,210,.55)}.ollama-button:disabled{opacity:.45;cursor:not-allowed}.ollama-button.selected{background:#3e8073;border-color:#8dd2c1}.ollama-button.danger{color:#ffb8b8;border-color:rgba(255,110,110,.3)}.ollama-toolbar{display:flex;justify-content:space-between;gap:16px;margin:22px 0 12px;align-items:center}.ollama-tabs{display:flex;gap:8px;flex-wrap:wrap}.ollama-search{min-width:280px;max-width:410px;width:100%;background:rgba(7,20,19,.65);border:1px solid rgba(155,205,194,.24);border-radius:7px;color:#e8f2ef;padding:10px 12px;font:inherit;font-size:12px}.ollama-search::placeholder{color:#78958e}.ollama-info-strip{display:flex;gap:18px;flex-wrap:wrap;color:#88aaa2;font-size:11px;padding:10px 0 18px}.ollama-notice{padding:10px 12px;margin-top:15px;border-radius:7px;font-size:12px}.ollama-notice.ok{background:rgba(42,141,98,.18);border:1px solid rgba(84,222,159,.32);color:#a9f1cf}.ollama-notice.warning{background:rgba(156,112,45,.2);border:1px solid rgba(255,193,86,.5);color:#ffe0a0}.ollama-notice.error{background:rgba(156,45,45,.18);border:1px solid rgba(255,116,116,.35);color:#ffc1c1}.ollama-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(390px,1fr));gap:15px}.ollama-model-card{border:1px solid rgba(155,205,194,.18);border-radius:11px;background:rgba(12,31,29,.78);padding:17px;box-shadow:0 12px 30px rgba(0,0,0,.12)}.ollama-card-top{display:flex;justify-content:space-between;gap:12px;align-items:flex-start}.ollama-model-title h3{margin:0;font-size:17px;word-break:break-word}.ollama-card-actions{display:flex;gap:6px;flex-wrap:wrap;justify-content:flex-end}.ollama-model-summary{display:grid;grid-template-columns:repeat(4,1fr);gap:8px;margin:17px 0 10px}.ollama-model-summary div{padding:9px;background:rgba(87,137,126,.09);border-radius:7px;min-width:0}.ollama-model-summary small{display:block;color:#779d95;font-size:9px;text-transform:uppercase;letter-spacing:.08em;margin-bottom:4px}.ollama-model-summary strong{display:block;color:#d8ebe6;font-size:12px;overflow-wrap:anywhere}.ollama-card-meta{display:flex;gap:12px;flex-wrap:wrap;color:#8eada6;font-size:10px}.ollama-strengths{margin:13px 0;color:#b1c9c3;font-size:12px;line-height:1.5}.ollama-strengths strong{color:#dfede9}.ollama-details-toggle{padding:6px 0;border:0;background:transparent;color:#8fd1bf}.ollama-details{border-top:1px solid rgba(155,205,194,.14);margin-top:10px;padding-top:13px;color:#a9c4bd;font-size:11px}.ollama-details h4{margin:0 0 8px;color:#d8ebe6;font-size:11px;text-transform:uppercase;letter-spacing:.08em}.ollama-details p{line-height:1.5}.ollama-capabilities{display:grid;gap:7px}.ollama-capability{display:flex;align-items:center;gap:8px}.ollama-capability span{color:#9cbab3}.ollama-detail-grid{display:grid;grid-template-columns:repeat(2,1fr);gap:7px;color:#8eada6}.ollama-detail-grid strong{color:#d2e7e1}.ollama-jobs{display:grid;gap:7px;margin:0 0 15px}.ollama-jobs>div{display:flex;justify-content:space-between;padding:10px 12px;border-radius:7px;background:rgba(50,113,99,.2);border:1px solid rgba(112,213,187,.23);font-size:11px;color:#b6dcd2}.ollama-empty{padding:48px 14px;text-align:center;color:#8aa8a1;font-size:13px}.ollama-jobs span{color:#a7e7d0}.ollama-badge.current{color:#a9f0d1;border-color:rgba(84,222,159,.42);background:rgba(35,137,91,.22)}.ollama-badge.popular{color:#f5d18c;border-color:rgba(244,182,80,.4);background:rgba(142,90,25,.2)}.ollama-variants{margin:15px 0 5px;border:1px solid rgba(155,205,194,.17);border-radius:8px;background:rgba(5,18,17,.34);overflow:hidden}.ollama-variants-heading{display:flex;justify-content:space-between;align-items:center;gap:10px;padding:11px 12px;border-bottom:1px solid rgba(155,205,194,.14)}.ollama-variants-heading h4{margin:0;color:#d8ebe6;font-size:11px;text-transform:uppercase;letter-spacing:.08em}.ollama-variants-heading span{color:#7fa69d;font-size:10px}.ollama-variant-table-wrap{overflow-x:auto}.ollama-variant-table{border-collapse:collapse;width:100%;font-size:11px;min-width:650px}.ollama-variant-table th{padding:9px 10px;text-align:left;color:#7fa69d;background:rgba(87,137,126,.08);font-size:9px;text-transform:uppercase;letter-spacing:.08em;font-weight:600}.ollama-variant-table td{padding:10px;border-top:1px solid rgba(155,205,194,.1);color:#a9c4bd;vertical-align:middle;white-space:nowrap}.ollama-variant-table tr.current{background:rgba(42,141,98,.12)}.ollama-variant-table td strong{color:#dcece8;display:block}.ollama-variant-table td small{display:block;color:#7fa69d;font-size:9px;margin-top:3px}.ollama-variant-table td .ollama-badge{margin-left:7px;vertical-align:middle}.ollama-current-label{color:#8fe1bf;font-size:10px}.ollama-variant-table .ollama-button{padding:6px 9px;font-size:10px} @media(max-width:760px){.ollama-page{padding:20px 16px 40px}.ollama-hero,.ollama-toolbar{display:block}.ollama-health{justify-content:flex-start;margin-top:15px}.ollama-search{margin-top:12px;max-width:none}.ollama-grid{grid-template-columns:1fr}.ollama-card-top{display:block}.ollama-card-actions{justify-content:flex-start;margin-top:12px}.ollama-model-summary{grid-template-columns:repeat(2,1fr)}} .ollama-popular-note{margin:14px 0;color:#a5bfba;font-size:12px;line-height:1.5} -.ollama-model-pool{margin-top:14px;padding:16px;border:1px solid rgba(164,211,199,.16);border-radius:12px;background:rgba(10,31,28,.7)}.ollama-pool-heading{display:flex;justify-content:space-between;gap:12px;align-items:flex-start}.ollama-pool-heading h3{margin:0 0 3px}.ollama-pool-heading p{margin:0;color:#8fa9a4;font-size:11px}.ollama-pool-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(230px,1fr));gap:8px;margin-top:12px}.ollama-pool-item,.ollama-chat-model-selection label{display:flex;gap:8px;align-items:flex-start;padding:9px;border:1px solid rgba(155,205,194,.16);border-radius:7px;background:rgba(71,117,108,.1);color:#c5ddd7;font-size:11px;cursor:pointer}.ollama-pool-item.loaded{border-color:rgba(117,210,183,.75);background:linear-gradient(135deg,rgba(48,113,96,.48),rgba(35,82,73,.42));box-shadow:0 0 0 1px rgba(117,210,183,.18),0 0 18px rgba(117,210,183,.12)}.ollama-pool-item.loaded strong{color:#effff9}.ollama-pool-item.loaded small{color:#a8ead8}.ollama-pool-item.selected{outline:1px solid rgba(117,210,183,.45)}.ollama-pool-item input,.ollama-chat-model-selection input{margin-top:2px;accent-color:#75d2b7}.ollama-pool-item span{display:flex;flex-direction:column;gap:3px}.ollama-pool-item small,.ollama-chat-model-selection small{color:#8fa9a4;font-size:10px}.ollama-chat-model-selection label.loaded{border-color:rgba(117,210,183,.48);background:rgba(48,113,96,.28);color:#effff9}.ollama-pool-actions{display:flex;gap:8px;flex-wrap:wrap;margin-top:10px}.ollama-chat-model-selection{display:flex;gap:8px;align-items:center;flex-wrap:wrap;margin-top:14px;padding-top:12px;border-top:1px solid rgba(164,211,199,.14);color:#d5e8e2}.ollama-chat-model-selection>strong{margin-right:4px}.ollama-chat-model-selection>small{margin-right:8px}.ollama-chat-model-selection label{align-items:center;padding:6px 8px}.ollama-loaded-capabilities{flex-basis:100%;color:#b8ead9;font-size:10px}.ollama-permanent-label{color:#9af1c7;font-size:10px} +.ollama-model-pool{margin-top:14px;padding:16px;border:1px solid rgba(164,211,199,.16);border-radius:12px;background:rgba(10,31,28,.7)}.ollama-pool-heading{display:flex;justify-content:space-between;gap:12px;align-items:flex-start}.ollama-pool-heading h3{margin:0 0 3px}.ollama-pool-heading p{margin:0;color:#8fa9a4;font-size:11px}.ollama-pool-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(230px,1fr));gap:8px;margin-top:12px}.ollama-pool-item,.ollama-chat-model-selection label{display:flex;gap:8px;align-items:flex-start;padding:9px;border:1px solid rgba(155,205,194,.16);border-radius:7px;background:rgba(71,117,108,.1);color:#c5ddd7;font-size:11px;cursor:pointer}.ollama-pool-item.loaded{border-color:rgba(117,210,183,.75);background:linear-gradient(135deg,rgba(48,113,96,.48),rgba(35,82,73,.42));box-shadow:0 0 0 1px rgba(117,210,183,.18),0 0 18px rgba(117,210,183,.12)}.ollama-pool-item.loaded strong{color:#effff9}.ollama-pool-item.loaded small{color:#a8ead8}.ollama-pool-item.selected{outline:1px solid rgba(117,210,183,.45)}.ollama-pool-item input,.ollama-chat-model-selection input{margin-top:2px;accent-color:#75d2b7}.ollama-pool-item span{display:flex;flex-direction:column;gap:3px}.ollama-pool-item small,.ollama-chat-model-selection small{color:#8fa9a4;font-size:10px}.ollama-placement-control{display:flex!important;flex-direction:row!important;align-items:center;gap:6px;margin-top:3px}.ollama-placement-control select{max-width:190px;border:1px solid rgba(155,205,194,.28);border-radius:5px;background:#102d29;color:#e8f2ef;padding:5px;font:inherit;font-size:10px}.ollama-chat-model-selection label.loaded{border-color:rgba(117,210,183,.48);background:rgba(48,113,96,.28);color:#effff9}.ollama-pool-actions{display:flex;gap:8px;flex-wrap:wrap;margin-top:10px}.ollama-chat-model-selection{display:flex;gap:8px;align-items:center;flex-wrap:wrap;margin-top:14px;padding-top:12px;border-top:1px solid rgba(164,211,199,.14);color:#d5e8e2}.ollama-chat-model-selection>strong{margin-right:4px}.ollama-chat-model-selection>small{margin-right:8px}.ollama-chat-model-selection label{align-items:center;padding:6px 8px}.ollama-loaded-capabilities{flex-basis:100%;color:#b8ead9;font-size:10px}.ollama-permanent-label{color:#9af1c7;font-size:10px} .ollama-chat{margin-top:18px}.ollama-chat-header{display:flex;justify-content:space-between;gap:20px;align-items:flex-start;padding:18px;border:1px solid rgba(164,211,199,.16);border-radius:12px;background:rgba(23,52,48,.42)}.ollama-chat-header h2{margin:6px 0;color:#effcf8}.ollama-chat-header p{margin:0;color:#a5bfba;font-size:12px;max-width:720px}.ollama-chat-model{display:flex;align-items:flex-end;gap:10px;min-width:260px}.ollama-chat-model label{display:flex;flex-direction:column;gap:6px;color:#a5bfba;font-size:11px}.ollama-chat-model select,.ollama-url-input{border:1px solid rgba(155,205,194,.28);border-radius:7px;background:#102d29;color:#e8f2ef;padding:9px;font:inherit;font-size:12px}.ollama-runtime-panel{margin-top:14px;padding:16px;border:1px solid rgba(164,211,199,.16);border-radius:12px;background:rgba(10,31,28,.7)}.ollama-runtime-heading{display:flex;justify-content:space-between;gap:12px;align-items:flex-start}.ollama-runtime-heading h3{margin:0 0 3px}.ollama-runtime-heading p{margin:0;color:#8fa9a4;font-size:11px}.ollama-badge.muted{color:#a7b7b4;border-color:rgba(167,183,180,.25)}.ollama-runtime-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:10px;margin-top:14px}.ollama-runtime-stat{display:flex;flex-direction:column;gap:5px;min-height:70px;padding:11px;border-radius:8px;background:rgba(71,117,108,.11);color:#a5bfba}.ollama-runtime-stat strong{color:#effcf8;font-size:15px}.ollama-runtime-stat small{font-size:10px}.ollama-meter{height:5px;border-radius:99px;background:#173c36;overflow:hidden}.ollama-meter span{display:block;height:100%;border-radius:inherit;background:#75d2b7}.ollama-memory-chart{height:70px;display:flex;align-items:flex-end;gap:2px;margin-top:14px;padding:5px 0;border-bottom:1px solid rgba(164,211,199,.18)}.ollama-memory-chart span{flex:1;min-width:2px;background:#55a995;border-radius:2px 2px 0 0;opacity:.8}.ollama-memory-chart.loading{height:70px;align-items:center;padding:0 16px;border:1px solid rgba(117,210,183,.55);border-radius:8px;background:linear-gradient(90deg,rgba(38,104,88,.35),rgba(25,70,62,.7),rgba(38,104,88,.35));background-size:200% 100%;animation:ollama-loading-sweep 1.8s linear infinite}.ollama-loading-progress{display:grid;gap:4px;width:100%;color:#bff7e2}.ollama-loading-progress strong{font-size:12px}.ollama-loading-progress span{font-size:11px;color:#e8fff7;overflow-wrap:anywhere}.ollama-loading-progress small{font-size:10px;color:#9dd7c5}.ollama-loading-track{height:5px;overflow:hidden;border-radius:99px;background:rgba(117,210,183,.18)}.ollama-loading-track span{display:block;width:35%;height:100%;border-radius:inherit;background:#75d2b7;animation:ollama-loading-indicator 1.2s ease-in-out infinite}.ollama-weight-stat .ollama-meter span{background:#86b9ff}@keyframes ollama-loading-sweep{0%{background-position:0 0}100%{background-position:200% 0}}@keyframes ollama-loading-indicator{0%{margin-left:-35%}100%{margin-left:100%}}.ollama-loaded-memory h4{margin:15px 0 8px}.ollama-loaded-row{display:flex;align-items:center;gap:14px;flex-wrap:wrap;padding:9px 0;border-top:1px solid rgba(164,211,199,.1);font-size:11px;color:#a5bfba}.ollama-loaded-row strong{color:#effcf8;min-width:180px}.ollama-chat-layout{display:flex;flex-direction:column;gap:14px;margin-top:14px}.ollama-conversation{min-height:360px;max-height:620px;overflow:auto;order:1}.ollama-composer{order:2}.ollama-conversation{min-height:320px;max-height:620px;overflow:auto;padding:14px;border:1px solid rgba(164,211,199,.16);border-radius:12px;background:rgba(10,25,23,.62)}.ollama-message{margin:0 0 14px;padding:11px 13px;border-radius:9px;white-space:pre-wrap;line-height:1.5;font-size:13px}.ollama-message small{display:block;margin-bottom:5px;color:#8fc7ba;font-size:10px;text-transform:uppercase;letter-spacing:.1em}.ollama-message.user{margin-left:12%;background:rgba(54,105,94,.3)}.ollama-message.assistant{margin-right:8%;background:rgba(34,63,62,.6)}.ollama-composer{display:flex;flex-direction:column;gap:10px;padding:14px;border:1px solid rgba(164,211,199,.16);border-radius:12px;background:rgba(23,52,48,.42)}.ollama-composer textarea{min-height:160px;resize:vertical;border:1px solid rgba(155,205,194,.25);border-radius:8px;background:#0d2522;color:#e8f2ef;padding:11px;font:inherit;font-size:13px}.ollama-attachment-actions{display:flex;align-items:center;gap:7px;flex-wrap:wrap}.ollama-file-button{display:inline-flex;align-items:center;border:1px solid rgba(155,205,194,.24);border-radius:7px;background:rgba(70,117,108,.18);color:#dbebe7;padding:8px 11px;cursor:pointer;font-size:11px}.ollama-file-button input{display:none}.ollama-url-input{flex:1;min-width:160px}.ollama-attachments{display:flex;gap:6px;flex-wrap:wrap}.ollama-attachment{display:inline-flex;align-items:center;gap:5px;padding:5px 8px;border-radius:999px;background:rgba(80,125,115,.16);color:#c5dfd8;font-size:10px;max-width:100%;overflow:hidden;text-overflow:ellipsis}.ollama-attachment button{border:0;background:none;color:#ffb3b3;cursor:pointer}.ollama-chat-footnote{margin:0;color:#819b96;font-size:10px;line-height:1.45} @media(max-width:900px){.ollama-chat-header,.ollama-chat-layout{display:block}.ollama-chat-model{margin-top:15px;align-items:center}.ollama-composer{margin-top:14px}.ollama-runtime-grid{grid-template-columns:1fr 1fr}} .ollama-persistence-panel{display:grid;gap:12px;margin:16px 0;padding:16px;border:1px solid rgba(164,211,199,.16);border-radius:12px;background:rgba(10,31,28,.7)}.ollama-persistence-heading{display:flex;justify-content:space-between;gap:16px;align-items:center}.ollama-persistence-heading h3{margin:0}.ollama-persistence-heading p{margin:4px 0 0;color:#8fa9a4;font-size:11px}.ollama-conversation-list{display:flex;flex-wrap:wrap;gap:8px}.ollama-conversation-list .ollama-button{font-size:11px}.ollama-conversation-list .selected{background:#3e8073;border-color:#8dd2c1}.ollama-metrics-summary,.ollama-metrics-detail{display:flex;flex-wrap:wrap;gap:12px;font-size:11px;color:#a5bfba}.ollama-metrics-summary strong{color:#effcf8}.ollama-metrics-detail{padding-top:8px;border-top:1px solid rgba(164,211,199,.14)} diff --git a/dashboard/manifest.json b/dashboard/manifest.json index 0e17f92..0b352d7 100644 --- a/dashboard/manifest.json +++ b/dashboard/manifest.json @@ -3,7 +3,7 @@ "label": "Ollama Models", "description": "Inspect, manage, and chat with local Ollama models, including shared persistent conversations, performance metrics, images, PDFs, URLs, and live memory telemetry.", "icon": "Cpu", - "version": "1.5.14", + "version": "1.5.15", "tab": {"path": "/ollama-manager", "position": "after:models"}, "entry": "dist/index.js", "css": "dist/style.css", diff --git a/dashboard/plugin_api.py b/dashboard/plugin_api.py index d026a85..88eef5c 100644 --- a/dashboard/plugin_api.py +++ b/dashboard/plugin_api.py @@ -1116,6 +1116,7 @@ def _new_job(name: str, action: str, target: str = "local") -> str: class ModelRequest(BaseModel): name: str target: str = "local" + placement: str = "gpu_ram" class ConnectionRequest(BaseModel): @@ -1125,6 +1126,7 @@ class ConnectionRequest(BaseModel): class ModelsRequest(BaseModel): names: list[str] = Field(default_factory=list) + placements: dict[str, str] = Field(default_factory=dict) class ChatAttachment(BaseModel): @@ -1140,6 +1142,7 @@ class ChatRequest(BaseModel): message: str = "" history: list[dict[str, Any]] = Field(default_factory=list) attachments: list[ChatAttachment] = Field(default_factory=list) + placements: dict[str, str] = Field(default_factory=dict) request_id: str = "" conversation_id: str = "" @@ -1172,18 +1175,29 @@ def _ollama_error(exc: HTTPError) -> HTTPException: return HTTPException(502, f"Ollama request failed: {detail}") -def _load_model(name: str) -> dict[str, Any]: +def _model_placement(value: str | None) -> str: + placement = str(value or "gpu_ram").strip().lower() + if placement not in {"gpu_ram", "ram_only"}: + raise HTTPException(400, "Model placement must be gpu_ram or ram_only") + return placement + + +def _load_model(name: str, placement: str = "gpu_ram") -> dict[str, Any]: name = _require_installed_model(name) + placement = _model_placement(placement) + options: dict[str, Any] = {"num_predict": 1} + if placement == "ram_only": + options["num_gpu"] = 0 try: result = _json_request( LOCAL_OLLAMA + "/api/generate", method="POST", - payload={"model": name, "prompt": "", "stream": False, "keep_alive": CHAT_KEEP_ALIVE, "options": {"num_predict": 1}}, + payload={"model": name, "prompt": "", "stream": False, "keep_alive": CHAT_KEEP_ALIVE, "options": options}, timeout=900, ) except HTTPError as exc: raise _ollama_error(exc) from exc - return {"ok": True, "model": name, "response": result.get("response", ""), "runtime": _runtime_snapshot()} + return {"ok": True, "model": name, "placement": placement, "response": result.get("response", ""), "runtime": _runtime_snapshot()} def _chat_payload(body: ChatRequest, model_name: str | None = None) -> dict[str, Any]: @@ -1208,7 +1222,10 @@ def _chat_payload(body: ChatRequest, model_name: str | None = None) -> dict[str, if images: user_message["images"] = images messages.append(user_message) - return {"model": model, "messages": messages, "stream": False, "keep_alive": CHAT_KEEP_ALIVE} + payload: dict[str, Any] = {"model": model, "messages": messages, "stream": False, "keep_alive": CHAT_KEEP_ALIVE} + if _model_placement(body.placements.get(model)) == "ram_only": + payload["options"] = {"num_gpu": 0} + return payload class _ChatStopped(Exception): @@ -1420,7 +1437,7 @@ def runtime() -> dict[str, Any]: @router.post("/chat/load") def chat_load(body: ModelRequest) -> dict[str, Any]: - return _load_model(body.name) + return _load_model(body.name, body.placement) @router.post("/models/load") @@ -1428,16 +1445,19 @@ def models_load(body: ModelsRequest) -> dict[str, Any]: names = list(dict.fromkeys(_valid_name(name) for name in body.names if str(name).strip()))[:12] if not names: raise HTTPException(400, "Select at least one model to load") + placements = {name: _model_placement(body.placements.get(name)) for name in names} results = [] for name in names: - _model_load_update(name, active=True, state="queued", stage="Waiting for Ollama", started_at=time.time(), error="") + _model_load_update(name, active=True, state="queued", stage="Waiting for Ollama", started_at=time.time(), error="", placement=placements[name]) for name in names: - _model_load_update(name, state="loading", stage="Loading model into Ollama memory") + placement = placements[name] + stage = "Loading into GPU + RAM" if placement == "gpu_ram" else "Loading into system RAM only" + _model_load_update(name, state="loading", stage=stage) try: - results.append({"name": name, "ok": True, "result": _load_model(name)}) + results.append({"name": name, "placement": placement, "ok": True, "result": _load_model(name, placement)}) _model_load_update(name, state="checking", stage="Checking Ollama resident state") except Exception as exc: - results.append({"name": name, "ok": False, "error": str(exc)}) + results.append({"name": name, "placement": placement, "ok": False, "error": str(exc)}) _model_load_update(name, active=False, state="failed", stage="Ollama load failed", finished_at=time.time(), error=str(exc)) resident_rows = _local_ps() resident_names = {str(row.get("name") or row.get("model")) for row in resident_rows} diff --git a/plugin.yaml b/plugin.yaml index 1de4c69..dcbfc8e 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -1,5 +1,5 @@ name: ollama-manager -version: 1.5.14 +version: 1.5.15 description: Native dashboard manager and chat interface for local Ollama models, attachments, URLs, shared persistent conversations, performance metrics, and live runtime telemetry. auto_install_dependencies: true python_dependencies: