feat: add per-model GPU and RAM placement
This commit is contained in:
Hermes Agent
Vendored
+19
-4
@@ -8,7 +8,18 @@
|
||||
var fetchJSON = SDK.fetchJSON;
|
||||
var API = "/api/plugins/ollama-manager";
|
||||
var CHAT_STORAGE_KEY = "hermes.ollama-manager.chat.v1";
|
||||
var PLACEMENT_STORAGE_KEY = "hermes.ollama-manager.placement.v1";
|
||||
|
||||
function readSavedPlacements() {
|
||||
try {
|
||||
var raw = window.localStorage.getItem(PLACEMENT_STORAGE_KEY);
|
||||
var value = raw ? JSON.parse(raw) : {};
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
||||
} catch (_) { return {}; }
|
||||
}
|
||||
function savePlacements(value) {
|
||||
try { window.localStorage.setItem(PLACEMENT_STORAGE_KEY, JSON.stringify(value || {})); } catch (_) {}
|
||||
}
|
||||
function readSavedChat() {
|
||||
try {
|
||||
var raw = window.localStorage.getItem(CHAT_STORAGE_KEY);
|
||||
@@ -210,7 +221,7 @@
|
||||
var models = props.models || [], loaded = models.filter(function (item) { return item.loaded; });
|
||||
return h("section", { className: "ollama-model-pool" },
|
||||
h("div", { className: "ollama-pool-heading" }, h("div", null, h("h3", null, "Model pool"), h("p", null, "Choose installed models to keep permanently loaded. Loaded models remain available to Hermes Agent through Local Ollama.")), h(Badge, { tone: loaded.length ? "live" : "muted" }, loaded.length + " loaded")),
|
||||
h("div", { className: "ollama-pool-grid" }, models.map(function (item) { var loaded = !!item.loaded, selected = props.poolSelection.indexOf(item.name) >= 0; return h("label", { className: "ollama-pool-item" + (loaded ? " loaded" : "") + (selected ? " selected" : ""), key: item.name, title: loaded ? "Loaded and resident in Ollama" : "Installed but not resident" }, h("input", { type: "checkbox", checked: selected, onChange: function () { props.onTogglePool(item.name); } }), h("span", null, h("strong", null, item.name), h("small", null, loaded ? "Loaded and resident · keep-alive active" : "Installed · not loaded", " · ", (item.capabilities || []).join(", ") || "capabilities unknown"))); })),
|
||||
h("div", { className: "ollama-pool-grid" }, models.map(function (item) { var loaded = !!item.loaded, selected = props.poolSelection.indexOf(item.name) >= 0, placement = props.placements[item.name] || "gpu_ram"; return h("label", { className: "ollama-pool-item" + (loaded ? " loaded" : "") + (selected ? " selected" : ""), key: item.name, title: loaded ? "Loaded and resident in Ollama" : "Installed but not resident" }, h("input", { type: "checkbox", checked: selected, onChange: function () { props.onTogglePool(item.name); } }), h("span", null, h("strong", null, item.name), h("small", null, loaded ? "Loaded and resident · keep-alive active" : "Installed · not loaded", " · ", (item.capabilities || []).join(", ") || "capabilities unknown"), h("span", { className: "ollama-placement-control" }, h("small", null, "Placement"), h("select", { value: placement, onClick: function (event) { event.stopPropagation(); }, onChange: function (event) { event.stopPropagation(); props.onPlacementChange(item.name, event.target.value); } }, h("option", { value: "gpu_ram" }, "GPU + RAM (automatic offload)"), h("option", { value: "ram_only" }, "RAM only (CPU)"))))); })),
|
||||
h("div", { className: "ollama-pool-actions" }, h(Button, { disabled: !props.poolSelection.length || !!props.busy, onClick: props.onLoad }, props.busy === "/models/load" ? "Loading " + props.poolSelection.length + " model" + (props.poolSelection.length === 1 ? "" : "s") + "…" : "Load selected permanently"), h(Button, { className: "secondary", disabled: !props.poolSelection.length || !!props.busy, onClick: props.onUnload }, props.busy === "/models/unload" ? "Unloading…" : "Unload selected")),
|
||||
h("div", { className: "ollama-chat-model-selection" }, h("strong", null, "Models for this answer"), h("small", null, "Select one or more loaded models for parallel perspectives."), loaded.length ? loaded.map(function (item) { return h("label", { className: "loaded", key: item.name }, h("input", { type: "checkbox", checked: props.selectedModels.indexOf(item.name) >= 0, onChange: function () { props.onToggleChat(item.name); } }), item.name, " · ", (item.capabilities || []).join(", ")); }) : h("span", null, "Load one or more models above first."))
|
||||
);
|
||||
@@ -225,6 +236,7 @@
|
||||
var poolState = React.useState(loadedModels.map(function (item) { return item.name; })), poolSelection = poolState[0], setPoolSelection = poolState[1];
|
||||
var poolLoadedSignature = React.useRef("");
|
||||
var chatLoadedSignature = React.useRef("");
|
||||
var placementState = React.useState(readSavedPlacements()), placements = placementState[0], setPlacements = placementState[1];
|
||||
var messageState = React.useState(""), message = messageState[0], setMessage = messageState[1];
|
||||
var urlState = React.useState(""), url = urlState[0], setUrl = urlState[1];
|
||||
var attachState = React.useState([]), attachments = attachState[0], setAttachments = attachState[1];
|
||||
@@ -312,12 +324,15 @@
|
||||
function pollRuntime() { fetchJSON(API + "/runtime").then(function (value) { setRuntime(value); setSamples(function (old) { return old.concat([{ used: Number(value.memory_used_bytes || 0), total: Number(value.memory_total_bytes || 0) }]).slice(-60); }); }).catch(function () {}); }
|
||||
React.useEffect(function () { pollRuntime(); var timer = setInterval(pollRuntime, 1000); return function () { clearInterval(timer); }; }, []);
|
||||
|
||||
React.useEffect(function () { savePlacements(placements); }, [placements]);
|
||||
function toggleIn(setter, name) { setter(function (old) { return old.indexOf(name) >= 0 ? old.filter(function (item) { return item !== name; }) : old.concat([name]); }); }
|
||||
function setPlacement(name, value) { setPlacements(function (old) { var next = Object.assign({}, old); next[name] = value === "ram_only" ? "ram_only" : "gpu_ram"; return next; }); }
|
||||
function manageModels(endpoint, label) {
|
||||
if (!poolSelection.length) { setNotice({ error: "Select one or more installed models first." }); return; }
|
||||
var requested = poolSelection.slice();
|
||||
var requestedPlacements = requested.reduce(function (result, name) { result[name] = placements[name] || "gpu_ram"; return result; }, {});
|
||||
setBusy(endpoint); setNotice({ ok: label + " in progress for " + requested.length + " model" + (requested.length === 1 ? "" : "s") + "…" });
|
||||
fetchJSON(API + endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ names: requested }) }).then(function (result) {
|
||||
fetchJSON(API + endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ names: requested, placements: requestedPlacements }) }).then(function (result) {
|
||||
if (endpoint === "/models/load") {
|
||||
var resident = Array.isArray(result.resident) ? result.resident : ((result.runtime && result.runtime.model_memory) || []).map(function (item) { return item.name; });
|
||||
var missing = Array.isArray(result.not_resident) ? result.not_resident : requested.filter(function (name) { return resident.indexOf(name) < 0; });
|
||||
@@ -362,7 +377,7 @@
|
||||
var requestId = makeRequestId();
|
||||
var controller = typeof AbortController === "function" ? new AbortController() : null;
|
||||
var current = { id: requestId, controller: controller, stopped: false };
|
||||
var outgoing = { role: "user", content: message.trim() || "[Attachments]" }, body = { model: selectedModels[0], models: selectedModels, message: message, history: history, attachments: attachments, request_id: requestId, conversation_id: conversationId };
|
||||
var outgoing = { role: "user", content: message.trim() || "[Attachments]" }, body = { model: selectedModels[0], models: selectedModels, placements: placements, message: message, history: history, attachments: attachments, request_id: requestId, conversation_id: conversationId };
|
||||
setHistory(function (old) { return old.concat([outgoing]); }); setMessage(""); setBusy("send"); setActiveRequest(current); setThinking({ request_id: requestId, startedAt: Date.now(), stage: attachments.length ? "Preparing attachments and sending request to Ollama" : "Sending request to Ollama" }); setThinkingDetails(null); setThinkingOpen(false); setNotice(null);
|
||||
var requestOptions = { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) };
|
||||
if (controller) requestOptions.signal = controller.signal;
|
||||
@@ -376,7 +391,7 @@
|
||||
aggregate && h("div", { className: "ollama-metrics-summary" }, h("strong", null, "Model performance · ", aggregate.sample_count || 0, " samples"), h("span", null, "TTFT avg: ", aggregate.avg_time_to_first_token_ms == null ? "n/a" : aggregate.avg_time_to_first_token_ms + " ms"), h("span", null, "Output: ", aggregate.avg_eval_tokens_per_second == null ? "n/a" : aggregate.avg_eval_tokens_per_second + " tok/s"), h("span", null, "Latency: ", aggregate.avg_total_latency_ms == null ? "n/a" : aggregate.avg_total_latency_ms + " ms"), h("span", null, "Errors: ", aggregate.error_count || 0)),
|
||||
metrics && metrics.length > 0 && h("div", { className: "ollama-metrics-detail" }, (metrics.slice(-3)).map(function (item, index) { return h("span", { key: index }, item.model || "model", " · TTFT ", item.time_to_first_token_ms == null ? "n/a" : item.time_to_first_token_ms + " ms", " · ", item.eval_count == null ? "n/a" : item.eval_count + " output tokens", " · ", item.eval_tokens_per_second == null ? "n/a" : item.eval_tokens_per_second + " tok/s"); }))
|
||||
),
|
||||
h(ModelPoolPanel, { models: models, loadedModels: loadedModels, selectedModels: selectedModels, poolSelection: poolSelection, busy: busy, onTogglePool: togglePoolModel, onToggleChat: toggleChatModel, onLoad: loadModel, onUnload: unloadModels }),
|
||||
h(ModelPoolPanel, { models: models, loadedModels: loadedModels, selectedModels: selectedModels, poolSelection: poolSelection, placements: placements, busy: busy, onTogglePool: togglePoolModel, onPlacementChange: setPlacement, onToggleChat: toggleChatModel, onLoad: loadModel, onUnload: unloadModels }),
|
||||
notice && h("div", { className: "ollama-notice " + (notice.error ? "error" : notice.warning ? "warning" : "ok") }, notice.error || notice.warning || notice.ok),
|
||||
thinking && h(ThinkingStatus, { stage: thinkingDetails && thinkingDetails.stage ? thinkingDetails.stage : (thinkingElapsed < 1 ? thinking.stage : "Ollama is generating the response"), elapsed: thinkingDetails && thinkingDetails.elapsed != null ? thinkingDetails.elapsed : thinkingElapsed, details: thinkingDetails, expanded: thinkingOpen, onToggle: function () { setThinkingOpen(!thinkingOpen); }, onStop: stop }),
|
||||
h(RuntimePanel, { runtime: runtime, samples: samples }),
|
||||
|
||||
Reference in New Issue
Block a user