diff --git a/README.md b/README.md index 18ce210..5996ff8 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Native-like Hermes dashboard plugin for local Ollama model management and chat. - Paste images directly into the composer and drag/drop images, PDFs, and text files - Streamed Ollama responses with a real Stop action that cancels the active request - Minimized-by-default expandable thinking/progress details with live stage, elapsed time, event, and character counters -- Validation harness mode: choose one primary model and one or more independent validator models; validators review the primary draft and the primary model compiles one final answer +- Validation harness mode: choose one primary model and one or more independent validator models; validators review the primary draft, and the primary model applies valid corrections to compile one final answer. If the primary returns a report instead of an answer, the plugin retries finalization and never exposes validator-only text as the final response - Server-owned chat jobs continue after the browser closes and persist final answers for later resume. A newly opened dashboard discovers queued/running jobs from the shared server store and resumes observing them automatically. - SQLite is the default chat store for new users - Optional native PostgreSQL storage can be installed and linked explicitly from the plugin diff --git a/dashboard/manifest.json b/dashboard/manifest.json index 666b3ea..c1c792c 100644 --- a/dashboard/manifest.json +++ b/dashboard/manifest.json @@ -3,7 +3,7 @@ "label": "Ollama Models", "description": "Inspect, manage, and chat with local Ollama models, including shared persistent conversations, performance metrics, images, PDFs, URLs, and live memory telemetry.", "icon": "Cpu", - "version": "1.7.8", + "version": "1.7.9", "tab": {"path": "/ollama-manager", "position": "after:models"}, "entry": "dist/index.js", "css": "dist/style.css", diff --git a/dashboard/plugin_api.py b/dashboard/plugin_api.py index 8cd609d..37feeaa 100644 --- a/dashboard/plugin_api.py +++ b/dashboard/plugin_api.py @@ -82,6 +82,7 @@ CHAT_HEARTBEAT_INTERVAL = 5.0 HARNESS_MIN_VALIDATORS = 1 HARNESS_MAX_DRAFT_CHARS = 24_000 HARNESS_MAX_VALIDATION_CHARS = 8_000 +HARNESS_MAX_FINAL_RETRY_CHARS = 24_000 PERFORMANCE_HISTORY_FILE = "performance-history.jsonl" PERFORMANCE_HISTORY_BUCKET_SECONDS = 60 PERFORMANCE_HISTORY_MAX_SAMPLES = 24 * 60 @@ -1931,6 +1932,26 @@ def _harness_compiler_prompt(question: str, draft: str, reports: list[tuple[str, ) +def _looks_like_validation_report(content: str) -> bool: + normalized = re.sub(r"\s+", " ", str(content or "").strip().lower()) + if not normalized: + return False + markers = ("validation report", "validator report", "narrative structure", "character consistency", "rating:") + return normalized.startswith("# validation") or normalized.startswith("# story validation") or sum(marker in normalized for marker in markers) >= 2 + + +def _harness_retry_prompt(question: str, draft: str, reports: list[tuple[str, str]]) -> str: + return ( + "IMPORTANT FINALIZATION RETRY. Return the actual finished answer to the original user request, not a " + "review, critique, score, validation report, plan, or commentary about other models. Rewrite and improve " + "the primary draft using valid corrections from the reports. Do not mention validation, validators, the " + "draft, this retry, or the harness. Return only the polished user-facing result.\n\n" + f"ORIGINAL USER REQUEST:\n{question[:MAX_ATTACHMENT_TEXT]}\n\n" + f"PRIMARY DRAFT TO IMPROVE:\n{draft[:HARNESS_MAX_DRAFT_CHARS]}\n\n" + f"CORRECTIONS TO APPLY:\n{_harness_compiler_prompt(question, draft, reports)[-HARNESS_MAX_FINAL_RETRY_CHARS:]}" + ) + + def _harness_models(body: ChatRequest) -> tuple[str, list[str], bool]: legacy_models = [str(name).strip() for name in body.models if str(name).strip()] primary_name = str(body.primary_model or body.model or (legacy_models[0] if legacy_models else "")).strip() @@ -2131,6 +2152,34 @@ def _run_validation_harness( final_content = str(final_message.get("content") or "").strip() if not final_content: raise HTTPException(502, "Primary model returned an empty compiled answer") + if _looks_like_validation_report(final_content): + retry_id = uuid.uuid4().hex + retry_started = time.time() + _persist_chat_stage(request_id, "compiler-retry", primary, "running", retry_started) + _persist_chat_event(request_id, conversation_id, "compiler-retry-started", level="warning", stage="Primary returned review text; requesting final answer", model=primary) + _chat_state(retry_id, state="preparing", stage="Preparing final-answer retry", model=primary, parent_id=request_id, conversation_id=conversation_id) + retry_payload = _chat_payload( + body, + primary, + message_override=_harness_retry_prompt(body.message, draft, [(item["model"], item["report"]) for item in reports]), + history_override=[], + attachment_parts=attachment_parts, + ) + retry_result = _stream_chat_request(retry_payload, retry_id, cancel_event=cancel_event, parent_id=request_id) + with _chat_requests_lock: + retry_state = dict(_chat_requests.get(retry_id, {})) + metrics.append(_persist_metric(conversation_id, retry_id, primary, retry_state, status="final-retry")) + retry_message = retry_result.get("message") if isinstance(retry_result.get("message"), dict) else {} + retry_content = str(retry_message.get("content") or "").strip() + retry_finished = float(retry_state.get("finished_at") or time.time()) + if retry_content and not _looks_like_validation_report(retry_content): + final_content = retry_content + _persist_chat_stage(request_id, "compiler-retry", primary, "completed", retry_started, finished_at=retry_finished, output_chars=len(final_content)) + _persist_chat_event(request_id, conversation_id, "compiler-retry-completed", stage="Final answer retry completed", model=primary, payload={"output_chars": len(final_content)}) + else: + final_content = draft + _persist_chat_stage(request_id, "compiler-retry", primary, "fallback", retry_started, finished_at=retry_finished, output_chars=len(final_content), error="Primary returned review text twice; preserved the primary draft") + _persist_chat_event(request_id, conversation_id, "compiler-retry-fallback", level="warning", stage="Preserved primary draft after invalid finalization", model=primary, payload={"output_chars": len(final_content)}) compiler_finished = time.time() _persist_chat_stage(request_id, "compiler", primary, "completed", compiler_started, finished_at=compiler_finished, output_chars=len(final_content)) _persist_chat_event(request_id, conversation_id, "compiler-completed", stage="Primary final answer compiled", model=primary, payload={"output_chars": len(final_content)}) diff --git a/plugin.yaml b/plugin.yaml index 9edee54..c01503a 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -1,5 +1,5 @@ name: ollama-manager -version: 1.7.8 +version: 1.7.9 description: Native dashboard manager and chat interface for local Ollama models, attachments, URLs, shared persistent conversations, performance metrics, and live runtime telemetry. auto_install_dependencies: true python_dependencies: diff --git a/tests/test_validation_harness.py b/tests/test_validation_harness.py index d9e53fa..4b40931 100644 --- a/tests/test_validation_harness.py +++ b/tests/test_validation_harness.py @@ -142,6 +142,42 @@ class ValidationHarnessTests(unittest.TestCase): self.assertIn("VALIDATOR 1", compiler_prompt) self.assertIn("VALIDATOR 2", compiler_prompt) self.assertNotIn("validator-a found no material issue\n\nvalidator-b found no material issue", final) + def test_validation_report_is_retried_and_never_returned_as_final_answer(self): + body = api.ChatRequest(primary_model="primary", validator_models=["validator"], harness=True, message="Write the requested result") + calls = [] + + def fake_stream(payload, request_id, cancel_event=None, parent_id=None): + model = payload["model"] + prompt = payload["messages"][-1]["content"] + calls.append((model, prompt)) + if model == "validator": + content = "The draft needs a stronger ending." + elif "IMPORTANT FINALIZATION RETRY" in prompt: + content = "refined final answer" + elif "VALIDATION REPORTS:" in prompt: + content = "# Story Validation Report\n## Narrative Structure\nRating: 8/10" + else: + content = "primary draft" + return {"message": {"role": "assistant", "content": content}, "done": True} + + def fake_metric(conversation_id, request_id, model, state, status=None, error=""): + return {"request_id": request_id, "model": model, "status": status} + + with patch.object(api, "_require_installed_model", side_effect=lambda name: name), patch.object( + api, "_stream_chat_request", side_effect=fake_stream + ), patch.object(api, "_persist_metric", side_effect=fake_metric), patch.object( + api, "_persist_chat_stage" + ), patch.object(api, "_persist_chat_event"), patch.object(api, "_chat_state"): + final, metrics, reports = api._run_validation_harness( + body, "root-request", "conversation", "primary", ["validator"], threading.Event(), [] + ) + + self.assertEqual(final, "refined final answer") + self.assertNotIn("Validation Report", final) + self.assertEqual(len(reports), 1) + self.assertEqual(len(metrics), 4) + self.assertTrue(any("IMPORTANT FINALIZATION RETRY" in prompt for _, prompt in calls)) + def test_status_omits_full_catalog_by_default(self): with patch.object(api, "_local_tags", return_value=[]), patch.object(api, "_local_ps", return_value=[]), patch.object( api, "_ensure_catalog", return_value={"models": [], "families": {}, "source": "test"}