fix: return initial and enhanced chat outputs
This commit is contained in:
Hermes Agent
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import threading
|
||||
import tempfile
|
||||
import time
|
||||
@@ -9,12 +10,12 @@ from dashboard import plugin_api as api
|
||||
|
||||
|
||||
class ValidationHarnessTests(unittest.TestCase):
|
||||
def test_legacy_multiple_models_map_to_primary_and_validators(self):
|
||||
body = api.ChatRequest(models=["primary", "validator-a", "validator-b"])
|
||||
def test_legacy_multiple_models_map_to_primary_and_single_enhancer(self):
|
||||
body = api.ChatRequest(models=["primary", "enhancer-a", "enhancer-b"])
|
||||
with patch.object(api, "_require_installed_model", side_effect=lambda name: name):
|
||||
primary, validators, harness = api._harness_models(body)
|
||||
self.assertEqual(primary, "primary")
|
||||
self.assertEqual(validators, ["validator-a", "validator-b"])
|
||||
self.assertEqual(validators, ["enhancer-a"])
|
||||
self.assertTrue(harness)
|
||||
|
||||
def test_sqlite_is_default_even_when_postgres_is_installed(self):
|
||||
@@ -85,6 +86,34 @@ class ValidationHarnessTests(unittest.TestCase):
|
||||
self.assertEqual(response["heartbeat_at"], 110.0)
|
||||
self.assertEqual(response["heartbeat_age"], 2.5)
|
||||
|
||||
def test_job_status_returns_durable_initial_and_enhanced_outputs(self):
|
||||
job = {
|
||||
"request_id": "request",
|
||||
"conversation_id": "conversation",
|
||||
"status": "completed",
|
||||
"mode": "harness",
|
||||
"primary_model": "primary",
|
||||
"validator_models_json": json.dumps(["enhancer"]),
|
||||
"result_json": json.dumps({
|
||||
"message": {"role": "assistant", "content": "enhanced"},
|
||||
"initial_output": "initial",
|
||||
"enhanced_output": "enhanced",
|
||||
"primary_model": "primary",
|
||||
"enhancement_model": "enhancer",
|
||||
}),
|
||||
"error": "",
|
||||
"attempt": 1,
|
||||
"started_at": 100.0,
|
||||
"heartbeat_at": 110.0,
|
||||
"finished_at": 111.0,
|
||||
"updated_at": 111.0,
|
||||
}
|
||||
response = api._job_status_response(job)
|
||||
self.assertTrue(response["done"])
|
||||
self.assertEqual(response["initial_output"], "initial")
|
||||
self.assertEqual(response["enhanced_output"], "enhanced")
|
||||
self.assertEqual(response["enhancement_model"], "enhancer")
|
||||
|
||||
def test_active_jobs_route_returns_server_owned_jobs(self):
|
||||
active = [{"request_id": "request", "status": "running"}]
|
||||
with patch.object(api, "_list_chat_jobs", return_value=active) as listed:
|
||||
@@ -92,12 +121,12 @@ class ValidationHarnessTests(unittest.TestCase):
|
||||
self.assertEqual(response, {"jobs": active})
|
||||
listed.assert_called_once_with(active_only=True, conversation_id="conversation", limit=20)
|
||||
|
||||
def test_primary_draft_validators_and_primary_compilation_produce_one_answer(self):
|
||||
def test_primary_draft_then_enhancer_returns_both_complete_outputs(self):
|
||||
body = api.ChatRequest(
|
||||
primary_model="primary",
|
||||
validator_models=["validator-a", "validator-b"],
|
||||
validator_models=["enhancer"],
|
||||
harness=True,
|
||||
message="What is the verified answer?",
|
||||
message="Write a short story with a complete ending.",
|
||||
)
|
||||
calls = []
|
||||
|
||||
@@ -105,114 +134,83 @@ class ValidationHarnessTests(unittest.TestCase):
|
||||
model = payload["model"]
|
||||
prompt = payload["messages"][-1]["content"]
|
||||
calls.append((model, prompt, parent_id))
|
||||
if model == "primary" and "VALIDATION REPORTS:" not in prompt:
|
||||
content = "primary draft"
|
||||
elif model.startswith("validator"):
|
||||
content = f"{model} found no material issue"
|
||||
else:
|
||||
content = "one compiled final answer"
|
||||
content = "initial story from primary" if model == "primary" else "enhanced complete story from enhancer"
|
||||
return {"message": {"role": "assistant", "content": content}, "done": True}
|
||||
|
||||
def fake_metric(conversation_id, request_id, model, state, status=None, error=""):
|
||||
return {"request_id": request_id, "model": model, "status": status}
|
||||
|
||||
with patch.object(api, "_require_installed_model", side_effect=lambda name: name), patch.object(
|
||||
api, "_stream_chat_request", side_effect=fake_stream
|
||||
), patch.object(api, "_persist_metric", side_effect=fake_metric), patch.object(
|
||||
api, "_persist_chat_stage"
|
||||
), patch.object(api, "_persist_chat_event"):
|
||||
final, metrics, reports = api._run_validation_harness(
|
||||
body,
|
||||
"root-request",
|
||||
"conversation",
|
||||
"primary",
|
||||
["validator-a", "validator-b"],
|
||||
threading.Event(),
|
||||
[],
|
||||
with patch.object(api, "_require_installed_model", side_effect=lambda name: name), patch.object(api, "_stream_chat_request", side_effect=fake_stream), patch.object(
|
||||
api, "_persist_metric", side_effect=fake_metric
|
||||
), patch.object(api, "_persist_chat_stage"), patch.object(api, "_persist_chat_event"), patch.object(
|
||||
api, "_chat_state"
|
||||
):
|
||||
initial, enhanced, metrics, enhancer = api._run_validation_harness(
|
||||
body, "root-request", "conversation", "primary", ["enhancer"], threading.Event(), []
|
||||
)
|
||||
|
||||
self.assertEqual(final, "one compiled final answer")
|
||||
self.assertEqual([item["model"] for item in reports], ["validator-a", "validator-b"])
|
||||
self.assertEqual(len(metrics), 4)
|
||||
self.assertEqual(len(calls), 4)
|
||||
self.assertEqual(initial, "initial story from primary")
|
||||
self.assertEqual(enhanced, "enhanced complete story from enhancer")
|
||||
self.assertEqual(enhancer, "enhancer")
|
||||
self.assertEqual(len(metrics), 2)
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(calls[0][0], "primary")
|
||||
self.assertEqual(calls[1][0], "enhancer")
|
||||
self.assertIn("initial story from primary", calls[1][1])
|
||||
self.assertIn("complete enhanced user-facing content", calls[1][1])
|
||||
self.assertTrue(all(call[2] == "root-request" for call in calls))
|
||||
compiler_prompt = calls[-1][1]
|
||||
self.assertIn("PRIMARY DRAFT:", compiler_prompt)
|
||||
self.assertIn("VALIDATOR 1", compiler_prompt)
|
||||
self.assertIn("VALIDATOR 2", compiler_prompt)
|
||||
self.assertNotIn("validator-a found no material issue\n\nvalidator-b found no material issue", final)
|
||||
def test_validation_report_is_retried_and_never_returned_as_final_answer(self):
|
||||
body = api.ChatRequest(primary_model="primary", validator_models=["validator"], harness=True, message="Write the requested result")
|
||||
|
||||
def test_enhancer_commentary_is_retried_and_never_returned_as_output(self):
|
||||
body = api.ChatRequest(primary_model="primary", validator_models=["enhancer"], harness=True, message="Write the requested story")
|
||||
calls = []
|
||||
|
||||
def fake_stream(payload, request_id, cancel_event=None, parent_id=None):
|
||||
model = payload["model"]
|
||||
prompt = payload["messages"][-1]["content"]
|
||||
calls.append((model, prompt))
|
||||
if model == "validator":
|
||||
content = "The draft needs a stronger ending."
|
||||
elif "IMPORTANT FINALIZATION RETRY" in prompt:
|
||||
content = "refined final answer"
|
||||
elif "VALIDATION REPORTS:" in prompt:
|
||||
if model == "primary":
|
||||
content = "initial story"
|
||||
elif "FINAL OUTPUT RETRY" in prompt:
|
||||
content = "full enhanced story"
|
||||
else:
|
||||
content = "# Story Validation Report\n## Narrative Structure\nRating: 8/10"
|
||||
else:
|
||||
content = "primary draft"
|
||||
return {"message": {"role": "assistant", "content": content}, "done": True}
|
||||
|
||||
def fake_metric(conversation_id, request_id, model, state, status=None, error=""):
|
||||
return {"request_id": request_id, "model": model, "status": status}
|
||||
|
||||
with patch.object(api, "_require_installed_model", side_effect=lambda name: name), patch.object(
|
||||
api, "_stream_chat_request", side_effect=fake_stream
|
||||
), patch.object(api, "_persist_metric", side_effect=fake_metric), patch.object(
|
||||
api, "_persist_chat_stage"
|
||||
), patch.object(api, "_persist_chat_event"), patch.object(api, "_chat_state"):
|
||||
final, metrics, reports = api._run_validation_harness(
|
||||
body, "root-request", "conversation", "primary", ["validator"], threading.Event(), []
|
||||
with patch.object(api, "_require_installed_model", side_effect=lambda name: name), patch.object(api, "_stream_chat_request", side_effect=fake_stream), patch.object(
|
||||
api, "_persist_metric", return_value={"status": "ok"}
|
||||
), patch.object(api, "_persist_chat_stage"), patch.object(api, "_persist_chat_event"), patch.object(
|
||||
api, "_chat_state"
|
||||
):
|
||||
initial, enhanced, metrics, enhancer = api._run_validation_harness(
|
||||
body, "root-request", "conversation", "primary", ["enhancer"], threading.Event(), []
|
||||
)
|
||||
|
||||
self.assertEqual(final, "refined final answer")
|
||||
self.assertNotIn("Validation Report", final)
|
||||
self.assertEqual(len(reports), 1)
|
||||
self.assertEqual(len(metrics), 4)
|
||||
self.assertTrue(any("IMPORTANT FINALIZATION RETRY" in prompt for _, prompt in calls))
|
||||
self.assertEqual(initial, "initial story")
|
||||
self.assertEqual(enhanced, "full enhanced story")
|
||||
self.assertEqual(enhancer, "enhancer")
|
||||
self.assertNotIn("Validation Report", enhanced)
|
||||
self.assertTrue(any("FINAL OUTPUT RETRY" in prompt for _, prompt in calls))
|
||||
self.assertEqual(len(metrics), 3)
|
||||
|
||||
def test_empty_compiler_output_is_retried_and_never_becomes_a_502(self):
|
||||
body = api.ChatRequest(primary_model="primary", validator_models=["validator"], harness=True, message="Write the requested result")
|
||||
calls = []
|
||||
def test_empty_enhancer_output_falls_back_to_initial_output(self):
|
||||
body = api.ChatRequest(primary_model="primary", validator_models=["enhancer"], harness=True, message="Write the requested result")
|
||||
|
||||
def fake_stream(payload, request_id, cancel_event=None, parent_id=None):
|
||||
model = payload["model"]
|
||||
prompt = payload["messages"][-1]["content"]
|
||||
calls.append((model, prompt))
|
||||
if model == "validator":
|
||||
content = "Make the result more specific."
|
||||
elif "IMPORTANT FINALIZATION RETRY" in prompt:
|
||||
content = "refined final answer after empty compiler output"
|
||||
elif "VALIDATION REPORTS:" in prompt:
|
||||
content = ""
|
||||
else:
|
||||
content = "primary draft"
|
||||
return {"message": {"role": "assistant", "content": content}, "done": True}
|
||||
return {"message": {"role": "assistant", "content": "initial draft" if model == "primary" else ""}, "done": True}
|
||||
|
||||
def fake_metric(conversation_id, request_id, model, state, status=None, error=""):
|
||||
return {"request_id": request_id, "model": model, "status": status}
|
||||
|
||||
with patch.object(api, "_require_installed_model", side_effect=lambda name: name), patch.object(
|
||||
api, "_stream_chat_request", side_effect=fake_stream
|
||||
), patch.object(api, "_persist_metric", side_effect=fake_metric), patch.object(
|
||||
api, "_persist_chat_stage"
|
||||
), patch.object(api, "_persist_chat_event"), patch.object(api, "_chat_state"):
|
||||
final, metrics, reports = api._run_validation_harness(
|
||||
body, "root-request", "conversation", "primary", ["validator"], threading.Event(), []
|
||||
with patch.object(api, "_require_installed_model", side_effect=lambda name: name), patch.object(api, "_stream_chat_request", side_effect=fake_stream), patch.object(
|
||||
api, "_persist_metric", return_value={"status": "ok"}
|
||||
), patch.object(api, "_persist_chat_stage"), patch.object(api, "_persist_chat_event"), patch.object(
|
||||
api, "_chat_state"
|
||||
):
|
||||
initial, enhanced, metrics, _ = api._run_validation_harness(
|
||||
body, "root-request", "conversation", "primary", ["enhancer"], threading.Event(), []
|
||||
)
|
||||
|
||||
self.assertEqual(final, "refined final answer after empty compiler output")
|
||||
self.assertEqual(len(reports), 1)
|
||||
self.assertEqual(len(metrics), 4)
|
||||
self.assertTrue(any("Primary returned empty output" in prompt for _, prompt in calls) or any("IMPORTANT FINALIZATION RETRY" in prompt for _, prompt in calls))
|
||||
|
||||
self.assertEqual(initial, "initial draft")
|
||||
self.assertEqual(enhanced, "initial draft")
|
||||
self.assertEqual(len(metrics), 3)
|
||||
|
||||
def test_status_omits_full_catalog_by_default(self):
|
||||
with patch.object(api, "_local_tags", return_value=[]), patch.object(api, "_local_ps", return_value=[]), patch.object(
|
||||
|
||||
Reference in New Issue
Block a user