fix: recover empty validation compilations

This commit is contained in:
Hermes Agent
2026-08-30 00:55:48 +10:00
parent 2c731f4982
commit c7358ac97d
4 changed files with 44 additions and 8 deletions
+36
View File
@@ -178,6 +178,42 @@ class ValidationHarnessTests(unittest.TestCase):
self.assertEqual(len(metrics), 4)
self.assertTrue(any("IMPORTANT FINALIZATION RETRY" in prompt for _, prompt in calls))
def test_empty_compiler_output_is_retried_and_never_becomes_a_502(self):
body = api.ChatRequest(primary_model="primary", validator_models=["validator"], harness=True, message="Write the requested result")
calls = []
def fake_stream(payload, request_id, cancel_event=None, parent_id=None):
model = payload["model"]
prompt = payload["messages"][-1]["content"]
calls.append((model, prompt))
if model == "validator":
content = "Make the result more specific."
elif "IMPORTANT FINALIZATION RETRY" in prompt:
content = "refined final answer after empty compiler output"
elif "VALIDATION REPORTS:" in prompt:
content = ""
else:
content = "primary draft"
return {"message": {"role": "assistant", "content": content}, "done": True}
def fake_metric(conversation_id, request_id, model, state, status=None, error=""):
return {"request_id": request_id, "model": model, "status": status}
with patch.object(api, "_require_installed_model", side_effect=lambda name: name), patch.object(
api, "_stream_chat_request", side_effect=fake_stream
), patch.object(api, "_persist_metric", side_effect=fake_metric), patch.object(
api, "_persist_chat_stage"
), patch.object(api, "_persist_chat_event"), patch.object(api, "_chat_state"):
final, metrics, reports = api._run_validation_harness(
body, "root-request", "conversation", "primary", ["validator"], threading.Event(), []
)
self.assertEqual(final, "refined final answer after empty compiler output")
self.assertEqual(len(reports), 1)
self.assertEqual(len(metrics), 4)
self.assertTrue(any("Primary returned empty output" in prompt for _, prompt in calls) or any("IMPORTANT FINALIZATION RETRY" in prompt for _, prompt in calls))
def test_status_omits_full_catalog_by_default(self):
with patch.object(api, "_local_tags", return_value=[]), patch.object(api, "_local_ps", return_value=[]), patch.object(
api, "_ensure_catalog", return_value={"models": [], "families": {}, "source": "test"}