Block publish when LLM rewrite quality degrades

This commit is contained in:
Mimikko-zeus
2026-06-04 16:29:40 +08:00
parent 5a98696255
commit f7e4c9722b
6 changed files with 132 additions and 1 deletions

View File

@@ -52,6 +52,19 @@ class LegacyScriptDelegationTests(unittest.TestCase):
self.assertEqual(calls[0]["source_mode"], "mock")
self.assertEqual(calls[0]["llm_mode"], "mock")
def test_main_exits_nonzero_when_new_pipeline_blocks_publish(self):
module = load_pipeline_module()
def fake_run_daily_report(**kwargs):
return {"reports": {"stage8": {"status": "blocked", "error": "rewrite_fallback_ratio_exceeded"}}}
with patch.object(module, "load_env", return_value={}):
with patch("ai_daily_report.runner.run_daily_report", side_effect=fake_run_daily_report):
with self.assertRaises(SystemExit) as raised:
module.main()
self.assertEqual(raised.exception.code, 2)
if __name__ == "__main__":
unittest.main()

View File

@@ -1,5 +1,6 @@
import json
import unittest
from urllib.error import HTTPError
from ai_daily_report.pipeline import run_stage0_to_stage8
@@ -74,6 +75,65 @@ class Stage0To8PipelineTests(unittest.TestCase):
self.assertIn("stage8", result["reports"])
self.assertEqual(result["reports"]["stage8"]["status"], "ok")
def test_run_stage0_to_stage8_blocks_publish_when_rewrite_quality_gate_fails(self):
configs = [{"name": "AI HOT", "type": "fake", "role": "primary", "priority": 10}]
def fetcher(config, run_date):
return [
{
"title_raw": f"News {index}",
"summary_raw": f"Summary {index}",
"url": f"https://example.com/{index}",
"source_label": "Example",
"section_hint": "模型发布/更新",
}
for index in range(6)
]
def semantic_llm_call(prompt):
return json.dumps({"duplicate_groups": [], "not_duplicates": [], "uncertain": []})
def rewrite_llm_call(prompt):
raise HTTPError(
url="https://llm.example/v1/chat/completions",
code=503,
msg="Service Unavailable",
hdrs=None,
fp=None,
)
def guide_llm_call(prompt):
payload = json.loads(prompt)
return json.dumps(
{
"theme": "模型能力继续更新。",
"threads": [
{
"title": "模型更新",
"text": "多条模型新闻更新。",
"item_ids": [payload["items"][0]["id"]],
"kind": "thread",
}
],
}
)
result = run_stage0_to_stage8(
configs,
"2026-06-04",
fetcher=fetcher,
semantic_llm_call=semantic_llm_call,
rewrite_llm_call=rewrite_llm_call,
guide_llm_call=guide_llm_call,
mode="publish",
base_url="https://blog.example",
client=None,
)
self.assertEqual(result["publish"].status, "blocked")
self.assertIn("rewrite_fallback_ratio_exceeded", result["reports"]["stage7"]["blocking_errors"])
self.assertIn("rewrite_fallback_ratio_exceeded", result["reports"]["stage8"]["error"])
if __name__ == "__main__":
unittest.main()

View File

@@ -1,5 +1,6 @@
import json
import unittest
from urllib.error import HTTPError
from ai_daily_report.models import NewsItem
from ai_daily_report.rewrite import rewrite_items
@@ -91,6 +92,29 @@ class Stage4RewriteTests(unittest.TestCase):
self.assertEqual(report["fallback_count"], 0)
self.assertEqual(calls, [["a", "b"], ["a"], ["b"]])
def test_rewrite_items_does_not_retry_single_items_after_transient_http_error(self):
items = [news_item("a"), news_item("b")]
calls = 0
def llm_call(prompt):
nonlocal calls
calls += 1
raise HTTPError(
url="https://llm.example/v1/chat/completions",
code=503,
msg="Service Unavailable",
hdrs=None,
fp=None,
)
rewritten, report = rewrite_items(items, llm_call=llm_call, batch_size=2)
self.assertEqual(calls, 1)
self.assertEqual([item.title for item in rewritten], ["OpenAI launches GPT-5 API", "OpenAI launches GPT-5 API"])
self.assertEqual(report["fallback_count"], 2)
self.assertTrue(report["quality_gate_failed"])
self.assertIn("rewrite_fallback_ratio_exceeded", report["blocking_errors"])
if __name__ == "__main__":
unittest.main()