Block publish when LLM rewrite quality degrades
This commit is contained in:
@@ -52,6 +52,19 @@ class LegacyScriptDelegationTests(unittest.TestCase):
|
||||
self.assertEqual(calls[0]["source_mode"], "mock")
|
||||
self.assertEqual(calls[0]["llm_mode"], "mock")
|
||||
|
||||
def test_main_exits_nonzero_when_new_pipeline_blocks_publish(self):
|
||||
module = load_pipeline_module()
|
||||
|
||||
def fake_run_daily_report(**kwargs):
|
||||
return {"reports": {"stage8": {"status": "blocked", "error": "rewrite_fallback_ratio_exceeded"}}}
|
||||
|
||||
with patch.object(module, "load_env", return_value={}):
|
||||
with patch("ai_daily_report.runner.run_daily_report", side_effect=fake_run_daily_report):
|
||||
with self.assertRaises(SystemExit) as raised:
|
||||
module.main()
|
||||
|
||||
self.assertEqual(raised.exception.code, 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import json
|
||||
import unittest
|
||||
from urllib.error import HTTPError
|
||||
|
||||
from ai_daily_report.pipeline import run_stage0_to_stage8
|
||||
|
||||
@@ -74,6 +75,65 @@ class Stage0To8PipelineTests(unittest.TestCase):
|
||||
self.assertIn("stage8", result["reports"])
|
||||
self.assertEqual(result["reports"]["stage8"]["status"], "ok")
|
||||
|
||||
def test_run_stage0_to_stage8_blocks_publish_when_rewrite_quality_gate_fails(self):
|
||||
configs = [{"name": "AI HOT", "type": "fake", "role": "primary", "priority": 10}]
|
||||
|
||||
def fetcher(config, run_date):
|
||||
return [
|
||||
{
|
||||
"title_raw": f"News {index}",
|
||||
"summary_raw": f"Summary {index}",
|
||||
"url": f"https://example.com/{index}",
|
||||
"source_label": "Example",
|
||||
"section_hint": "模型发布/更新",
|
||||
}
|
||||
for index in range(6)
|
||||
]
|
||||
|
||||
def semantic_llm_call(prompt):
|
||||
return json.dumps({"duplicate_groups": [], "not_duplicates": [], "uncertain": []})
|
||||
|
||||
def rewrite_llm_call(prompt):
|
||||
raise HTTPError(
|
||||
url="https://llm.example/v1/chat/completions",
|
||||
code=503,
|
||||
msg="Service Unavailable",
|
||||
hdrs=None,
|
||||
fp=None,
|
||||
)
|
||||
|
||||
def guide_llm_call(prompt):
|
||||
payload = json.loads(prompt)
|
||||
return json.dumps(
|
||||
{
|
||||
"theme": "模型能力继续更新。",
|
||||
"threads": [
|
||||
{
|
||||
"title": "模型更新",
|
||||
"text": "多条模型新闻更新。",
|
||||
"item_ids": [payload["items"][0]["id"]],
|
||||
"kind": "thread",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
result = run_stage0_to_stage8(
|
||||
configs,
|
||||
"2026-06-04",
|
||||
fetcher=fetcher,
|
||||
semantic_llm_call=semantic_llm_call,
|
||||
rewrite_llm_call=rewrite_llm_call,
|
||||
guide_llm_call=guide_llm_call,
|
||||
mode="publish",
|
||||
base_url="https://blog.example",
|
||||
client=None,
|
||||
)
|
||||
|
||||
self.assertEqual(result["publish"].status, "blocked")
|
||||
self.assertIn("rewrite_fallback_ratio_exceeded", result["reports"]["stage7"]["blocking_errors"])
|
||||
self.assertIn("rewrite_fallback_ratio_exceeded", result["reports"]["stage8"]["error"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import json
|
||||
import unittest
|
||||
from urllib.error import HTTPError
|
||||
|
||||
from ai_daily_report.models import NewsItem
|
||||
from ai_daily_report.rewrite import rewrite_items
|
||||
@@ -91,6 +92,29 @@ class Stage4RewriteTests(unittest.TestCase):
|
||||
self.assertEqual(report["fallback_count"], 0)
|
||||
self.assertEqual(calls, [["a", "b"], ["a"], ["b"]])
|
||||
|
||||
def test_rewrite_items_does_not_retry_single_items_after_transient_http_error(self):
|
||||
items = [news_item("a"), news_item("b")]
|
||||
calls = 0
|
||||
|
||||
def llm_call(prompt):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise HTTPError(
|
||||
url="https://llm.example/v1/chat/completions",
|
||||
code=503,
|
||||
msg="Service Unavailable",
|
||||
hdrs=None,
|
||||
fp=None,
|
||||
)
|
||||
|
||||
rewritten, report = rewrite_items(items, llm_call=llm_call, batch_size=2)
|
||||
|
||||
self.assertEqual(calls, 1)
|
||||
self.assertEqual([item.title for item in rewritten], ["OpenAI launches GPT-5 API", "OpenAI launches GPT-5 API"])
|
||||
self.assertEqual(report["fallback_count"], 2)
|
||||
self.assertTrue(report["quality_gate_failed"])
|
||||
self.assertIn("rewrite_fallback_ratio_exceeded", report["blocking_errors"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user