Improve AI daily report operations and dedupe observability

config/pipeline.json

@@ -16,5 +16,37 @@
     "enabled": true,
     "max_age_days": 7,
     "history_path": "~/.hermes/scripts/ai_morning_out/published_urls.json"
+  },
+  "semantic_candidate_recall": {
+    "enabled": true,
+    "max_pairs": 80,
+    "max_pairs_per_item": 5,
+    "title_similarity_threshold": 0.45,
+    "title_jaccard_threshold": 0.25,
+    "summary_jaccard_threshold": 0.18,
+    "strong_entity_overlap_threshold": 2
+  },
+  "quality_gate": {
+    "required_source_failure_policy": "block",
+    "block_on_required_source_failure": true,
+    "warn_on_enabled_source_failure": true,
+    "warn_when_stage3_candidates_zero_min_items": 30,
+    "warn_on_final_title_similarity": 0.55,
+    "warn_on_entity_frequency": 3,
+    "required_sources": ["AI HOT"]
+  },
+  "publish_idempotency": {
+    "enabled": true,
+    "allow_republish": false,
+    "slug_lookup_paths": [
+      "/api/service/posts/{slug}",
+      "/api/service/posts?slug={slug}",
+      "/api/service/posts/slug/{slug}"
+    ]
+  },
+  "llm_observability": {
+    "enabled": true,
+    "prompt_preview_chars": 500,
+    "response_preview_chars": 500
   }
 }

config/sources.json

@@ -4,21 +4,50 @@
     "type": "aihot",
     "role": "primary",
     "required": true,
+    "failure_policy": "block",
     "priority": 10,
     "timeout_seconds": 25,
     "retries": 2,
     "min_items": 10,
     "enabled": true
   },
+  {
+    "name": "橘鸦AI早报",
+    "type": "juya_rss",
+    "url": "https://imjuya.github.io/juya-ai-daily/rss.xml",
+    "role": "supplement",
+    "required": false,
+    "failure_policy": "warn",
+    "priority": 20,
+    "timeout_seconds": 45,
+    "retries": 2,
+    "min_items": 0,
+    "enabled": true
+  },
+  {
+    "name": "量子位",
+    "type": "rss",
+    "url": "https://www.qbitai.com/feed",
+    "role": "supplement",
+    "required": false,
+    "failure_policy": "warn",
+    "priority": 30,
+    "timeout_seconds": 25,
+    "retries": 1,
+    "min_items": 0,
+    "enabled": true
+  },
   {
     "name": "InfoQ AI",
     "type": "rss",
     "url": "https://feed.infoq.com/ai-ml-data-eng/",
     "role": "supplement",
     "required": false,
+    "failure_policy": "warn",
     "priority": 40,
     "timeout_seconds": 25,
     "retries": 1,
+    "min_items": 0,
     "max_item_age_days": 3,
     "enabled": true
   },
@@ -28,32 +57,12 @@
     "url": "https://www.technologyreview.com/topic/artificial-intelligence/feed",
     "role": "supplement",
     "required": false,
+    "failure_policy": "warn",
     "priority": 50,
     "timeout_seconds": 25,
     "retries": 1,
+    "min_items": 0,
     "max_item_age_days": 5,
     "enabled": true
-  },
-  {
-    "name": "量子位",
-    "type": "rss",
-    "url": "https://www.qbitai.com/feed",
-    "role": "supplement",
-    "required": false,
-    "priority": 30,
-    "timeout_seconds": 25,
-    "retries": 1,
-    "enabled": true
-  },
-  {
-    "name": "橘鸦AI早报",
-    "type": "juya_rss",
-    "url": "https://imjuya.github.io/juya-ai-daily/rss.xml",
-    "role": "supplement",
-    "required": false,
-    "priority": 20,
-    "timeout_seconds": 45,
-    "retries": 2,
-    "enabled": true
   }
 ]