形式;把 style 插到 class 后面 inner_with_p_style = inner_html.replace( f'
', f'
', ) return f'
{inner_with_p_style}
'
# === 单任务:classify (含黑名单 drop gate) ===
async def _enrich_classify(
article: Article,
setting: LlmSetting,
client: LlmClient,
blocklist: list[str],
) -> tuple[bool, list[str]]:
"""分类 + 黑名单判断。
返回 (drop, categories):
- drop=True → 整篇文章应删除(分类命中 blocklist)
- categories → 写入 article.category 的多标签列表
"""
template = setting.classify_prompt or get_default_prompts()["classify_prompt"]
# 老 prompt 可能只支持 {title}/{summary},不支持 {body} / {blocklist} —— _safe_format 兜底
vars_ = {
"title": (article.title_zh or article.title)[:200],
"summary": (article.summary_zh or "")[:400],
"body": (article.body_zh_text or "")[:1500],
"blocklist": "、".join(blocklist) if blocklist else "(无)",
}
prompt = _safe_format(template, vars_)
result = await client.classify_json(
system="你是新闻分类助手,只返回 JSON。",
user=prompt,
)
cats_raw = result.get("categories") or result.get("tags") or []
cats: list[str] = []
if isinstance(cats_raw, list):
cats = [str(c).strip() for c in cats_raw[:DEFAULT_IMAGE_MAX_TAGS] if str(c).strip()]
# 兼容两种来源:LLM 自己判断的 drop 字段,或后端兜底检查命中
drop_flag = bool(result.get("drop"))
if not drop_flag and blocklist:
# 兜底:即使 LLM 没正确返回 drop 字段,我们也用本地匹配兜底
bl_set = {b.strip() for b in blocklist if b and b.strip()}
drop_flag = any(c in bl_set for c in cats)
return drop_flag, cats
def _merge_blocklist(setting: LlmSetting, source: Source | None) -> list[str]:
"""合并全局 + per-source blocklist,去重保序。"""
out: list[str] = []
seen: set[str] = set()
sources: list[list[str]] = [setting.blocklist_tags or []]
if source is not None:
sources.append(source.blocklist_tags or [])
for src in sources:
for t in src:
t = (t or "").strip()
if t and t not in seen:
seen.add(t)
out.append(t)
return out
# === 单任务:image ===
async def _enrich_image(article: Article, setting: LlmSetting, client: LlmClient) -> None:
template = setting.image_prompt_template or get_default_prompts()["image_prompt_template"]
# 用正文第一段作为 prompt(英文 prompt 走 title 仍可工作,所以 title 也带上作 fallback)
first_para = _first_paragraph(article.body_zh_text or "", max_chars=DEFAULT_IMAGE_FIRST_PARA_CHARS)
if not first_para:
first_para = (article.title_zh or article.title or "")[:200]
title_for_prompt = (article.title_zh or article.title or "")[:200]
# template 同时支持 {body} 和 {title} 两种占位符;老的只支持 {title} 也能跑
prompt = _safe_format(template, {"body": first_para, "title": title_for_prompt})
url = await client.generate_image(prompt, size=DEFAULT_IMAGE_SIZE)
article.image_ai_url = url
article.image_ai_status = "ok"
def _first_paragraph(text: str, max_chars: int) -> str:
"""取正文第一段(按 \\n\\n 切)。如果首段超长就截断。"""
if not text:
return ""
for p in text.split("\n\n"):
p = p.strip()
if p:
return p[:max_chars]
return ""
# === 单任务:commentary ===
async def _enrich_commentary(article: Article, setting: LlmSetting, client: LlmClient) -> None:
template = setting.commentary_prompt or get_default_prompts()["commentary_prompt"]
prompt = _safe_format(
template,
{
"title": (article.title_zh or article.title)[:200],
"body": (article.body_zh_text or "")[:3000],
},
)
text = await client.chat(
system="你是资深新闻评论员。",
user=prompt,
temperature=0.6,
max_tokens=600,
)
article.commentary = text or None
article.commentary_status = "ok"
# === 总编排:enrich_article ===
async def enrich_article(article_id: int) -> dict[str, str]:
"""对单篇文章做 4 项 LLM 增强。
顺序:classify(黑名单 gate) → format → image → commentary
- classify 命中 blocklist → 整篇文章 DELETE,后续任务直接 return
- 任一任务失败,只标 status 不影响其他任务
返回 {task: status} 字典(用于日志)。
"""
async with AsyncSessionLocal() as session:
art = (
await session.execute(select(Article).where(Article.id == article_id))
).scalar_one_or_none()
if not art:
logger.warning("enrich_article: id=%s not found", article_id)
return {}
if not (art.title_zh or art.body_zh_text):
logger.info("enrich_article: id=%s no translation yet, skip", article_id)
return {}
# 拉取设置
setting = await get_setting()
if not setting.enabled:
logger.info("enrich_article: llm disabled, skip id=%s", article_id)
return {"format": "skipped", "classify": "skipped", "image": "skipped", "commentary": "skipped"}
# 用配置生成 client(允许热改设置)
client = LlmClient(
chat_model=setting.chat_model,
image_model=setting.image_model,
interval_sec=setting.interval_sec,
)
results: dict[str, str] = {}
async with AsyncSessionLocal() as session:
art = (
await session.execute(select(Article).where(Article.id == article_id))
).scalar_one_or_none()
if not art:
return {}
# === 1) classify(黑名单 gate,优先执行)===
blocklist = _merge_blocklist(setting, art.source if art.source_id else None)
try:
drop, cats = await _enrich_classify(art, setting, client, blocklist)
art.classify_status = "ok"
if cats:
art.category = ",".join(cats)[:64] or None
if drop:
# 命中 blocklist → 删文章,后续 3 步全跳
logger.info(
"enrich_article id=%s dropped (blocklist hit, cats=%s, blocklist=%s)",
article_id, cats, blocklist,
)
await session.delete(art)
await session.commit()
return {"classify": "dropped", "format": "skipped", "image": "skipped", "commentary": "skipped"}
except Exception as e:
logger.exception("enrich classify failed for article %s: %s", article_id, e)
art.classify_status = "failed"
results["classify"] = f"failed:{type(e).__name__}"
# classify 失败也继续(format/image/commentary 还能跑)
# === 2) format ===
try:
await _enrich_format(art, setting, client)
results["format"] = "ok"
except Exception as e:
logger.exception("enrich format failed for article %s: %s", article_id, e)
art.format_status = "failed"
results["format"] = f"failed:{type(e).__name__}"
# === 3) image ===
try:
await _enrich_image(art, setting, client)
results["image"] = "ok"
except Exception as e:
logger.exception("enrich image failed for article %s: %s", article_id, e)
art.image_ai_status = "failed"
results["image"] = f"failed:{type(e).__name__}"
# === 4) commentary ===
try:
await _enrich_commentary(art, setting, client)
results["commentary"] = "ok"
except Exception as e:
logger.exception("enrich commentary failed for article %s: %s", article_id, e)
art.commentary_status = "failed"
results["commentary"] = f"failed:{type(e).__name__}"
await session.commit()
logger.info("enrich_article id=%s: %s", article_id, results)
return results
# === 后台循环 ===
# 与 translation_loop 一样,常驻从队列里取文章
ENRICHMENT_INTERVAL_SEC = 5.0 # 没活时等待
ENRICHMENT_BATCH_SIZE = 3 # 每轮并发拉取候选,然后顺序处理(LLM 客户端本身有节流)
async def enrichment_loop() -> None:
"""扫描已翻译但未 enrich 的文章(任一 *_status 为 pending/n/a 且 translation_status=ok)。
跟 translation_loop 一样常驻。
"""
logger.info("enrichment_loop started")
# 等一下让翻译先跑
await asyncio.sleep(10)
while True:
try:
async with AsyncSessionLocal() as session:
# 已翻译完成 + 4 个状态中至少有一个是 pending
# 关键:不能按 translated_at 升序 — 老文章已 enrich,新文章 translated_at=NULL(被排到最后)
# 改为按 id 升序(新文章 id 大),循环里再过滤 status
rows = (
await session.execute(
select(Article)
.where(
Article.translation_status == "ok",
Article.title_zh.is_not(None),
)
.order_by(Article.id.asc())
.limit(ENRICHMENT_BATCH_SIZE * 20) # 多取一些找需要 enrich 的
)
).scalars()
candidates = list(rows)
# 过滤:任一 *_status 是 pending
todo_ids: list[int] = []
for a in candidates:
statuses = [
a.format_status or "pending",
a.classify_status or "pending",
a.image_ai_status or "pending",
a.commentary_status or "pending",
]
if any(s in ("pending", "failed", "n/a") for s in statuses):
todo_ids.append(a.id)
if len(todo_ids) >= ENRICHMENT_BATCH_SIZE:
break
if not todo_ids:
await asyncio.sleep(ENRICHMENT_INTERVAL_SEC)
continue
for aid in todo_ids:
try:
await enrich_article(aid)
except Exception as e:
logger.exception("enrich_article %s in loop failed: %s", aid, e)
await asyncio.sleep(0.2) # 文章间轻节流(LLM 内部还有 interval_sec)
except Exception as e:
logger.exception("enrichment_loop error: %s", e)
await asyncio.sleep(ENRICHMENT_INTERVAL_SEC)