feat(llm): 新增 LLM 智能增强服务(Agnes client + 4 项 enrichment 任务 + admin API + migration)

backend/app/services/llm/__init__.py

@@ -0,0 +1,7 @@
+"""LLM 服务:客户端 + 智能增强。"""
+from app.services.llm.client import LlmClient, client  # noqa: F401
+from app.services.llm.enrichment import (  # noqa: F401
+    enrich_article,
+    enrichment_loop,
+    get_setting,
+)

backend/app/services/llm/client.py

@@ -0,0 +1,147 @@
+"""Agnes(及任意 OpenAI 兼容端点)的 LLM 客户端。
+
+设计:
+- 内部持 chat 和 image 两个 Semaphore(各 1 个并发),互不阻塞
+- 每次调用后 await asyncio.sleep(interval_sec) 节流
+- 失败重试 1 次,再失败抛异常由上层标记 status=failed
+- 用 httpx.AsyncClient,超时 60s
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+import httpx
+
+from app.config import settings as app_settings
+
+logger = logging.getLogger("news.llm.client")
+
+
+class LlmClient:
+    """单一客户端,所有 LLM 调用都过它。"""
+
+    def __init__(
+        self,
+        base_url: str | None = None,
+        api_key: str | None = None,
+        chat_model: str | None = None,
+        image_model: str | None = None,
+        interval_sec: float | None = None,
+    ):
+        self.base_url = (base_url or app_settings.agnes_base_url).rstrip("/")
+        self.api_key = api_key or app_settings.agnes_api_key
+        self.chat_model = chat_model or app_settings.agnes_chat_model
+        self.image_model = image_model or app_settings.agnes_image_model
+        self.interval_sec = (
+            interval_sec if interval_sec is not None else app_settings.llm_interval_sec
+        )
+        # chat 和 image 各一个串行信号
+        self._chat_sem = asyncio.Semaphore(1)
+        self._image_sem = asyncio.Semaphore(1)
+
+    def is_configured(self) -> bool:
+        return bool(self.api_key)
+
+    async def chat(
+        self,
+        system: str,
+        user: str,
+        *,
+        temperature: float = 0.4,
+        max_tokens: int = 1500,
+        model: str | None = None,
+    ) -> str:
+        """调 chat/completions,返回 assistant 文本。"""
+        if not self.is_configured():
+            raise RuntimeError("AGNES_API_KEY 未配置")
+        url = f"{self.base_url}/chat/completions"
+        payload = {
+            "model": model or self.chat_model,
+            "messages": [
+                {"role": "system", "content": system},
+                {"role": "user", "content": user},
+            ],
+            "temperature": temperature,
+            "max_tokens": max_tokens,
+        }
+        async with self._chat_sem:
+            res = await self._post_with_retry(url, payload)
+            await asyncio.sleep(self.interval_sec)
+        return res["choices"][0]["message"]["content"].strip()
+
+    async def classify_json(
+        self,
+        system: str,
+        user: str,
+        *,
+        max_tokens: int = 200,
+    ) -> dict[str, Any]:
+        """调 chat 并尝试解析 JSON。失败时回退:返回空 dict。"""
+        text = await self.chat(system, user, temperature=0.2, max_tokens=max_tokens)
+        # 容错解析:可能被 ```json ... ``` 包裹
+        text = text.strip()
+        if text.startswith("```"):
+            # 去掉代码块围栏
+            lines = text.split("\n")
+            text = "\n".join(l for l in lines if not l.strip().startswith("```"))
+            text = text.strip()
+        import json
+        try:
+            return json.loads(text)
+        except Exception as e:
+            logger.warning("classify_json 解析失败: %s; raw=%r", e, text[:200])
+            return {}
+
+    async def generate_image(
+        self,
+        prompt: str,
+        *,
+        size: str = "1024x768",
+        model: str | None = None,
+    ) -> str:
+        """调 images/generations,返回图片 URL。"""
+        if not self.is_configured():
+            raise RuntimeError("AGNES_API_KEY 未配置")
+        url = f"{self.base_url}/images/generations"
+        payload = {
+            "model": model or self.image_model,
+            "prompt": prompt,
+            "size": size,
+        }
+        async with self._image_sem:
+            res = await self._post_with_retry(url, payload, timeout=120)
+            await asyncio.sleep(self.interval_sec)
+        return res["data"][0]["url"]
+
+    async def _post_with_retry(
+        self, url: str, payload: dict, *, timeout: float = 60.0, retries: int = 1
+    ) -> dict:
+        """POST + 简单重试(对 5xx / 超时)。"""
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+        last_exc: Exception | None = None
+        for attempt in range(retries + 1):
+            try:
+                async with httpx.AsyncClient(timeout=timeout) as client:
+                    r = await client.post(url, json=payload, headers=headers)
+                if r.status_code >= 500:
+                    raise RuntimeError(f"LLM 5xx: {r.status_code} {r.text[:200]}")
+                if r.status_code != 200:
+                    raise RuntimeError(f"LLM {r.status_code}: {r.text[:300]}")
+                return r.json()
+            except Exception as e:
+                last_exc = e
+                if attempt < retries:
+                    wait = 2 ** attempt
+                    logger.warning("LLM 调用失败,%.1fs 后重试: %s", wait, e)
+                    await asyncio.sleep(wait)
+        assert last_exc is not None
+        raise last_exc
+
+
+# 全局单例(读环境变量 + 启动时初始化)
+client = LlmClient()

backend/app/services/llm/enrichment.py

@@ -0,0 +1,238 @@
+"""LLM 智能增强服务(翻译后调)。
+
+4 个独立任务:
+  1. format   — 排版译文(写入 body_zh_formatted)
+  2. classify — 分类(写入 category)
+  3. image    — 生成插图(写入 image_ai_url)
+  4. commentary — 写点评(写入 commentary)
+
+设计:
+- 任务入口: enrich_article(article_id, settings_row)
+- 任务间互不影响:每个任务独立 try/except + 写 status
+- 全部任务共走 LlmClient 的全局限速
+- 若设置 enabled=False,只跳过(不调 LLM)
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+from sqlalchemy import select
+
+from app.database import AsyncSessionLocal
+from app.models.article import Article
+from app.models.llm_setting import LlmSetting
+from app.schemas.llm import get_default_prompts
+from app.services.llm.client import LlmClient
+
+logger = logging.getLogger("news.llm.enrichment")
+
+
+# === 获取当前设置(行锁 + 缓存刷新)===
+async def get_setting() -> LlmSetting:
+    """读 llm_settings 单行;不存在则用默认值插入。"""
+    async with AsyncSessionLocal() as session:
+        row = (await session.execute(select(LlmSetting).where(LlmSetting.id == 1))).scalar_one_or_none()
+        if row is None:
+            defaults = get_default_prompts()
+            row = LlmSetting(
+                id=1,
+                format_prompt=defaults["format_prompt"],
+                classify_prompt=defaults["classify_prompt"],
+                commentary_prompt=defaults["commentary_prompt"],
+                image_prompt_template=defaults["image_prompt_template"],
+            )
+            session.add(row)
+            await session.commit()
+            await session.refresh(row)
+        return row
+
+
+# === 单任务:format ===
+async def _enrich_format(article: Article, setting: LlmSetting, client: LlmClient) -> None:
+    prompt = (setting.format_prompt or get_default_prompts()["format_prompt"]).format(
+        body=(article.body_zh_text or "")[:6000]
+    )
+    text = await client.chat(
+        system="你是中文新闻排版助手,只输出排版后的纯文本。",
+        user=prompt,
+        temperature=0.3,
+        max_tokens=2000,
+    )
+    # 极简 HTML 包裹:按段切 + <p>
+    parts = [f"<p>{p.strip()}</p>" for p in text.split("\n\n") if p.strip()]
+    article.body_zh_formatted = "\n".join(parts) or None
+    article.format_status = "ok"
+
+
+# === 单任务:classify ===
+async def _enrich_classify(article: Article, setting: LlmSetting, client: LlmClient) -> None:
+    prompt = (setting.classify_prompt or get_default_prompts()["classify_prompt"]).format(
+        title=(article.title_zh or article.title)[:200],
+        summary=(article.summary_zh or "")[:400],
+    )
+    result = await client.classify_json(
+        system="你是新闻分类助手,只返回 JSON。",
+        user=prompt,
+    )
+    cats = result.get("categories") or []
+    if isinstance(cats, list) and cats:
+        article.category = ",".join(str(c).strip() for c in cats[:3])[:32]
+    article.classify_status = "ok"
+
+
+# === 单任务:image ===
+async def _enrich_image(article: Article, setting: LlmSetting, client: LlmClient) -> None:
+    template = (setting.image_prompt_template or get_default_prompts()["image_prompt_template"])
+    # 默认用 title_zh(若有),否则用原文 title
+    title_for_prompt = (article.title_zh or article.title or "")[:200]
+    prompt = template.format(title=title_for_prompt)
+    url = await client.generate_image(prompt, size=setting.image_size)
+    article.image_ai_url = url
+    article.image_ai_status = "ok"
+
+
+# === 单任务:commentary ===
+async def _enrich_commentary(article: Article, setting: LlmSetting, client: LlmClient) -> None:
+    prompt = (setting.commentary_prompt or get_default_prompts()["commentary_prompt"]).format(
+        title=(article.title_zh or article.title)[:200],
+        body=(article.body_zh_text or "")[:3000],
+    )
+    text = await client.chat(
+        system="你是资深新闻评论员。",
+        user=prompt,
+        temperature=0.6,
+        max_tokens=600,
+    )
+    article.commentary = text or None
+    article.commentary_status = "ok"
+
+
+# === 总编排:enrich_article ===
+async def enrich_article(article_id: int) -> dict[str, str]:
+    """对单篇文章做 4 项 LLM 增强。
+
+    返回 {task: status} 字典(用于日志)。
+    """
+    async with AsyncSessionLocal() as session:
+        art = (
+            await session.execute(select(Article).where(Article.id == article_id))
+        ).scalar_one_or_none()
+        if not art:
+            logger.warning("enrich_article: id=%s not found", article_id)
+            return {}
+        if not (art.title_zh or art.body_zh_text):
+            logger.info("enrich_article: id=%s no translation yet, skip", article_id)
+            return {}
+
+    # 拉取设置
+    setting = await get_setting()
+    if not setting.enabled:
+        logger.info("enrich_article: llm disabled, skip id=%s", article_id)
+        return {"format": "skipped", "classify": "skipped", "image": "skipped", "commentary": "skipped"}
+
+    # 用配置生成 client(允许热改设置)
+    client = LlmClient(
+        chat_model=setting.chat_model,
+        image_model=setting.image_model,
+        interval_sec=setting.interval_sec,
+    )
+
+    results: dict[str, str] = {}
+
+    async with AsyncSessionLocal() as session:
+        art = (
+            await session.execute(select(Article).where(Article.id == article_id))
+        ).scalar_one_or_none()
+        if not art:
+            return {}
+
+        # 4 个任务(互不影响);format / classify / commentary 是 chat,image 是 image
+        # 串行执行(已经过 client 内部 Semaphore),但每个 try/except 独立
+        tasks: list[tuple[str, Any]] = [
+            ("format", _enrich_format(art, setting, client)),
+            ("classify", _enrich_classify(art, setting, client)),
+            ("image", _enrich_image(art, setting, client)),
+            ("commentary", _enrich_commentary(art, setting, client)),
+        ]
+        for name, coro in tasks:
+            try:
+                await coro
+                results[name] = "ok"
+            except Exception as e:
+                logger.exception("enrich %s failed for article %s: %s", name, article_id, e)
+                results[name] = f"failed:{type(e).__name__}"
+                # 标 status
+                if name == "format":
+                    art.format_status = "failed"
+                elif name == "classify":
+                    art.classify_status = "failed"
+                elif name == "image":
+                    art.image_ai_status = "failed"
+                elif name == "commentary":
+                    art.commentary_status = "failed"
+
+        await session.commit()
+    logger.info("enrich_article id=%s: %s", article_id, results)
+    return results
+
+
+# === 后台循环 ===
+# 与 translation_loop 一样,常驻从队列里取文章
+ENRICHMENT_INTERVAL_SEC = 5.0  # 没活时等待
+ENRICHMENT_BATCH_SIZE = 1
+
+
+async def enrichment_loop() -> None:
+    """扫描已翻译但未 enrich 的文章(任一 *_status 为 pending/n/a 且 translation_status=ok)。
+
+    跟 translation_loop 一样常驻。
+    """
+    logger.info("enrichment_loop started")
+    # 等一下让翻译先跑
+    await asyncio.sleep(10)
+    while True:
+        try:
+            async with AsyncSessionLocal() as session:
+                # 已翻译完成 + 4 个状态中至少有一个是 pending
+                rows = (
+                    await session.execute(
+                        select(Article)
+                        .where(
+                            Article.translation_status == "ok",
+                            Article.title_zh.is_not(None),
+                        )
+                        .order_by(Article.translated_at.asc().nullslast(), Article.id.asc())
+                        .limit(ENRICHMENT_BATCH_SIZE * 5)  # 多取几个找需要 enrich 的
+                    )
+                ).scalars()
+                candidates = list(rows)
+
+            # 过滤:任一 *_status 是 pending
+            todo_ids: list[int] = []
+            for a in candidates:
+                statuses = [
+                    a.format_status or "pending",
+                    a.classify_status or "pending",
+                    a.image_ai_status or "pending",
+                    a.commentary_status or "pending",
+                ]
+                if any(s in ("pending", "failed", "n/a") for s in statuses):
+                    todo_ids.append(a.id)
+                if len(todo_ids) >= ENRICHMENT_BATCH_SIZE:
+                    break
+
+            if not todo_ids:
+                await asyncio.sleep(ENRICHMENT_INTERVAL_SEC)
+                continue
+
+            for aid in todo_ids:
+                try:
+                    await enrich_article(aid)
+                except Exception as e:
+                    logger.exception("enrich_article %s in loop failed: %s", aid, e)
+                await asyncio.sleep(0.5)  # 文章间轻节流
+        except Exception as e:
+            logger.exception("enrichment_loop error: %s", e)
+            await asyncio.sleep(ENRICHMENT_INTERVAL_SEC)