feat(ingest): API Push 短新闻接口层
- POST /api/v1/ingest:鉴权(X-Ingest-Token) + 限速(每 token 2 篇/秒,
Redis 滑动桶,INGEST_RATE_PER_SEC 可调) + 三层去重(L1 external_id /
L2 content_hash / L3 DB UNIQUE 兜底,均带 reason)
- 写入字段:is_short_news=True、translation/format/image_ai_status='n/a'、
classify_status=(有 tags?'ok':'pending')、commentary_{angel,meituan}_status='pending'、
body_zh_text=body_text(走统一路径,前端/prompt 不用改)
- services/fetchers/api_push.py:compute_content_hash + synthesize_url +
normalize_published_at + build_initial_status 纯函数
- schemas/ingest.py:IngestPayload(title 1-200/body 1-5000/tags 去重去空) +
IngestResponse(article_id/content_hash/status/reason/matched_external_id)
- admin.py:POST/GET/DELETE /admin/sources/{id}/ingest-tokens — owner 生成
(raw_token 仅一次性返回)、列出、撤销
- schemas/article.py:ArticleListItem 加 is_short_news/source_ref;
ArticleDetail 加 is_short_news/source_ref/external_id
- main.py:挂 ingest router;config.py + .env.example:ingest_rate_per_sec 默认 2
短新闻由 commit 1 enrichment_loop 自动接管 classify + 双 provider commentary,
跳过 format/image。
This commit is contained in:
86
backend/app/services/fetchers/api_push.py
Normal file
86
backend/app/services/fetchers/api_push.py
Normal file
@@ -0,0 +1,86 @@
|
||||
"""API Push 短新闻 — normalize 工具。
|
||||
|
||||
不走 fetcher 抽象(那是"周期拉取"语义),API Push 是"被动接收"。
|
||||
提供两个纯函数,供 ingest 路由调用:
|
||||
|
||||
- compute_content_hash(external_id, title, body) -> str
|
||||
- normalize_payload(payload: dict) -> dict(供入库时使用)
|
||||
|
||||
设计要点:
|
||||
- external_id 存在时,作为主幂等 key(L1)
|
||||
- external_id 缺失时,title+body[:500] 作为兜底指纹(L2)
|
||||
- url 可选;缺失时合成 api-push://{source_slug}/{hash[:16]} 占位
|
||||
- 字段长度校验集中在路由里(返回 400),这里只做归一化
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
|
||||
def compute_content_hash(
|
||||
*,
|
||||
external_id: str | None,
|
||||
title: str,
|
||||
body: str,
|
||||
) -> str:
|
||||
"""三层去重核心 key。
|
||||
|
||||
- external_id 存在:`sha1("ext:" + external_id)` —— 调用方幂等保证,最强
|
||||
- external_id 缺失:`sha1(title.strip() + "|" + body[:500])` —— 兜底,防尾部噪声
|
||||
|
||||
注:body 取原始字符串的前 500 字符,不做 strip。
|
||||
因为不同长度的 body(200字 vs 2000字)前 500 字符一定相等,这是设计意图 —
|
||||
仅靠"前 N 字符"判断重复,避免被尾部噪声(URL尾巴/HTML 注释)误判。
|
||||
"""
|
||||
if external_id:
|
||||
raw = f"ext:{external_id.strip()}"
|
||||
else:
|
||||
raw = f"{title.strip()}|{body[:500]}"
|
||||
return hashlib.sha1(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def synthesize_url(source_slug: str, content_hash: str) -> str:
|
||||
"""短新闻 url 占位(articles.url NOT NULL,需要合成)。"""
|
||||
return f"api-push://{source_slug}/{content_hash[:16]}"
|
||||
|
||||
|
||||
def normalize_published_at(value: Any) -> datetime:
|
||||
"""published_at 兜底:无值 → now(UTC)。"""
|
||||
if value is None:
|
||||
return datetime.now(timezone.utc)
|
||||
if isinstance(value, datetime):
|
||||
# 入参可能是 naive datetime(没带 tz),按 UTC 兜底
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
# ISO8601 解析;失败的让 pydantic 在路由层报 400
|
||||
try:
|
||||
# fromisoformat 在 3.11+ 支持 'Z' 后缀;3.12 没问题
|
||||
dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
except ValueError:
|
||||
# 解析失败兜底为 now;路由层校验会先于 normalize 跑,pydantic 应该已经报 400 了
|
||||
return datetime.now(timezone.utc)
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def build_initial_status(*, has_tags: bool) -> dict[str, str]:
|
||||
"""返回 enrich 状态字段的初始值。
|
||||
|
||||
- has_tags=True → classify_status='ok'(直接用 tags 当分类,不浪费 LLM 调用)
|
||||
- has_tags=False → classify_status='pending'(enrichment_loop 会跑 classify)
|
||||
- 其他:*_status='n/a' 或 'pending',具体见 commit 1 enrichment_article 的跳过逻辑
|
||||
"""
|
||||
return {
|
||||
"translation_status": "n/a", # 跳过翻译(中文原生)
|
||||
"format_status": "n/a", # 跳过排版(短文不需要)
|
||||
"image_ai_status": "n/a", # 跳过插图(用户明确不要)
|
||||
"classify_status": "ok" if has_tags else "pending",
|
||||
"commentary_status": "pending", # 双 provider 评论都跑
|
||||
"commentary_meituan_status": "pending",
|
||||
}
|
||||
Reference in New Issue
Block a user