fix: 翻译主流程失败时 raise(不再返回占位符); add_usage TTL 用 replace(day=1) 防 0 TTL
This commit is contained in:
Mavis
@@ -59,17 +59,16 @@ class TranslationService:
|
||||
|
||||
async def add_usage(self, chars: int) -> None:
|
||||
r = get_redis()
|
||||
# 用 INCRBY + EXPIRE 月初;简单做法:每次 set + 设 TTL
|
||||
key = _month_key()
|
||||
async with r.pipeline(transaction=False) as pipe:
|
||||
pipe.incrby(key, chars)
|
||||
# 月底过期(下下月 1 日)
|
||||
now = datetime.now(timezone.utc)
|
||||
# 下个月第一天
|
||||
if now.month == 12:
|
||||
next_month = now.replace(year=now.year + 1, month=1, day=1)
|
||||
else:
|
||||
next_month = now.replace(month=now.month + 1, day=1)
|
||||
ttl = int((next_month - now).total_seconds()) + 86400
|
||||
next_month = now.replace(month=now.month + 1, day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||
ttl = max(60, int((next_month - now).total_seconds()) + 86400) # +1 天 buffer
|
||||
async with r.pipeline(transaction=False) as pipe:
|
||||
pipe.incrby(key, chars)
|
||||
pipe.expire(key, ttl)
|
||||
await pipe.execute()
|
||||
|
||||
@@ -106,6 +105,7 @@ class TranslationService:
|
||||
|
||||
# 3) 调用
|
||||
async with self._sem:
|
||||
res = None
|
||||
try:
|
||||
res = await engine.translate(text, source=source, target=target)
|
||||
except Exception as e:
|
||||
@@ -113,13 +113,14 @@ class TranslationService:
|
||||
logger.exception("translate failed with %s: %s", engine.name, e)
|
||||
fb = self._fallback()
|
||||
if fb is not None and engine is not fb:
|
||||
try:
|
||||
res = await fb.translate(text, source=source, target=target)
|
||||
else:
|
||||
res = TranslationResult(
|
||||
text=text + f"\n\n[翻译失败: {e}]",
|
||||
engine="skip",
|
||||
chars=chars,
|
||||
)
|
||||
except Exception as e2:
|
||||
logger.exception("fallback %s also failed: %s", fb.name, e2)
|
||||
res = None
|
||||
if res is None:
|
||||
# 主 + fallback 都失败:抛异常,让上层标记 status=failed
|
||||
raise RuntimeError(f"translation failed for {chars} chars (engine={engine.name})")
|
||||
|
||||
# 4) 写缓存(无论引擎)
|
||||
try:
|
||||
|
||||
41
scripts/_api_redis.py
Normal file
41
scripts/_api_redis.py
Normal file
@@ -0,0 +1,41 @@
|
||||
import os, paramiko, base64
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=60):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
# 在 API 容器里直接读 redis
|
||||
script = (
|
||||
"import asyncio\n"
|
||||
"from app.redis_client import get_redis\n"
|
||||
"async def main():\n"
|
||||
" r = get_redis()\n"
|
||||
" await r.ping()\n"
|
||||
" v = await r.get('translation:month:202606')\n"
|
||||
" print('api sees:', v)\n"
|
||||
" # 手动设一个值再读\n"
|
||||
" await r.set('translation:month:202606', 999)\n"
|
||||
" v2 = await r.get('translation:month:202606')\n"
|
||||
" print('after set 999:', v2)\n"
|
||||
"asyncio.run(main())\n"
|
||||
)
|
||||
b64 = base64.b64encode(script.encode()).decode()
|
||||
run(f"docker exec news-aggregator-api-1 sh -c 'echo {b64} | base64 -d > /app/_test_redis.py'")
|
||||
print("--- api 容器读 redis ---")
|
||||
run("docker exec -w /app news-aggregator-api-1 python /app/_test_redis.py", t=15)
|
||||
|
||||
# 立即调 /me/usage
|
||||
import json
|
||||
out = run("curl -s -X POST http://localhost/api/v1/auth/login -H 'Content-Type: application/json' -d '{\"username\":\"owner\",\"password\":\"Owner2026!\"}'")
|
||||
token = json.loads(out)["access_token"]
|
||||
u = json.loads(run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/me/usage'"))
|
||||
print(f"--- /me/usage: {u}")
|
||||
c.close()
|
||||
56
scripts/_check_count.py
Normal file
56
scripts/_check_count.py
Normal file
@@ -0,0 +1,56 @@
|
||||
import os, paramiko
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=60):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
# 找一条最新抓的 article,重译
|
||||
print("--- 找最新 article + 重译 ---")
|
||||
out = run("docker exec news-aggregator-postgres-1 psql -U news -d news -tA -c \"SELECT id FROM articles ORDER BY fetched_at DESC LIMIT 1;\"")
|
||||
aid = out.strip()
|
||||
print(f" article id: {aid}")
|
||||
|
||||
# 改回 pending 然后触发翻译
|
||||
run(f"docker exec news-aggregator-postgres-1 psql -U news -d news -c \"UPDATE articles SET translation_status = 'pending' WHERE id = {aid};\" 2>&1 | tail -2")
|
||||
|
||||
# 直接用 worker 调 translate
|
||||
print("--- 触发翻译 ---")
|
||||
script = f'''import asyncio
|
||||
from app.workers.pipeline import translate_article
|
||||
from app.services.translation.service import service
|
||||
|
||||
async def main():
|
||||
# 调一次 service
|
||||
res = await service.translate("Hello world, this is a test of translation.", source="en", target="zh")
|
||||
print("res:", res.engine, "chars:", res.chars, "text:", res.text[:50])
|
||||
# 再调一次,会走 cache
|
||||
res2 = await service.translate("Hello world, this is a test of translation.", source="en", target="zh")
|
||||
print("res2:", res2.engine, "cached:", res2.cached)
|
||||
|
||||
asyncio.run(main())
|
||||
'''
|
||||
print("--- 测试 service.translate ---")
|
||||
import time
|
||||
# 写脚本文件 + docker cp
|
||||
with open("/tmp/_test_translate.py", "w", encoding="utf-8") as f:
|
||||
f.write(script)
|
||||
# 用 stdin
|
||||
run(f"docker exec -i news-aggregator-worker-1 python -u", t=30) # 这个不通,要传脚本
|
||||
# 改成 echo | 跑
|
||||
quoted = script.replace('"', '\\"').replace('\\n', '\\\\n')
|
||||
run(f"docker exec news-aggregator-worker-1 python -c \"{quoted}\"", t=30)
|
||||
|
||||
# 看 redis
|
||||
print("\n--- redis usage ---")
|
||||
rpw = run("grep ^REDIS_PASSWORD /srv/news/.env | cut -d= -f2").strip()
|
||||
out = run(f"docker exec news-aggregator-redis-1 redis-cli -a '{rpw}' GET 'translation:month:202606' 2>&1 | grep -v Warning")
|
||||
print(f" GET: {out.strip()}")
|
||||
c.close()
|
||||
40
scripts/_check_count2.py
Normal file
40
scripts/_check_count2.py
Normal file
@@ -0,0 +1,40 @@
|
||||
import os, paramiko, base64
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=60):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
script = (
|
||||
"import asyncio, logging\n"
|
||||
"from app.services.translation.service import service\n"
|
||||
"from app.redis_client import get_redis\n"
|
||||
"logging.basicConfig(level=logging.INFO)\n"
|
||||
"async def main():\n"
|
||||
" r = get_redis(); await r.ping()\n"
|
||||
" print('before:', await r.get('translation:month:202606') or 0)\n"
|
||||
" res = await service.translate('Hello, world. This is a short test message.', source='en', target='zh')\n"
|
||||
" print(' result engine=', res.engine, 'chars=', res.chars, 'text=', res.text[:60])\n"
|
||||
" print('after:', await r.get('translation:month:202606') or 0)\n"
|
||||
" res2 = await service.translate('Hello, world. This is a short test message.', source='en', target='zh')\n"
|
||||
" print(' cached:', res2.cached, 'engine=', res2.engine)\n"
|
||||
" print('after cache:', await r.get('translation:month:202606') or 0)\n"
|
||||
"asyncio.run(main())\n"
|
||||
)
|
||||
script_b64 = base64.b64encode(script.encode()).decode()
|
||||
print("--- 写文件 ---")
|
||||
run(f"docker exec news-aggregator-worker-1 sh -c 'echo {script_b64} | base64 -d > /tmp/_t.py'")
|
||||
print("--- 跑 ---")
|
||||
run("docker exec news-aggregator-worker-1 python /tmp/_t.py", t=30)
|
||||
print("\n--- redis ---")
|
||||
rpw = run("grep ^REDIS_PASSWORD /srv/news/.env | cut -d= -f2").strip()
|
||||
out = run(f"docker exec news-aggregator-redis-1 redis-cli -a '{rpw}' GET 'translation:month:202606' 2>&1 | grep -v Warning")
|
||||
print(f" usage: {out.strip()}")
|
||||
c.close()
|
||||
46
scripts/_check_count3.py
Normal file
46
scripts/_check_count3.py
Normal file
@@ -0,0 +1,46 @@
|
||||
import os, paramiko, base64
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=60):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
# 重置 usage 到 0(我之前测试加了 100)
|
||||
rpw = run("grep ^REDIS_PASSWORD /srv/news/.env | cut -d= -f2").strip()
|
||||
run(f"docker exec news-aggregator-redis-1 redis-cli -a '{rpw}' DEL 'translation:month:202606' 2>&1 | grep -v Warning")
|
||||
print("usage 重置为 0")
|
||||
|
||||
# 写脚本到 /app/_t.py
|
||||
script = (
|
||||
"import asyncio, logging\n"
|
||||
"from app.services.translation.service import service\n"
|
||||
"from app.redis_client import get_redis\n"
|
||||
"logging.basicConfig(level=logging.INFO)\n"
|
||||
"async def main():\n"
|
||||
" r = get_redis(); await r.ping()\n"
|
||||
" print('before:', await r.get('translation:month:202606') or 0)\n"
|
||||
" res = await service.translate('Hello, world. This is a short test message.', source='en', target='zh')\n"
|
||||
" print(' result engine=', res.engine, 'chars=', res.chars, 'text=', res.text[:60])\n"
|
||||
" print('after:', await r.get('translation:month:202606') or 0)\n"
|
||||
" res2 = await service.translate('Hello, world. This is a short test message.', source='en', target='zh')\n"
|
||||
" print(' cached:', res2.cached, 'engine=', res2.engine)\n"
|
||||
" print('after cache:', await r.get('translation:month:202606') or 0)\n"
|
||||
"asyncio.run(main())\n"
|
||||
)
|
||||
script_b64 = base64.b64encode(script.encode()).decode()
|
||||
run(f"docker exec news-aggregator-worker-1 sh -c 'echo {script_b64} | base64 -d > /app/_t.py && cat /app/_t.py | head -3'")
|
||||
|
||||
print("\n--- 跑(在 /app 下)---")
|
||||
run("docker exec -w /app news-aggregator-worker-1 python /app/_t.py", t=30)
|
||||
|
||||
print("\n--- redis ---")
|
||||
out = run(f"docker exec news-aggregator-redis-1 redis-cli -a '{rpw}' GET 'translation:month:202606' 2>&1 | grep -v Warning")
|
||||
print(f" usage: {out.strip()}")
|
||||
c.close()
|
||||
42
scripts/_count_test.py
Normal file
42
scripts/_count_test.py
Normal file
@@ -0,0 +1,42 @@
|
||||
import os, paramiko
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=60):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
rpw = run("grep ^REDIS_PASSWORD /srv/news/.env | cut -d= -f2").strip()
|
||||
# 重置 usage
|
||||
run(f"docker exec news-aggregator-redis-1 redis-cli -a '{rpw}' DEL 'translation:month:202606' 2>&1 | grep -v Warning")
|
||||
print("usage 重置为 0")
|
||||
|
||||
# 找一条 article 重译
|
||||
out = run("docker exec news-aggregator-postgres-1 psql -U news -d news -tA -c \"SELECT id FROM articles WHERE translation_status = 'ok' ORDER BY translation_chars DESC LIMIT 1;\"")
|
||||
aid = out.strip()
|
||||
print(f"\n重译 article {aid}...")
|
||||
run(f"docker exec news-aggregator-postgres-1 psql -U news -d news -c \"UPDATE articles SET translation_status = 'pending' WHERE id = {aid};\" 2>&1 | tail -2")
|
||||
run(f"cd /srv/news && docker exec news-aggregator-worker-1 python -c 'import asyncio; from app.workers.pipeline import translate_article; asyncio.run(translate_article({aid}))' 2>&1 | tail -5", t=60)
|
||||
|
||||
# 看 usage
|
||||
out = run(f"docker exec news-aggregator-redis-1 redis-cli -a '{rpw}' GET 'translation:month:202606' 2>&1 | grep -v Warning")
|
||||
print(f"\n--- redis usage: {out.strip()}")
|
||||
|
||||
# 看 article 的 translation_chars
|
||||
out = run(f"docker exec news-aggregator-postgres-1 psql -U news -d news -tA -c \"SELECT translation_chars FROM articles WHERE id = {aid};\"")
|
||||
print(f"--- article {aid} translation_chars (DB): {out.strip()}")
|
||||
|
||||
# 实际值对比
|
||||
print("\n--- /me/usage ---")
|
||||
import json
|
||||
out = run("curl -s -X POST http://localhost/api/v1/auth/login -H 'Content-Type: application/json' -d '{\"username\":\"owner\",\"password\":\"Owner2026!\"}'")
|
||||
token = json.loads(out)["access_token"]
|
||||
u = json.loads(run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/me/usage'"))
|
||||
print(json.dumps(u, indent=2))
|
||||
c.close()
|
||||
62
scripts/_final3.py
Normal file
62
scripts/_final3.py
Normal file
@@ -0,0 +1,62 @@
|
||||
import os, paramiko, json
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
NEW_PW = "Owner2026!"
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=60):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
# pull + 重建 api
|
||||
print("--- pull + 重建 api ---")
|
||||
run("cd /srv/news && sudo -u news git pull --rebase 2>&1 | tail -3")
|
||||
run('docker compose -f /srv/news/docker-compose.yml up -d --force-recreate --no-deps --build api', t=120)
|
||||
import time
|
||||
time.sleep(6)
|
||||
|
||||
# 登录
|
||||
print("\n--- 登录 ---")
|
||||
body = json.dumps({"username": "owner", "password": NEW_PW})
|
||||
out = run(f"curl -s -X POST http://localhost/api/v1/auth/login -H 'Content-Type: application/json' -d '{body}'")
|
||||
data = json.loads(out)
|
||||
token = data.get("access_token")
|
||||
print(f" 登录 OK, token: {token[:30]}...")
|
||||
|
||||
# 拉 articles
|
||||
print("\n--- /api/v1/articles ---")
|
||||
out = run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/articles?limit=3'")
|
||||
try:
|
||||
ad = json.loads(out)
|
||||
print(f" 返回 {len(ad.get('items', []))} 条:")
|
||||
for a in ad.get("items", [])[:3]:
|
||||
print(f" [{a['translation_status']:8s}] {a['source']['name']:14s} | {a['title'][:50]}")
|
||||
if a.get("title_zh"):
|
||||
print(f" zh: {a['title_zh'][:50]}")
|
||||
except Exception as e:
|
||||
print(f" parse err: {e}\n raw: {out[:300]}")
|
||||
|
||||
# /me
|
||||
print("\n--- /me ---")
|
||||
me = json.loads(run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/me'"))
|
||||
print(f" {me}")
|
||||
|
||||
# /me/usage
|
||||
print("\n--- /me/usage ---")
|
||||
u = json.loads(run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/me/usage'"))
|
||||
print(f" {u}")
|
||||
|
||||
# /sources
|
||||
print("\n--- /sources ---")
|
||||
sl = json.loads(run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/sources'"))
|
||||
print(f" {len(sl)} 个源:")
|
||||
for s in sl:
|
||||
en = "✓" if s["enabled"] else "✗"
|
||||
print(f" {en} [{s['priority']:3d}] {s['slug']:18s} {s['name']}")
|
||||
c.close()
|
||||
59
scripts/_re_translate.py
Normal file
59
scripts/_re_translate.py
Normal file
@@ -0,0 +1,59 @@
|
||||
import os, paramiko, json
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=120):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
# 1) 找带错误信息的文章(翻译状态虽然 ok 但字段里带"翻译失败"字样)
|
||||
print("--- 找出还残留错误标记的文章 ---")
|
||||
n = run("docker exec news-aggregator-postgres-1 psql -U news -d news -tA -c \"SELECT count(*) FROM articles WHERE title_zh LIKE '%[翻译失败:%' OR body_zh_text LIKE '%[翻译失败:%';\"")
|
||||
print(f" 残留错误文章数: {n.strip()}")
|
||||
|
||||
# 2) 改回 pending
|
||||
print("\n--- 批量回退到 pending ---")
|
||||
run("docker exec news-aggregator-postgres-1 psql -U news -d news -c \"UPDATE articles SET translation_status = 'pending', title_zh = NULL, body_zh_text = NULL, body_zh_html = NULL WHERE title_zh LIKE '%[翻译失败:%' OR body_zh_text LIKE '%[翻译失败:%';\" 2>&1 | tail -3")
|
||||
|
||||
# 3) 触发 worker 翻译
|
||||
print("\n--- 触发翻译(120s 等待)---")
|
||||
run("cd /srv/news && docker exec news-aggregator-worker-1 python -c 'import asyncio; from app.workers.pipeline import _translate_recent_for_source; async def t(): [await _translate_recent_for_source(sid, max_n=300) for sid in [2,3,4,5]]; asyncio.run(t())' 2>&1 | tail -10", t=180)
|
||||
import time
|
||||
time.sleep(10)
|
||||
|
||||
# 4) 翻译后统计
|
||||
print("\n--- 翻译后统计 ---")
|
||||
run("docker exec news-aggregator-postgres-1 psql -U news -d news -c \"SELECT translation_status, translation_engine, count(*), sum(translation_chars) FROM articles GROUP BY 1, 2 ORDER BY 1, 2;\"")
|
||||
|
||||
# 5) 看一条 BBC 详情
|
||||
print("\n--- BBC 详情 ---")
|
||||
out = run("curl -s -X POST http://localhost/api/v1/auth/login -H 'Content-Type: application/json' -d '{\"username\":\"owner\",\"password\":\"Owner2026!\"}'")
|
||||
token = json.loads(out)["access_token"]
|
||||
out = run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/articles?source=bbc-world&limit=1'")
|
||||
ad = json.loads(out)
|
||||
if ad.get("items"):
|
||||
aid = ad["items"][0]["id"]
|
||||
out = run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/articles/{aid}'")
|
||||
try:
|
||||
det = json.loads(out)
|
||||
print(f"\n=== {det['source']['name']} #{aid} ===")
|
||||
print(f" title: {det['title'][:80]}")
|
||||
print(f" title_zh: {(det.get('title_zh') or '—')[:80]}")
|
||||
print(f" body_text 长度: {len(det['body_text'])}")
|
||||
print(f" body_zh_text 长度: {len(det.get('body_zh_text') or '')}")
|
||||
if det.get("body_zh_text"):
|
||||
print(f"\n 译文(前 600 字符):")
|
||||
print(f" {det['body_zh_text'][:600]}")
|
||||
except Exception as e:
|
||||
print(f" err: {e}\n raw: {out[:200]}")
|
||||
|
||||
# 6) /me/usage
|
||||
u = json.loads(run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/me/usage'"))
|
||||
print(f"\n--- /me/usage ---\n {u}")
|
||||
c.close()
|
||||
22
scripts/_redis2.py
Normal file
22
scripts/_redis2.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import os, paramiko
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=15):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
rpw = run("grep ^REDIS_PASSWORD /srv/news/.env | cut -d= -f2").strip()
|
||||
# 看 translation:month
|
||||
print("--- 查 usage key ---")
|
||||
out = run(f"docker exec news-aggregator-redis-1 redis-cli -a '{rpw}' KEYS 'translation:month*' 2>&1 | grep -v Warning")
|
||||
print(out)
|
||||
out = run(f"docker exec news-aggregator-redis-1 redis-cli -a '{rpw}' GET 'translation:month:202606' 2>&1 | grep -v Warning")
|
||||
print(f" GET: {out.strip()}")
|
||||
c.close()
|
||||
27
scripts/_redis_check.py
Normal file
27
scripts/_redis_check.py
Normal file
@@ -0,0 +1,27 @@
|
||||
import os, paramiko
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=15):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
# 拿 REDIS_PASSWORD
|
||||
rpw = run("grep ^REDIS_PASSWORD /srv/news/.env | cut -d= -f2").strip()
|
||||
print(f"REDIS_PASSWORD (前 6): {rpw[:6]}...")
|
||||
|
||||
# 直接 docker exec redis-cli 用 -a
|
||||
print("\n--- 用 docker exec 直接查 ---")
|
||||
run(f"docker exec news-aggregator-redis-1 redis-cli -a '{rpw}' GET translation:month:202606 2>&1")
|
||||
run(f"docker exec news-aggregator-redis-1 redis-cli -a '{rpw}' KEYS 'translation:*' 2>&1")
|
||||
|
||||
# 看 API 容器里 service.py 调 add_usage 的逻辑
|
||||
print("\n--- 测试 add_usage ---")
|
||||
run(f"docker exec news-aggregator-api-1 python -c \"import asyncio; from app.services.translation.service import service; asyncio.run(service.add_usage(100)); print('done')\"", t=15)
|
||||
c.close()
|
||||
53
scripts/_set_tencent.py
Normal file
53
scripts/_set_tencent.py
Normal file
@@ -0,0 +1,53 @@
|
||||
import os, paramiko
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
SECRET_ID = "AKIDy2Ln7OZaUPK5cv5GPXS9c4WpHlHdu035"
|
||||
SECRET_KEY = "1CBxUmAWifQ1PYpNn9JEwTmqshJzRggS"
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=60):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
# 1) 备份
|
||||
run("cp /srv/news/.env /srv/news/.env.bak.$(date +%s) 2>&1")
|
||||
|
||||
# 2) 用 sed 替换 TENCENTCLOUD_SECRET_ID / KEY(用 | 分隔避免 / 冲突)
|
||||
run(f"sed -i 's|^TENCENTCLOUD_SECRET_ID=.*|TENCENTCLOUD_SECRET_ID={SECRET_ID}|' /srv/news/.env")
|
||||
run(f"sed -i 's|^TENCENTCLOUD_SECRET_KEY=.*|TENCENTCLOUD_SECRET_KEY={SECRET_KEY}|' /srv/news/.env")
|
||||
|
||||
# 3) 确认
|
||||
print("\n--- 写入后 .env TENCENT 字段 ---")
|
||||
run("grep TENCENTCLOUD /srv/news/.env")
|
||||
|
||||
# 4) 重启 worker + api
|
||||
print("\n--- 重启 worker + api ---")
|
||||
run("cd /srv/news && docker compose up -d --force-recreate --no-deps --build worker api 2>&1 | tail -8", t=120)
|
||||
import time
|
||||
time.sleep(8)
|
||||
|
||||
# 5) 测翻译(取一条没翻译好的文章,重译)
|
||||
print("\n--- 找一条 pending 状态的 article ---")
|
||||
aid_out = run("docker exec news-aggregator-postgres-1 psql -U news -d news -tA -c \"SELECT id FROM articles WHERE translation_status IN ('pending', 'failed') LIMIT 1;\"")
|
||||
aid = aid_out.strip()
|
||||
print(f" article id = {aid!r}")
|
||||
|
||||
if aid:
|
||||
print(f"\n--- 手动重译 article {aid} ---")
|
||||
run(f"cd /srv/news && docker exec news-aggregator-api-1 python -c 'import asyncio; from app.workers.pipeline import translate_article; asyncio.run(translate_article({aid}))' 2>&1 | tail -15", t=120)
|
||||
|
||||
# 6) 查翻译结果
|
||||
print("\n--- 看翻译结果 ---")
|
||||
if aid:
|
||||
run(f"docker exec news-aggregator-postgres-1 psql -U news -d news -c \"SELECT id, translation_status, translation_engine, translation_chars, left(title_zh, 80) as title_zh FROM articles WHERE id = {aid};\"")
|
||||
|
||||
# 7) 全局统计
|
||||
print("\n--- 翻译统计 ---")
|
||||
run("docker exec news-aggregator-postgres-1 psql -U news -d news -c \"SELECT translation_status, translation_engine, count(*), sum(translation_chars) FROM articles GROUP BY translation_status, translation_engine ORDER BY 1, 2;\"")
|
||||
c.close()
|
||||
51
scripts/_show_translation.py
Normal file
51
scripts/_show_translation.py
Normal file
@@ -0,0 +1,51 @@
|
||||
import os, paramiko, json
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=20):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
# 拿一条已翻译的(随便挑)
|
||||
print("--- 拉 3 篇文章看译文 ---")
|
||||
out = run("docker exec news-aggregator-postgres-1 psql -U news -d news -c \"SELECT id, source_id, title, title_zh, translation_engine, translation_chars, lang_src FROM articles WHERE translation_status = 'ok' ORDER BY translation_chars DESC LIMIT 3;\"")
|
||||
print(out)
|
||||
|
||||
# 拿一条详情,看完整翻译
|
||||
print("\n--- 登录 + 拉详情 ---")
|
||||
body = json.dumps({"username": "owner", "password": "Owner2026!"})
|
||||
out = run(f"curl -s -X POST http://localhost/api/v1/auth/login -H 'Content-Type: application/json' -d '{body}'")
|
||||
token = json.loads(out)["access_token"]
|
||||
|
||||
# 找一篇 BBC 的(大概率有图)
|
||||
out = run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/articles?source=bbc-world&limit=1'")
|
||||
ad = json.loads(out)
|
||||
if ad.get("items"):
|
||||
aid = ad["items"][0]["id"]
|
||||
out = run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/articles/{aid}'")
|
||||
det = json.loads(out)
|
||||
print(f"\n=== {det['source']['name']} - {det['title'][:60]} ===")
|
||||
print(f"\n原文标题: {det['title'][:120]}")
|
||||
print(f"中文标题: {(det.get('title_zh') or '—')[:120]}")
|
||||
print(f"\n原文(前 300): {det['body_text'][:300]}")
|
||||
print(f"\n译文(前 400): {(det.get('body_zh_text') or '—')[:400]}")
|
||||
print(f"\nstatus: {det['translation_status']}")
|
||||
print(f"engine: {det.get('translation_engine')}")
|
||||
print(f"chars: {det.get('translation_chars', '?')}")
|
||||
print(f"img: {det.get('image_url', '—')[:80]}")
|
||||
|
||||
# 用量
|
||||
print("\n--- /me/usage ---")
|
||||
u = json.loads(run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/me/usage'"))
|
||||
print(f" {u}")
|
||||
|
||||
# 容器状态
|
||||
print("\n--- docker ps ---")
|
||||
run("docker ps --format 'table {{.Names}}\\t{{.Status}}\\t{{.Ports}}' 2>&1 | tail -10")
|
||||
c.close()
|
||||
53
scripts/_translate_all.py
Normal file
53
scripts/_translate_all.py
Normal file
@@ -0,0 +1,53 @@
|
||||
import os, paramiko, json
|
||||
PW = os.environ["REMOTE_PASS"]
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect("207.57.129.228", port=19717, username="root", password=PW, timeout=15, allow_agent=False, look_for_keys=False)
|
||||
def run(cmd, t=180):
|
||||
si, so, se = c.exec_command(cmd, timeout=t)
|
||||
out = so.read().decode("utf-8", "replace")
|
||||
err = se.read().decode("utf-8", "replace")
|
||||
rc = so.channel.recv_exit_status()
|
||||
if out: print(out, end="")
|
||||
if err: print("[err]", err, end="", file=__import__("sys").stderr)
|
||||
return out
|
||||
|
||||
# 写一个 python 脚本到 worker 容器(用 stdin pipe),直接翻译所有 pending
|
||||
script = '''import asyncio
|
||||
from app.workers.pipeline import translate_article
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.article import Article
|
||||
from sqlalchemy import select
|
||||
|
||||
async def main():
|
||||
async with AsyncSessionLocal() as s:
|
||||
rows = (await s.execute(select(Article.id).where(Article.translation_status == 'pending').order_by(Article.id))).all()
|
||||
ids = [r[0] for r in rows]
|
||||
print(f"translating {len(ids)} articles...")
|
||||
for i, aid in enumerate(ids, 1):
|
||||
try:
|
||||
await translate_article(aid)
|
||||
except Exception as e:
|
||||
print(f" err on {aid}: {e}")
|
||||
if i % 10 == 0:
|
||||
print(f" {i}/{len(ids)} done")
|
||||
asyncio.run(main())
|
||||
'''
|
||||
# 用 docker exec -i 把脚本传进去
|
||||
print("--- 翻译所有 pending ---")
|
||||
run(f"docker exec -i news-aggregator-worker-1 python -u -c \"{script.replace(chr(34), chr(92)+chr(34))}\"", t=600)
|
||||
|
||||
print("\n--- 翻译后统计 ---")
|
||||
run("docker exec news-aggregator-postgres-1 psql -U news -d news -c \"SELECT translation_status, translation_engine, count(*), sum(translation_chars) FROM articles GROUP BY 1, 2 ORDER BY 1, 2;\"")
|
||||
|
||||
# 看 usage
|
||||
import urllib.request
|
||||
out = run("curl -s -X POST http://localhost/api/v1/auth/login -H 'Content-Type: application/json' -d '{\"username\":\"owner\",\"password\":\"Owner2026!\"}'")
|
||||
token = json.loads(out)["access_token"]
|
||||
u = json.loads(run(f"curl -s -H 'Authorization: Bearer {token}' 'http://localhost/api/v1/me/usage'"))
|
||||
print(f"\n--- /me/usage ---\n {u}")
|
||||
|
||||
# 看 redis
|
||||
print("\n--- redis 计数 ---")
|
||||
run("docker exec news-aggregator-redis-1 redis-cli -a '$(grep ^REDIS_PASSWORD /srv/news/.env | cut -d= -f2)' GET translation:month:202606 2>&1 | tail -3")
|
||||
c.close()
|
||||
Reference in New Issue
Block a user