feat(spider): 增强Playwright截图功能以支持打包环境

添加对打包环境的支持,自动检测运行环境并设置正确的浏览器路径
移除重复的截图方法,优化错误日志记录
更新requirements.txt添加playwright依赖
This commit is contained in:
2026-01-19 18:58:50 +08:00
parent 4f64ef61f3
commit f23feaf140
6 changed files with 32 additions and 69 deletions

View File

@@ -281,6 +281,7 @@ class SpiderManager:
"""
from playwright.sync_api import sync_playwright
import os
import sys
url = "https://www.sse.com.cn/"
xpath = "//div[contains(@class,'gray_bg')]//div[contains(@class,'col-md-7')]"
@@ -288,12 +289,34 @@ class SpiderManager:
logger.info(f"开始爬取上证所网页截图: {url}")
logger.info(f"目标XPath: {xpath}")
output_dir = os.path.dirname(os.path.abspath(__file__))
# 获取当前脚本目录
if getattr(sys, 'frozen', False):
# 打包后的环境
current_dir = os.path.dirname(sys.executable)
# 设置Playwright浏览器路径
playwright_dir = os.path.join(current_dir, '_internal', 'ms-playwright')
logger.info(f"打包环境Playwright浏览器路径: {playwright_dir}")
else:
# 开发环境
current_dir = os.path.dirname(os.path.abspath(__file__))
playwright_dir = None
logger.info(f"开发环境,当前目录: {current_dir}")
output_dir = current_dir
screenshot_path = os.path.join(output_dir, "sse_screenshot.png")
try:
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
# 设置浏览器路径
browser_launch_options = {
'headless': True
}
if playwright_dir:
browser_launch_options['executable_path'] = os.path.join(playwright_dir, 'chromium-1091', 'chrome-win', 'chrome.exe')
logger.info(f"使用自定义浏览器路径: {browser_launch_options['executable_path']}")
browser = p.chromium.launch(**browser_launch_options)
page = browser.new_page()
page.set_default_timeout(60000)
@@ -330,62 +353,7 @@ class SpiderManager:
except Exception as e:
logger.error(f"爬取上证所截图失败: {e}")
logger.exception(e) # 记录详细异常
return ""
def fetch_sse_screenshot(self) -> str:
"""
爬取上证所网页指定元素截图
返回截图文件路径
"""
from playwright.sync_api import sync_playwright
import os
url = "https://www.sse.com.cn/"
xpath = "//div[contains(@class,'gray_bg')]//div[contains(@class,'col-md-7')]"
logger.info(f"开始爬取上证所网页截图: {url}")
logger.info(f"目标XPath: {xpath}")
output_dir = os.path.dirname(os.path.abspath(__file__))
screenshot_path = os.path.join(output_dir, "sse_screenshot.png")
try:
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
page.set_default_timeout(60000)
logger.info("正在访问页面...")
page.goto(url, wait_until="networkidle")
logger.info("等待页面加载完成...")
page.wait_for_load_state("domcontentloaded")
page.wait_for_timeout(5000)
logger.info(f"查找XPath元素: {xpath}")
element = page.locator(f"xpath={xpath}")
if element.count() > 0:
logger.info("✓ 找到目标元素")
is_visible = element.is_visible()
logger.info(f"元素可见: {is_visible}")
if not is_visible:
logger.info("元素不可见,尝试滚动到可见区域...")
element.scroll_into_view_if_needed()
page.wait_for_timeout(2000)
logger.info(f"正在截取元素截图到: {screenshot_path}")
element.screenshot(path=screenshot_path)
logger.info("✓ 截屏成功")
browser.close()
return screenshot_path
else:
logger.warning("✗ 未找到目标元素")
browser.close()
return ""
except Exception as e:
logger.error(f"爬取上证所截图失败: {e}")
return ""