feat(spider): 增强Playwright截图功能以支持打包环境
添加对打包环境的支持,自动检测运行环境并设置正确的浏览器路径 移除重复的截图方法,优化错误日志记录 更新requirements.txt添加playwright依赖
This commit is contained in:
84
spider.py
84
spider.py
@@ -281,6 +281,7 @@ class SpiderManager:
|
||||
"""
|
||||
from playwright.sync_api import sync_playwright
|
||||
import os
|
||||
import sys
|
||||
|
||||
url = "https://www.sse.com.cn/"
|
||||
xpath = "//div[contains(@class,'gray_bg')]//div[contains(@class,'col-md-7')]"
|
||||
@@ -288,12 +289,34 @@ class SpiderManager:
|
||||
logger.info(f"开始爬取上证所网页截图: {url}")
|
||||
logger.info(f"目标XPath: {xpath}")
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
# 获取当前脚本目录
|
||||
if getattr(sys, 'frozen', False):
|
||||
# 打包后的环境
|
||||
current_dir = os.path.dirname(sys.executable)
|
||||
# 设置Playwright浏览器路径
|
||||
playwright_dir = os.path.join(current_dir, '_internal', 'ms-playwright')
|
||||
logger.info(f"打包环境,Playwright浏览器路径: {playwright_dir}")
|
||||
else:
|
||||
# 开发环境
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
playwright_dir = None
|
||||
logger.info(f"开发环境,当前目录: {current_dir}")
|
||||
|
||||
output_dir = current_dir
|
||||
screenshot_path = os.path.join(output_dir, "sse_screenshot.png")
|
||||
|
||||
try:
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
# 设置浏览器路径
|
||||
browser_launch_options = {
|
||||
'headless': True
|
||||
}
|
||||
|
||||
if playwright_dir:
|
||||
browser_launch_options['executable_path'] = os.path.join(playwright_dir, 'chromium-1091', 'chrome-win', 'chrome.exe')
|
||||
logger.info(f"使用自定义浏览器路径: {browser_launch_options['executable_path']}")
|
||||
|
||||
browser = p.chromium.launch(**browser_launch_options)
|
||||
page = browser.new_page()
|
||||
|
||||
page.set_default_timeout(60000)
|
||||
@@ -330,62 +353,7 @@ class SpiderManager:
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"爬取上证所截图失败: {e}")
|
||||
logger.exception(e) # 记录详细异常
|
||||
return ""
|
||||
|
||||
def fetch_sse_screenshot(self) -> str:
|
||||
"""
|
||||
爬取上证所网页指定元素截图
|
||||
返回截图文件路径
|
||||
"""
|
||||
from playwright.sync_api import sync_playwright
|
||||
import os
|
||||
|
||||
url = "https://www.sse.com.cn/"
|
||||
xpath = "//div[contains(@class,'gray_bg')]//div[contains(@class,'col-md-7')]"
|
||||
|
||||
logger.info(f"开始爬取上证所网页截图: {url}")
|
||||
logger.info(f"目标XPath: {xpath}")
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
screenshot_path = os.path.join(output_dir, "sse_screenshot.png")
|
||||
|
||||
try:
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
page = browser.new_page()
|
||||
|
||||
page.set_default_timeout(60000)
|
||||
logger.info("正在访问页面...")
|
||||
page.goto(url, wait_until="networkidle")
|
||||
|
||||
logger.info("等待页面加载完成...")
|
||||
page.wait_for_load_state("domcontentloaded")
|
||||
page.wait_for_timeout(5000)
|
||||
|
||||
logger.info(f"查找XPath元素: {xpath}")
|
||||
element = page.locator(f"xpath={xpath}")
|
||||
|
||||
if element.count() > 0:
|
||||
logger.info("✓ 找到目标元素")
|
||||
|
||||
is_visible = element.is_visible()
|
||||
logger.info(f"元素可见: {is_visible}")
|
||||
|
||||
if not is_visible:
|
||||
logger.info("元素不可见,尝试滚动到可见区域...")
|
||||
element.scroll_into_view_if_needed()
|
||||
page.wait_for_timeout(2000)
|
||||
|
||||
logger.info(f"正在截取元素截图到: {screenshot_path}")
|
||||
element.screenshot(path=screenshot_path)
|
||||
logger.info("✓ 截屏成功")
|
||||
browser.close()
|
||||
return screenshot_path
|
||||
else:
|
||||
logger.warning("✗ 未找到目标元素")
|
||||
browser.close()
|
||||
return ""
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"爬取上证所截图失败: {e}")
|
||||
return ""
|
||||
|
||||
Reference in New Issue
Block a user