363 lines
14 KiB
Python
363 lines
14 KiB
Python
|
import json
|
|||
|
|
import asyncio
|
|||
|
|
from loguru import logger
|
|||
|
|
from playwright.async_api import async_playwright
|
|||
|
|
from playwright_stealth.stealth import Stealth
|
|||
|
|
|
|||
|
|
class ProductHuntScraper:
|
|||
|
|
def __init__(self):
|
|||
|
|
self.browser = None
|
|||
|
|
self.page = None
|
|||
|
|
self.product_url = "https://www.producthunt.com/products/elsie-ai-beta"
|
|||
|
|
|
|||
|
|
async def start_browser(self):
|
|||
|
|
"""启动浏览器"""
|
|||
|
|
try:
|
|||
|
|
logger.info("正在启动Playwright浏览器...")
|
|||
|
|
playwright = await async_playwright().start()
|
|||
|
|
|
|||
|
|
# 使用更真实的浏览器配置
|
|||
|
|
self.browser = await playwright.chromium.launch(
|
|||
|
|
headless=True, # 设置为True避免显示浏览器窗口
|
|||
|
|
args=[
|
|||
|
|
'--disable-blink-features=AutomationControlled',
|
|||
|
|
'--disable-web-security',
|
|||
|
|
'--disable-features=VizDisplayCompositor',
|
|||
|
|
'--no-sandbox',
|
|||
|
|
'--disable-setuid-sandbox',
|
|||
|
|
'--disable-infobars',
|
|||
|
|
'--disable-dev-shm-usage',
|
|||
|
|
'--disable-accelerated-2d-canvas',
|
|||
|
|
'--no-first-run',
|
|||
|
|
'--no-zygote',
|
|||
|
|
'--disable-gpu'
|
|||
|
|
]
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# 创建页面上下文，使用更真实的用户代理
|
|||
|
|
context = await self.browser.new_context(
|
|||
|
|
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
|
|||
|
|
viewport={'width': 1920, 'height': 1080},
|
|||
|
|
locale='en-US',
|
|||
|
|
timezone_id='America/New_York'
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
self.page = await context.new_page()
|
|||
|
|
|
|||
|
|
# 应用stealth设置，使浏览器看起来更像真实用户
|
|||
|
|
stealth = Stealth()
|
|||
|
|
await stealth.apply_stealth_async(self.page)
|
|||
|
|
|
|||
|
|
# 设置额外的请求头
|
|||
|
|
await self.page.set_extra_http_headers({
|
|||
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|||
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|||
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
|
|||
|
|
'Cache-Control': 'no-cache',
|
|||
|
|
'Pragma': 'no-cache'
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
logger.success("浏览器启动成功")
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"启动浏览器失败: {str(e)}")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
async def wait_for_cloudflare(self, timeout=120000):
|
|||
|
|
"""等待Cloudflare验证完成"""
|
|||
|
|
try:
|
|||
|
|
logger.info("等待Cloudflare验证完成...")
|
|||
|
|
|
|||
|
|
# 等待页面标题不再是"Just a moment..."或者验证成功元素出现
|
|||
|
|
await self.page.wait_for_function(
|
|||
|
|
"""() => {
|
|||
|
|
return document.title !== "Just a moment..." &&
|
|||
|
|
!document.querySelector('.lds-ring') &&
|
|||
|
|
!document.querySelector('#challenge-error-text');
|
|||
|
|
}""",
|
|||
|
|
timeout=timeout
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
logger.success("Cloudflare验证完成")
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"等待Cloudflare验证超时: {str(e)}")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
async def navigate_to_product(self):
|
|||
|
|
"""导航到产品页面"""
|
|||
|
|
try:
|
|||
|
|
logger.info(f"正在导航到产品页面: {self.product_url}")
|
|||
|
|
|
|||
|
|
# 先访问主页建立会话
|
|||
|
|
logger.info("先访问ProductHunt主页...")
|
|||
|
|
await self.page.goto("https://www.producthunt.com", {"waitUntil": "networkidle", "timeout": 60000})
|
|||
|
|
|
|||
|
|
# 等待一下，模拟真实用户行为
|
|||
|
|
await asyncio.sleep(3)
|
|||
|
|
|
|||
|
|
# 再访问产品页面
|
|||
|
|
logger.info("访问产品页面...")
|
|||
|
|
await self.page.goto(self.product_url, {"waitUntil": "networkidle", "timeout": 60000})
|
|||
|
|
|
|||
|
|
# 等待Cloudflare验证
|
|||
|
|
if not await self.wait_for_cloudflare():
|
|||
|
|
logger.error("Cloudflare验证失败")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
# 等待页面加载完成
|
|||
|
|
await asyncio.sleep(5)
|
|||
|
|
|
|||
|
|
logger.success("成功导航到产品页面")
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"导航到产品页面失败: {str(e)}")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
async def extract_product_info(self):
|
|||
|
|
"""提取产品信息"""
|
|||
|
|
try:
|
|||
|
|
logger.info("正在提取产品信息...")
|
|||
|
|
|
|||
|
|
# 尝试多种选择器来获取产品名称
|
|||
|
|
name_selectors = [
|
|||
|
|
'h1[data-test="post-name"]',
|
|||
|
|
'h1[data-test="post-title"]',
|
|||
|
|
'h1[class*="styles_name"]',
|
|||
|
|
'h1',
|
|||
|
|
'[data-test="post-name"]',
|
|||
|
|
'[data-test="post-title"]',
|
|||
|
|
'.styles_name__',
|
|||
|
|
'.styles_title__',
|
|||
|
|
'h1[class*="name"]',
|
|||
|
|
'h1[class*="title"]'
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
product_name = "未找到产品名称"
|
|||
|
|
for selector in name_selectors:
|
|||
|
|
try:
|
|||
|
|
element = await self.page.wait_for_selector(selector, {"timeout": 5000})
|
|||
|
|
if element:
|
|||
|
|
product_name = await element.inner_text()
|
|||
|
|
if product_name and product_name.strip():
|
|||
|
|
logger.info(f"使用选择器 {selector} 找到产品名称: {product_name}")
|
|||
|
|
break
|
|||
|
|
except:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# 尝试多种选择器来获取产品简介
|
|||
|
|
description_selectors = [
|
|||
|
|
'[data-test="post-description"]',
|
|||
|
|
'[data-test="post-tagline"]',
|
|||
|
|
'.styles_tagline__',
|
|||
|
|
'.styles_description__',
|
|||
|
|
'div[class*="tagline"]',
|
|||
|
|
'div[class*="description"]',
|
|||
|
|
'p[class*="tagline"]',
|
|||
|
|
'p[class*="description"]'
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
product_description = "未找到产品简介"
|
|||
|
|
for selector in description_selectors:
|
|||
|
|
try:
|
|||
|
|
element = await self.page.wait_for_selector(selector, {"timeout": 5000})
|
|||
|
|
if element:
|
|||
|
|
product_description = await element.inner_text()
|
|||
|
|
if product_description and product_description.strip():
|
|||
|
|
logger.info(f"使用选择器 {selector} 找到产品简介: {product_description}")
|
|||
|
|
break
|
|||
|
|
except:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# 尝试获取评论
|
|||
|
|
comments_selectors = [
|
|||
|
|
'[data-test="comment-item"]',
|
|||
|
|
'.styles_comment__',
|
|||
|
|
'div[class*="comment"]',
|
|||
|
|
'article[class*="comment"]'
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
comments = []
|
|||
|
|
for selector in comments_selectors:
|
|||
|
|
try:
|
|||
|
|
elements = await self.page.query_selector_all(selector)
|
|||
|
|
if elements:
|
|||
|
|
for element in elements[:5]: # 只获取前5条评论
|
|||
|
|
try:
|
|||
|
|
comment_text = await element.inner_text()
|
|||
|
|
if comment_text and comment_text.strip():
|
|||
|
|
comments.append(comment_text.strip())
|
|||
|
|
except:
|
|||
|
|
continue
|
|||
|
|
if comments:
|
|||
|
|
logger.info(f"使用选择器 {selector} 找到 {len(comments)} 条评论")
|
|||
|
|
break
|
|||
|
|
except:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
if not comments:
|
|||
|
|
comments = ["未找到评论"]
|
|||
|
|
|
|||
|
|
# 尝试获取标签
|
|||
|
|
tags_selectors = [
|
|||
|
|
'[data-test="post-topic"]',
|
|||
|
|
'.styles_topic__',
|
|||
|
|
'a[class*="topic"]',
|
|||
|
|
'span[class*="topic"]'
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
tags = []
|
|||
|
|
for selector in tags_selectors:
|
|||
|
|
try:
|
|||
|
|
elements = await self.page.query_selector_all(selector)
|
|||
|
|
if elements:
|
|||
|
|
for element in elements:
|
|||
|
|
try:
|
|||
|
|
tag_text = await element.inner_text()
|
|||
|
|
if tag_text and tag_text.strip():
|
|||
|
|
tags.append(tag_text.strip())
|
|||
|
|
except:
|
|||
|
|
continue
|
|||
|
|
if tags:
|
|||
|
|
logger.info(f"使用选择器 {selector} 找到 {len(tags)} 个标签")
|
|||
|
|
break
|
|||
|
|
except:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
if not tags:
|
|||
|
|
tags = ["未找到标签"]
|
|||
|
|
|
|||
|
|
# 尝试获取点赞数和评论数
|
|||
|
|
upvotes = "未找到"
|
|||
|
|
comments_count = "未找到"
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
upvotes_element = await self.page.query_selector('[data-test="vote-button"]')
|
|||
|
|
if upvotes_element:
|
|||
|
|
upvotes_text = await upvotes_element.inner_text()
|
|||
|
|
if upvotes_text and upvotes_text.strip():
|
|||
|
|
upvotes = upvotes_text.strip()
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
comments_count_element = await self.page.query_selector('[data-test="comment-count"]')
|
|||
|
|
if comments_count_element:
|
|||
|
|
comments_count_text = await comments_count_element.inner_text()
|
|||
|
|
if comments_count_text and comments_count_text.strip():
|
|||
|
|
comments_count = comments_count_text.strip()
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
# 尝试获取产品图片
|
|||
|
|
image_url = "未找到图片"
|
|||
|
|
try:
|
|||
|
|
image_element = await self.page.query_selector('img[data-test="product-image"]')
|
|||
|
|
if image_element:
|
|||
|
|
image_url = await image_element.get_attribute('src')
|
|||
|
|
if not image_url:
|
|||
|
|
image_url = await image_element.get_attribute('data-src')
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
product_info = {
|
|||
|
|
"name": product_name,
|
|||
|
|
"description": product_description,
|
|||
|
|
"tags": tags,
|
|||
|
|
"upvotes": upvotes,
|
|||
|
|
"comments_count": comments_count,
|
|||
|
|
"comments": comments,
|
|||
|
|
"image_url": image_url,
|
|||
|
|
"url": self.product_url
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
logger.success("产品信息提取完成")
|
|||
|
|
return product_info
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"提取产品信息失败: {str(e)}")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
async def save_data(self, data):
|
|||
|
|
"""保存数据到JSON文件"""
|
|||
|
|
try:
|
|||
|
|
with open('product_info_stealth.json', 'w', encoding='utf-8') as f:
|
|||
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|||
|
|
logger.success("数据已保存到 product_info_stealth.json")
|
|||
|
|
return True
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"保存数据失败: {str(e)}")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
async def take_screenshot(self):
|
|||
|
|
"""保存页面截图"""
|
|||
|
|
try:
|
|||
|
|
await self.page.screenshot(path='product_screenshot_stealth.png', full_page=True)
|
|||
|
|
logger.success("页面截图已保存到 product_screenshot_stealth.png")
|
|||
|
|
return True
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"保存截图失败: {str(e)}")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
async def save_html(self):
|
|||
|
|
"""保存页面HTML内容"""
|
|||
|
|
try:
|
|||
|
|
html_content = await self.page.content()
|
|||
|
|
with open('product_page_stealth.html', 'w', encoding='utf-8') as f:
|
|||
|
|
f.write(html_content)
|
|||
|
|
logger.success("页面HTML已保存到 product_page_stealth.html")
|
|||
|
|
return True
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"保存HTML失败: {str(e)}")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
async def close_browser(self):
|
|||
|
|
"""关闭浏览器"""
|
|||
|
|
if self.browser:
|
|||
|
|
await self.browser.close()
|
|||
|
|
logger.info("浏览器已关闭")
|
|||
|
|
|
|||
|
|
async def scrape(self):
|
|||
|
|
"""执行完整的抓取流程"""
|
|||
|
|
try:
|
|||
|
|
if not await self.start_browser():
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
if not await self.navigate_to_product():
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
# 保存HTML和截图用于调试
|
|||
|
|
await self.save_html()
|
|||
|
|
await self.take_screenshot()
|
|||
|
|
|
|||
|
|
product_info = await self.extract_product_info()
|
|||
|
|
if product_info:
|
|||
|
|
await self.save_data(product_info)
|
|||
|
|
logger.info(f"抓取完成: {product_info['name']}")
|
|||
|
|
return True
|
|||
|
|
else:
|
|||
|
|
logger.error("未能提取产品信息")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"抓取过程中发生错误: {str(e)}")
|
|||
|
|
return False
|
|||
|
|
finally:
|
|||
|
|
await self.close_browser()
|
|||
|
|
|
|||
|
|
async def main():
|
|||
|
|
"""主函数"""
|
|||
|
|
logger.info("开始ProductHunt产品信息抓取（使用Stealth模式）")
|
|||
|
|
scraper = ProductHuntScraper()
|
|||
|
|
success = await scraper.scrape()
|
|||
|
|
|
|||
|
|
if success:
|
|||
|
|
logger.success("抓取成功完成")
|
|||
|
|
else:
|
|||
|
|
logger.error("抓取失败")
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
asyncio.run(main())
|