407 lines
17 KiB
Python
407 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
ProductHunt数据抓取器
|
||
从tophub_data.db查询包含producthunt.com的链接，然后使用Playwright抓取产品信息并保存到product.db
|
||
"""
|
||
|
||
import sqlite3
|
||
import asyncio
|
||
import os
|
||
from datetime import datetime
|
||
from loguru import logger
|
||
from tqdm import tqdm
|
||
import sys
|
||
|
||
# 配置日志
|
||
logger.remove()
|
||
logger.add(sys.stderr, level="INFO", format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>")
|
||
|
||
class ProductHuntScraper:
|
||
"""ProductHunt数据抓取器"""
|
||
|
||
def __init__(self):
|
||
self.tophub_db_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tophub_data.db")
|
||
self.product_db_path = os.path.join(os.path.dirname(__file__), "product.db")
|
||
self.product_urls = []
|
||
|
||
def query_producthunt_urls(self):
|
||
"""查询包含producthunt.com的链接"""
|
||
logger.info("正在查询tophub_data.db数据库...")
|
||
|
||
try:
|
||
conn = sqlite3.connect(self.tophub_db_path)
|
||
cursor = conn.cursor()
|
||
|
||
# 查询包含producthunt.com的链接
|
||
cursor.execute("SELECT url FROM articles WHERE url LIKE '%producthunt.com%'")
|
||
urls = [row[0] for row in cursor.fetchall()]
|
||
|
||
conn.close()
|
||
|
||
logger.success(f"找到 {len(urls)} 个包含producthunt.com的链接")
|
||
return urls
|
||
|
||
except Exception as e:
|
||
logger.error(f"查询数据库失败: {e}")
|
||
return []
|
||
|
||
def init_product_database(self):
|
||
"""初始化product.db数据库"""
|
||
logger.info("正在初始化product.db数据库...")
|
||
|
||
try:
|
||
conn = sqlite3.connect(self.product_db_path)
|
||
cursor = conn.cursor()
|
||
|
||
# 创建产品信息表
|
||
cursor.execute('''
|
||
CREATE TABLE IF NOT EXISTS products (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
url TEXT NOT NULL UNIQUE,
|
||
name TEXT,
|
||
introduction TEXT,
|
||
user_count TEXT,
|
||
maker_link TEXT,
|
||
maker_statement TEXT,
|
||
created_at TEXT NOT NULL,
|
||
updated_at TEXT NOT NULL
|
||
)
|
||
''')
|
||
|
||
conn.commit()
|
||
conn.close()
|
||
logger.success("product.db数据库初始化完成")
|
||
|
||
except Exception as e:
|
||
logger.error(f"初始化数据库失败: {e}")
|
||
|
||
def check_duplicate(self, url):
|
||
"""检查URL是否已存在"""
|
||
try:
|
||
conn = sqlite3.connect(self.product_db_path)
|
||
cursor = conn.cursor()
|
||
|
||
cursor.execute("SELECT COUNT(*) FROM products WHERE url = ?", (url,))
|
||
count = cursor.fetchone()[0]
|
||
|
||
conn.close()
|
||
return count > 0
|
||
|
||
except Exception as e:
|
||
logger.error(f"检查重复失败: {e}")
|
||
return False
|
||
|
||
def save_product_info(self, product_info):
|
||
"""保存产品信息到数据库"""
|
||
try:
|
||
conn = sqlite3.connect(self.product_db_path)
|
||
cursor = conn.cursor()
|
||
|
||
current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||
|
||
# 检查是否已存在
|
||
cursor.execute("SELECT id FROM products WHERE url = ?", (product_info['url'],))
|
||
existing = cursor.fetchone()
|
||
|
||
if existing:
|
||
# 更新现有记录
|
||
cursor.execute('''
|
||
UPDATE products SET
|
||
name = ?, introduction = ?, user_count = ?,
|
||
maker_link = ?, maker_statement = ?, updated_at = ?
|
||
WHERE url = ?
|
||
''', (
|
||
product_info.get('name'),
|
||
product_info.get('introduction'),
|
||
product_info.get('user_count'),
|
||
product_info.get('maker_link'),
|
||
product_info.get('maker_statement'),
|
||
current_time,
|
||
product_info['url']
|
||
))
|
||
logger.info(f"更新产品信息: {product_info.get('name', '未知')}")
|
||
else:
|
||
# 插入新记录
|
||
cursor.execute('''
|
||
INSERT INTO products
|
||
(url, name, introduction, user_count, maker_link, maker_statement, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||
''', (
|
||
product_info['url'],
|
||
product_info.get('name'),
|
||
product_info.get('introduction'),
|
||
product_info.get('user_count'),
|
||
product_info.get('maker_link'),
|
||
product_info.get('maker_statement'),
|
||
current_time,
|
||
current_time
|
||
))
|
||
logger.info(f"新增产品信息: {product_info.get('name', '未知')}")
|
||
|
||
conn.commit()
|
||
conn.close()
|
||
return True
|
||
|
||
except Exception as e:
|
||
logger.error(f"保存产品信息失败: {e}")
|
||
return False
|
||
|
||
async def scrape_product_info(self, url):
|
||
"""使用Playwright抓取产品信息"""
|
||
try:
|
||
# 导入Playwright相关模块
|
||
from playwright.async_api import async_playwright
|
||
|
||
logger.info(f"开始抓取: {url}")
|
||
|
||
# 创建Playwright实例
|
||
playwright = await async_playwright().start()
|
||
browser = await playwright.chromium.launch(headless=True)
|
||
page = await browser.new_page()
|
||
|
||
# 设置超时时间
|
||
page.set_default_timeout(120000) # 增加超时时间以处理Cloudflare挑战
|
||
|
||
# 导航到页面
|
||
await page.goto(url, wait_until="domcontentloaded")
|
||
|
||
# 检查是否是Cloudflare挑战页面
|
||
page_title = await page.title()
|
||
if "请稍候" in page_title or "Checking" in page_title or "Verifying" in page_title:
|
||
logger.info("检测到Cloudflare挑战页面，等待验证完成...")
|
||
|
||
# 等待Cloudflare挑战完成
|
||
try:
|
||
# 等待页面标题变化或特定元素出现
|
||
await page.wait_for_function(
|
||
"""() => {
|
||
const title = document.title;
|
||
return !title.includes('请稍候') &&
|
||
!title.includes('Checking') &&
|
||
!title.includes('Verifying') &&
|
||
title !== '请稍候…';
|
||
}""",
|
||
timeout=300000 # 5分钟
|
||
)
|
||
logger.info("Cloudflare挑战已完成")
|
||
except Exception as e:
|
||
logger.warning(f"等待Cloudflare挑战超时: {e}")
|
||
|
||
# 等待页面加载
|
||
await page.wait_for_timeout(3000)
|
||
|
||
product_info = {'url': url}
|
||
|
||
# 提取产品名称 - 改进的XPath选择器
|
||
try:
|
||
# 尝试多种选择器
|
||
name_selectors = [
|
||
"xpath=//h1",
|
||
"xpath=//h1[@data-test='product-name']",
|
||
"xpath=//h1[contains(@class, 'text')]",
|
||
"xpath=//title"
|
||
]
|
||
|
||
for selector in name_selectors:
|
||
name_element = await page.query_selector(selector)
|
||
if name_element:
|
||
name_text = (await name_element.text_content()).strip()
|
||
# 过滤掉页面标题中的无关内容
|
||
if name_text and 'Product Hunt' not in name_text and len(name_text) > 5:
|
||
product_info['name'] = name_text
|
||
logger.info(f"提取到产品名称: {product_info['name']}")
|
||
break
|
||
|
||
if 'name' not in product_info:
|
||
logger.warning("未找到有效的产品名称元素")
|
||
|
||
except Exception as e:
|
||
logger.warning(f"提取产品名称失败: {e}")
|
||
|
||
# 提取产品简介 - 改进的XPath选择器
|
||
try:
|
||
intro_selectors = [
|
||
"xpath=//*[@class='relative text-16 font-normal text-gray-700']//div",
|
||
"xpath=//p[contains(@class, 'description')]",
|
||
"xpath=//div[contains(@class, 'description')]",
|
||
"xpath=//meta[@name='description']"
|
||
]
|
||
|
||
for selector in intro_selectors:
|
||
intro_element = await page.query_selector(selector)
|
||
if intro_element:
|
||
intro_text = (await intro_element.text_content()).strip()
|
||
if intro_text:
|
||
product_info['introduction'] = intro_text
|
||
logger.info(f"提取到产品简介: {product_info['introduction'][:100]}...")
|
||
break
|
||
|
||
if 'introduction' not in product_info:
|
||
logger.warning("未找到产品简介元素")
|
||
|
||
except Exception as e:
|
||
logger.warning(f"提取产品简介失败: {e}")
|
||
|
||
# 提取用户数 - 改进的XPath选择器
|
||
try:
|
||
user_count_selectors = [
|
||
"xpath=//*[@class='flex flex-row gap-2']//div/div[2]/span/p",
|
||
"xpath=//span[contains(text(), 'users')]",
|
||
"xpath=//span[contains(text(), 'upvotes')]",
|
||
"xpath=//div[contains(@class, 'stats')]"
|
||
]
|
||
|
||
for selector in user_count_selectors:
|
||
user_count_element = await page.query_selector(selector)
|
||
if user_count_element:
|
||
user_count_text = (await user_count_element.text_content()).strip()
|
||
if user_count_text:
|
||
product_info['user_count'] = user_count_text
|
||
logger.info(f"提取到用户数: {product_info['user_count']}")
|
||
break
|
||
|
||
if 'user_count' not in product_info:
|
||
logger.warning("未找到用户数元素")
|
||
|
||
except Exception as e:
|
||
logger.warning(f"提取用户数失败: {e}")
|
||
|
||
# 提取制作人链接 - 改进的XPath选择器
|
||
try:
|
||
maker_link_selectors = [
|
||
"xpath=//span[contains(@class, 'absolute')]",
|
||
"xpath=//a[contains(@href, 'hunter')]",
|
||
"xpath=//a[contains(text(), 'hunter')]",
|
||
"xpath=//a[contains(@class, 'maker')]"
|
||
]
|
||
|
||
for selector in maker_link_selectors:
|
||
maker_element = await page.query_selector(selector)
|
||
if maker_element:
|
||
# 如果是span，找父级a标签
|
||
if 'span' in selector:
|
||
a_element = await maker_element.evaluate_handle('(element) => element.closest("a")')
|
||
if a_element:
|
||
maker_link = await a_element.get_attribute('href')
|
||
else:
|
||
maker_link = await maker_element.get_attribute('href')
|
||
|
||
if maker_link and not maker_link.startswith('http'):
|
||
base_url = "https://www.producthunt.com"
|
||
if maker_link.startswith('/'):
|
||
maker_link = base_url + maker_link
|
||
else:
|
||
maker_link = base_url + '/' + maker_link
|
||
|
||
if maker_link:
|
||
product_info['maker_link'] = maker_link
|
||
logger.info(f"提取到制作人链接: {maker_link}")
|
||
break
|
||
|
||
if 'maker_link' not in product_info:
|
||
logger.warning("未找到制作人链接元素")
|
||
|
||
except Exception as e:
|
||
logger.warning(f"提取制作人链接失败: {e}")
|
||
|
||
# 提取制作人发言（简化版本）
|
||
try:
|
||
if product_info.get('maker_link'):
|
||
# 在新页面中打开制作人链接
|
||
new_page = await browser.new_page()
|
||
await new_page.goto(product_info['maker_link'], wait_until="domcontentloaded")
|
||
await new_page.wait_for_timeout(5000)
|
||
|
||
# 尝试多种选择器提取发言内容
|
||
statement_selectors = [
|
||
"xpath=//*[@id='comment-4597755']/div/div[2]/div/div/div",
|
||
"xpath=//div[contains(@class, 'comment')]",
|
||
"xpath=//p[contains(@class, 'comment')]",
|
||
"xpath=//article"
|
||
]
|
||
|
||
for selector in statement_selectors:
|
||
comment_element = await new_page.query_selector(selector)
|
||
if comment_element:
|
||
statement_text = (await comment_element.text_content()).strip()
|
||
if statement_text and len(statement_text) > 10:
|
||
product_info['maker_statement'] = statement_text
|
||
logger.info(f"提取到制作人发言: {product_info['maker_statement'][:100]}...")
|
||
break
|
||
|
||
await new_page.close()
|
||
else:
|
||
logger.warning("没有制作人链接，跳过提取制作人发言")
|
||
except Exception as e:
|
||
logger.warning(f"提取制作人发言失败: {e}")
|
||
|
||
# 关闭浏览器
|
||
await browser.close()
|
||
await playwright.stop()
|
||
|
||
logger.success(f"抓取完成: {product_info.get('name', '未知')}")
|
||
return product_info
|
||
|
||
except Exception as e:
|
||
logger.error(f"抓取产品信息失败: {e}")
|
||
return {'url': url}
|
||
|
||
async def process_urls(self):
|
||
"""处理所有URL"""
|
||
# 查询URL
|
||
self.product_urls = self.query_producthunt_urls()
|
||
|
||
if not self.product_urls:
|
||
logger.warning("未找到包含producthunt.com的链接")
|
||
return
|
||
|
||
# 初始化数据库
|
||
self.init_product_database()
|
||
|
||
logger.info(f"开始处理 {len(self.product_urls)} 个产品链接")
|
||
|
||
# 创建进度条
|
||
with tqdm(total=len(self.product_urls), desc="处理进度") as pbar:
|
||
for url in self.product_urls:
|
||
try:
|
||
# 检查是否已存在
|
||
if self.check_duplicate(url):
|
||
logger.info(f"跳过已存在的链接: {url}")
|
||
pbar.update(1)
|
||
continue
|
||
|
||
# 抓取产品信息
|
||
product_info = await self.scrape_product_info(url)
|
||
|
||
# 保存到数据库
|
||
if product_info:
|
||
self.save_product_info(product_info)
|
||
|
||
pbar.update(1)
|
||
|
||
except Exception as e:
|
||
logger.error(f"处理链接失败 {url}: {e}")
|
||
pbar.update(1)
|
||
|
||
def run(self):
|
||
"""运行主程序"""
|
||
logger.info("开始ProductHunt数据抓取任务")
|
||
|
||
try:
|
||
# 运行异步任务
|
||
asyncio.run(self.process_urls())
|
||
logger.success("任务完成")
|
||
|
||
except Exception as e:
|
||
logger.error(f"程序执行失败: {e}")
|
||
|
||
|
||
def main():
|
||
"""主函数"""
|
||
scraper = ProductHuntScraper()
|
||
scraper.run()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main() |