增加抓取producthunt的数据
This commit is contained in:
356
product/integrated_scraper.py
Normal file
356
product/integrated_scraper.py
Normal file
@@ -0,0 +1,356 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
全功能ProductHunt数据抓取器
|
||||
使用playwright-get-data.py中的专业功能绕过Cloudflare挑战
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import asyncio
|
||||
import os
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
from loguru import logger
|
||||
from tqdm import tqdm
|
||||
import sys
|
||||
|
||||
# 导入playwright-get-data.py中的功能
|
||||
import importlib.util
|
||||
|
||||
# 动态导入playwright-get-data.py
|
||||
playwright_data_path = os.path.join(os.path.dirname(__file__), "playwright-get-data.py")
|
||||
spec = importlib.util.spec_from_file_location("playwright_get_data", playwright_data_path)
|
||||
playwright_get_data = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(playwright_get_data)
|
||||
ProductHuntScraper = playwright_get_data.ProductHuntScraper
|
||||
|
||||
# 配置日志
|
||||
logger.remove()
|
||||
logger.add(sys.stderr, level="INFO", format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>")
|
||||
|
||||
class ProductHuntScraperFull:
|
||||
"""全功能ProductHunt数据抓取器"""
|
||||
|
||||
def __init__(self, tophub_db_path=None, product_db_path=None, debug_port=9222, limit=10, skip_duplicates=True):
|
||||
"""
|
||||
初始化抓取器
|
||||
|
||||
Args:
|
||||
tophub_db_path: tophub数据库路径
|
||||
product_db_path: 产品数据库路径
|
||||
debug_port: Chrome调试端口
|
||||
limit: 抓取链接数量限制
|
||||
skip_duplicates: 是否跳过已存在的URL
|
||||
"""
|
||||
if tophub_db_path:
|
||||
self.tophub_db_path = tophub_db_path
|
||||
else:
|
||||
self.tophub_db_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tophub_data.db")
|
||||
|
||||
if product_db_path:
|
||||
self.product_db_path = product_db_path
|
||||
else:
|
||||
self.product_db_path = os.path.join(os.path.dirname(__file__), "products.db")
|
||||
|
||||
self.debug_port = debug_port
|
||||
self.limit = limit
|
||||
self.skip_duplicates = skip_duplicates
|
||||
self.product_urls = []
|
||||
|
||||
def query_producthunt_urls(self, limit=None):
|
||||
"""查询包含producthunt.com的链接"""
|
||||
if limit is None:
|
||||
limit = self.limit
|
||||
|
||||
logger.info(f"正在查询tophub_data.db数据库，限制: {limit}条")
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(self.tophub_db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 查询包含producthunt.com的链接
|
||||
if limit > 0:
|
||||
cursor.execute("SELECT url FROM articles WHERE url LIKE '%producthunt.com%' LIMIT ?", (limit,))
|
||||
else:
|
||||
cursor.execute("SELECT url FROM articles WHERE url LIKE '%producthunt.com%'")
|
||||
|
||||
urls = [row[0] for row in cursor.fetchall()]
|
||||
|
||||
conn.close()
|
||||
|
||||
logger.success(f"找到 {len(urls)} 个包含producthunt.com的链接")
|
||||
return urls
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"查询数据库失败: {e}")
|
||||
return []
|
||||
|
||||
def init_product_database(self):
|
||||
"""初始化产品数据库"""
|
||||
logger.info("正在初始化产品数据库...")
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(self.product_db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 创建产品信息表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS products (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
name TEXT,
|
||||
introduction TEXT,
|
||||
user_count TEXT,
|
||||
maker_link TEXT,
|
||||
maker_statement TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)
|
||||
''')
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
logger.success("产品数据库初始化完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"初始化数据库失败: {e}")
|
||||
|
||||
def check_duplicate(self, url):
|
||||
"""检查URL是否已存在"""
|
||||
try:
|
||||
conn = sqlite3.connect(self.product_db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("SELECT COUNT(*) FROM products WHERE url = ?", (url,))
|
||||
count = cursor.fetchone()[0]
|
||||
|
||||
conn.close()
|
||||
return count > 0
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"检查重复失败: {e}")
|
||||
return False
|
||||
|
||||
def save_product_info(self, product_info):
|
||||
"""保存产品信息到数据库"""
|
||||
try:
|
||||
conn = sqlite3.connect(self.product_db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# 检查是否已存在
|
||||
cursor.execute("SELECT id FROM products WHERE url = ?", (product_info['url'],))
|
||||
existing = cursor.fetchone()
|
||||
|
||||
if existing:
|
||||
# 更新现有记录
|
||||
cursor.execute('''
|
||||
UPDATE products SET
|
||||
name = ?, introduction = ?, user_count = ?,
|
||||
maker_link = ?, maker_statement = ?, updated_at = ?
|
||||
WHERE url = ?
|
||||
''', (
|
||||
product_info.get('name'),
|
||||
product_info.get('introduction'),
|
||||
product_info.get('user_count'),
|
||||
product_info.get('maker_link'),
|
||||
product_info.get('maker_statement'),
|
||||
current_time,
|
||||
product_info['url']
|
||||
))
|
||||
logger.info(f"更新产品信息: {product_info.get('name', '未知')}")
|
||||
else:
|
||||
# 插入新记录
|
||||
cursor.execute('''
|
||||
INSERT INTO products
|
||||
(url, name, introduction, user_count, maker_link, maker_statement, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
''', (
|
||||
product_info['url'],
|
||||
product_info.get('name'),
|
||||
product_info.get('introduction'),
|
||||
product_info.get('user_count'),
|
||||
product_info.get('maker_link'),
|
||||
product_info.get('maker_statement'),
|
||||
current_time,
|
||||
current_time
|
||||
))
|
||||
logger.info(f"新增产品信息: {product_info.get('name', '未知')}")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"保存产品信息失败: {e}")
|
||||
return False
|
||||
|
||||
async def scrape_product_info(self, url):
|
||||
"""使用playwright-get-data.py中的专业功能抓取产品信息"""
|
||||
try:
|
||||
logger.info(f"开始抓取: {url}")
|
||||
|
||||
# 创建ProductHuntScraper实例
|
||||
scraper = ProductHuntScraper(debug_port=self.debug_port)
|
||||
|
||||
# 连接到已运行的Chrome实例
|
||||
connected = await scraper.connect_to_existing_chrome()
|
||||
if not connected:
|
||||
logger.error("连接Chrome失败，跳过此URL")
|
||||
return None
|
||||
|
||||
# 导航到ProductHunt页面
|
||||
navigated = await scraper.navigate_to_producthunt(url)
|
||||
if not navigated:
|
||||
logger.error("导航到页面失败，跳过此URL")
|
||||
await scraper.close()
|
||||
return None
|
||||
|
||||
# 提取产品信息
|
||||
product_info = await scraper.extract_product_info()
|
||||
if product_info:
|
||||
product_info['url'] = url
|
||||
logger.success(f"成功提取产品信息: {product_info.get('name', '未知')}")
|
||||
else:
|
||||
logger.error("提取产品信息失败")
|
||||
|
||||
# 关闭连接
|
||||
await scraper.close()
|
||||
|
||||
return product_info
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"抓取产品信息失败: {e}")
|
||||
return None
|
||||
|
||||
async def run_scraping(self, urls=None):
|
||||
"""运行抓取任务"""
|
||||
logger.info("=== 开始ProductHunt数据抓取 ===")
|
||||
|
||||
# 初始化数据库
|
||||
self.init_product_database()
|
||||
|
||||
# 获取要抓取的URL列表
|
||||
if urls is None:
|
||||
self.product_urls = self.query_producthunt_urls()
|
||||
else:
|
||||
self.product_urls = urls
|
||||
|
||||
if not self.product_urls:
|
||||
logger.error("未找到要抓取的ProductHunt链接")
|
||||
return False
|
||||
|
||||
logger.info(f"找到 {len(self.product_urls)} 个ProductHunt链接")
|
||||
|
||||
# 统计抓取结果
|
||||
success_count = 0
|
||||
skip_count = 0
|
||||
error_count = 0
|
||||
|
||||
# 使用进度条显示处理进度
|
||||
with tqdm(total=len(self.product_urls), desc="抓取ProductHunt链接") as pbar:
|
||||
for url in self.product_urls:
|
||||
logger.info(f"处理URL: {url}")
|
||||
|
||||
# 检查是否已存在
|
||||
if self.skip_duplicates and self.check_duplicate(url):
|
||||
logger.info(f"URL已存在，跳过: {url}")
|
||||
skip_count += 1
|
||||
pbar.update(1)
|
||||
continue
|
||||
|
||||
# 抓取产品信息
|
||||
product_info = await self.scrape_product_info(url)
|
||||
|
||||
if product_info:
|
||||
# 保存到数据库
|
||||
success = self.save_product_info(product_info)
|
||||
if success:
|
||||
logger.success(f"成功保存产品信息: {product_info.get('name', '未知')}")
|
||||
success_count += 1
|
||||
else:
|
||||
logger.error(f"保存产品信息失败: {url}")
|
||||
error_count += 1
|
||||
else:
|
||||
logger.error(f"抓取产品信息失败: {url}")
|
||||
error_count += 1
|
||||
|
||||
pbar.update(1)
|
||||
|
||||
# 显示抓取结果统计
|
||||
self.show_scraping_results(success_count, skip_count, error_count)
|
||||
|
||||
logger.success("=== ProductHunt数据抓取完成 ===")
|
||||
return True
|
||||
|
||||
def show_scraping_results(self, success_count, skip_count, error_count):
|
||||
"""显示抓取结果统计"""
|
||||
try:
|
||||
conn = sqlite3.connect(self.product_db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 统计数据库中的产品数量
|
||||
cursor.execute("SELECT COUNT(*) FROM products")
|
||||
total_count = cursor.fetchone()[0]
|
||||
|
||||
# 获取最新抓取的产品信息
|
||||
cursor.execute("SELECT name, url FROM products ORDER BY updated_at DESC LIMIT 10")
|
||||
recent_products = cursor.fetchall()
|
||||
|
||||
conn.close()
|
||||
|
||||
logger.info("=== 抓取结果统计 ===")
|
||||
logger.info(f"成功抓取: {success_count} 个产品")
|
||||
logger.info(f"跳过重复: {skip_count} 个链接")
|
||||
logger.info(f"抓取失败: {error_count} 个链接")
|
||||
logger.info(f"数据库中的产品总数: {total_count}")
|
||||
|
||||
if recent_products:
|
||||
logger.info("最新抓取的产品:")
|
||||
for name, url in recent_products:
|
||||
logger.info(f" - {name}: {url}")
|
||||
else:
|
||||
logger.info("数据库中暂无产品记录")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"显示抓取结果失败: {e}")
|
||||
|
||||
def parse_arguments():
|
||||
"""解析命令行参数"""
|
||||
parser = argparse.ArgumentParser(description="全功能ProductHunt数据抓取器")
|
||||
parser.add_argument("--tophub-db", help="tophub数据库路径", default=None)
|
||||
parser.add_argument("--product-db", help="产品数据库路径", default=None)
|
||||
parser.add_argument("--debug-port", type=int, help="Chrome调试端口", default=9222)
|
||||
parser.add_argument("--limit", type=int, help="抓取链接数量限制", default=10)
|
||||
parser.add_argument("--no-skip-duplicates", action="store_true", help="不跳过重复URL")
|
||||
parser.add_argument("--urls", nargs="+", help="指定要抓取的URL列表")
|
||||
parser.add_argument("--log-file", help="日志文件路径", default="producthunt_scraper.log")
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
async def main():
|
||||
"""主函数"""
|
||||
args = parse_arguments()
|
||||
|
||||
# 配置日志文件输出
|
||||
logger.add(args.log_file, level="INFO", rotation="10 MB")
|
||||
|
||||
# 创建抓取器实例
|
||||
scraper = ProductHuntScraperFull(
|
||||
tophub_db_path=args.tophub_db,
|
||||
product_db_path=args.product_db,
|
||||
debug_port=args.debug_port,
|
||||
limit=args.limit,
|
||||
skip_duplicates=not args.no_skip_duplicates
|
||||
)
|
||||
|
||||
# 运行抓取任务
|
||||
if args.urls:
|
||||
await scraper.run_scraping(urls=args.urls)
|
||||
else:
|
||||
await scraper.run_scraping()
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 运行异步主函数
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user