#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
ProductHunt数据抓取器
从tophub_data.db查询包含producthunt.com的链接，然后使用Playwright抓取产品信息并保存到product.db
"""

import sqlite3
import asyncio
import os
from datetime import datetime
from loguru import logger
from tqdm import tqdm
import sys

# 配置日志
logger.remove()
logger.add(sys.stderr, level="INFO", format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>")

class ProductHuntScraper:
    """ProductHunt数据抓取器"""
    
    def __init__(self):
        self.tophub_db_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tophub_data.db")
        self.product_db_path = os.path.join(os.path.dirname(__file__), "product.db")
        self.product_urls = []
        
    def query_producthunt_urls(self):
        """查询包含producthunt.com的链接"""
        logger.info("正在查询tophub_data.db数据库...")
        
        try:
            conn = sqlite3.connect(self.tophub_db_path)
            cursor = conn.cursor()
            
            # 查询包含producthunt.com的链接
            cursor.execute("SELECT url FROM articles WHERE url LIKE '%producthunt.com%'")
            urls = [row[0] for row in cursor.fetchall()]
            
            conn.close()
            
            logger.success(f"找到 {len(urls)} 个包含producthunt.com的链接")
            return urls
            
        except Exception as e:
            logger.error(f"查询数据库失败: {e}")
            return []
    
    def init_product_database(self):
        """初始化product.db数据库"""
        logger.info("正在初始化product.db数据库...")
        
        try:
            conn = sqlite3.connect(self.product_db_path)
            cursor = conn.cursor()
            
            # 创建产品信息表
            cursor.execute('''
                CREATE TABLE IF NOT EXISTS products (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    url TEXT NOT NULL UNIQUE,
                    name TEXT,
                    introduction TEXT,
                    user_count TEXT,
                    maker_link TEXT,
                    maker_statement TEXT,
                    created_at TEXT NOT NULL,
                    updated_at TEXT NOT NULL
                )
            ''')
            
            conn.commit()
            conn.close()
            logger.success("product.db数据库初始化完成")
            
        except Exception as e:
            logger.error(f"初始化数据库失败: {e}")
    
    def check_duplicate(self, url):
        """检查URL是否已存在"""
        try:
            conn = sqlite3.connect(self.product_db_path)
            cursor = conn.cursor()
            
            cursor.execute("SELECT COUNT(*) FROM products WHERE url = ?", (url,))
            count = cursor.fetchone()[0]
            
            conn.close()
            return count > 0
            
        except Exception as e:
            logger.error(f"检查重复失败: {e}")
            return False
    
    def save_product_info(self, product_info):
        """保存产品信息到数据库"""
        try:
            conn = sqlite3.connect(self.product_db_path)
            cursor = conn.cursor()
            
            current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            
            # 检查是否已存在
            cursor.execute("SELECT id FROM products WHERE url = ?", (product_info['url'],))
            existing = cursor.fetchone()
            
            if existing:
                # 更新现有记录
                cursor.execute('''
                    UPDATE products SET 
                    name = ?, introduction = ?, user_count = ?, 
                    maker_link = ?, maker_statement = ?, updated_at = ?
                    WHERE url = ?
                ''', (
                    product_info.get('name'),
                    product_info.get('introduction'),
                    product_info.get('user_count'),
                    product_info.get('maker_link'),
                    product_info.get('maker_statement'),
                    current_time,
                    product_info['url']
                ))
                logger.info(f"更新产品信息: {product_info.get('name', '未知')}")
            else:
                # 插入新记录
                cursor.execute('''
                    INSERT INTO products 
                    (url, name, introduction, user_count, maker_link, maker_statement, created_at, updated_at)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                ''', (
                    product_info['url'],
                    product_info.get('name'),
                    product_info.get('introduction'),
                    product_info.get('user_count'),
                    product_info.get('maker_link'),
                    product_info.get('maker_statement'),
                    current_time,
                    current_time
                ))
                logger.info(f"新增产品信息: {product_info.get('name', '未知')}")
            
            conn.commit()
            conn.close()
            return True
            
        except Exception as e:
            logger.error(f"保存产品信息失败: {e}")
            return False
    
    async def scrape_product_info(self, url):
        """使用Playwright抓取产品信息"""
        try:
            # 导入Playwright相关模块
            from playwright.async_api import async_playwright
            
            logger.info(f"开始抓取: {url}")
            
            # 创建Playwright实例
            playwright = await async_playwright().start()
            browser = await playwright.chromium.launch(headless=True)
            page = await browser.new_page()
            
            # 设置超时时间
            page.set_default_timeout(120000)  # 增加超时时间以处理Cloudflare挑战
            
            # 导航到页面
            await page.goto(url, wait_until="domcontentloaded")
            
            # 检查是否是Cloudflare挑战页面
            page_title = await page.title()
            if "请稍候" in page_title or "Checking" in page_title or "Verifying" in page_title:
                logger.info("检测到Cloudflare挑战页面，等待验证完成...")
                
                # 等待Cloudflare挑战完成
                try:
                    # 等待页面标题变化或特定元素出现
                    await page.wait_for_function(
                        """() => {
                            const title = document.title;
                            return !title.includes('请稍候') && 
                                   !title.includes('Checking') && 
                                   !title.includes('Verifying') &&
                                   title !== '请稍候…';
                        }""",
                        timeout=300000  # 5分钟
                    )
                    logger.info("Cloudflare挑战已完成")
                except Exception as e:
                    logger.warning(f"等待Cloudflare挑战超时: {e}")
            
            # 等待页面加载
            await page.wait_for_timeout(3000)
            
            product_info = {'url': url}
            
            # 提取产品名称 - 改进的XPath选择器
            try:
                # 尝试多种选择器
                name_selectors = [
                    "xpath=//h1",
                    "xpath=//h1[@data-test='product-name']",
                    "xpath=//h1[contains(@class, 'text')]",
                    "xpath=//title"
                ]
                
                for selector in name_selectors:
                    name_element = await page.query_selector(selector)
                    if name_element:
                        name_text = (await name_element.text_content()).strip()
                        # 过滤掉页面标题中的无关内容
                        if name_text and 'Product Hunt' not in name_text and len(name_text) > 5:
                            product_info['name'] = name_text
                            logger.info(f"提取到产品名称: {product_info['name']}")
                            break
                
                if 'name' not in product_info:
                    logger.warning("未找到有效的产品名称元素")
                    
            except Exception as e:
                logger.warning(f"提取产品名称失败: {e}")
            
            # 提取产品简介 - 改进的XPath选择器
            try:
                intro_selectors = [
                    "xpath=//*[@class='relative text-16 font-normal text-gray-700']//div",
                    "xpath=//p[contains(@class, 'description')]",
                    "xpath=//div[contains(@class, 'description')]",
                    "xpath=//meta[@name='description']"
                ]
                
                for selector in intro_selectors:
                    intro_element = await page.query_selector(selector)
                    if intro_element:
                        intro_text = (await intro_element.text_content()).strip()
                        if intro_text:
                            product_info['introduction'] = intro_text
                            logger.info(f"提取到产品简介: {product_info['introduction'][:100]}...")
                            break
                
                if 'introduction' not in product_info:
                    logger.warning("未找到产品简介元素")
                    
            except Exception as e:
                logger.warning(f"提取产品简介失败: {e}")
            
            # 提取用户数 - 改进的XPath选择器
            try:
                user_count_selectors = [
                    "xpath=//*[@class='flex flex-row gap-2']//div/div[2]/span/p",
                    "xpath=//span[contains(text(), 'users')]",
                    "xpath=//span[contains(text(), 'upvotes')]",
                    "xpath=//div[contains(@class, 'stats')]"
                ]
                
                for selector in user_count_selectors:
                    user_count_element = await page.query_selector(selector)
                    if user_count_element:
                        user_count_text = (await user_count_element.text_content()).strip()
                        if user_count_text:
                            product_info['user_count'] = user_count_text
                            logger.info(f"提取到用户数: {product_info['user_count']}")
                            break
                
                if 'user_count' not in product_info:
                    logger.warning("未找到用户数元素")
                    
            except Exception as e:
                logger.warning(f"提取用户数失败: {e}")
            
            # 提取制作人链接 - 改进的XPath选择器
            try:
                maker_link_selectors = [
                    "xpath=//span[contains(@class, 'absolute')]",
                    "xpath=//a[contains(@href, 'hunter')]",
                    "xpath=//a[contains(text(), 'hunter')]",
                    "xpath=//a[contains(@class, 'maker')]"
                ]
                
                for selector in maker_link_selectors:
                    maker_element = await page.query_selector(selector)
                    if maker_element:
                        # 如果是span，找父级a标签
                        if 'span' in selector:
                            a_element = await maker_element.evaluate_handle('(element) => element.closest("a")')
                            if a_element:
                                maker_link = await a_element.get_attribute('href')
                        else:
                            maker_link = await maker_element.get_attribute('href')
                        
                        if maker_link and not maker_link.startswith('http'):
                            base_url = "https://www.producthunt.com"
                            if maker_link.startswith('/'):
                                maker_link = base_url + maker_link
                            else:
                                maker_link = base_url + '/' + maker_link
                        
                        if maker_link:
                            product_info['maker_link'] = maker_link
                            logger.info(f"提取到制作人链接: {maker_link}")
                            break
                
                if 'maker_link' not in product_info:
                    logger.warning("未找到制作人链接元素")
                    
            except Exception as e:
                logger.warning(f"提取制作人链接失败: {e}")
            
            # 提取制作人发言（简化版本）
            try:
                if product_info.get('maker_link'):
                    # 在新页面中打开制作人链接
                    new_page = await browser.new_page()
                    await new_page.goto(product_info['maker_link'], wait_until="domcontentloaded")
                    await new_page.wait_for_timeout(5000)
                    
                    # 尝试多种选择器提取发言内容
                    statement_selectors = [
                        "xpath=//*[@id='comment-4597755']/div/div[2]/div/div/div",
                        "xpath=//div[contains(@class, 'comment')]",
                        "xpath=//p[contains(@class, 'comment')]",
                        "xpath=//article"
                    ]
                    
                    for selector in statement_selectors:
                        comment_element = await new_page.query_selector(selector)
                        if comment_element:
                            statement_text = (await comment_element.text_content()).strip()
                            if statement_text and len(statement_text) > 10:
                                product_info['maker_statement'] = statement_text
                                logger.info(f"提取到制作人发言: {product_info['maker_statement'][:100]}...")
                                break
                    
                    await new_page.close()
                else:
                    logger.warning("没有制作人链接，跳过提取制作人发言")
            except Exception as e:
                logger.warning(f"提取制作人发言失败: {e}")
            
            # 关闭浏览器
            await browser.close()
            await playwright.stop()
            
            logger.success(f"抓取完成: {product_info.get('name', '未知')}")
            return product_info
            
        except Exception as e:
            logger.error(f"抓取产品信息失败: {e}")
            return {'url': url}
    
    async def process_urls(self):
        """处理所有URL"""
        # 查询URL
        self.product_urls = self.query_producthunt_urls()
        
        if not self.product_urls:
            logger.warning("未找到包含producthunt.com的链接")
            return
        
        # 初始化数据库
        self.init_product_database()
        
        logger.info(f"开始处理 {len(self.product_urls)} 个产品链接")
        
        # 创建进度条
        with tqdm(total=len(self.product_urls), desc="处理进度") as pbar:
            for url in self.product_urls:
                try:
                    # 检查是否已存在
                    if self.check_duplicate(url):
                        logger.info(f"跳过已存在的链接: {url}")
                        pbar.update(1)
                        continue
                    
                    # 抓取产品信息
                    product_info = await self.scrape_product_info(url)
                    
                    # 保存到数据库
                    if product_info:
                        self.save_product_info(product_info)
                    
                    pbar.update(1)
                    
                except Exception as e:
                    logger.error(f"处理链接失败 {url}: {e}")
                    pbar.update(1)
    
    def run(self):
        """运行主程序"""
        logger.info("开始ProductHunt数据抓取任务")
        
        try:
            # 运行异步任务
            asyncio.run(self.process_urls())
            logger.success("任务完成")
            
        except Exception as e:
            logger.error(f"程序执行失败: {e}")


def main():
    """主函数"""
    scraper = ProductHuntScraper()
    scraper.run()


if __name__ == "__main__":
    main()