import os import json import time from datetime import datetime import undetected_chromedriver as uc from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.common.exceptions import TimeoutException, NoSuchElementException from loguru import logger class ProductHuntScraper: def __init__(self): self.driver = None self.product_url = "https://www.producthunt.com/products/elsie-ai-beta" def connect_to_chrome(self): """连接到Chrome实例""" try: logger.info("正在初始化未检测的Chrome驱动...") # 使用undetected-chromedriver创建驱动实例 options = uc.ChromeOptions() options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") options.add_argument("--disable-gpu") options.add_argument("--window-size=1920,1080") # 创建驱动 self.driver = uc.Chrome(options=options, version_main=142) logger.info("成功连接到未检测的Chrome实例") return True except Exception as e: logger.error(f"连接Chrome实例失败: {str(e)}") return False def navigate_to_product(self): """导航到产品页面""" try: logger.info(f"正在导航到产品页面: {self.product_url}") self.driver.get(self.product_url) # 等待页面加载 logger.info("等待页面加载...") WebDriverWait(self.driver, 20).until( EC.presence_of_element_located((By.TAG_NAME, "body")) ) # 额外等待，确保动态内容加载 logger.info("等待动态内容加载...") time.sleep(10) logger.info("页面加载完成") return True except TimeoutException: logger.error("页面加载超时") return False except Exception as e: logger.error(f"导航到产品页面失败: {str(e)}") return False def extract_product_info(self): """提取产品信息""" try: logger.info("开始提取产品信息") product_info = { "url": self.product_url, "scraped_at": datetime.now().isoformat() } # 提取产品名称 (h1标签) try: name_element = self.driver.find_element(By.TAG_NAME, "h1") product_info["name"] = name_element.text.strip() logger.info(f"产品名称: {product_info['name']}") except NoSuchElementException: logger.warning("未找到产品名称 (h1标签)") product_info["name"] = "未找到" # 提取产品简介 (class为"relative text-16 font-normal text-gray-700"的div) try: desc_selector = "div.relative.text-16.font-normal.text-gray-700" desc_element = self.driver.find_element(By.CSS_SELECTOR, desc_selector) product_info["description"] = desc_element.text.strip() logger.info(f"产品简介: {product_info['description'][:50]}...") except NoSuchElementException: logger.warning("未找到产品简介 (div.relative.text-16.font-normal.text-gray-700)") product_info["description"] = "未找到" # 提取第一个评论 (class为"flex flex-1 flex-col gap-2"的div) try: comment_selector = "div.flex.flex-1.flex-col.gap-2" comment_element = self.driver.find_element(By.CSS_SELECTOR, comment_selector) product_info["first_comment"] = comment_element.text.strip() logger.info(f"第一个评论: {product_info['first_comment'][:50]}...") except NoSuchElementException: logger.warning("未找到第一个评论 (div.flex.flex-1.flex-col.gap-2)") product_info["first_comment"] = "未找到" # 尝试提取其他有用信息 try: # 尝试获取产品标签 tag_elements = self.driver.find_elements(By.CSS_SELECTOR, "[class*='tag'], [class*='category'], [class*='topic']") if tag_elements: tags = [tag.text.strip() for tag in tag_elements if tag.text.strip()] product_info["tags"] = tags[:5] # 最多取5个标签 logger.info(f"找到标签: {tags[:3]}") except Exception as e: logger.debug(f"提取标签时出错: {str(e)}") # 尝试获取点赞数 try: like_elements = self.driver.find_elements(By.CSS_SELECTOR, "[class*='vote'], [class*='like'], [class*='upvote']") if like_elements: product_info["likes"] = like_elements[0].text.strip() logger.info(f"点赞数: {product_info['likes']}") except Exception as e: logger.debug(f"提取点赞数时出错: {str(e)}") # 尝试获取评论数 try: comment_count_elements = self.driver.find_elements(By.CSS_SELECTOR, "[class*='comment-count'], [class*='comments']") if comment_count_elements: product_info["comment_count"] = comment_count_elements[0].text.strip() logger.info(f"评论数: {product_info['comment_count']}") except Exception as e: logger.debug(f"提取评论数时出错: {str(e)}") return product_info except Exception as e: logger.error(f"提取产品信息失败: {str(e)}") return None def save_to_file(self, data, filename="product_info.json"): """保存数据到文件""" try: with open(filename, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) logger.info(f"数据已保存到 {filename}") return True except Exception as e: logger.error(f"保存数据失败: {str(e)}") return False def save_screenshot(self, filename="product_screenshot.png"): """保存页面截图，用于调试""" try: self.driver.save_screenshot(filename) logger.info(f"页面截图已保存到 {filename}") return True except Exception as e: logger.error(f"保存页面截图失败: {str(e)}") return False def close(self): """关闭浏览器""" if self.driver: self.driver.quit() logger.info("浏览器已关闭") def scrape_product(self): """执行完整的抓取流程""" if not self.connect_to_chrome(): logger.error("无法连接到Chrome实例") return False try: if not self.navigate_to_product(): logger.error("无法导航到产品页面") return False # 保存截图用于调试 self.save_screenshot() product_info = self.extract_product_info() if product_info: self.save_to_file(product_info) return True else: logger.error("未能提取产品信息") return False finally: self.close() def main(): logger.info("开始ProductHunt产品信息抓取") scraper = ProductHuntScraper() # 可以修改product_url来抓取其他产品 # scraper.product_url = "https://www.producthunt.com/products/your-product" success = scraper.scrape_product() if success: logger.info("产品信息抓取完成") else: logger.error("产品信息抓取失败") if __name__ == "__main__": main()