xiaji/tophux_scrape
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

更新了web方式查看products的效果

Browse Source
This commit is contained in:
xiaji
2025-11-30 12:36:48 +08:00
parent 1c91dd45ed
commit ff7e114324
18 changed files with 21092 additions and 31986 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5785
2025年11月30日91634.txt Normal file
View File

File diff suppressed because it is too large Load Diff

106
check_product_db.log
View File

@@ -1,106 +0,0 @@
2025-11-26 23:10:42.930 | INFO | __main__:check_database_structure:31 - 找到数据库文件: ['product\\product.db', 'product\\products.db']
2025-11-26 23:10:42.931 | INFO | __main__:check_database_structure:34 -
检查数据库: product\product.db
2025-11-26 23:10:42.932 | INFO | __main__:check_database_structure:48 - 数据库中的表:
2025-11-26 23:10:42.932 | INFO | __main__:check_database_structure:51 - - products
2025-11-26 23:10:42.932 | INFO | __main__:check_database_structure:57 - 表结构:
2025-11-26 23:10:42.932 | INFO | __main__:check_database_structure:59 - id (INTEGER)
2025-11-26 23:10:42.934 | INFO | __main__:check_database_structure:59 - url (TEXT)
2025-11-26 23:10:42.935 | INFO | __main__:check_database_structure:59 - name (TEXT)
2025-11-26 23:10:42.935 | INFO | __main__:check_database_structure:59 - introduction (TEXT)
2025-11-26 23:10:42.935 | INFO | __main__:check_database_structure:59 - user_count (TEXT)
2025-11-26 23:10:42.935 | INFO | __main__:check_database_structure:59 - maker_link (TEXT)
2025-11-26 23:10:42.935 | INFO | __main__:check_database_structure:59 - maker_statement (TEXT)
2025-11-26 23:10:42.935 | INFO | __main__:check_database_structure:59 - created_at (TEXT)
2025-11-26 23:10:42.935 | INFO | __main__:check_database_structure:59 - updated_at (TEXT)
2025-11-26 23:10:42.935 | SUCCESS | __main__:check_database_structure:64 - 表 products 包含name和introduction字段
2025-11-26 23:10:42.935 | INFO | __main__:check_database_structure:69 - 示例数据:
2025-11-26 23:10:42.935 | ERROR | __main__:check_database_structure:78 - 检查数据库 product\product.db 时出错: 'NoneType' object is not subscriptable
2025-11-26 23:10:42.935 | INFO | __main__:check_database_structure:34 -
检查数据库: product\products.db
2025-11-26 23:10:42.936 | INFO | __main__:check_database_structure:48 - 数据库中的表:
2025-11-26 23:10:42.936 | INFO | __main__:check_database_structure:51 - - products
2025-11-26 23:10:42.936 | INFO | __main__:check_database_structure:57 - 表结构:
2025-11-26 23:10:42.937 | INFO | __main__:check_database_structure:59 - id (INTEGER)
2025-11-26 23:10:42.937 | INFO | __main__:check_database_structure:59 - url (TEXT)
2025-11-26 23:10:42.937 | INFO | __main__:check_database_structure:59 - name (TEXT)
2025-11-26 23:10:42.937 | INFO | __main__:check_database_structure:59 - introduction (TEXT)
2025-11-26 23:10:42.937 | INFO | __main__:check_database_structure:59 - user_count (TEXT)
2025-11-26 23:10:42.937 | INFO | __main__:check_database_structure:59 - maker_link (TEXT)
2025-11-26 23:10:42.937 | INFO | __main__:check_database_structure:59 - maker_statement (TEXT)
2025-11-26 23:10:42.937 | INFO | __main__:check_database_structure:59 - created_at (TEXT)
2025-11-26 23:10:42.937 | INFO | __main__:check_database_structure:59 - updated_at (TEXT)
2025-11-26 23:10:42.937 | SUCCESS | __main__:check_database_structure:64 - 表 products 包含name和introduction字段
2025-11-26 23:10:42.937 | INFO | __main__:check_database_structure:69 - 示例数据:
2025-11-26 23:10:42.938 | INFO | __main__:check_database_structure:71 - 示例1: name='Pixley AI', introduction='Pixley is the first platform that lets children tu...'
2025-11-26 23:10:42.938 | INFO | __main__:check_database_structure:71 - 示例2: name='Burner', introduction='Burner is a small, secure computer that keeps your...'
2025-11-26 23:10:42.938 | INFO | __main__:check_database_structure:71 - 示例3: name='American Ratings Lead Magnet Portal', introduction='Build verified business credibility with the Ameri...'
2025-11-26 23:10:42.938 | INFO | __main__:check_database_structure:51 - - sqlite_sequence
2025-11-26 23:10:42.938 | INFO | __main__:check_database_structure:57 - 表结构:
2025-11-26 23:10:42.938 | INFO | __main__:check_database_structure:59 - name ()
2025-11-26 23:10:42.938 | INFO | __main__:check_database_structure:59 - seq ()
2025-11-26 23:10:42.938 | WARNING | __main__:check_database_structure:73 - 表 sqlite_sequence 缺少name或introduction字段
2025-11-26 23:10:42.938 | INFO | __main__:check_database_structure:51 - - product_analysis
2025-11-26 23:10:42.938 | INFO | __main__:check_database_structure:57 - 表结构:
2025-11-26 23:10:42.939 | INFO | __main__:check_database_structure:59 - id (INTEGER)
2025-11-26 23:10:42.939 | INFO | __main__:check_database_structure:59 - original_id (INTEGER)
2025-11-26 23:10:42.939 | INFO | __main__:check_database_structure:59 - original_name (TEXT)
2025-11-26 23:10:42.939 | INFO | __main__:check_database_structure:59 - product_name (TEXT)
2025-11-26 23:10:42.939 | INFO | __main__:check_database_structure:59 - product_intro (TEXT)
2025-11-26 23:10:42.939 | INFO | __main__:check_database_structure:59 - development_difficulty (TEXT)
2025-11-26 23:10:42.939 | INFO | __main__:check_database_structure:59 - ai_response (TEXT)
2025-11-26 23:10:42.939 | INFO | __main__:check_database_structure:59 - created_at (TIMESTAMP)
2025-11-26 23:10:42.939 | WARNING | __main__:check_database_structure:73 - 表 product_analysis 缺少name或introduction字段
2025-11-26 23:10:49.516 | INFO | __main__:check_database_structure:31 - 找到数据库文件: ['product\\product.db', 'product\\products.db']
2025-11-26 23:10:49.516 | INFO | __main__:check_database_structure:34 -
检查数据库: product\product.db
2025-11-26 23:10:49.517 | INFO | __main__:check_database_structure:48 - 数据库中的表:
2025-11-26 23:10:49.518 | INFO | __main__:check_database_structure:51 - - products
2025-11-26 23:10:49.519 | INFO | __main__:check_database_structure:57 - 表结构:
2025-11-26 23:10:49.519 | INFO | __main__:check_database_structure:59 - id (INTEGER)
2025-11-26 23:10:49.519 | INFO | __main__:check_database_structure:59 - url (TEXT)
2025-11-26 23:10:49.519 | INFO | __main__:check_database_structure:59 - name (TEXT)
2025-11-26 23:10:49.520 | INFO | __main__:check_database_structure:59 - introduction (TEXT)
2025-11-26 23:10:49.521 | INFO | __main__:check_database_structure:59 - user_count (TEXT)
2025-11-26 23:10:49.521 | INFO | __main__:check_database_structure:59 - maker_link (TEXT)
2025-11-26 23:10:49.521 | INFO | __main__:check_database_structure:59 - maker_statement (TEXT)
2025-11-26 23:10:49.521 | INFO | __main__:check_database_structure:59 - created_at (TEXT)
2025-11-26 23:10:49.521 | INFO | __main__:check_database_structure:59 - updated_at (TEXT)
2025-11-26 23:10:49.521 | SUCCESS | __main__:check_database_structure:64 - 表 products 包含name和introduction字段
2025-11-26 23:10:49.522 | INFO | __main__:check_database_structure:69 - 示例数据:
2025-11-26 23:10:49.522 | ERROR | __main__:check_database_structure:78 - 检查数据库 product\product.db 时出错: 'NoneType' object is not subscriptable
2025-11-26 23:10:49.522 | INFO | __main__:check_database_structure:34 -
检查数据库: product\products.db
2025-11-26 23:10:49.523 | INFO | __main__:check_database_structure:48 - 数据库中的表:
2025-11-26 23:10:49.523 | INFO | __main__:check_database_structure:51 - - products
2025-11-26 23:10:49.523 | INFO | __main__:check_database_structure:57 - 表结构:
2025-11-26 23:10:49.523 | INFO | __main__:check_database_structure:59 - id (INTEGER)
2025-11-26 23:10:49.523 | INFO | __main__:check_database_structure:59 - url (TEXT)
2025-11-26 23:10:49.523 | INFO | __main__:check_database_structure:59 - name (TEXT)
2025-11-26 23:10:49.523 | INFO | __main__:check_database_structure:59 - introduction (TEXT)
2025-11-26 23:10:49.523 | INFO | __main__:check_database_structure:59 - user_count (TEXT)
2025-11-26 23:10:49.523 | INFO | __main__:check_database_structure:59 - maker_link (TEXT)
2025-11-26 23:10:49.524 | INFO | __main__:check_database_structure:59 - maker_statement (TEXT)
2025-11-26 23:10:49.524 | INFO | __main__:check_database_structure:59 - created_at (TEXT)
2025-11-26 23:10:49.524 | INFO | __main__:check_database_structure:59 - updated_at (TEXT)
2025-11-26 23:10:49.524 | SUCCESS | __main__:check_database_structure:64 - 表 products 包含name和introduction字段
2025-11-26 23:10:49.524 | INFO | __main__:check_database_structure:69 - 示例数据:
2025-11-26 23:10:49.524 | INFO | __main__:check_database_structure:71 - 示例1: name='Pixley AI', introduction='Pixley is the first platform that lets children tu...'
2025-11-26 23:10:49.524 | INFO | __main__:check_database_structure:71 - 示例2: name='Burner', introduction='Burner is a small, secure computer that keeps your...'
2025-11-26 23:10:49.524 | INFO | __main__:check_database_structure:71 - 示例3: name='American Ratings Lead Magnet Portal', introduction='Build verified business credibility with the Ameri...'
2025-11-26 23:10:49.524 | INFO | __main__:check_database_structure:51 - - sqlite_sequence
2025-11-26 23:10:49.525 | INFO | __main__:check_database_structure:57 - 表结构:
2025-11-26 23:10:49.525 | INFO | __main__:check_database_structure:59 - name ()
2025-11-26 23:10:49.525 | INFO | __main__:check_database_structure:59 - seq ()
2025-11-26 23:10:49.525 | WARNING | __main__:check_database_structure:73 - 表 sqlite_sequence 缺少name或introduction字段
2025-11-26 23:10:49.525 | INFO | __main__:check_database_structure:51 - - product_analysis
2025-11-26 23:10:49.525 | INFO | __main__:check_database_structure:57 - 表结构:
2025-11-26 23:10:49.525 | INFO | __main__:check_database_structure:59 - id (INTEGER)
2025-11-26 23:10:49.525 | INFO | __main__:check_database_structure:59 - original_id (INTEGER)
2025-11-26 23:10:49.525 | INFO | __main__:check_database_structure:59 - original_name (TEXT)
2025-11-26 23:10:49.525 | INFO | __main__:check_database_structure:59 - product_name (TEXT)
2025-11-26 23:10:49.526 | INFO | __main__:check_database_structure:59 - product_intro (TEXT)
2025-11-26 23:10:49.526 | INFO | __main__:check_database_structure:59 - development_difficulty (TEXT)
2025-11-26 23:10:49.526 | INFO | __main__:check_database_structure:59 - ai_response (TEXT)
2025-11-26 23:10:49.526 | INFO | __main__:check_database_structure:59 - created_at (TIMESTAMP)
2025-11-26 23:10:49.526 | WARNING | __main__:check_database_structure:73 - 表 product_analysis 缺少name或introduction字段

1669
db_modify_zhipu.log
View File

File diff suppressed because it is too large Load Diff

8372
integrated_product_system.log
View File

File diff suppressed because it is too large Load Diff

BIN
product/__pycache__/web_sqlite_viewer.cpython-313.pyc Normal file
View File

Binary file not shown.

146
product/integrated_product_system.py
View File

@@ -322,27 +322,59 @@ class IntegratedProductSystem:
logger.error(f"调用Ollama AI API时出错: {e}")
return None
def parse_ai_response(self, response: str) -> Tuple[str, str, str]:
"""解析AI响应内容"""
def parse_ai_response(self, response: str) -> Tuple[str, str, str, int]:
"""解析AI响应内容，提取产品名称、简介、难度描述和难度分数"""
try:
# 使用/分割响应内容
parts = response.split('/')
product_name = ""
product_intro = ""
difficulty = ""
difficulty_score = None
if len(parts) >= 3:
product_name = parts[0].strip()
product_intro = parts[1].strip()
difficulty = parts[2].strip()
logger.info(f"解析结果: 名称='{product_name}', 简介='{product_intro[:30]}...', 难度='{difficulty}'")
return product_name, product_intro, difficulty
# 从难度描述中提取分数
import re
# 尝试匹配数字分数
score_match = re.search(r'\b(\d+)\b分|\b难度(\d+)\b|\b(\d+)\b', difficulty)
if score_match:
# 获取第一个匹配的数字
for group in score_match.groups():
if group:
difficulty_score = int(group)
break
# 如果没有提取到分数，根据关键词设置默认分数
if difficulty_score is None:
difficulty_lower = difficulty.lower()
if any(keyword in difficulty_lower for keyword in ['高', '很难', '非常难', '复杂']):
difficulty_score = 85
elif any(keyword in difficulty_lower for keyword in ['中', '一般', '适中', '普通']):
difficulty_score = 60
elif any(keyword in difficulty_lower for keyword in ['低', '简单', '容易']):
difficulty_score = 35
else:
difficulty_score = 50 # 默认中等难度
logger.info(f"提取到难度分数: {difficulty_score}")
else:
logger.warning(f"响应格式不符合预期: {response}")
# 如果格式不符合，返回原始内容
return "", response, ""
difficulty = response
difficulty_score = 50 # 默认中等难度
return product_name, product_intro, difficulty, difficulty_score
except Exception as e:
logger.error(f"解析AI响应失败: {e}")
return "", response, ""
return "", response, "", 50
def check_product_exists_in_analysis(self, conn: sqlite3.Connection, original_name: str) -> bool:
"""检查产品是否已存在于分析结果表中"""
@@ -367,19 +399,23 @@ class IntegratedProductSystem:
def save_analysis_result(self, conn: sqlite3.Connection,
original_id: int, original_name: str,
product_name: str, difficulty: str, ai_response: str):
"""保存分析结果到数据库"""
product_name: str, difficulty: str, ai_response: str, difficulty_score: int = None):
"""保存分析结果到数据库，包括难度分数"""
try:
cursor = conn.cursor()
# 如果没有提供难度分数，设置默认值50
if difficulty_score is None:
difficulty_score = 50
cursor.execute("""
INSERT INTO product_analysis
(original_id, original_name, product_name, development_difficulty, ai_response)
VALUES (?, ?, ?, ?, ?)
""", (original_id, original_name, product_name, difficulty, ai_response))
(original_id, original_name, product_name, development_difficulty, difficulty_score, ai_response)
VALUES (?, ?, ?, ?, ?, ?)
""", (original_id, original_name, product_name, difficulty, difficulty_score, ai_response))
conn.commit()
logger.success(f"保存分析结果成功: {product_name}")
logger.success(f"保存分析结果成功: {product_name}, 难度分数: {difficulty_score}")
except Exception as e:
logger.error(f"保存分析结果失败: {e}")
@@ -435,11 +471,11 @@ class IntegratedProductSystem:
logger.info(f"API调用成功，正在处理数据...")
# 解析响应
product_name, product_intro, difficulty = self.parse_ai_response(ai_response)
product_name, product_intro, difficulty, difficulty_score = self.parse_ai_response(ai_response)
# 保存结果（不再保存product_intro，避免与ai_response重复）
self.save_analysis_result(conn, original_id, name,
product_name, difficulty, ai_response)
product_name, difficulty, ai_response, difficulty_score)
success_count += 1
# 显示完成状态
@@ -549,8 +585,86 @@ class IntegratedProductSystem:
except Exception as e:
logger.error(f"显示抓取结果失败: {e}")
def analyze_missing_scores(self):
"""分析并补充缺失难度分数的产品"""
logger.info("=== 开始分析缺失难度分数的产品 ===")
conn = None
try:
# 连接数据库
conn = self.connect_to_database()
cursor = conn.cursor()
# 查询缺失难度分数的产品
cursor.execute("""
SELECT pa.id, p.name, p.introduction, pa.ai_response
FROM product_analysis pa
JOIN products p ON pa.original_id = p.id
WHERE pa.difficulty_score IS NULL OR pa.difficulty_score = ''
""")
products_with_missing_scores = cursor.fetchall()
logger.info(f"找到 {len(products_with_missing_scores)} 个缺失难度分数的产品")
if not products_with_missing_scores:
logger.info("没有发现缺失难度分数的产品")
return
# 为每个缺失分数的产品分析并更新分数
updated_count = 0
for i, (analysis_id, name, introduction, ai_response) in enumerate(products_with_missing_scores, 1):
logger.info(f"处理缺失分数的产品 {i}/{len(products_with_missing_scores)}: {name}")
# 如果已有AI响应，从响应中重新提取分数
difficulty_score = None
if ai_response:
try:
_, _, _, difficulty_score = self.parse_ai_response(ai_response)
logger.info(f"从现有AI响应中提取分数: {difficulty_score}")
except Exception as e:
logger.error(f"从现有响应提取分数失败: {e}")
# 如果无法从现有响应提取，重新调用API
if difficulty_score is None:
logger.info(f"重新调用API分析产品: {name}")
ai_response = self.call_ollama_ai_api(name, introduction)
if ai_response:
_, _, _, difficulty_score = self.parse_ai_response(ai_response)
# 更新AI响应
cursor.execute("""
UPDATE product_analysis
SET ai_response = ?
WHERE id = ?
""", (ai_response, analysis_id))
# 更新难度分数
if difficulty_score is not None:
cursor.execute("""
UPDATE product_analysis
SET difficulty_score = ?
WHERE id = ?
""", (difficulty_score, analysis_id))
conn.commit()
updated_count += 1
logger.success(f"成功更新产品 '{name}' 的难度分数为 {difficulty_score}")
else:
logger.warning(f"无法为产品 '{name}' 确定难度分数")
# 避免API调用过于频繁
if i < len(products_with_missing_scores):
time.sleep(2)
logger.success(f"缺失分数分析完成! 成功更新 {updated_count} 个产品的难度分数")
except Exception as e:
logger.error(f"分析缺失分数过程中出错: {e}")
finally:
if conn:
conn.close()
logger.info("数据库连接已关闭")
async def run_full_workflow_async(self, max_products=None, analyze_only=False):
"""异步运行完整工作流程：抓取+分析"""
"""异步运行完整工作流程：抓取+分析+补充缺失分数"""
logger.info("=== 开始全功能产品系统工作流程 ===")
# 初始化数据库
@@ -567,6 +681,10 @@ class IntegratedProductSystem:
logger.info("步骤2: 开始AI分析产品数据...")
self.analyze_products(max_products)
# 步骤3: 分析并补充缺失的难度分数
logger.info("步骤3: 开始分析并补充缺失的难度分数...")
self.analyze_missing_scores()
logger.success("=== 全功能产品系统工作流程完成 ===")
def run_full_workflow(self, max_products=None, analyze_only=False):

BIN
product/product.db Normal file
View File

Binary file not shown.

BIN
product/products.db
View File

Binary file not shown.

532
product/templates/index.html Normal file
View File

@@ -0,0 +1,532 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>SQLite数据库查看器</title>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
min-height: 100vh;
color: #333;
}
.container {
width: 100%;
margin: 0 auto;
padding: 20px;
}
.header {
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(10px);
border-radius: 15px;
padding: 25px;
margin-bottom: 25px;
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
}
.header h1 {
color: #2c3e50;
font-size: 2.5em;
margin-bottom: 10px;
text-align: center;
}
.controls {
display: flex;
gap: 20px;
align-items: center;
flex-wrap: wrap;
margin-top: 20px;
}
.control-group {
display: flex;
flex-direction: column;
gap: 8px;
}
.control-group label {
font-weight: 600;
color: #34495e;
font-size: 0.9em;
}
select, input {
padding: 12px 15px;
border: 2px solid #e0e6ed;
border-radius: 8px;
font-size: 14px;
transition: all 0.3s ease;
background: white;
}
select:focus, input:focus {
outline: none;
border-color: #667eea;
box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.1);
}
.btn {
padding: 12px 24px;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
border: none;
border-radius: 8px;
cursor: pointer;
font-weight: 600;
transition: all 0.3s ease;
text-decoration: none;
display: inline-block;
}
.btn:hover {
transform: translateY(-2px);
box-shadow: 0 5px 15px rgba(0, 0, 0, 0.2);
}
.data-container {
width: 100%;
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(10px);
border-radius: 15px;
padding: 25px;
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
}
.table-info {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 20px;
padding: 15px;
background: #f8f9fa;
border-radius: 10px;
}
.table-info h2 {
color: #2c3e50;
font-size: 1.5em;
}
.stats {
display: flex;
gap: 20px;
font-size: 0.9em;
color: #7f8c8d;
}
.table-wrapper {
width: 100%;
overflow-x: auto;
border-radius: 10px;
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.1);
}
table {
width: 100%;
border-collapse: collapse;
background: white;
font-size: 14px;
}
th {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
padding: 15px 12px;
text-align: left;
font-weight: 600;
position: sticky;
top: 0;
z-index: 10;
}
td {
padding: 12px;
border-bottom: 1px solid #ecf0f1;
vertical-align: top;
}
tr:nth-child(even) {
background-color: #f8f9fa;
}
tr:hover {
background-color: #e3f2fd;
transition: background-color 0.3s ease;
}
.multiline-cell {
white-space: pre-wrap;
line-height: 1.6;
max-height: 200px;
overflow-y: auto;
padding: 8px;
background: #fff3cd;
border-radius: 6px;
border-left: 4px solid #ffc107;
}
.normal-cell {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
max-width: 300px;
}
.empty-cell {
color: #95a5a6;
font-style: italic;
}
.pagination {
display: flex;
justify-content: center;
align-items: center;
gap: 10px;
margin-top: 25px;
flex-wrap: wrap;
}
.page-info {
color: #7f8c8d;
font-weight: 600;
}
.page-btn {
padding: 8px 12px;
border: 2px solid #e0e6ed;
background: white;
border-radius: 6px;
cursor: pointer;
transition: all 0.3s ease;
min-width: 40px;
text-align: center;
}
.page-btn:hover {
border-color: #667eea;
background: #667eea;
color: white;
}
.page-btn.active {
background: #667eea;
color: white;
border-color: #667eea;
}
.page-btn:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.loading {
text-align: center;
padding: 40px;
color: #7f8c8d;
font-size: 1.1em;
}
.error {
background: #f8d7da;
color: #721c24;
padding: 15px;
border-radius: 8px;
border: 1px solid #f5c6cb;
margin: 20px 0;
}
.no-data {
text-align: center;
padding: 40px;
color: #7f8c8d;
font-size: 1.1em;
}
@media (max-width: 768px) {
.controls {
flex-direction: column;
align-items: stretch;
}
.table-info {
flex-direction: column;
gap: 15px;
text-align: center;
}
.stats {
justify-content: center;
}
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>🗄️ SQLite数据库查看器</h1>
<div class="controls">
<div class="control-group">
<label for="tableSelect">选择数据表:</label>
<select id="tableSelect">
<option value="">加载中...</option>
</select>
</div>
<div class="control-group">
<label for="searchField">筛选字段:</label>
<select id="searchField" multiple disabled style="min-height: 80px;">
<option value="">所有文本字段</option>
</select>
</div>
<div class="control-group">
<label for="searchValue">筛选内容:</label>
<input type="text" id="searchValue" placeholder="输入筛选内容..." disabled>
</div>
<button class="btn" onclick="loadData()">刷新数据</button>
</div>
</div>
<div class="data-container">
<div class="table-info">
<h2 id="tableName">请选择数据表</h2>
<div class="stats">
<span id="recordCount">记录数: 0</span>
<span id="pageInfo">第 0 页，共 0 页</span>
</div>
</div>
<div id="dataContainer">
<div class="no-data">请选择数据表以查看内容</div>
</div>
<div id="pagination" class="pagination" style="display: none;">
<button class="page-btn" onclick="changePage('prev')" id="prevBtn">上一页</button>
<span class="page-info" id="pageInfoDetail"></span>
<button class="page-btn" onclick="changePage('next')" id="nextBtn">下一页</button>
</div>
</div>
</div>
<script>
let currentTable = '';
let currentPage = 1;
let perPage = 50;
let totalPages = 1;
let currentData = null;
// 绑定事件
document.addEventListener('DOMContentLoaded', function() {
loadTables();
// 绑定事件
document.getElementById('tableSelect').addEventListener('change', function() {
currentTable = this.value;
currentPage = 1;
if (currentTable) {
loadTableStructure();
loadData();
}
});
document.getElementById('searchField').addEventListener('change', loadData);
document.getElementById('searchValue').addEventListener('input', debounce(loadData, 500));
});
// 防抖函数
function debounce(func, wait) {
let timeout;
return function executedFunction(...args) {
const later = () => {
clearTimeout(timeout);
func(...args);
};
clearTimeout(timeout);
timeout = setTimeout(later, wait);
};
}
// 加载表列表
async function loadTables() {
try {
const response = await fetch('/api/tables');
const data = await response.json();
const select = document.getElementById('tableSelect');
select.innerHTML = '<option value="">选择数据表...</option>';
data.tables.forEach(table => {
const option = document.createElement('option');
option.value = table;
option.textContent = table;
select.appendChild(option);
});
} catch (error) {
console.error('加载表列表失败:', error);
showError('加载表列表失败: ' + error.message);
}
}
// 加载表结构
async function loadTableStructure() {
if (!currentTable) return;
try {
const response = await fetch(`/api/table/${currentTable}/structure`);
const data = await response.json();
const searchField = document.getElementById('searchField');
searchField.innerHTML = '<option value="">所有文本字段</option>';
data.structure.forEach(field => {
const option = document.createElement('option');
option.value = field.name;
option.textContent = field.name;
searchField.appendChild(option);
});
searchField.disabled = false;
document.getElementById('searchValue').disabled = false;
} catch (error) {
console.error('加载表结构失败:', error);
}
}
// 加载数据
async function loadData() {
if (!currentTable) return;
const container = document.getElementById('dataContainer');
container.innerHTML = '<div class="loading">📊 数据加载中...</div>';
const searchFieldSelect = document.getElementById('searchField');
const searchValue = document.getElementById('searchValue').value;
try {
let url = `/api/table/${currentTable}/data?page=${currentPage}&per_page=${perPage}`;
if (searchValue) {
// 获取所有选中的字段
const selectedFields = Array.from(searchFieldSelect.selectedOptions)
.map(option => option.value)
.filter(value => value !== '');
if (selectedFields.length > 0) {
// 如果选择了特定字段，传递所有选中的字段
selectedFields.forEach(field => {
url += `&search_field=${encodeURIComponent(field)}`;
});
} else {
// 否则使用"all"表示所有文本字段
url += '&search_field=all';
}
url += `&search_value=${encodeURIComponent(searchValue)}`;
}
const response = await fetch(url);
currentData = await response.json();
displayData(currentData);
updatePagination();
} catch (error) {
console.error('加载数据失败:', error);
showError('加载数据失败: ' + error.message);
}
}
// 显示数据
function displayData(data) {
const container = document.getElementById('dataContainer');
if (!data.rows || data.rows.length === 0) {
container.innerHTML = '<div class="no-data">📭 没有找到数据</div>';
return;
}
let html = '<div class="table-wrapper"><table><thead><tr>';
// 表头
data.columns.forEach(col => {
html += `<th>${col}</th>`;
});
html += '</tr></thead><tbody>';
// 数据行
data.rows.forEach(row => {
html += '<tr>';
row.forEach(cell => {
if (cell.type === 'multiline') {
html += `<td><div class="multiline-cell">${escapeHtml(cell.value)}</div></td>`;
} else if (cell.type === 'empty') {
html += '<td><div class="empty-cell">空</div></td>';
} else {
html += `<td><div class="normal-cell">${escapeHtml(cell.value)}</div></td>`;
}
});
html += '</tr>';
});
html += '</tbody></table></div>';
container.innerHTML = html;
// 更新统计信息
document.getElementById('tableName').textContent = `📋 ${currentTable}`;
document.getElementById('recordCount').textContent = `记录数: ${data.total_count}`;
document.getElementById('pageInfo').textContent = `第 ${currentPage} 页，共 ${data.total_pages} 页`;
}
// 更新分页
function updatePagination() {
if (!currentData) return;
totalPages = currentData.total_pages;
const pagination = document.getElementById('pagination');
const prevBtn = document.getElementById('prevBtn');
const nextBtn = document.getElementById('nextBtn');
const pageInfo = document.getElementById('pageInfoDetail');
if (totalPages <= 1) {
pagination.style.display = 'none';
return;
}
pagination.style.display = 'flex';
prevBtn.disabled = currentPage <= 1;
nextBtn.disabled = currentPage >= totalPages;
pageInfo.textContent = `${currentPage} / ${totalPages}`;
}
// 翻页
function changePage(direction) {
if (direction === 'prev' && currentPage > 1) {
currentPage--;
loadData();
} else if (direction === 'next' && currentPage < totalPages) {
currentPage++;
loadData();
}
}
// HTML转义
function escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
// 显示错误
function showError(message) {
const container = document.getElementById('dataContainer');
container.innerHTML = `<div class="error">❌ ${message}</div>`;
}
</script>
</body>
</html>

166
product/web_sqlite_comparison.md
View File

@@ -1,166 +0,0 @@
# Web SQLite查看器对比文档
## 概述
我为您创建了两个不同风格的Web SQLite数据库查看器，都支持现代化的界面、内容筛选和动态行高调整功能。
## 🚀 现代化版本 (modern_sqlite_viewer.py)
**访问地址**: http://localhost:5001
### 特点
- **技术栈**: Flask + Bootstrap 5 + DataTables + jQuery
- **界面风格**: 现代化渐变设计，卡片式布局
- **功能特性**:
- 服务器端分页处理，支持大数据集
- 高级搜索和筛选功能
- 响应式设计，完美适配移动端
- 动态加载指示器
- 专业的数据统计面板
- 列排序和分页控制
- 多行内容智能识别和美化显示
### 适用场景
- 需要处理大量数据（数千条记录以上）
- 需要专业级的数据分析和浏览功能
- 需要移动端友好的界面
- 需要高级的数据操作功能
## 🎯 轻量级版本 (simple_sqlite_viewer.py)
**访问地址**: http://localhost:5002
### 特点
- **技术栈**: Flask + 纯原生HTML/CSS/JS（无外部依赖）
- **界面风格**: 简洁优雅，内联样式
- **功能特性**:
- 客户端数据处理，快速响应
- 轻量级搜索功能
- 无外部依赖，加载速度快
- 简洁的统计信息
- 自适应行高显示
- 移动端适配
### 适用场景
- 数据量较小（几百条记录以内）
- 需要快速部署和访问
- 网络环境较差或需要离线使用
- 偏好简洁无依赖的解决方案
## 🎨 共同特性
### 动态行高调整
两个版本都实现了智能的行高调整：
- **自动识别**: 自动检测多行文本内容
- **美观显示**: 多行内容使用渐变背景和高亮边框
- **自然变化**: 行高根据内容长度自然调整，无突兀感
### 内容筛选功能
- **全局搜索**: 在所有列中搜索匹配内容
- **实时筛选**: 输入时即时显示结果
- **高亮显示**: 搜索结果清晰标识
### 数据库支持
- **自动创建**: 如果product.db不存在，自动创建示例数据库
- **表结构识别**: 自动识别所有表和列结构
- **数据类型处理**: 智能处理各种数据类型（文本、数字、日期等）
## 🔧 技术实现亮点
### 多行内容处理
```python
# 智能检测和处理多行文本
if isinstance(cell, str):
if '\n' in cell:
# 多行文本，用<br>替换换行符
formatted_cell = cell.replace('\n', '<br>')
row_data[col] = f'<div class="multiline-content">{formatted_cell}</div>'
```
### 响应式设计
- 使用CSS Grid和Flexbox布局
- 媒体查询适配不同屏幕尺寸
- 触摸友好的交互设计
### 性能优化
- **防抖处理**: 搜索输入使用防抖技术
- **异步加载**: 数据异步加载，界面不卡顿
- **内存管理**: 合理的数据结构和内存使用
## 🚀 使用方法
### 启动应用
```bash
# 现代化版本
python modern_sqlite_viewer.py
# 轻量级版本
python simple_sqlite_viewer.py
```
### 访问界面
- 打开浏览器访问对应地址
- 选择要查看的数据表
- 使用搜索框筛选内容
- 观察行高随内容自然变化
### 数据文件
- 默认读取 `product.db` 文件
- 如果文件不存在，自动创建示例数据
- 支持任何SQLite数据库文件
## 🎨 界面预览
### 现代化版本界面特色
- 渐变背景和毛玻璃效果
- 卡片式布局设计
- 专业的数据统计面板
- 高级DataTables功能
### 轻量级版本界面特色
- 简洁的渐变设计
- 内联CSS，无外部依赖
- 快速响应的用户体验
- 清晰的统计信息
## 📊 性能对比
| 特性 | 现代化版本 | 轻量级版本 |
|------|------------|------------|
| 加载速度 | 中等（依赖CDN） | 快速（内联资源） |
| 数据处理能力 | 强（服务器端分页） | 中（客户端处理） |
| 界面美观度 | 高（Bootstrap 5） | 中（自定义样式） |
| 功能丰富度 | 高（DataTables） | 中（基础功能） |
| 外部依赖 | 有（CDN资源） | 无（纯内联） |
| 移动端适配 | 优秀 | 良好 |
## 🔧 自定义和扩展
### 修改数据库路径
在代码中修改 `DB_PATH` 变量即可：
```python
DB_PATH = "path/to/your/database.db"
```
### 添加新功能
两个版本都基于模块化设计，易于扩展：
- 添加新的API端点
- 自定义界面样式
- 增加数据处理逻辑
### 部署到生产环境
建议使用生产级WSGI服务器：
```bash
pip install gunicorn
gunicorn -w 4 -b 0.0.0.0:5000 your_app:app
```
## 🎯 选择建议
- **需要专业功能** → 选择现代化版本
- **需要快速部署** → 选择轻量级版本
- **数据量较大** → 选择现代化版本
- **网络环境差** → 选择轻量级版本
- **需要离线使用** → 选择轻量级版本
两个版本都提供了优秀的用户体验和现代化的界面设计，您可以根据具体需求选择合适的版本。

451
product/web_sqlite_viewer.py
View File

@@ -11,11 +11,17 @@ import os
import json
from datetime import datetime
from loguru import logger
import threading
import time
import requests
app = Flask(__name__)
# 数据库路径
DB_PATH = os.path.join(os.path.dirname(__file__), 'product.db')
DB_PATH = os.path.join(os.path.dirname(__file__), 'products.db')
# 任务状态存储
analysis_tasks = {}
class SQLiteWebViewer:
def __init__(self, db_path):
@@ -68,25 +74,75 @@ class SQLiteWebViewer:
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# 获取总记录数
if search_field and search_value:
count_query = f"SELECT COUNT(*) FROM {table_name} WHERE {search_field} LIKE ?"
cursor.execute(count_query, (f'%{search_value}%',))
else:
count_query = f"SELECT COUNT(*) FROM {table_name}"
cursor.execute(count_query)
# 获取字段类型信息
field_types = {}
text_fields = []
cursor.execute(f"PRAGMA table_info({table_name});")
columns_info = cursor.fetchall()
for col_info in columns_info:
field_name = col_info[1]
field_type = col_info[2]
field_types[field_name] = field_type # 字段名 -> 字段类型
# 收集文本类型字段
if field_type.upper() not in ['INTEGER', 'REAL', 'FLOAT', 'NUMERIC']:
text_fields.append(field_name)
# 解析搜索条件
query_params = []
where_clause = ""
if search_value:
# 获取要搜索的字段列表
search_fields = []
if isinstance(search_field, list):
# 如果是列表，使用所有提供的字段
search_fields = [f for f in search_field if f in field_types]
elif search_field == "all":
# 如果是"all"，使用所有文本字段
search_fields = text_fields
elif search_field and search_field in field_types:
# 如果是单个字段，直接使用
search_fields = [search_field]
if search_fields:
conditions = []
for field in search_fields:
# 检查是否为数值比较操作符
import re
numeric_op_pattern = re.compile(r'^(<=?|>=?|=)(\d+(\.\d+)?)$')
match = numeric_op_pattern.match(search_value)
# 检查字段是否为数值类型
is_numeric_field = field_types.get(field, '').upper() in ['INTEGER', 'REAL', 'FLOAT', 'NUMERIC']
if match and is_numeric_field:
# 数值比较操作
operator = match.group(1)
value = match.group(2)
conditions.append(f"{field} {operator} ?")
query_params.append(float(value) if '.' in value else int(value))
logger.info(f"应用数值比较筛选: {field} {operator} {value}")
else:
# 默认文本模糊匹配
conditions.append(f"{field} LIKE ?")
query_params.append(f'%{search_value}%')
logger.info(f"应用文本模糊匹配: {field} LIKE '%{search_value}%'")
if conditions:
where_clause = " WHERE " + " OR ".join(conditions)
# 获取总记录数
count_query = f"SELECT COUNT(*) FROM {table_name}{where_clause}"
cursor.execute(count_query, query_params)
total_count = cursor.fetchone()[0]
# 获取分页数据
offset = (page - 1) * per_page
query_params.extend([per_page, offset])
if search_field and search_value:
data_query = f"SELECT * FROM {table_name} WHERE {search_field} LIKE ? LIMIT ? OFFSET ?"
cursor.execute(data_query, (f'%{search_value}%', per_page, offset))
else:
data_query = f"SELECT * FROM {table_name} LIMIT ? OFFSET ?"
cursor.execute(data_query, (per_page, offset))
data_query = f"SELECT * FROM {table_name}{where_clause} LIMIT ? OFFSET ?"
cursor.execute(data_query, query_params)
rows = cursor.fetchall()
@@ -100,8 +156,12 @@ class SQLiteWebViewer:
processed_rows = []
for row in rows:
processed_row = []
for cell in row:
if cell is None:
for i, cell in enumerate(row):
col_name = columns[i]
# 处理difficulty_score字段的缺失值
if col_name == "difficulty_score" and cell is None:
processed_row.append({'value': '未评分', 'type': 'empty'})
elif cell is None:
processed_row.append({'value': '', 'type': 'empty'})
elif isinstance(cell, str) and ('\n' in cell or len(cell) > 100):
# 多行文本或长文本
@@ -156,12 +216,156 @@ def get_table_data(table_name):
"""获取表数据"""
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', 50))
search_field = request.args.get('search_field')
# 获取所有search_field参数（可能是多个）
search_field = request.args.getlist('search_field')
# 如果只有一个且为空，使用单个值
if len(search_field) == 1:
search_field = search_field[0]
search_value = request.args.get('search_value')
data = viewer.get_table_data(table_name, page, per_page, search_field, search_value)
return jsonify(data)
@app.route('/api/analyze_missing_scores')
def analyze_missing_scores():
"""触发缺失分数分析任务"""
task_id = str(int(time.time()))
analysis_tasks[task_id] = {
'status': 'running',
'progress': 0,
'total': 0,
'completed': 0,
'error': None,
'start_time': datetime.now().isoformat()
}
# 在后台线程中执行分析
def run_analysis():
try:
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
# 检查product_analysis表是否存在difficulty_score字段
cursor.execute("PRAGMA table_info(product_analysis)")
columns = [col[1] for col in cursor.fetchall()]
if 'difficulty_score' not in columns:
# 如果不存在，添加该字段
cursor.execute("ALTER TABLE product_analysis ADD COLUMN difficulty_score REAL")
conn.commit()
logger.info("添加了difficulty_score字段")
# 查询缺失分数的产品
cursor.execute("""
SELECT pa.id, p.name, p.description
FROM product_analysis pa
JOIN products p ON pa.product_id = p.id
WHERE pa.difficulty_score IS NULL OR pa.difficulty_score = ''
""")
missing_scores = cursor.fetchall()
total = len(missing_scores)
analysis_tasks[task_id]['total'] = total
logger.info(f"找到 {total} 个缺失分数的产品")
for i, (analysis_id, product_name, introduction) in enumerate(missing_scores):
try:
# 调用Ollama API分析难度分数
score = analyze_product_difficulty(product_name, introduction)
# 更新数据库
cursor.execute(
"UPDATE product_analysis SET difficulty_score = ? WHERE id = ?",
(score, analysis_id)
)
conn.commit()
analysis_tasks[task_id]['completed'] = i + 1
analysis_tasks[task_id]['progress'] = int((i + 1) / total * 100)
logger.info(f"已分析产品 {i+1}/{total}: {product_name}, 分数: {score}")
# 避免频繁调用API
time.sleep(1)
except Exception as e:
logger.error(f"分析产品 {product_name} 失败: {e}")
# 继续处理下一个产品
continue
analysis_tasks[task_id]['status'] = 'completed'
logger.info("所有缺失分数分析完成")
except Exception as e:
logger.error(f"分析任务失败: {e}")
analysis_tasks[task_id]['status'] = 'failed'
analysis_tasks[task_id]['error'] = str(e)
finally:
conn.close()
analysis_tasks[task_id]['end_time'] = datetime.now().isoformat()
threading.Thread(target=run_analysis).start()
return jsonify({
'task_id': task_id,
'status': 'started',
'message': '分析任务已启动，请通过task_id查询进度'
})
@app.route('/api/update_task_status/<task_id>')
def update_task_status(task_id):
"""获取任务状态"""
if task_id in analysis_tasks:
return jsonify(analysis_tasks[task_id])
else:
return jsonify({
'error': 'Task not found',
'message': '找不到指定的任务ID'
}), 404
def analyze_product_difficulty(product_name, introduction):
"""调用Ollama API分析产品难度分数"""
try:
# 构建提示词
prompt = f"""请基于以下产品信息，分析其技术实现的难度分数（1-100分）：
产品名称：{product_name}
产品简介：{introduction}
请只返回一个整数分数，不需要其他解释。分数越高表示技术实现难度越大。
"""
# 调用Ollama API
response = requests.post(
'http://localhost:11434/api/generate',
json={
'model': 'llama3',
'prompt': prompt,
'format': 'json',
'stream': False
},
timeout=30
)
if response.status_code == 200:
data = response.json()
# 提取分数
score_text = data.get('response', '50').strip()
# 尝试从文本中提取数字
import re
numbers = re.findall(r'\d+', score_text)
if numbers:
score = int(numbers[0])
# 确保分数在1-100范围内
return max(1, min(100, score))
# 如果API调用失败或无法提取分数，返回默认值50
logger.warning(f"无法从Ollama API获取有效分数，返回默认值")
return 50
except Exception as e:
logger.error(f"调用Ollama API失败: {e}")
# 出错时返回默认值
return 50
@app.route('/static/<path:filename>')
def static_files(filename):
"""静态文件"""
@@ -194,7 +398,7 @@ def create_html_template():
}
.container {
max-width: 1400px;
width: 100%;
margin: 0 auto;
padding: 20px;
}
@@ -425,6 +629,60 @@ def create_html_template():
font-size: 1.1em;
}
.analyze-btn {
background: linear-gradient(135deg, #28a745, #20c997);
color: white;
border: none;
padding: 8px 16px;
border-radius: 6px;
font-size: 1em;
cursor: pointer;
transition: all 0.3s ease;
}
.analyze-btn:hover {
transform: translateY(-1px);
box-shadow: 0 4px 8px rgba(40, 167, 69, 0.3);
}
.analyze-btn:disabled {
background: #6c757d;
cursor: not-allowed;
transform: none;
box-shadow: none;
}
.progress-container {
margin-top: 20px;
padding: 15px;
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(10px);
border-radius: 12px;
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.1);
display: none;
}
.progress-bar {
width: 100%;
height: 20px;
background: #e9ecef;
border-radius: 10px;
overflow: hidden;
margin: 10px 0;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, #667eea, #764ba2);
transition: width 0.3s ease;
display: flex;
align-items: center;
justify-content: center;
color: white;
font-size: 0.8em;
font-weight: 600;
}
@media (max-width: 768px) {
.controls {
flex-direction: column;
@@ -449,15 +707,19 @@ def create_html_template():
<h1>🗄️ SQLite数据库查看器</h1>
<div class="controls">
<div class="control-group">
<label for="tableSelect">选择数据表:</label>
<select id="tableSelect">
<option value="">加载中...</option>
</select>
<label for="tableSelect">选择数据表:</label>
<select id="tableSelect">
<option value="">加载中...</option>
</select>
</div>
<div class="control-group">
<label for="analyzeBtn">分析:</label>
<button id="analyzeScoresBtn" class="analyze-btn">📊 分析缺失分数</button>
</div>
<div class="control-group">
<label for="searchField">筛选字段:</label>
<select id="searchField" disabled>
<option value="">选择字段...</option>
<select id="searchField" multiple disabled style="min-height: 80px;">
<option value="">所有文本字段</option>
</select>
</div>
<div class="control-group">
@@ -466,6 +728,7 @@ def create_html_template():
</div>
<button class="btn" onclick="loadData()">刷新数据</button>
</div>
</div>
</div>
<div class="data-container">
@@ -488,6 +751,15 @@ def create_html_template():
</div>
</div>
</div>
<div id="progressSection" class="progress-container">
<h3>📊 分数分析进度</h3>
<div class="progress-bar">
<div id="progressFill" class="progress-fill" style="width: 0%;">0%</div>
</div>
<p id="progressText">等待分析开始...</p>
</div>
</div>
<script>
let currentTable = '';
@@ -496,20 +768,88 @@ def create_html_template():
let totalPages = 1;
let currentData = null;
// 初始化
document.addEventListener('DOMContentLoaded', function() {
loadTables();
// 绑定事件
document.getElementById('tableSelect').addEventListener('change', function() {
currentTable = this.value;
currentPage = 1;
if (currentTable) {
loadTableStructure();
loadData();
}
// 绑定事件
document.addEventListener('DOMContentLoaded', function() {
loadTables();
// 绑定事件
document.getElementById('tableSelect').addEventListener('change', function() {
currentTable = this.value;
currentPage = 1;
if (currentTable) {
loadTableStructure();
loadData();
}
});
// 分析缺失分数按钮事件
document.getElementById('analyzeScoresBtn').addEventListener('click', analyzeMissingScores);
});
// 分析缺失分数
async function analyzeMissingScores() {
const analyzeBtn = document.getElementById('analyzeScoresBtn');
const progressSection = document.getElementById('progressSection');
const progressFill = document.getElementById('progressFill');
const progressText = document.getElementById('progressText');
try {
// 禁用按钮
analyzeBtn.disabled = true;
analyzeBtn.textContent = '分析中...';
// 显示进度条
progressSection.style.display = 'block';
progressFill.style.width = '0%';
progressFill.textContent = '0%';
progressText.textContent = '正在启动分析任务...';
// 启动分析任务
const response = await fetch('/api/analyze_missing_scores');
const data = await response.json();
if (data.task_id) {
// 定期查询任务状态
const interval = setInterval(async () => {
try {
const statusResponse = await fetch(`/api/update_task_status/${data.task_id}`);
const statusData = await statusResponse.json();
// 更新进度
progressFill.style.width = `${statusData.progress}%`;
progressFill.textContent = `${statusData.progress}%`;
if (statusData.status === 'running') {
progressText.textContent = `正在分析: ${statusData.completed}/${statusData.total} 个产品`;
} else if (statusData.status === 'completed') {
progressText.textContent = '🎉 所有缺失分数分析完成！';
clearInterval(interval);
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
// 如果当前正在查看product_analysis表，自动刷新
if (currentTable === 'product_analysis') {
loadData();
}
} else if (statusData.status === 'failed') {
progressText.textContent = `❌ 分析失败: ${statusData.error}`;
clearInterval(interval);
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
}
} catch (error) {
console.error('查询任务状态失败:', error);
progressText.textContent = '查询任务状态失败';
}
}, 2000);
}
} catch (error) {
console.error('启动分析任务失败:', error);
progressText.textContent = `启动分析失败: ${error.message}`;
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
}
document.getElementById('searchField').addEventListener('change', loadData);
document.getElementById('searchValue').addEventListener('input', debounce(loadData, 500));
});
@@ -556,7 +896,7 @@ def create_html_template():
const data = await response.json();
const searchField = document.getElementById('searchField');
searchField.innerHTML = '<option value="">所有字段</option>';
searchField.innerHTML = '<option value="">所有文本字段</option>';
data.structure.forEach(field => {
const option = document.createElement('option');
option.value = field.name;
@@ -578,13 +918,27 @@ def create_html_template():
const container = document.getElementById('dataContainer');
container.innerHTML = '<div class="loading">📊 数据加载中...</div>';
const searchField = document.getElementById('searchField').value;
const searchFieldSelect = document.getElementById('searchField');
const searchValue = document.getElementById('searchValue').value;
try {
let url = `/api/table/${currentTable}/data?page=${currentPage}&per_page=${perPage}`;
if (searchField && searchValue) {
url += `&search_field=${searchField}&search_value=${encodeURIComponent(searchValue)}`;
if (searchValue) {
// 获取所有选中的字段
const selectedFields = Array.from(searchFieldSelect.selectedOptions)
.map(option => option.value)
.filter(value => value !== '');
if (selectedFields.length > 0) {
// 如果选择了特定字段，传递所有选中的字段
selectedFields.forEach(field => {
url += `&search_field=${encodeURIComponent(field)}`;
});
} else {
// 否则使用"all"表示所有文本字段
url += '&search_field=all';
}
url += `&search_value=${encodeURIComponent(searchValue)}`;
}
const response = await fetch(url);
@@ -722,6 +1076,18 @@ if __name__ == '__main__':
)
''')
# 创建product_analysis表
cursor.execute('''
CREATE TABLE IF NOT EXISTS product_analysis (
id INTEGER PRIMARY KEY AUTOINCREMENT,
product_id INTEGER,
analysis TEXT,
difficulty_score REAL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (product_id) REFERENCES products(id)
)
''')
# 插入示例数据
sample_data = [
('产品A', '这是一个非常优秀的产品\n具有多种实用功能\n用户反馈很好', '高性能\n易用性\n稳定性', 99.99),
@@ -730,6 +1096,11 @@ if __name__ == '__main__':
]
cursor.executemany('INSERT INTO products (name, description, features, price) VALUES (?, ?, ?, ?)', sample_data)
# 插入一些分析数据
cursor.execute('INSERT INTO product_analysis (product_id, analysis, difficulty_score) VALUES (1, "产品分析示例", 75)')
cursor.execute('INSERT INTO product_analysis (product_id, analysis) VALUES (2, "产品分析示例，无分数")')
conn.commit()
conn.close()

7
product_info.json
View File

@@ -1,7 +0,0 @@
{
"name": "PaletteBrain",
"introduction": "PaletteBrain is a macOS application that lets you use ChatGPT with any application by using a shortcut. Create custom templates with your own shortcuts or use the default templates provided.",
"user_count": "225 followers",
"maker_link": "https://www.producthunt.com/products/raycast",
"maker_statement": "Hey Product Hunt 👋If you're new to Raycast: Think of it as your command center for Windows. Hit \"Alt + Space\", type what you need, and go. Launch apps, search files, run extensions like GitHub or Notion, ask AI. All without touching your mouse.Why we built this:We spent the last 5 years building Raycast for Mac. Hundreds of thousands of people now use it daily to cut through the noise. But Windows users deserve the same. Your tools shouldn't get in the way. They should help you focus and get things done. So today, we bring the same experience to your PC.What makes Raycast different:• Custom file search - Windows doesn't have proper indexing, so we built our own. Fast and accurate.• Hundreds of extensions - Control your smart home, search Notion, manage GitHub, translate text, find GIFs. Or build your own!• Free AI during beta - Ask questions to get answers with citations without a subscription needed.• Built for Windows - Familiar keyboard shortcuts, design that fits right in, and you can even launch your favorite games.What's coming next:We'll add AI Chat, Notes, and more features in the months ahead. We ship fast and want your feedback. So please let us know what you miss.Yesterday was Windows' 40th anniversary. It felt like the right moment to launch this. Try it and let us know what you think. What should we build next?"
}

1570
producthunt_scraper.log
View File

File diff suppressed because it is too large Load Diff

8
requirements_gui.txt
View File

@@ -1,8 +0,0 @@
# SQLite数据库查看器依赖包
PySide6>=6.5.0
loguru>=0.7.0
pyqt-test>=1.0.0
# 可选：用于GUI测试的额外依赖
pytest>=7.0.0
pytest-qt>=4.0.0

666
templates/index.html Normal file
View File

@@ -0,0 +1,666 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>SQLite数据库查看器</title>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
min-height: 100vh;
color: #333;
}
.container {
width: 100%;
margin: 0 auto;
padding: 20px;
}
.header {
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(10px);
border-radius: 15px;
padding: 25px;
margin-bottom: 25px;
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
}
.header h1 {
color: #2c3e50;
font-size: 2.5em;
margin-bottom: 10px;
text-align: center;
}
.controls {
display: flex;
gap: 20px;
align-items: center;
flex-wrap: wrap;
margin-top: 20px;
}
.control-group {
display: flex;
flex-direction: column;
gap: 8px;
}
.control-group label {
font-weight: 600;
color: #34495e;
font-size: 0.9em;
}
select, input {
padding: 12px 15px;
border: 2px solid #e0e6ed;
border-radius: 8px;
font-size: 14px;
transition: all 0.3s ease;
background: white;
}
select:focus, input:focus {
outline: none;
border-color: #667eea;
box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.1);
}
.btn {
padding: 12px 24px;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
border: none;
border-radius: 8px;
cursor: pointer;
font-weight: 600;
transition: all 0.3s ease;
text-decoration: none;
display: inline-block;
}
.btn:hover {
transform: translateY(-2px);
box-shadow: 0 5px 15px rgba(0, 0, 0, 0.2);
}
.data-container {
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(10px);
border-radius: 15px;
padding: 25px;
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
}
.table-info {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 20px;
padding: 15px;
background: #f8f9fa;
border-radius: 10px;
}
.table-info h2 {
color: #2c3e50;
font-size: 1.5em;
}
.stats {
display: flex;
gap: 20px;
font-size: 0.9em;
color: #7f8c8d;
}
.table-wrapper {
overflow-x: auto;
border-radius: 10px;
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.1);
}
table {
width: 100%;
border-collapse: collapse;
background: white;
font-size: 14px;
}
th {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
padding: 15px 12px;
text-align: left;
font-weight: 600;
position: sticky;
top: 0;
z-index: 10;
}
td {
padding: 12px;
border-bottom: 1px solid #ecf0f1;
vertical-align: top;
}
tr:nth-child(even) {
background-color: #f8f9fa;
}
tr:hover {
background-color: #e3f2fd;
transition: background-color 0.3s ease;
}
.multiline-cell {
white-space: pre-wrap;
line-height: 1.6;
max-height: 200px;
overflow-y: auto;
padding: 8px;
background: #fff3cd;
border-radius: 6px;
border-left: 4px solid #ffc107;
}
.normal-cell {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
max-width: 300px;
}
.empty-cell {
color: #95a5a6;
font-style: italic;
}
.pagination {
display: flex;
justify-content: center;
align-items: center;
gap: 10px;
margin-top: 25px;
flex-wrap: wrap;
}
.page-info {
color: #7f8c8d;
font-weight: 600;
}
.page-btn {
padding: 8px 12px;
border: 2px solid #e0e6ed;
background: white;
border-radius: 6px;
cursor: pointer;
transition: all 0.3s ease;
min-width: 40px;
text-align: center;
}
.page-btn:hover {
border-color: #667eea;
background: #667eea;
color: white;
}
.page-btn.active {
background: #667eea;
color: white;
border-color: #667eea;
}
.page-btn:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.loading {
text-align: center;
padding: 40px;
color: #7f8c8d;
font-size: 1.1em;
}
.error {
background: #f8d7da;
color: #721c24;
padding: 15px;
border-radius: 8px;
border: 1px solid #f5c6cb;
margin: 20px 0;
}
.no-data {
text-align: center;
padding: 40px;
color: #7f8c8d;
font-size: 1.1em;
}
.analyze-btn {
background: linear-gradient(135deg, #28a745, #20c997);
color: white;
border: none;
padding: 8px 16px;
border-radius: 6px;
font-size: 1em;
cursor: pointer;
transition: all 0.3s ease;
}
.analyze-btn:hover {
transform: translateY(-1px);
box-shadow: 0 4px 8px rgba(40, 167, 69, 0.3);
}
.analyze-btn:disabled {
background: #6c757d;
cursor: not-allowed;
transform: none;
box-shadow: none;
}
.progress-container {
margin-top: 20px;
padding: 15px;
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(10px);
border-radius: 12px;
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.1);
display: none;
}
.progress-bar {
width: 100%;
height: 20px;
background: #e9ecef;
border-radius: 10px;
overflow: hidden;
margin: 10px 0;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, #667eea, #764ba2);
transition: width 0.3s ease;
display: flex;
align-items: center;
justify-content: center;
color: white;
font-size: 0.8em;
font-weight: 600;
}
@media (max-width: 768px) {
.controls {
flex-direction: column;
align-items: stretch;
}
.table-info {
flex-direction: column;
gap: 15px;
text-align: center;
}
.stats {
justify-content: center;
}
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>🗄️ SQLite数据库查看器</h1>
<div class="controls">
<div class="control-group">
<label for="tableSelect">选择数据表:</label>
<select id="tableSelect">
<option value="">加载中...</option>
</select>
</div>
<div class="control-group">
<label for="analyzeBtn">分析:</label>
<button id="analyzeScoresBtn" class="analyze-btn">📊 分析缺失分数</button>
</div>
<div class="control-group">
<label for="searchField">筛选字段:</label>
<select id="searchField" multiple disabled style="min-height: 80px;">
<option value="">所有文本字段</option>
</select>
</div>
<div class="control-group">
<label for="searchValue">筛选内容:</label>
<input type="text" id="searchValue" placeholder="输入筛选内容..." disabled>
</div>
<button class="btn" onclick="loadData()">刷新数据</button>
</div>
</div>
</div>
<div class="data-container">
<div class="table-info">
<h2 id="tableName">请选择数据表</h2>
<div class="stats">
<span id="recordCount">记录数: 0</span>
<span id="pageInfo">第 0 页，共 0 页</span>
</div>
</div>
<div id="dataContainer">
<div class="no-data">请选择数据表以查看内容</div>
</div>
<div id="pagination" class="pagination" style="display: none;">
<button class="page-btn" onclick="changePage('prev')" id="prevBtn">上一页</button>
<span class="page-info" id="pageInfoDetail"></span>
<button class="page-btn" onclick="changePage('next')" id="nextBtn">下一页</button>
</div>
</div>
</div>
<div id="progressSection" class="progress-container">
<h3>📊 分数分析进度</h3>
<div class="progress-bar">
<div id="progressFill" class="progress-fill" style="width: 0%;">0%</div>
</div>
<p id="progressText">等待分析开始...</p>
</div>
</div>
<script>
let currentTable = '';
let currentPage = 1;
let perPage = 50;
let totalPages = 1;
let currentData = null;
// 绑定事件
document.addEventListener('DOMContentLoaded', function() {
loadTables();
// 绑定事件
document.getElementById('tableSelect').addEventListener('change', function() {
currentTable = this.value;
currentPage = 1;
if (currentTable) {
loadTableStructure();
loadData();
}
});
// 分析缺失分数按钮事件
document.getElementById('analyzeScoresBtn').addEventListener('click', analyzeMissingScores);
});
// 分析缺失分数
async function analyzeMissingScores() {
const analyzeBtn = document.getElementById('analyzeScoresBtn');
const progressSection = document.getElementById('progressSection');
const progressFill = document.getElementById('progressFill');
const progressText = document.getElementById('progressText');
try {
// 禁用按钮
analyzeBtn.disabled = true;
analyzeBtn.textContent = '分析中...';
// 显示进度条
progressSection.style.display = 'block';
progressFill.style.width = '0%';
progressFill.textContent = '0%';
progressText.textContent = '正在启动分析任务...';
// 启动分析任务
const response = await fetch('/api/analyze_missing_scores');
const data = await response.json();
if (data.task_id) {
// 定期查询任务状态
const interval = setInterval(async () => {
try {
const statusResponse = await fetch(`/api/update_task_status/${data.task_id}`);
const statusData = await statusResponse.json();
// 更新进度
progressFill.style.width = `${statusData.progress}%`;
progressFill.textContent = `${statusData.progress}%`;
if (statusData.status === 'running') {
progressText.textContent = `正在分析: ${statusData.completed}/${statusData.total} 个产品`;
} else if (statusData.status === 'completed') {
progressText.textContent = '🎉 所有缺失分数分析完成！';
clearInterval(interval);
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
// 如果当前正在查看product_analysis表，自动刷新
if (currentTable === 'product_analysis') {
loadData();
}
} else if (statusData.status === 'failed') {
progressText.textContent = `❌ 分析失败: ${statusData.error}`;
clearInterval(interval);
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
}
} catch (error) {
console.error('查询任务状态失败:', error);
progressText.textContent = '查询任务状态失败';
}
}, 2000);
}
} catch (error) {
console.error('启动分析任务失败:', error);
progressText.textContent = `启动分析失败: ${error.message}`;
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
}
document.getElementById('searchField').addEventListener('change', loadData);
document.getElementById('searchValue').addEventListener('input', debounce(loadData, 500));
});
// 防抖函数
function debounce(func, wait) {
let timeout;
return function executedFunction(...args) {
const later = () => {
clearTimeout(timeout);
func(...args);
};
clearTimeout(timeout);
timeout = setTimeout(later, wait);
};
}
// 加载表列表
async function loadTables() {
try {
const response = await fetch('/api/tables');
const data = await response.json();
const select = document.getElementById('tableSelect');
select.innerHTML = '<option value="">选择数据表...</option>';
data.tables.forEach(table => {
const option = document.createElement('option');
option.value = table;
option.textContent = table;
select.appendChild(option);
});
} catch (error) {
console.error('加载表列表失败:', error);
showError('加载表列表失败: ' + error.message);
}
}
// 加载表结构
async function loadTableStructure() {
if (!currentTable) return;
try {
const response = await fetch(`/api/table/${currentTable}/structure`);
const data = await response.json();
const searchField = document.getElementById('searchField');
searchField.innerHTML = '<option value="">所有文本字段</option>';
data.structure.forEach(field => {
const option = document.createElement('option');
option.value = field.name;
option.textContent = field.name;
searchField.appendChild(option);
});
searchField.disabled = false;
document.getElementById('searchValue').disabled = false;
} catch (error) {
console.error('加载表结构失败:', error);
}
}
// 加载数据
async function loadData() {
if (!currentTable) return;
const container = document.getElementById('dataContainer');
container.innerHTML = '<div class="loading">📊 数据加载中...</div>';
const searchFieldSelect = document.getElementById('searchField');
const searchValue = document.getElementById('searchValue').value;
try {
let url = `/api/table/${currentTable}/data?page=${currentPage}&per_page=${perPage}`;
if (searchValue) {
// 获取所有选中的字段
const selectedFields = Array.from(searchFieldSelect.selectedOptions)
.map(option => option.value)
.filter(value => value !== '');
if (selectedFields.length > 0) {
// 如果选择了特定字段，传递所有选中的字段
selectedFields.forEach(field => {
url += `&search_field=${encodeURIComponent(field)}`;
});
} else {
// 否则使用"all"表示所有文本字段
url += '&search_field=all';
}
url += `&search_value=${encodeURIComponent(searchValue)}`;
}
const response = await fetch(url);
currentData = await response.json();
displayData(currentData);
updatePagination();
} catch (error) {
console.error('加载数据失败:', error);
showError('加载数据失败: ' + error.message);
}
}
// 显示数据
function displayData(data) {
const container = document.getElementById('dataContainer');
if (!data.rows || data.rows.length === 0) {
container.innerHTML = '<div class="no-data">📭 没有找到数据</div>';
return;
}
let html = '<div class="table-wrapper"><table><thead><tr>';
// 表头
data.columns.forEach(col => {
html += `<th>${col}</th>`;
});
html += '</tr></thead><tbody>';
// 数据行
data.rows.forEach(row => {
html += '<tr>';
row.forEach(cell => {
if (cell.type === 'multiline') {
html += `<td><div class="multiline-cell">${escapeHtml(cell.value)}</div></td>`;
} else if (cell.type === 'empty') {
html += '<td><div class="empty-cell">空</div></td>';
} else {
html += `<td><div class="normal-cell">${escapeHtml(cell.value)}</div></td>`;
}
});
html += '</tr>';
});
html += '</tbody></table></div>';
container.innerHTML = html;
// 更新统计信息
document.getElementById('tableName').textContent = `📋 ${currentTable}`;
document.getElementById('recordCount').textContent = `记录数: ${data.total_count}`;
document.getElementById('pageInfo').textContent = `第 ${currentPage} 页，共 ${data.total_pages} 页`;
}
// 更新分页
function updatePagination() {
if (!currentData) return;
totalPages = currentData.total_pages;
const pagination = document.getElementById('pagination');
const prevBtn = document.getElementById('prevBtn');
const nextBtn = document.getElementById('nextBtn');
const pageInfo = document.getElementById('pageInfoDetail');
if (totalPages <= 1) {
pagination.style.display = 'none';
return;
}
pagination.style.display = 'flex';
prevBtn.disabled = currentPage <= 1;
nextBtn.disabled = currentPage >= totalPages;
pageInfo.textContent = `${currentPage} / ${totalPages}`;
}
// 翻页
function changePage(direction) {
if (direction === 'prev' && currentPage > 1) {
currentPage--;
loadData();
} else if (direction === 'next' && currentPage < totalPages) {
currentPage++;
loadData();
}
}
// HTML转义
function escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
// 显示错误
function showError(message) {
const container = document.getElementById('dataContainer');
container.innerHTML = `<div class="error">❌ ${message}</div>`;
}
</script>
</body>
</html>

31401
tophub_add_data_to_db.log
View File

File diff suppressed because it is too large Load Diff

BIN
tophub_data.db
View File

Binary file not shown.

2199
tophub_scraper.log
View File

File diff suppressed because it is too large Load Diff