xiaji/tophux_scrape
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

更新了web方式查看products的效果

Browse Source
This commit is contained in:
xiaji
2025-11-30 12:36:48 +08:00
parent 1c91dd45ed
commit ff7e114324
18 changed files with 21092 additions and 31986 deletions
Download Patch File Download Diff File Expand all files Collapse all files

451
product/web_sqlite_viewer.py
View File

@@ -11,11 +11,17 @@ import os
import json
from datetime import datetime
from loguru import logger
import threading
import time
import requests
app = Flask(__name__)
# 数据库路径
DB_PATH = os.path.join(os.path.dirname(__file__), 'product.db')
DB_PATH = os.path.join(os.path.dirname(__file__), 'products.db')
# 任务状态存储
analysis_tasks = {}
class SQLiteWebViewer:
def __init__(self, db_path):
@@ -68,25 +74,75 @@ class SQLiteWebViewer:
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# 获取总记录数
if search_field and search_value:
count_query = f"SELECT COUNT(*) FROM {table_name} WHERE {search_field} LIKE ?"
cursor.execute(count_query, (f'%{search_value}%',))
else:
count_query = f"SELECT COUNT(*) FROM {table_name}"
cursor.execute(count_query)
# 获取字段类型信息
field_types = {}
text_fields = []
cursor.execute(f"PRAGMA table_info({table_name});")
columns_info = cursor.fetchall()
for col_info in columns_info:
field_name = col_info[1]
field_type = col_info[2]
field_types[field_name] = field_type # 字段名 -> 字段类型
# 收集文本类型字段
if field_type.upper() not in ['INTEGER', 'REAL', 'FLOAT', 'NUMERIC']:
text_fields.append(field_name)
# 解析搜索条件
query_params = []
where_clause = ""
if search_value:
# 获取要搜索的字段列表
search_fields = []
if isinstance(search_field, list):
# 如果是列表，使用所有提供的字段
search_fields = [f for f in search_field if f in field_types]
elif search_field == "all":
# 如果是"all"，使用所有文本字段
search_fields = text_fields
elif search_field and search_field in field_types:
# 如果是单个字段，直接使用
search_fields = [search_field]
if search_fields:
conditions = []
for field in search_fields:
# 检查是否为数值比较操作符
import re
numeric_op_pattern = re.compile(r'^(<=?|>=?|=)(\d+(\.\d+)?)$')
match = numeric_op_pattern.match(search_value)
# 检查字段是否为数值类型
is_numeric_field = field_types.get(field, '').upper() in ['INTEGER', 'REAL', 'FLOAT', 'NUMERIC']
if match and is_numeric_field:
# 数值比较操作
operator = match.group(1)
value = match.group(2)
conditions.append(f"{field} {operator} ?")
query_params.append(float(value) if '.' in value else int(value))
logger.info(f"应用数值比较筛选: {field} {operator} {value}")
else:
# 默认文本模糊匹配
conditions.append(f"{field} LIKE ?")
query_params.append(f'%{search_value}%')
logger.info(f"应用文本模糊匹配: {field} LIKE '%{search_value}%'")
if conditions:
where_clause = " WHERE " + " OR ".join(conditions)
# 获取总记录数
count_query = f"SELECT COUNT(*) FROM {table_name}{where_clause}"
cursor.execute(count_query, query_params)
total_count = cursor.fetchone()[0]
# 获取分页数据
offset = (page - 1) * per_page
query_params.extend([per_page, offset])
if search_field and search_value:
data_query = f"SELECT * FROM {table_name} WHERE {search_field} LIKE ? LIMIT ? OFFSET ?"
cursor.execute(data_query, (f'%{search_value}%', per_page, offset))
else:
data_query = f"SELECT * FROM {table_name} LIMIT ? OFFSET ?"
cursor.execute(data_query, (per_page, offset))
data_query = f"SELECT * FROM {table_name}{where_clause} LIMIT ? OFFSET ?"
cursor.execute(data_query, query_params)
rows = cursor.fetchall()
@@ -100,8 +156,12 @@ class SQLiteWebViewer:
processed_rows = []
for row in rows:
processed_row = []
for cell in row:
if cell is None:
for i, cell in enumerate(row):
col_name = columns[i]
# 处理difficulty_score字段的缺失值
if col_name == "difficulty_score" and cell is None:
processed_row.append({'value': '未评分', 'type': 'empty'})
elif cell is None:
processed_row.append({'value': '', 'type': 'empty'})
elif isinstance(cell, str) and ('\n' in cell or len(cell) > 100):
# 多行文本或长文本
@@ -156,12 +216,156 @@ def get_table_data(table_name):
"""获取表数据"""
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', 50))
search_field = request.args.get('search_field')
# 获取所有search_field参数（可能是多个）
search_field = request.args.getlist('search_field')
# 如果只有一个且为空，使用单个值
if len(search_field) == 1:
search_field = search_field[0]
search_value = request.args.get('search_value')
data = viewer.get_table_data(table_name, page, per_page, search_field, search_value)
return jsonify(data)
@app.route('/api/analyze_missing_scores')
def analyze_missing_scores():
"""触发缺失分数分析任务"""
task_id = str(int(time.time()))
analysis_tasks[task_id] = {
'status': 'running',
'progress': 0,
'total': 0,
'completed': 0,
'error': None,
'start_time': datetime.now().isoformat()
}
# 在后台线程中执行分析
def run_analysis():
try:
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
# 检查product_analysis表是否存在difficulty_score字段
cursor.execute("PRAGMA table_info(product_analysis)")
columns = [col[1] for col in cursor.fetchall()]
if 'difficulty_score' not in columns:
# 如果不存在，添加该字段
cursor.execute("ALTER TABLE product_analysis ADD COLUMN difficulty_score REAL")
conn.commit()
logger.info("添加了difficulty_score字段")
# 查询缺失分数的产品
cursor.execute("""
SELECT pa.id, p.name, p.description
FROM product_analysis pa
JOIN products p ON pa.product_id = p.id
WHERE pa.difficulty_score IS NULL OR pa.difficulty_score = ''
""")
missing_scores = cursor.fetchall()
total = len(missing_scores)
analysis_tasks[task_id]['total'] = total
logger.info(f"找到 {total} 个缺失分数的产品")
for i, (analysis_id, product_name, introduction) in enumerate(missing_scores):
try:
# 调用Ollama API分析难度分数
score = analyze_product_difficulty(product_name, introduction)
# 更新数据库
cursor.execute(
"UPDATE product_analysis SET difficulty_score = ? WHERE id = ?",
(score, analysis_id)
)
conn.commit()
analysis_tasks[task_id]['completed'] = i + 1
analysis_tasks[task_id]['progress'] = int((i + 1) / total * 100)
logger.info(f"已分析产品 {i+1}/{total}: {product_name}, 分数: {score}")
# 避免频繁调用API
time.sleep(1)
except Exception as e:
logger.error(f"分析产品 {product_name} 失败: {e}")
# 继续处理下一个产品
continue
analysis_tasks[task_id]['status'] = 'completed'
logger.info("所有缺失分数分析完成")
except Exception as e:
logger.error(f"分析任务失败: {e}")
analysis_tasks[task_id]['status'] = 'failed'
analysis_tasks[task_id]['error'] = str(e)
finally:
conn.close()
analysis_tasks[task_id]['end_time'] = datetime.now().isoformat()
threading.Thread(target=run_analysis).start()
return jsonify({
'task_id': task_id,
'status': 'started',
'message': '分析任务已启动，请通过task_id查询进度'
})
@app.route('/api/update_task_status/<task_id>')
def update_task_status(task_id):
"""获取任务状态"""
if task_id in analysis_tasks:
return jsonify(analysis_tasks[task_id])
else:
return jsonify({
'error': 'Task not found',
'message': '找不到指定的任务ID'
}), 404
def analyze_product_difficulty(product_name, introduction):
"""调用Ollama API分析产品难度分数"""
try:
# 构建提示词
prompt = f"""请基于以下产品信息，分析其技术实现的难度分数（1-100分）：
产品名称：{product_name}
产品简介：{introduction}
请只返回一个整数分数，不需要其他解释。分数越高表示技术实现难度越大。
"""
# 调用Ollama API
response = requests.post(
'http://localhost:11434/api/generate',
json={
'model': 'llama3',
'prompt': prompt,
'format': 'json',
'stream': False
},
timeout=30
)
if response.status_code == 200:
data = response.json()
# 提取分数
score_text = data.get('response', '50').strip()
# 尝试从文本中提取数字
import re
numbers = re.findall(r'\d+', score_text)
if numbers:
score = int(numbers[0])
# 确保分数在1-100范围内
return max(1, min(100, score))
# 如果API调用失败或无法提取分数，返回默认值50
logger.warning(f"无法从Ollama API获取有效分数，返回默认值")
return 50
except Exception as e:
logger.error(f"调用Ollama API失败: {e}")
# 出错时返回默认值
return 50
@app.route('/static/<path:filename>')
def static_files(filename):
"""静态文件"""
@@ -194,7 +398,7 @@ def create_html_template():
}
.container {
max-width: 1400px;
width: 100%;
margin: 0 auto;
padding: 20px;
}
@@ -425,6 +629,60 @@ def create_html_template():
font-size: 1.1em;
}
.analyze-btn {
background: linear-gradient(135deg, #28a745, #20c997);
color: white;
border: none;
padding: 8px 16px;
border-radius: 6px;
font-size: 1em;
cursor: pointer;
transition: all 0.3s ease;
}
.analyze-btn:hover {
transform: translateY(-1px);
box-shadow: 0 4px 8px rgba(40, 167, 69, 0.3);
}
.analyze-btn:disabled {
background: #6c757d;
cursor: not-allowed;
transform: none;
box-shadow: none;
}
.progress-container {
margin-top: 20px;
padding: 15px;
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(10px);
border-radius: 12px;
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.1);
display: none;
}
.progress-bar {
width: 100%;
height: 20px;
background: #e9ecef;
border-radius: 10px;
overflow: hidden;
margin: 10px 0;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, #667eea, #764ba2);
transition: width 0.3s ease;
display: flex;
align-items: center;
justify-content: center;
color: white;
font-size: 0.8em;
font-weight: 600;
}
@media (max-width: 768px) {
.controls {
flex-direction: column;
@@ -449,15 +707,19 @@ def create_html_template():
<h1>🗄️ SQLite数据库查看器</h1>
<div class="controls">
<div class="control-group">
<label for="tableSelect">选择数据表:</label>
<select id="tableSelect">
<option value="">加载中...</option>
</select>
<label for="tableSelect">选择数据表:</label>
<select id="tableSelect">
<option value="">加载中...</option>
</select>
</div>
<div class="control-group">
<label for="analyzeBtn">分析:</label>
<button id="analyzeScoresBtn" class="analyze-btn">📊 分析缺失分数</button>
</div>
<div class="control-group">
<label for="searchField">筛选字段:</label>
<select id="searchField" disabled>
<option value="">选择字段...</option>
<select id="searchField" multiple disabled style="min-height: 80px;">
<option value="">所有文本字段</option>
</select>
</div>
<div class="control-group">
@@ -466,6 +728,7 @@ def create_html_template():
</div>
<button class="btn" onclick="loadData()">刷新数据</button>
</div>
</div>
</div>
<div class="data-container">
@@ -488,6 +751,15 @@ def create_html_template():
</div>
</div>
</div>
<div id="progressSection" class="progress-container">
<h3>📊 分数分析进度</h3>
<div class="progress-bar">
<div id="progressFill" class="progress-fill" style="width: 0%;">0%</div>
</div>
<p id="progressText">等待分析开始...</p>
</div>
</div>
<script>
let currentTable = '';
@@ -496,20 +768,88 @@ def create_html_template():
let totalPages = 1;
let currentData = null;
// 初始化
document.addEventListener('DOMContentLoaded', function() {
loadTables();
// 绑定事件
document.getElementById('tableSelect').addEventListener('change', function() {
currentTable = this.value;
currentPage = 1;
if (currentTable) {
loadTableStructure();
loadData();
}
// 绑定事件
document.addEventListener('DOMContentLoaded', function() {
loadTables();
// 绑定事件
document.getElementById('tableSelect').addEventListener('change', function() {
currentTable = this.value;
currentPage = 1;
if (currentTable) {
loadTableStructure();
loadData();
}
});
// 分析缺失分数按钮事件
document.getElementById('analyzeScoresBtn').addEventListener('click', analyzeMissingScores);
});
// 分析缺失分数
async function analyzeMissingScores() {
const analyzeBtn = document.getElementById('analyzeScoresBtn');
const progressSection = document.getElementById('progressSection');
const progressFill = document.getElementById('progressFill');
const progressText = document.getElementById('progressText');
try {
// 禁用按钮
analyzeBtn.disabled = true;
analyzeBtn.textContent = '分析中...';
// 显示进度条
progressSection.style.display = 'block';
progressFill.style.width = '0%';
progressFill.textContent = '0%';
progressText.textContent = '正在启动分析任务...';
// 启动分析任务
const response = await fetch('/api/analyze_missing_scores');
const data = await response.json();
if (data.task_id) {
// 定期查询任务状态
const interval = setInterval(async () => {
try {
const statusResponse = await fetch(`/api/update_task_status/${data.task_id}`);
const statusData = await statusResponse.json();
// 更新进度
progressFill.style.width = `${statusData.progress}%`;
progressFill.textContent = `${statusData.progress}%`;
if (statusData.status === 'running') {
progressText.textContent = `正在分析: ${statusData.completed}/${statusData.total} 个产品`;
} else if (statusData.status === 'completed') {
progressText.textContent = '🎉 所有缺失分数分析完成！';
clearInterval(interval);
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
// 如果当前正在查看product_analysis表，自动刷新
if (currentTable === 'product_analysis') {
loadData();
}
} else if (statusData.status === 'failed') {
progressText.textContent = `❌ 分析失败: ${statusData.error}`;
clearInterval(interval);
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
}
} catch (error) {
console.error('查询任务状态失败:', error);
progressText.textContent = '查询任务状态失败';
}
}, 2000);
}
} catch (error) {
console.error('启动分析任务失败:', error);
progressText.textContent = `启动分析失败: ${error.message}`;
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
}
document.getElementById('searchField').addEventListener('change', loadData);
document.getElementById('searchValue').addEventListener('input', debounce(loadData, 500));
});
@@ -556,7 +896,7 @@ def create_html_template():
const data = await response.json();
const searchField = document.getElementById('searchField');
searchField.innerHTML = '<option value="">所有字段</option>';
searchField.innerHTML = '<option value="">所有文本字段</option>';
data.structure.forEach(field => {
const option = document.createElement('option');
option.value = field.name;
@@ -578,13 +918,27 @@ def create_html_template():
const container = document.getElementById('dataContainer');
container.innerHTML = '<div class="loading">📊 数据加载中...</div>';
const searchField = document.getElementById('searchField').value;
const searchFieldSelect = document.getElementById('searchField');
const searchValue = document.getElementById('searchValue').value;
try {
let url = `/api/table/${currentTable}/data?page=${currentPage}&per_page=${perPage}`;
if (searchField && searchValue) {
url += `&search_field=${searchField}&search_value=${encodeURIComponent(searchValue)}`;
if (searchValue) {
// 获取所有选中的字段
const selectedFields = Array.from(searchFieldSelect.selectedOptions)
.map(option => option.value)
.filter(value => value !== '');
if (selectedFields.length > 0) {
// 如果选择了特定字段，传递所有选中的字段
selectedFields.forEach(field => {
url += `&search_field=${encodeURIComponent(field)}`;
});
} else {
// 否则使用"all"表示所有文本字段
url += '&search_field=all';
}
url += `&search_value=${encodeURIComponent(searchValue)}`;
}
const response = await fetch(url);
@@ -722,6 +1076,18 @@ if __name__ == '__main__':
)
''')
# 创建product_analysis表
cursor.execute('''
CREATE TABLE IF NOT EXISTS product_analysis (
id INTEGER PRIMARY KEY AUTOINCREMENT,
product_id INTEGER,
analysis TEXT,
difficulty_score REAL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (product_id) REFERENCES products(id)
)
''')
# 插入示例数据
sample_data = [
('产品A', '这是一个非常优秀的产品\n具有多种实用功能\n用户反馈很好', '高性能\n易用性\n稳定性', 99.99),
@@ -730,6 +1096,11 @@ if __name__ == '__main__':
]
cursor.executemany('INSERT INTO products (name, description, features, price) VALUES (?, ?, ?, ?)', sample_data)
# 插入一些分析数据
cursor.execute('INSERT INTO product_analysis (product_id, analysis, difficulty_score) VALUES (1, "产品分析示例", 75)')
cursor.execute('INSERT INTO product_analysis (product_id, analysis) VALUES (2, "产品分析示例，无分数")')
conn.commit()
conn.close()