xiaji/tophux_scrape
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

更新了web方式查看products的效果

Browse Source
This commit is contained in:
xiaji
2025-11-30 12:36:48 +08:00
parent 1c91dd45ed
commit ff7e114324
18 changed files with 21092 additions and 31986 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
product/__pycache__/web_sqlite_viewer.cpython-313.pyc Normal file
View File

Binary file not shown.

146
product/integrated_product_system.py
View File

@@ -322,27 +322,59 @@ class IntegratedProductSystem:
logger.error(f"调用Ollama AI API时出错: {e}")
return None
def parse_ai_response(self, response: str) -> Tuple[str, str, str]:
"""解析AI响应内容"""
def parse_ai_response(self, response: str) -> Tuple[str, str, str, int]:
"""解析AI响应内容，提取产品名称、简介、难度描述和难度分数"""
try:
# 使用/分割响应内容
parts = response.split('/')
product_name = ""
product_intro = ""
difficulty = ""
difficulty_score = None
if len(parts) >= 3:
product_name = parts[0].strip()
product_intro = parts[1].strip()
difficulty = parts[2].strip()
logger.info(f"解析结果: 名称='{product_name}', 简介='{product_intro[:30]}...', 难度='{difficulty}'")
return product_name, product_intro, difficulty
# 从难度描述中提取分数
import re
# 尝试匹配数字分数
score_match = re.search(r'\b(\d+)\b分|\b难度(\d+)\b|\b(\d+)\b', difficulty)
if score_match:
# 获取第一个匹配的数字
for group in score_match.groups():
if group:
difficulty_score = int(group)
break
# 如果没有提取到分数，根据关键词设置默认分数
if difficulty_score is None:
difficulty_lower = difficulty.lower()
if any(keyword in difficulty_lower for keyword in ['高', '很难', '非常难', '复杂']):
difficulty_score = 85
elif any(keyword in difficulty_lower for keyword in ['中', '一般', '适中', '普通']):
difficulty_score = 60
elif any(keyword in difficulty_lower for keyword in ['低', '简单', '容易']):
difficulty_score = 35
else:
difficulty_score = 50 # 默认中等难度
logger.info(f"提取到难度分数: {difficulty_score}")
else:
logger.warning(f"响应格式不符合预期: {response}")
# 如果格式不符合，返回原始内容
return "", response, ""
difficulty = response
difficulty_score = 50 # 默认中等难度
return product_name, product_intro, difficulty, difficulty_score
except Exception as e:
logger.error(f"解析AI响应失败: {e}")
return "", response, ""
return "", response, "", 50
def check_product_exists_in_analysis(self, conn: sqlite3.Connection, original_name: str) -> bool:
"""检查产品是否已存在于分析结果表中"""
@@ -367,19 +399,23 @@ class IntegratedProductSystem:
def save_analysis_result(self, conn: sqlite3.Connection,
original_id: int, original_name: str,
product_name: str, difficulty: str, ai_response: str):
"""保存分析结果到数据库"""
product_name: str, difficulty: str, ai_response: str, difficulty_score: int = None):
"""保存分析结果到数据库，包括难度分数"""
try:
cursor = conn.cursor()
# 如果没有提供难度分数，设置默认值50
if difficulty_score is None:
difficulty_score = 50
cursor.execute("""
INSERT INTO product_analysis
(original_id, original_name, product_name, development_difficulty, ai_response)
VALUES (?, ?, ?, ?, ?)
""", (original_id, original_name, product_name, difficulty, ai_response))
(original_id, original_name, product_name, development_difficulty, difficulty_score, ai_response)
VALUES (?, ?, ?, ?, ?, ?)
""", (original_id, original_name, product_name, difficulty, difficulty_score, ai_response))
conn.commit()
logger.success(f"保存分析结果成功: {product_name}")
logger.success(f"保存分析结果成功: {product_name}, 难度分数: {difficulty_score}")
except Exception as e:
logger.error(f"保存分析结果失败: {e}")
@@ -435,11 +471,11 @@ class IntegratedProductSystem:
logger.info(f"API调用成功，正在处理数据...")
# 解析响应
product_name, product_intro, difficulty = self.parse_ai_response(ai_response)
product_name, product_intro, difficulty, difficulty_score = self.parse_ai_response(ai_response)
# 保存结果（不再保存product_intro，避免与ai_response重复）
self.save_analysis_result(conn, original_id, name,
product_name, difficulty, ai_response)
product_name, difficulty, ai_response, difficulty_score)
success_count += 1
# 显示完成状态
@@ -549,8 +585,86 @@ class IntegratedProductSystem:
except Exception as e:
logger.error(f"显示抓取结果失败: {e}")
def analyze_missing_scores(self):
"""分析并补充缺失难度分数的产品"""
logger.info("=== 开始分析缺失难度分数的产品 ===")
conn = None
try:
# 连接数据库
conn = self.connect_to_database()
cursor = conn.cursor()
# 查询缺失难度分数的产品
cursor.execute("""
SELECT pa.id, p.name, p.introduction, pa.ai_response
FROM product_analysis pa
JOIN products p ON pa.original_id = p.id
WHERE pa.difficulty_score IS NULL OR pa.difficulty_score = ''
""")
products_with_missing_scores = cursor.fetchall()
logger.info(f"找到 {len(products_with_missing_scores)} 个缺失难度分数的产品")
if not products_with_missing_scores:
logger.info("没有发现缺失难度分数的产品")
return
# 为每个缺失分数的产品分析并更新分数
updated_count = 0
for i, (analysis_id, name, introduction, ai_response) in enumerate(products_with_missing_scores, 1):
logger.info(f"处理缺失分数的产品 {i}/{len(products_with_missing_scores)}: {name}")
# 如果已有AI响应，从响应中重新提取分数
difficulty_score = None
if ai_response:
try:
_, _, _, difficulty_score = self.parse_ai_response(ai_response)
logger.info(f"从现有AI响应中提取分数: {difficulty_score}")
except Exception as e:
logger.error(f"从现有响应提取分数失败: {e}")
# 如果无法从现有响应提取，重新调用API
if difficulty_score is None:
logger.info(f"重新调用API分析产品: {name}")
ai_response = self.call_ollama_ai_api(name, introduction)
if ai_response:
_, _, _, difficulty_score = self.parse_ai_response(ai_response)
# 更新AI响应
cursor.execute("""
UPDATE product_analysis
SET ai_response = ?
WHERE id = ?
""", (ai_response, analysis_id))
# 更新难度分数
if difficulty_score is not None:
cursor.execute("""
UPDATE product_analysis
SET difficulty_score = ?
WHERE id = ?
""", (difficulty_score, analysis_id))
conn.commit()
updated_count += 1
logger.success(f"成功更新产品 '{name}' 的难度分数为 {difficulty_score}")
else:
logger.warning(f"无法为产品 '{name}' 确定难度分数")
# 避免API调用过于频繁
if i < len(products_with_missing_scores):
time.sleep(2)
logger.success(f"缺失分数分析完成! 成功更新 {updated_count} 个产品的难度分数")
except Exception as e:
logger.error(f"分析缺失分数过程中出错: {e}")
finally:
if conn:
conn.close()
logger.info("数据库连接已关闭")
async def run_full_workflow_async(self, max_products=None, analyze_only=False):
"""异步运行完整工作流程：抓取+分析"""
"""异步运行完整工作流程：抓取+分析+补充缺失分数"""
logger.info("=== 开始全功能产品系统工作流程 ===")
# 初始化数据库
@@ -567,6 +681,10 @@ class IntegratedProductSystem:
logger.info("步骤2: 开始AI分析产品数据...")
self.analyze_products(max_products)
# 步骤3: 分析并补充缺失的难度分数
logger.info("步骤3: 开始分析并补充缺失的难度分数...")
self.analyze_missing_scores()
logger.success("=== 全功能产品系统工作流程完成 ===")
def run_full_workflow(self, max_products=None, analyze_only=False):

BIN
product/product.db Normal file
View File

Binary file not shown.

BIN
product/products.db
View File

Binary file not shown.

532
product/templates/index.html Normal file
View File

@@ -0,0 +1,532 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>SQLite数据库查看器</title>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
min-height: 100vh;
color: #333;
}
.container {
width: 100%;
margin: 0 auto;
padding: 20px;
}
.header {
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(10px);
border-radius: 15px;
padding: 25px;
margin-bottom: 25px;
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
}
.header h1 {
color: #2c3e50;
font-size: 2.5em;
margin-bottom: 10px;
text-align: center;
}
.controls {
display: flex;
gap: 20px;
align-items: center;
flex-wrap: wrap;
margin-top: 20px;
}
.control-group {
display: flex;
flex-direction: column;
gap: 8px;
}
.control-group label {
font-weight: 600;
color: #34495e;
font-size: 0.9em;
}
select, input {
padding: 12px 15px;
border: 2px solid #e0e6ed;
border-radius: 8px;
font-size: 14px;
transition: all 0.3s ease;
background: white;
}
select:focus, input:focus {
outline: none;
border-color: #667eea;
box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.1);
}
.btn {
padding: 12px 24px;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
border: none;
border-radius: 8px;
cursor: pointer;
font-weight: 600;
transition: all 0.3s ease;
text-decoration: none;
display: inline-block;
}
.btn:hover {
transform: translateY(-2px);
box-shadow: 0 5px 15px rgba(0, 0, 0, 0.2);
}
.data-container {
width: 100%;
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(10px);
border-radius: 15px;
padding: 25px;
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
}
.table-info {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 20px;
padding: 15px;
background: #f8f9fa;
border-radius: 10px;
}
.table-info h2 {
color: #2c3e50;
font-size: 1.5em;
}
.stats {
display: flex;
gap: 20px;
font-size: 0.9em;
color: #7f8c8d;
}
.table-wrapper {
width: 100%;
overflow-x: auto;
border-radius: 10px;
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.1);
}
table {
width: 100%;
border-collapse: collapse;
background: white;
font-size: 14px;
}
th {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
padding: 15px 12px;
text-align: left;
font-weight: 600;
position: sticky;
top: 0;
z-index: 10;
}
td {
padding: 12px;
border-bottom: 1px solid #ecf0f1;
vertical-align: top;
}
tr:nth-child(even) {
background-color: #f8f9fa;
}
tr:hover {
background-color: #e3f2fd;
transition: background-color 0.3s ease;
}
.multiline-cell {
white-space: pre-wrap;
line-height: 1.6;
max-height: 200px;
overflow-y: auto;
padding: 8px;
background: #fff3cd;
border-radius: 6px;
border-left: 4px solid #ffc107;
}
.normal-cell {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
max-width: 300px;
}
.empty-cell {
color: #95a5a6;
font-style: italic;
}
.pagination {
display: flex;
justify-content: center;
align-items: center;
gap: 10px;
margin-top: 25px;
flex-wrap: wrap;
}
.page-info {
color: #7f8c8d;
font-weight: 600;
}
.page-btn {
padding: 8px 12px;
border: 2px solid #e0e6ed;
background: white;
border-radius: 6px;
cursor: pointer;
transition: all 0.3s ease;
min-width: 40px;
text-align: center;
}
.page-btn:hover {
border-color: #667eea;
background: #667eea;
color: white;
}
.page-btn.active {
background: #667eea;
color: white;
border-color: #667eea;
}
.page-btn:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.loading {
text-align: center;
padding: 40px;
color: #7f8c8d;
font-size: 1.1em;
}
.error {
background: #f8d7da;
color: #721c24;
padding: 15px;
border-radius: 8px;
border: 1px solid #f5c6cb;
margin: 20px 0;
}
.no-data {
text-align: center;
padding: 40px;
color: #7f8c8d;
font-size: 1.1em;
}
@media (max-width: 768px) {
.controls {
flex-direction: column;
align-items: stretch;
}
.table-info {
flex-direction: column;
gap: 15px;
text-align: center;
}
.stats {
justify-content: center;
}
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>🗄️ SQLite数据库查看器</h1>
<div class="controls">
<div class="control-group">
<label for="tableSelect">选择数据表:</label>
<select id="tableSelect">
<option value="">加载中...</option>
</select>
</div>
<div class="control-group">
<label for="searchField">筛选字段:</label>
<select id="searchField" multiple disabled style="min-height: 80px;">
<option value="">所有文本字段</option>
</select>
</div>
<div class="control-group">
<label for="searchValue">筛选内容:</label>
<input type="text" id="searchValue" placeholder="输入筛选内容..." disabled>
</div>
<button class="btn" onclick="loadData()">刷新数据</button>
</div>
</div>
<div class="data-container">
<div class="table-info">
<h2 id="tableName">请选择数据表</h2>
<div class="stats">
<span id="recordCount">记录数: 0</span>
<span id="pageInfo">第 0 页，共 0 页</span>
</div>
</div>
<div id="dataContainer">
<div class="no-data">请选择数据表以查看内容</div>
</div>
<div id="pagination" class="pagination" style="display: none;">
<button class="page-btn" onclick="changePage('prev')" id="prevBtn">上一页</button>
<span class="page-info" id="pageInfoDetail"></span>
<button class="page-btn" onclick="changePage('next')" id="nextBtn">下一页</button>
</div>
</div>
</div>
<script>
let currentTable = '';
let currentPage = 1;
let perPage = 50;
let totalPages = 1;
let currentData = null;
// 绑定事件
document.addEventListener('DOMContentLoaded', function() {
loadTables();
// 绑定事件
document.getElementById('tableSelect').addEventListener('change', function() {
currentTable = this.value;
currentPage = 1;
if (currentTable) {
loadTableStructure();
loadData();
}
});
document.getElementById('searchField').addEventListener('change', loadData);
document.getElementById('searchValue').addEventListener('input', debounce(loadData, 500));
});
// 防抖函数
function debounce(func, wait) {
let timeout;
return function executedFunction(...args) {
const later = () => {
clearTimeout(timeout);
func(...args);
};
clearTimeout(timeout);
timeout = setTimeout(later, wait);
};
}
// 加载表列表
async function loadTables() {
try {
const response = await fetch('/api/tables');
const data = await response.json();
const select = document.getElementById('tableSelect');
select.innerHTML = '<option value="">选择数据表...</option>';
data.tables.forEach(table => {
const option = document.createElement('option');
option.value = table;
option.textContent = table;
select.appendChild(option);
});
} catch (error) {
console.error('加载表列表失败:', error);
showError('加载表列表失败: ' + error.message);
}
}
// 加载表结构
async function loadTableStructure() {
if (!currentTable) return;
try {
const response = await fetch(`/api/table/${currentTable}/structure`);
const data = await response.json();
const searchField = document.getElementById('searchField');
searchField.innerHTML = '<option value="">所有文本字段</option>';
data.structure.forEach(field => {
const option = document.createElement('option');
option.value = field.name;
option.textContent = field.name;
searchField.appendChild(option);
});
searchField.disabled = false;
document.getElementById('searchValue').disabled = false;
} catch (error) {
console.error('加载表结构失败:', error);
}
}
// 加载数据
async function loadData() {
if (!currentTable) return;
const container = document.getElementById('dataContainer');
container.innerHTML = '<div class="loading">📊 数据加载中...</div>';
const searchFieldSelect = document.getElementById('searchField');
const searchValue = document.getElementById('searchValue').value;
try {
let url = `/api/table/${currentTable}/data?page=${currentPage}&per_page=${perPage}`;
if (searchValue) {
// 获取所有选中的字段
const selectedFields = Array.from(searchFieldSelect.selectedOptions)
.map(option => option.value)
.filter(value => value !== '');
if (selectedFields.length > 0) {
// 如果选择了特定字段，传递所有选中的字段
selectedFields.forEach(field => {
url += `&search_field=${encodeURIComponent(field)}`;
});
} else {
// 否则使用"all"表示所有文本字段
url += '&search_field=all';
}
url += `&search_value=${encodeURIComponent(searchValue)}`;
}
const response = await fetch(url);
currentData = await response.json();
displayData(currentData);
updatePagination();
} catch (error) {
console.error('加载数据失败:', error);
showError('加载数据失败: ' + error.message);
}
}
// 显示数据
function displayData(data) {
const container = document.getElementById('dataContainer');
if (!data.rows || data.rows.length === 0) {
container.innerHTML = '<div class="no-data">📭 没有找到数据</div>';
return;
}
let html = '<div class="table-wrapper"><table><thead><tr>';
// 表头
data.columns.forEach(col => {
html += `<th>${col}</th>`;
});
html += '</tr></thead><tbody>';
// 数据行
data.rows.forEach(row => {
html += '<tr>';
row.forEach(cell => {
if (cell.type === 'multiline') {
html += `<td><div class="multiline-cell">${escapeHtml(cell.value)}</div></td>`;
} else if (cell.type === 'empty') {
html += '<td><div class="empty-cell">空</div></td>';
} else {
html += `<td><div class="normal-cell">${escapeHtml(cell.value)}</div></td>`;
}
});
html += '</tr>';
});
html += '</tbody></table></div>';
container.innerHTML = html;
// 更新统计信息
document.getElementById('tableName').textContent = `📋 ${currentTable}`;
document.getElementById('recordCount').textContent = `记录数: ${data.total_count}`;
document.getElementById('pageInfo').textContent = `第 ${currentPage} 页，共 ${data.total_pages} 页`;
}
// 更新分页
function updatePagination() {
if (!currentData) return;
totalPages = currentData.total_pages;
const pagination = document.getElementById('pagination');
const prevBtn = document.getElementById('prevBtn');
const nextBtn = document.getElementById('nextBtn');
const pageInfo = document.getElementById('pageInfoDetail');
if (totalPages <= 1) {
pagination.style.display = 'none';
return;
}
pagination.style.display = 'flex';
prevBtn.disabled = currentPage <= 1;
nextBtn.disabled = currentPage >= totalPages;
pageInfo.textContent = `${currentPage} / ${totalPages}`;
}
// 翻页
function changePage(direction) {
if (direction === 'prev' && currentPage > 1) {
currentPage--;
loadData();
} else if (direction === 'next' && currentPage < totalPages) {
currentPage++;
loadData();
}
}
// HTML转义
function escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
// 显示错误
function showError(message) {
const container = document.getElementById('dataContainer');
container.innerHTML = `<div class="error">❌ ${message}</div>`;
}
</script>
</body>
</html>

166
product/web_sqlite_comparison.md
View File

@@ -1,166 +0,0 @@
# Web SQLite查看器对比文档
## 概述
我为您创建了两个不同风格的Web SQLite数据库查看器，都支持现代化的界面、内容筛选和动态行高调整功能。
## 🚀 现代化版本 (modern_sqlite_viewer.py)
**访问地址**: http://localhost:5001
### 特点
- **技术栈**: Flask + Bootstrap 5 + DataTables + jQuery
- **界面风格**: 现代化渐变设计，卡片式布局
- **功能特性**:
- 服务器端分页处理，支持大数据集
- 高级搜索和筛选功能
- 响应式设计，完美适配移动端
- 动态加载指示器
- 专业的数据统计面板
- 列排序和分页控制
- 多行内容智能识别和美化显示
### 适用场景
- 需要处理大量数据（数千条记录以上）
- 需要专业级的数据分析和浏览功能
- 需要移动端友好的界面
- 需要高级的数据操作功能
## 🎯 轻量级版本 (simple_sqlite_viewer.py)
**访问地址**: http://localhost:5002
### 特点
- **技术栈**: Flask + 纯原生HTML/CSS/JS（无外部依赖）
- **界面风格**: 简洁优雅，内联样式
- **功能特性**:
- 客户端数据处理，快速响应
- 轻量级搜索功能
- 无外部依赖，加载速度快
- 简洁的统计信息
- 自适应行高显示
- 移动端适配
### 适用场景
- 数据量较小（几百条记录以内）
- 需要快速部署和访问
- 网络环境较差或需要离线使用
- 偏好简洁无依赖的解决方案
## 🎨 共同特性
### 动态行高调整
两个版本都实现了智能的行高调整：
- **自动识别**: 自动检测多行文本内容
- **美观显示**: 多行内容使用渐变背景和高亮边框
- **自然变化**: 行高根据内容长度自然调整，无突兀感
### 内容筛选功能
- **全局搜索**: 在所有列中搜索匹配内容
- **实时筛选**: 输入时即时显示结果
- **高亮显示**: 搜索结果清晰标识
### 数据库支持
- **自动创建**: 如果product.db不存在，自动创建示例数据库
- **表结构识别**: 自动识别所有表和列结构
- **数据类型处理**: 智能处理各种数据类型（文本、数字、日期等）
## 🔧 技术实现亮点
### 多行内容处理
```python
# 智能检测和处理多行文本
if isinstance(cell, str):
if '\n' in cell:
# 多行文本，用<br>替换换行符
formatted_cell = cell.replace('\n', '<br>')
row_data[col] = f'<div class="multiline-content">{formatted_cell}</div>'
```
### 响应式设计
- 使用CSS Grid和Flexbox布局
- 媒体查询适配不同屏幕尺寸
- 触摸友好的交互设计
### 性能优化
- **防抖处理**: 搜索输入使用防抖技术
- **异步加载**: 数据异步加载，界面不卡顿
- **内存管理**: 合理的数据结构和内存使用
## 🚀 使用方法
### 启动应用
```bash
# 现代化版本
python modern_sqlite_viewer.py
# 轻量级版本
python simple_sqlite_viewer.py
```
### 访问界面
- 打开浏览器访问对应地址
- 选择要查看的数据表
- 使用搜索框筛选内容
- 观察行高随内容自然变化
### 数据文件
- 默认读取 `product.db` 文件
- 如果文件不存在，自动创建示例数据
- 支持任何SQLite数据库文件
## 🎨 界面预览
### 现代化版本界面特色
- 渐变背景和毛玻璃效果
- 卡片式布局设计
- 专业的数据统计面板
- 高级DataTables功能
### 轻量级版本界面特色
- 简洁的渐变设计
- 内联CSS，无外部依赖
- 快速响应的用户体验
- 清晰的统计信息
## 📊 性能对比
| 特性 | 现代化版本 | 轻量级版本 |
|------|------------|------------|
| 加载速度 | 中等（依赖CDN） | 快速（内联资源） |
| 数据处理能力 | 强（服务器端分页） | 中（客户端处理） |
| 界面美观度 | 高（Bootstrap 5） | 中（自定义样式） |
| 功能丰富度 | 高（DataTables） | 中（基础功能） |
| 外部依赖 | 有（CDN资源） | 无（纯内联） |
| 移动端适配 | 优秀 | 良好 |
## 🔧 自定义和扩展
### 修改数据库路径
在代码中修改 `DB_PATH` 变量即可：
```python
DB_PATH = "path/to/your/database.db"
```
### 添加新功能
两个版本都基于模块化设计，易于扩展：
- 添加新的API端点
- 自定义界面样式
- 增加数据处理逻辑
### 部署到生产环境
建议使用生产级WSGI服务器：
```bash
pip install gunicorn
gunicorn -w 4 -b 0.0.0.0:5000 your_app:app
```
## 🎯 选择建议
- **需要专业功能** → 选择现代化版本
- **需要快速部署** → 选择轻量级版本
- **数据量较大** → 选择现代化版本
- **网络环境差** → 选择轻量级版本
- **需要离线使用** → 选择轻量级版本
两个版本都提供了优秀的用户体验和现代化的界面设计，您可以根据具体需求选择合适的版本。

451
product/web_sqlite_viewer.py
View File

@@ -11,11 +11,17 @@ import os
import json
from datetime import datetime
from loguru import logger
import threading
import time
import requests
app = Flask(__name__)
# 数据库路径
DB_PATH = os.path.join(os.path.dirname(__file__), 'product.db')
DB_PATH = os.path.join(os.path.dirname(__file__), 'products.db')
# 任务状态存储
analysis_tasks = {}
class SQLiteWebViewer:
def __init__(self, db_path):
@@ -68,25 +74,75 @@ class SQLiteWebViewer:
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# 获取总记录数
if search_field and search_value:
count_query = f"SELECT COUNT(*) FROM {table_name} WHERE {search_field} LIKE ?"
cursor.execute(count_query, (f'%{search_value}%',))
else:
count_query = f"SELECT COUNT(*) FROM {table_name}"
cursor.execute(count_query)
# 获取字段类型信息
field_types = {}
text_fields = []
cursor.execute(f"PRAGMA table_info({table_name});")
columns_info = cursor.fetchall()
for col_info in columns_info:
field_name = col_info[1]
field_type = col_info[2]
field_types[field_name] = field_type # 字段名 -> 字段类型
# 收集文本类型字段
if field_type.upper() not in ['INTEGER', 'REAL', 'FLOAT', 'NUMERIC']:
text_fields.append(field_name)
# 解析搜索条件
query_params = []
where_clause = ""
if search_value:
# 获取要搜索的字段列表
search_fields = []
if isinstance(search_field, list):
# 如果是列表，使用所有提供的字段
search_fields = [f for f in search_field if f in field_types]
elif search_field == "all":
# 如果是"all"，使用所有文本字段
search_fields = text_fields
elif search_field and search_field in field_types:
# 如果是单个字段，直接使用
search_fields = [search_field]
if search_fields:
conditions = []
for field in search_fields:
# 检查是否为数值比较操作符
import re
numeric_op_pattern = re.compile(r'^(<=?|>=?|=)(\d+(\.\d+)?)$')
match = numeric_op_pattern.match(search_value)
# 检查字段是否为数值类型
is_numeric_field = field_types.get(field, '').upper() in ['INTEGER', 'REAL', 'FLOAT', 'NUMERIC']
if match and is_numeric_field:
# 数值比较操作
operator = match.group(1)
value = match.group(2)
conditions.append(f"{field} {operator} ?")
query_params.append(float(value) if '.' in value else int(value))
logger.info(f"应用数值比较筛选: {field} {operator} {value}")
else:
# 默认文本模糊匹配
conditions.append(f"{field} LIKE ?")
query_params.append(f'%{search_value}%')
logger.info(f"应用文本模糊匹配: {field} LIKE '%{search_value}%'")
if conditions:
where_clause = " WHERE " + " OR ".join(conditions)
# 获取总记录数
count_query = f"SELECT COUNT(*) FROM {table_name}{where_clause}"
cursor.execute(count_query, query_params)
total_count = cursor.fetchone()[0]
# 获取分页数据
offset = (page - 1) * per_page
query_params.extend([per_page, offset])
if search_field and search_value:
data_query = f"SELECT * FROM {table_name} WHERE {search_field} LIKE ? LIMIT ? OFFSET ?"
cursor.execute(data_query, (f'%{search_value}%', per_page, offset))
else:
data_query = f"SELECT * FROM {table_name} LIMIT ? OFFSET ?"
cursor.execute(data_query, (per_page, offset))
data_query = f"SELECT * FROM {table_name}{where_clause} LIMIT ? OFFSET ?"
cursor.execute(data_query, query_params)
rows = cursor.fetchall()
@@ -100,8 +156,12 @@ class SQLiteWebViewer:
processed_rows = []
for row in rows:
processed_row = []
for cell in row:
if cell is None:
for i, cell in enumerate(row):
col_name = columns[i]
# 处理difficulty_score字段的缺失值
if col_name == "difficulty_score" and cell is None:
processed_row.append({'value': '未评分', 'type': 'empty'})
elif cell is None:
processed_row.append({'value': '', 'type': 'empty'})
elif isinstance(cell, str) and ('\n' in cell or len(cell) > 100):
# 多行文本或长文本
@@ -156,12 +216,156 @@ def get_table_data(table_name):
"""获取表数据"""
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', 50))
search_field = request.args.get('search_field')
# 获取所有search_field参数（可能是多个）
search_field = request.args.getlist('search_field')
# 如果只有一个且为空，使用单个值
if len(search_field) == 1:
search_field = search_field[0]
search_value = request.args.get('search_value')
data = viewer.get_table_data(table_name, page, per_page, search_field, search_value)
return jsonify(data)
@app.route('/api/analyze_missing_scores')
def analyze_missing_scores():
"""触发缺失分数分析任务"""
task_id = str(int(time.time()))
analysis_tasks[task_id] = {
'status': 'running',
'progress': 0,
'total': 0,
'completed': 0,
'error': None,
'start_time': datetime.now().isoformat()
}
# 在后台线程中执行分析
def run_analysis():
try:
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
# 检查product_analysis表是否存在difficulty_score字段
cursor.execute("PRAGMA table_info(product_analysis)")
columns = [col[1] for col in cursor.fetchall()]
if 'difficulty_score' not in columns:
# 如果不存在，添加该字段
cursor.execute("ALTER TABLE product_analysis ADD COLUMN difficulty_score REAL")
conn.commit()
logger.info("添加了difficulty_score字段")
# 查询缺失分数的产品
cursor.execute("""
SELECT pa.id, p.name, p.description
FROM product_analysis pa
JOIN products p ON pa.product_id = p.id
WHERE pa.difficulty_score IS NULL OR pa.difficulty_score = ''
""")
missing_scores = cursor.fetchall()
total = len(missing_scores)
analysis_tasks[task_id]['total'] = total
logger.info(f"找到 {total} 个缺失分数的产品")
for i, (analysis_id, product_name, introduction) in enumerate(missing_scores):
try:
# 调用Ollama API分析难度分数
score = analyze_product_difficulty(product_name, introduction)
# 更新数据库
cursor.execute(
"UPDATE product_analysis SET difficulty_score = ? WHERE id = ?",
(score, analysis_id)
)
conn.commit()
analysis_tasks[task_id]['completed'] = i + 1
analysis_tasks[task_id]['progress'] = int((i + 1) / total * 100)
logger.info(f"已分析产品 {i+1}/{total}: {product_name}, 分数: {score}")
# 避免频繁调用API
time.sleep(1)
except Exception as e:
logger.error(f"分析产品 {product_name} 失败: {e}")
# 继续处理下一个产品
continue
analysis_tasks[task_id]['status'] = 'completed'
logger.info("所有缺失分数分析完成")
except Exception as e:
logger.error(f"分析任务失败: {e}")
analysis_tasks[task_id]['status'] = 'failed'
analysis_tasks[task_id]['error'] = str(e)
finally:
conn.close()
analysis_tasks[task_id]['end_time'] = datetime.now().isoformat()
threading.Thread(target=run_analysis).start()
return jsonify({
'task_id': task_id,
'status': 'started',
'message': '分析任务已启动，请通过task_id查询进度'
})
@app.route('/api/update_task_status/<task_id>')
def update_task_status(task_id):
"""获取任务状态"""
if task_id in analysis_tasks:
return jsonify(analysis_tasks[task_id])
else:
return jsonify({
'error': 'Task not found',
'message': '找不到指定的任务ID'
}), 404
def analyze_product_difficulty(product_name, introduction):
"""调用Ollama API分析产品难度分数"""
try:
# 构建提示词
prompt = f"""请基于以下产品信息，分析其技术实现的难度分数（1-100分）：
产品名称：{product_name}
产品简介：{introduction}
请只返回一个整数分数，不需要其他解释。分数越高表示技术实现难度越大。
"""
# 调用Ollama API
response = requests.post(
'http://localhost:11434/api/generate',
json={
'model': 'llama3',
'prompt': prompt,
'format': 'json',
'stream': False
},
timeout=30
)
if response.status_code == 200:
data = response.json()
# 提取分数
score_text = data.get('response', '50').strip()
# 尝试从文本中提取数字
import re
numbers = re.findall(r'\d+', score_text)
if numbers:
score = int(numbers[0])
# 确保分数在1-100范围内
return max(1, min(100, score))
# 如果API调用失败或无法提取分数，返回默认值50
logger.warning(f"无法从Ollama API获取有效分数，返回默认值")
return 50
except Exception as e:
logger.error(f"调用Ollama API失败: {e}")
# 出错时返回默认值
return 50
@app.route('/static/<path:filename>')
def static_files(filename):
"""静态文件"""
@@ -194,7 +398,7 @@ def create_html_template():
}
.container {
max-width: 1400px;
width: 100%;
margin: 0 auto;
padding: 20px;
}
@@ -425,6 +629,60 @@ def create_html_template():
font-size: 1.1em;
}
.analyze-btn {
background: linear-gradient(135deg, #28a745, #20c997);
color: white;
border: none;
padding: 8px 16px;
border-radius: 6px;
font-size: 1em;
cursor: pointer;
transition: all 0.3s ease;
}
.analyze-btn:hover {
transform: translateY(-1px);
box-shadow: 0 4px 8px rgba(40, 167, 69, 0.3);
}
.analyze-btn:disabled {
background: #6c757d;
cursor: not-allowed;
transform: none;
box-shadow: none;
}
.progress-container {
margin-top: 20px;
padding: 15px;
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(10px);
border-radius: 12px;
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.1);
display: none;
}
.progress-bar {
width: 100%;
height: 20px;
background: #e9ecef;
border-radius: 10px;
overflow: hidden;
margin: 10px 0;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, #667eea, #764ba2);
transition: width 0.3s ease;
display: flex;
align-items: center;
justify-content: center;
color: white;
font-size: 0.8em;
font-weight: 600;
}
@media (max-width: 768px) {
.controls {
flex-direction: column;
@@ -449,15 +707,19 @@ def create_html_template():
<h1>🗄️ SQLite数据库查看器</h1>
<div class="controls">
<div class="control-group">
<label for="tableSelect">选择数据表:</label>
<select id="tableSelect">
<option value="">加载中...</option>
</select>
<label for="tableSelect">选择数据表:</label>
<select id="tableSelect">
<option value="">加载中...</option>
</select>
</div>
<div class="control-group">
<label for="analyzeBtn">分析:</label>
<button id="analyzeScoresBtn" class="analyze-btn">📊 分析缺失分数</button>
</div>
<div class="control-group">
<label for="searchField">筛选字段:</label>
<select id="searchField" disabled>
<option value="">选择字段...</option>
<select id="searchField" multiple disabled style="min-height: 80px;">
<option value="">所有文本字段</option>
</select>
</div>
<div class="control-group">
@@ -466,6 +728,7 @@ def create_html_template():
</div>
<button class="btn" onclick="loadData()">刷新数据</button>
</div>
</div>
</div>
<div class="data-container">
@@ -488,6 +751,15 @@ def create_html_template():
</div>
</div>
</div>
<div id="progressSection" class="progress-container">
<h3>📊 分数分析进度</h3>
<div class="progress-bar">
<div id="progressFill" class="progress-fill" style="width: 0%;">0%</div>
</div>
<p id="progressText">等待分析开始...</p>
</div>
</div>
<script>
let currentTable = '';
@@ -496,20 +768,88 @@ def create_html_template():
let totalPages = 1;
let currentData = null;
// 初始化
document.addEventListener('DOMContentLoaded', function() {
loadTables();
// 绑定事件
document.getElementById('tableSelect').addEventListener('change', function() {
currentTable = this.value;
currentPage = 1;
if (currentTable) {
loadTableStructure();
loadData();
}
// 绑定事件
document.addEventListener('DOMContentLoaded', function() {
loadTables();
// 绑定事件
document.getElementById('tableSelect').addEventListener('change', function() {
currentTable = this.value;
currentPage = 1;
if (currentTable) {
loadTableStructure();
loadData();
}
});
// 分析缺失分数按钮事件
document.getElementById('analyzeScoresBtn').addEventListener('click', analyzeMissingScores);
});
// 分析缺失分数
async function analyzeMissingScores() {
const analyzeBtn = document.getElementById('analyzeScoresBtn');
const progressSection = document.getElementById('progressSection');
const progressFill = document.getElementById('progressFill');
const progressText = document.getElementById('progressText');
try {
// 禁用按钮
analyzeBtn.disabled = true;
analyzeBtn.textContent = '分析中...';
// 显示进度条
progressSection.style.display = 'block';
progressFill.style.width = '0%';
progressFill.textContent = '0%';
progressText.textContent = '正在启动分析任务...';
// 启动分析任务
const response = await fetch('/api/analyze_missing_scores');
const data = await response.json();
if (data.task_id) {
// 定期查询任务状态
const interval = setInterval(async () => {
try {
const statusResponse = await fetch(`/api/update_task_status/${data.task_id}`);
const statusData = await statusResponse.json();
// 更新进度
progressFill.style.width = `${statusData.progress}%`;
progressFill.textContent = `${statusData.progress}%`;
if (statusData.status === 'running') {
progressText.textContent = `正在分析: ${statusData.completed}/${statusData.total} 个产品`;
} else if (statusData.status === 'completed') {
progressText.textContent = '🎉 所有缺失分数分析完成！';
clearInterval(interval);
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
// 如果当前正在查看product_analysis表，自动刷新
if (currentTable === 'product_analysis') {
loadData();
}
} else if (statusData.status === 'failed') {
progressText.textContent = `❌ 分析失败: ${statusData.error}`;
clearInterval(interval);
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
}
} catch (error) {
console.error('查询任务状态失败:', error);
progressText.textContent = '查询任务状态失败';
}
}, 2000);
}
} catch (error) {
console.error('启动分析任务失败:', error);
progressText.textContent = `启动分析失败: ${error.message}`;
analyzeBtn.disabled = false;
analyzeBtn.textContent = '📊 分析缺失分数';
}
document.getElementById('searchField').addEventListener('change', loadData);
document.getElementById('searchValue').addEventListener('input', debounce(loadData, 500));
});
@@ -556,7 +896,7 @@ def create_html_template():
const data = await response.json();
const searchField = document.getElementById('searchField');
searchField.innerHTML = '<option value="">所有字段</option>';
searchField.innerHTML = '<option value="">所有文本字段</option>';
data.structure.forEach(field => {
const option = document.createElement('option');
option.value = field.name;
@@ -578,13 +918,27 @@ def create_html_template():
const container = document.getElementById('dataContainer');
container.innerHTML = '<div class="loading">📊 数据加载中...</div>';
const searchField = document.getElementById('searchField').value;
const searchFieldSelect = document.getElementById('searchField');
const searchValue = document.getElementById('searchValue').value;
try {
let url = `/api/table/${currentTable}/data?page=${currentPage}&per_page=${perPage}`;
if (searchField && searchValue) {
url += `&search_field=${searchField}&search_value=${encodeURIComponent(searchValue)}`;
if (searchValue) {
// 获取所有选中的字段
const selectedFields = Array.from(searchFieldSelect.selectedOptions)
.map(option => option.value)
.filter(value => value !== '');
if (selectedFields.length > 0) {
// 如果选择了特定字段，传递所有选中的字段
selectedFields.forEach(field => {
url += `&search_field=${encodeURIComponent(field)}`;
});
} else {
// 否则使用"all"表示所有文本字段
url += '&search_field=all';
}
url += `&search_value=${encodeURIComponent(searchValue)}`;
}
const response = await fetch(url);
@@ -722,6 +1076,18 @@ if __name__ == '__main__':
)
''')
# 创建product_analysis表
cursor.execute('''
CREATE TABLE IF NOT EXISTS product_analysis (
id INTEGER PRIMARY KEY AUTOINCREMENT,
product_id INTEGER,
analysis TEXT,
difficulty_score REAL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (product_id) REFERENCES products(id)
)
''')
# 插入示例数据
sample_data = [
('产品A', '这是一个非常优秀的产品\n具有多种实用功能\n用户反馈很好', '高性能\n易用性\n稳定性', 99.99),
@@ -730,6 +1096,11 @@ if __name__ == '__main__':
]
cursor.executemany('INSERT INTO products (name, description, features, price) VALUES (?, ?, ?, ?)', sample_data)
# 插入一些分析数据
cursor.execute('INSERT INTO product_analysis (product_id, analysis, difficulty_score) VALUES (1, "产品分析示例", 75)')
cursor.execute('INSERT INTO product_analysis (product_id, analysis) VALUES (2, "产品分析示例，无分数")')
conn.commit()
conn.close()