facat/bidding_watcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 全面优化爬虫系统和数据处理能力

Browse Source 
- 增强数据重试机制：对数据为0的爬虫自动重试，提高数据完整性
- 优化前端筛选逻辑：改进日期筛选，只限制开始时间，更灵活的数据查看
- 新增最近数据接口：添加 /api/bids/recent 获取30天内最新招标数据
- 改进统计展示：实时显示筛选结果数量，优化用户体验
- 完善日志系统：确保日志目录自动创建，避免启动错误
- 增强独立脚本：使用自定义logger，完善错误处理和程序关闭
- 优化主程序：集成自定义日志服务，统一日志格式
- 扩展npm脚本：新增 web 命令用于构建前端
- 改进大唐爬虫：延长等待时间到60秒，提高页面加载成功率
- 优化数据筛选：今日招标改为使用独立接口，提升性能
This commit is contained in:
dmy
2026-01-12 12:28:37 +08:00
parent 3e6456e120
commit 1b28a3462a
10 changed files with 104 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
frontend/index.html
View File

@@ -4,7 +4,7 @@
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>frontend</title>
<title>投标</title>
</head>
<body>
<div id="app"></div>

45
frontend/src/App.vue
View File

@@ -188,7 +188,7 @@
</template>
<script setup lang="ts">
import { ref, onMounted, reactive, computed, watch, nextTick } from 'vue'
import { ref, onMounted, reactive, computed, watch } from 'vue'
import axios from 'axios'
import { ElMessage } from 'element-plus'
import { DataBoard, Document, Setting, Refresh } from '@element-plus/icons-vue'
@@ -285,26 +285,22 @@ const setLast3Days = () => {
console.log('setLast3Days called, todayBids:', todayBids.value.length, 'dateRange:', dateRange.value)
// 直接计算筛选结果并显示提示
// 直接计算筛选结果并显示提示（只限制开始时间，不限制结束时间）
const start = new Date(startDate)
start.setHours(0, 0, 0, 0)
const end = new Date(endDate)
end.setHours(23, 59, 59, 999)
let result = todayBids.value
result = result.filter(bid => {
if (!bid.publishDate) return false
const bidDate = new Date(bid.publishDate)
return bidDate >= start && bidDate <= end
return bidDate >= start
})
const totalBids = todayBids.value.length
const filteredCount = result.length
console.log('setLast3Days result, totalBids:', totalBids, 'filteredCount:', filteredCount)
if (totalBids > 0) {
ElMessage.info(`筛选结果：共 ${filteredCount} 条数据（总共 ${totalBids} 条）`)
} else {
if (totalBids === 0) {
ElMessage.warning('暂无数据，请先抓取数据')
}
}
@@ -326,26 +322,22 @@ const setLast7Days = () => {
console.log('setLast7Days called, todayBids:', todayBids.value.length, 'dateRange:', dateRange.value)
// 直接计算筛选结果并显示提示
// 直接计算筛选结果并显示提示（只限制开始时间，不限制结束时间）
const start = new Date(startDate)
start.setHours(0, 0, 0, 0)
const end = new Date(endDate)
end.setHours(23, 59, 59, 999)
let result = todayBids.value
result = result.filter(bid => {
if (!bid.publishDate) return false
const bidDate = new Date(bid.publishDate)
return bidDate >= start && bidDate <= end
return bidDate >= start
})
const totalBids = todayBids.value.length
const filteredCount = result.length
console.log('setLast7Days result, totalBids:', totalBids, 'filteredCount:', filteredCount)
if (totalBids > 0) {
ElMessage.info(`筛选结果：共 ${filteredCount} 条数据（总共 ${totalBids} 条）`)
} else {
if (totalBids === 0) {
ElMessage.warning('暂无数据，请先抓取数据')
}
}
@@ -368,18 +360,16 @@ const filteredTodayBids = computed(() => {
})
}
// 按日期范围筛选
// 按日期范围筛选（只限制开始时间，不限制结束时间）
if (dateRange.value && dateRange.value.length === 2) {
const [startDate, endDate] = dateRange.value
const [startDate] = dateRange.value
result = result.filter(bid => {
if (!bid.publishDate) return false
const bidDate = new Date(bid.publishDate)
const start = new Date(startDate)
const end = new Date(endDate)
// 设置时间为当天的开始和结束
// 设置时间为当天的开始
start.setHours(0, 0, 0, 0)
end.setHours(23, 59, 59, 999)
return bidDate >= start && bidDate <= end
return bidDate >= start
})
}
@@ -399,7 +389,7 @@ watch(filteredTodayBids, (newFilteredBids) => {
const fetchData = async () => {
loading.value = true
try {
const [bidsRes, highRes, kwRes, sourcesRes, statusRes] = await Promise.all([
const [bidsRes, recentRes, highRes, kwRes, sourcesRes, statusRes] = await Promise.all([
axios.get('/api/bids', {
params: {
page: currentPage.value,
@@ -407,6 +397,7 @@ const fetchData = async () => {
source: selectedSource.value || undefined
}
}),
axios.get('/api/bids/recent'),
axios.get('/api/bids/high-priority'),
axios.get('/api/keywords'),
axios.get('/api/bids/sources'),
@@ -414,19 +405,11 @@ const fetchData = async () => {
])
bids.value = bidsRes.data.items
total.value = bidsRes.data.total
todayBids.value = recentRes.data
highPriorityBids.value = highRes.data
keywords.value = kwRes.data
sourceOptions.value = sourcesRes.data
isCrawling.value = statusRes.data.isCrawling
// 过滤今天的数据用于 Today's Bids
const today = new Date()
today.setHours(0, 0, 0, 0)
todayBids.value = bidsRes.data.items.filter((bid: any) => {
if (!bid.publishDate) return false
const bidDate = new Date(bid.publishDate)
return bidDate >= today
})
} catch (error) {
ElMessage.error('Failed to fetch data')
} finally {