feat: 全面优化爬虫系统和数据处理能力
- 增强数据重试机制:对数据为0的爬虫自动重试,提高数据完整性 - 优化前端筛选逻辑:改进日期筛选,只限制开始时间,更灵活的数据查看 - 新增最近数据接口:添加 /api/bids/recent 获取30天内最新招标数据 - 改进统计展示:实时显示筛选结果数量,优化用户体验 - 完善日志系统:确保日志目录自动创建,避免启动错误 - 增强独立脚本:使用自定义logger,完善错误处理和程序关闭 - 优化主程序:集成自定义日志服务,统一日志格式 - 扩展npm脚本:新增 web 命令用于构建前端 - 改进大唐爬虫:延长等待时间到60秒,提高页面加载成功率 - 优化数据筛选:今日招标改为使用独立接口,提升性能
This commit is contained in:
@@ -188,7 +188,7 @@
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, onMounted, reactive, computed, watch, nextTick } from 'vue'
|
||||
import { ref, onMounted, reactive, computed, watch } from 'vue'
|
||||
import axios from 'axios'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { DataBoard, Document, Setting, Refresh } from '@element-plus/icons-vue'
|
||||
@@ -285,26 +285,22 @@ const setLast3Days = () => {
|
||||
|
||||
console.log('setLast3Days called, todayBids:', todayBids.value.length, 'dateRange:', dateRange.value)
|
||||
|
||||
// 直接计算筛选结果并显示提示
|
||||
// 直接计算筛选结果并显示提示(只限制开始时间,不限制结束时间)
|
||||
const start = new Date(startDate)
|
||||
start.setHours(0, 0, 0, 0)
|
||||
const end = new Date(endDate)
|
||||
end.setHours(23, 59, 59, 999)
|
||||
|
||||
let result = todayBids.value
|
||||
result = result.filter(bid => {
|
||||
if (!bid.publishDate) return false
|
||||
const bidDate = new Date(bid.publishDate)
|
||||
return bidDate >= start && bidDate <= end
|
||||
return bidDate >= start
|
||||
})
|
||||
|
||||
const totalBids = todayBids.value.length
|
||||
const filteredCount = result.length
|
||||
|
||||
console.log('setLast3Days result, totalBids:', totalBids, 'filteredCount:', filteredCount)
|
||||
if (totalBids > 0) {
|
||||
ElMessage.info(`筛选结果:共 ${filteredCount} 条数据(总共 ${totalBids} 条)`)
|
||||
} else {
|
||||
if (totalBids === 0) {
|
||||
ElMessage.warning('暂无数据,请先抓取数据')
|
||||
}
|
||||
}
|
||||
@@ -326,26 +322,22 @@ const setLast7Days = () => {
|
||||
|
||||
console.log('setLast7Days called, todayBids:', todayBids.value.length, 'dateRange:', dateRange.value)
|
||||
|
||||
// 直接计算筛选结果并显示提示
|
||||
// 直接计算筛选结果并显示提示(只限制开始时间,不限制结束时间)
|
||||
const start = new Date(startDate)
|
||||
start.setHours(0, 0, 0, 0)
|
||||
const end = new Date(endDate)
|
||||
end.setHours(23, 59, 59, 999)
|
||||
|
||||
let result = todayBids.value
|
||||
result = result.filter(bid => {
|
||||
if (!bid.publishDate) return false
|
||||
const bidDate = new Date(bid.publishDate)
|
||||
return bidDate >= start && bidDate <= end
|
||||
return bidDate >= start
|
||||
})
|
||||
|
||||
const totalBids = todayBids.value.length
|
||||
const filteredCount = result.length
|
||||
|
||||
console.log('setLast7Days result, totalBids:', totalBids, 'filteredCount:', filteredCount)
|
||||
if (totalBids > 0) {
|
||||
ElMessage.info(`筛选结果:共 ${filteredCount} 条数据(总共 ${totalBids} 条)`)
|
||||
} else {
|
||||
if (totalBids === 0) {
|
||||
ElMessage.warning('暂无数据,请先抓取数据')
|
||||
}
|
||||
}
|
||||
@@ -368,18 +360,16 @@ const filteredTodayBids = computed(() => {
|
||||
})
|
||||
}
|
||||
|
||||
// 按日期范围筛选
|
||||
// 按日期范围筛选(只限制开始时间,不限制结束时间)
|
||||
if (dateRange.value && dateRange.value.length === 2) {
|
||||
const [startDate, endDate] = dateRange.value
|
||||
const [startDate] = dateRange.value
|
||||
result = result.filter(bid => {
|
||||
if (!bid.publishDate) return false
|
||||
const bidDate = new Date(bid.publishDate)
|
||||
const start = new Date(startDate)
|
||||
const end = new Date(endDate)
|
||||
// 设置时间为当天的开始和结束
|
||||
// 设置时间为当天的开始
|
||||
start.setHours(0, 0, 0, 0)
|
||||
end.setHours(23, 59, 59, 999)
|
||||
return bidDate >= start && bidDate <= end
|
||||
return bidDate >= start
|
||||
})
|
||||
}
|
||||
|
||||
@@ -399,7 +389,7 @@ watch(filteredTodayBids, (newFilteredBids) => {
|
||||
const fetchData = async () => {
|
||||
loading.value = true
|
||||
try {
|
||||
const [bidsRes, highRes, kwRes, sourcesRes, statusRes] = await Promise.all([
|
||||
const [bidsRes, recentRes, highRes, kwRes, sourcesRes, statusRes] = await Promise.all([
|
||||
axios.get('/api/bids', {
|
||||
params: {
|
||||
page: currentPage.value,
|
||||
@@ -407,6 +397,7 @@ const fetchData = async () => {
|
||||
source: selectedSource.value || undefined
|
||||
}
|
||||
}),
|
||||
axios.get('/api/bids/recent'),
|
||||
axios.get('/api/bids/high-priority'),
|
||||
axios.get('/api/keywords'),
|
||||
axios.get('/api/bids/sources'),
|
||||
@@ -414,19 +405,11 @@ const fetchData = async () => {
|
||||
])
|
||||
bids.value = bidsRes.data.items
|
||||
total.value = bidsRes.data.total
|
||||
todayBids.value = recentRes.data
|
||||
highPriorityBids.value = highRes.data
|
||||
keywords.value = kwRes.data
|
||||
sourceOptions.value = sourcesRes.data
|
||||
isCrawling.value = statusRes.data.isCrawling
|
||||
|
||||
// 过滤今天的数据用于 Today's Bids
|
||||
const today = new Date()
|
||||
today.setHours(0, 0, 0, 0)
|
||||
todayBids.value = bidsRes.data.items.filter((bid: any) => {
|
||||
if (!bid.publishDate) return false
|
||||
const bidDate = new Date(bid.publishDate)
|
||||
return bidDate >= today
|
||||
})
|
||||
} catch (error) {
|
||||
ElMessage.error('Failed to fetch data')
|
||||
} finally {
|
||||
|
||||
Reference in New Issue
Block a user