refactor: 重构前端代码，拆分组件并优化README文档
This commit is contained in:
dmy
100
README.md
100
README.md
@@ -97,26 +97,86 @@ Nest is an MIT-licensed open source project. It can grow thanks to the sponsors
|
|||||||
|
|
||||||
Nest is [MIT licensed](https://github.com/nestjs/nest/blob/master/LICENSE).
|
Nest is [MIT licensed](https://github.com/nestjs/nest/blob/master/LICENSE).
|
||||||
|
|
||||||
How to Run:
|
## How to Run
|
||||||
1. Database Setup: Update the .env file with your PostgreSQL credentials.
|
|
||||||
|
|
||||||
1 DATABASE_TYPE=postgres
|
### 1. Database Setup
|
||||||
2 DATABASE_HOST=localhost
|
Update the `.env` file with your PostgreSQL credentials:
|
||||||
3 DATABASE_PORT=5432
|
|
||||||
4 DATABASE_USERNAME=your_username
|
|
||||||
5 DATABASE_PASSWORD=your_password
|
|
||||||
6 DATABASE_NAME=bidding
|
|
||||||
7 DATABASE_SYNCHRONIZE=true
|
|
||||||
2. Install Dependencies:
|
|
||||||
1 npm install
|
|
||||||
2 cd frontend && npm install
|
|
||||||
3. Build and Start:
|
|
||||||
|
|
||||||
1 # From the root directory
|
```env
|
||||||
2 cd frontend && npm run build
|
DATABASE_TYPE=postgres
|
||||||
3 cd ..
|
DATABASE_HOST=localhost
|
||||||
4 npm run build
|
DATABASE_PORT=5432
|
||||||
5 npm run start
|
DATABASE_USERNAME=your_username
|
||||||
|
DATABASE_PASSWORD=your_password
|
||||||
|
DATABASE_NAME=bidding
|
||||||
|
DATABASE_SYNCHRONIZE=true
|
||||||
|
```
|
||||||
|
|
||||||
The system will automatically initialize with the preset keywords: "山东", "海", "建设", "工程", "采购". You can
|
### 2. Install Dependencies
|
||||||
manage these and view crawled bidding information at http://localhost:3000.
|
|
||||||
|
```bash
|
||||||
|
npm install
|
||||||
|
cd frontend && npm install
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Build and Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# From the root directory
|
||||||
|
cd frontend && npm run build
|
||||||
|
cd ..
|
||||||
|
npm run build
|
||||||
|
npm run start
|
||||||
|
```
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
### Frontend Features
|
||||||
|
|
||||||
|
- **Dashboard**: View high priority bids and today's bids
|
||||||
|
- **Date Filtering**:
|
||||||
|
- Click "3天" or "7天" buttons to filter bids from the last 3 or 7 days
|
||||||
|
- The filter only limits the start date, showing all data from the selected start date onwards (including data newer than the end date)
|
||||||
|
- **Keyword Filtering**: Filter bids by keywords (saved in localStorage)
|
||||||
|
- **All Bids**: View all bids with pagination and source filtering
|
||||||
|
- **Keyword Management**: Add and delete keywords with weight-based priority
|
||||||
|
|
||||||
|
### Backend Features
|
||||||
|
|
||||||
|
- **Multi-Source Crawling**: Crawls bidding information from multiple sources:
|
||||||
|
- ChdtpCrawler
|
||||||
|
- ChngCrawler
|
||||||
|
- SzecpCrawler
|
||||||
|
- CdtCrawler
|
||||||
|
- EpsCrawler
|
||||||
|
- CnncecpCrawler
|
||||||
|
- CgnpcCrawler
|
||||||
|
- CeicCrawler
|
||||||
|
- EspicCrawler
|
||||||
|
- PowerbeijingCrawler
|
||||||
|
- **Automatic Retry**: If a crawler returns 0 items, it will be retried after all crawlers complete
|
||||||
|
- **Proxy Support**: Configurable proxy settings via environment variables
|
||||||
|
- **Scheduled Tasks**: Automatic crawling at scheduled intervals
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
|
||||||
|
```env
|
||||||
|
# Database
|
||||||
|
DATABASE_TYPE=postgres
|
||||||
|
DATABASE_HOST=localhost
|
||||||
|
DATABASE_PORT=5432
|
||||||
|
DATABASE_USERNAME=your_username
|
||||||
|
DATABASE_PASSWORD=your_password
|
||||||
|
DATABASE_NAME=bidding
|
||||||
|
DATABASE_SYNCHRONIZE=true
|
||||||
|
|
||||||
|
# Proxy (optional)
|
||||||
|
PROXY_HOST=your_proxy_host
|
||||||
|
PROXY_PORT=your_proxy_port
|
||||||
|
PROXY_USERNAME=your_proxy_username
|
||||||
|
PROXY_PASSWORD=your_proxy_password
|
||||||
|
```
|
||||||
|
|
||||||
|
## Initial Setup
|
||||||
|
|
||||||
|
The system will automatically initialize with the preset keywords: "山东", "海", "建设", "工程", "采购". You can manage these and view crawled bidding information at http://localhost:3000.
|
||||||
@@ -31,167 +31,43 @@
|
|||||||
</el-header>
|
</el-header>
|
||||||
|
|
||||||
<el-main>
|
<el-main>
|
||||||
<div v-if="activeIndex === '1'">
|
<Dashboard
|
||||||
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px;">
|
v-if="activeIndex === '1'"
|
||||||
<h2 style="margin: 0;">Dashboard</h2>
|
:today-bids="todayBids"
|
||||||
<el-button type="primary" :loading="crawling" :disabled="isCrawling" @click="handleCrawl">
|
:high-priority-bids="highPriorityBids"
|
||||||
<el-icon style="margin-right: 5px"><Refresh /></el-icon>
|
:keywords="keywords"
|
||||||
立刻抓取
|
:loading="loading"
|
||||||
</el-button>
|
:is-crawling="isCrawling"
|
||||||
</div>
|
@refresh="fetchData"
|
||||||
<el-row :gutter="20">
|
|
||||||
<el-col :span="24">
|
|
||||||
<el-card class="box-card" shadow="hover">
|
|
||||||
<template #header>
|
|
||||||
<div class="card-header">
|
|
||||||
<span>High Priority Bids</span>
|
|
||||||
<el-tag type="danger">Top 10</el-tag>
|
|
||||||
</div>
|
|
||||||
</template>
|
|
||||||
<el-table :data="highPriorityBids" style="width: 100%" size="small">
|
|
||||||
<el-table-column prop="title" label="Title">
|
|
||||||
<template #default="scope">
|
|
||||||
<a :href="scope.row.url" target="_blank">{{ scope.row.title }}</a>
|
|
||||||
</template>
|
|
||||||
</el-table-column>
|
|
||||||
<el-table-column prop="source" label="Source" width="240" />
|
|
||||||
<el-table-column prop="publishDate" label="Date" width="120">
|
|
||||||
<template #default="scope">{{ formatDate(scope.row.publishDate) }}</template>
|
|
||||||
</el-table-column>
|
|
||||||
</el-table>
|
|
||||||
</el-card>
|
|
||||||
</el-col>
|
|
||||||
</el-row>
|
|
||||||
<el-divider />
|
|
||||||
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px;">
|
|
||||||
<h3 style="margin: 0;">Today's Bids</h3>
|
|
||||||
<div style="display: flex; gap: 10px;">
|
|
||||||
<el-date-picker
|
|
||||||
v-model="dateRange"
|
|
||||||
type="daterange"
|
|
||||||
range-separator="To"
|
|
||||||
start-placeholder="Start Date"
|
|
||||||
end-placeholder="End Date"
|
|
||||||
format="YYYY-MM-DD"
|
|
||||||
value-format="YYYY-MM-DD"
|
|
||||||
clearable
|
|
||||||
style="width: 240px;"
|
|
||||||
/>
|
/>
|
||||||
<el-button type="primary" @click="setLast3Days">3天</el-button>
|
|
||||||
<el-button type="primary" @click="setLast7Days">7天</el-button>
|
|
||||||
<el-select
|
|
||||||
v-model="selectedKeywords"
|
|
||||||
multiple
|
|
||||||
collapse-tags
|
|
||||||
collapse-tags-tooltip
|
|
||||||
placeholder="Filter by Keywords"
|
|
||||||
clearable
|
|
||||||
style="width: 300px;"
|
|
||||||
>
|
|
||||||
<el-option
|
|
||||||
v-for="keyword in keywords"
|
|
||||||
:key="keyword.id"
|
|
||||||
:label="keyword.word"
|
|
||||||
:value="keyword.word"
|
|
||||||
/>
|
|
||||||
</el-select>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<el-table :data="filteredTodayBids" v-loading="loading" style="width: 100%">
|
|
||||||
<el-table-column prop="title" label="Title">
|
|
||||||
<template #default="scope">
|
|
||||||
<a :href="scope.row.url" target="_blank">{{ scope.row.title }}</a>
|
|
||||||
</template>
|
|
||||||
</el-table-column>
|
|
||||||
<el-table-column prop="source" label="Source" width="220" />
|
|
||||||
<el-table-column prop="publishDate" label="Date" width="150">
|
|
||||||
<template #default="scope">{{ formatDate(scope.row.publishDate) }}</template>
|
|
||||||
</el-table-column>
|
|
||||||
</el-table>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div v-if="activeIndex === '2'">
|
<Bids
|
||||||
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px;">
|
v-if="activeIndex === '2'"
|
||||||
<h2 style="margin: 0;">All Bids</h2>
|
:bids="bids"
|
||||||
<el-select v-model="selectedSource" placeholder="Filter by Source" clearable style="width: 200px" @change="currentPage = 1; fetchData()">
|
:source-options="sourceOptions"
|
||||||
<el-option
|
:loading="loading"
|
||||||
v-for="source in sourceOptions"
|
|
||||||
:key="source"
|
|
||||||
:label="source"
|
|
||||||
:value="source"
|
|
||||||
/>
|
|
||||||
</el-select>
|
|
||||||
</div>
|
|
||||||
<el-table :data="bids" v-loading="loading" style="width: 100%">
|
|
||||||
<el-table-column prop="title" label="Title">
|
|
||||||
<template #default="scope">
|
|
||||||
<a :href="scope.row.url" target="_blank">{{ scope.row.title }}</a>
|
|
||||||
</template>
|
|
||||||
</el-table-column>
|
|
||||||
<el-table-column prop="source" label="Source" width="200" />
|
|
||||||
<el-table-column prop="publishDate" label="Date" width="150">
|
|
||||||
<template #default="scope">{{ formatDate(scope.row.publishDate) }}</template>
|
|
||||||
</el-table-column>
|
|
||||||
</el-table>
|
|
||||||
<el-pagination
|
|
||||||
v-model:current-page="currentPage"
|
|
||||||
v-model:page-size="pageSize"
|
|
||||||
:page-sizes="[10, 20, 50, 100]"
|
|
||||||
:total="total"
|
:total="total"
|
||||||
layout="total, sizes, prev, pager, next, jumper"
|
@fetch="handleFetchBids"
|
||||||
@current-change="handlePageChange"
|
|
||||||
@size-change="handleSizeChange"
|
|
||||||
style="margin-top: 20px; justify-content: flex-end;"
|
|
||||||
/>
|
/>
|
||||||
</div>
|
|
||||||
|
|
||||||
<div v-if="activeIndex === '3'">
|
<Keywords
|
||||||
<div class="card-header" style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px;">
|
v-if="activeIndex === '3'"
|
||||||
<h2>Keyword Management</h2>
|
:keywords="keywords"
|
||||||
<el-button type="primary" @click="dialogVisible = true">Add Keyword</el-button>
|
:loading="loading"
|
||||||
</div>
|
@refresh="fetchData"
|
||||||
|
/>
|
||||||
<div v-loading="loading" style="min-height: 200px;">
|
|
||||||
<el-tag
|
|
||||||
v-for="keyword in keywords"
|
|
||||||
:key="keyword.id"
|
|
||||||
closable
|
|
||||||
:type="getTagType(keyword.weight)"
|
|
||||||
@close="handleDeleteKeyword(keyword.id)"
|
|
||||||
style="margin: 5px;"
|
|
||||||
>
|
|
||||||
{{ keyword.word }}
|
|
||||||
</el-tag>
|
|
||||||
<el-empty v-if="keywords.length === 0" description="No keywords" />
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</el-main>
|
</el-main>
|
||||||
</el-container>
|
</el-container>
|
||||||
|
|
||||||
<el-dialog v-model="dialogVisible" title="Add Keyword" width="30%">
|
|
||||||
<el-form :model="form" label-width="120px">
|
|
||||||
<el-form-item label="Keyword">
|
|
||||||
<el-input v-model="form.word" />
|
|
||||||
</el-form-item>
|
|
||||||
<el-form-item label="Weight">
|
|
||||||
<el-input-number v-model="form.weight" :min="1" :max="5" />
|
|
||||||
</el-form-item>
|
|
||||||
</el-form>
|
|
||||||
<template #footer>
|
|
||||||
<span class="dialog-footer">
|
|
||||||
<el-button @click="dialogVisible = false">Cancel</el-button>
|
|
||||||
<el-button type="primary" @click="handleAddKeyword">Confirm</el-button>
|
|
||||||
</span>
|
|
||||||
</template>
|
|
||||||
</el-dialog>
|
|
||||||
</el-container>
|
</el-container>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { ref, onMounted, reactive, computed, watch } from 'vue'
|
import { ref, onMounted } from 'vue'
|
||||||
import axios from 'axios'
|
import axios from 'axios'
|
||||||
import { ElMessage } from 'element-plus'
|
import { DataBoard, Document, Setting } from '@element-plus/icons-vue'
|
||||||
import { DataBoard, Document, Setting, Refresh } from '@element-plus/icons-vue'
|
import Dashboard from './components/Dashboard.vue'
|
||||||
|
import Bids from './components/Bids.vue'
|
||||||
|
import Keywords from './components/Keywords.vue'
|
||||||
|
|
||||||
const activeIndex = ref('1')
|
const activeIndex = ref('1')
|
||||||
const bids = ref<any[]>([])
|
const bids = ref<any[]>([])
|
||||||
@@ -199,202 +75,41 @@ const todayBids = ref<any[]>([])
|
|||||||
const highPriorityBids = ref<any[]>([])
|
const highPriorityBids = ref<any[]>([])
|
||||||
const keywords = ref<any[]>([])
|
const keywords = ref<any[]>([])
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
const crawling = ref(false)
|
const isCrawling = ref(false)
|
||||||
const dialogVisible = ref(false)
|
|
||||||
const selectedSource = ref('')
|
|
||||||
const currentPage = ref(1)
|
|
||||||
const pageSize = ref(10)
|
|
||||||
const total = ref(0)
|
const total = ref(0)
|
||||||
const sourceOptions = ref<string[]>([])
|
const sourceOptions = ref<string[]>([])
|
||||||
const isCrawling = ref(false)
|
|
||||||
const selectedKeywords = ref<string[]>([])
|
|
||||||
const dateRange = ref<[string, string] | null>(null)
|
|
||||||
|
|
||||||
// 从 localStorage 加载保存的关键字
|
|
||||||
const loadSavedKeywords = () => {
|
|
||||||
const saved = localStorage.getItem('selectedKeywords')
|
|
||||||
if (saved) {
|
|
||||||
try {
|
|
||||||
selectedKeywords.value = JSON.parse(saved)
|
|
||||||
} catch (e) {
|
|
||||||
console.error('Failed to parse saved keywords:', e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 监听关键字变化并保存到 localStorage
|
|
||||||
watch(selectedKeywords, (newKeywords) => {
|
|
||||||
localStorage.setItem('selectedKeywords', JSON.stringify(newKeywords))
|
|
||||||
}, { deep: true })
|
|
||||||
|
|
||||||
// 监听日期范围变化并显示提示
|
|
||||||
watch(dateRange, () => {
|
|
||||||
const totalBids = bids.value.length
|
|
||||||
const filteredCount = filteredTodayBids.value.length
|
|
||||||
|
|
||||||
if (totalBids > 0 && filteredCount < totalBids) {
|
|
||||||
ElMessage.info(`筛选结果：共 ${filteredCount} 条数据（总共 ${totalBids} 条）`)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
const form = reactive({
|
|
||||||
word: '',
|
|
||||||
weight: 1
|
|
||||||
})
|
|
||||||
|
|
||||||
// 根据 weight 获取 tag 类型
|
|
||||||
const getTagType = (weight: number) => {
|
|
||||||
if (weight >= 5) return 'danger'
|
|
||||||
if (weight >= 4) return 'warning'
|
|
||||||
if (weight >= 3) return 'primary'
|
|
||||||
if (weight >= 2) return 'success'
|
|
||||||
return 'info'
|
|
||||||
}
|
|
||||||
|
|
||||||
const handleSelect = (key: string) => {
|
const handleSelect = (key: string) => {
|
||||||
activeIndex.value = key
|
activeIndex.value = key
|
||||||
}
|
}
|
||||||
|
|
||||||
// 处理分页变化
|
const handleFetchBids = async (page: number, limit: number, source?: string) => {
|
||||||
const handlePageChange = (page: number) => {
|
loading.value = true
|
||||||
currentPage.value = page
|
try {
|
||||||
fetchData()
|
const res = await axios.get('/api/bids', {
|
||||||
}
|
params: {
|
||||||
|
page,
|
||||||
// 处理每页数量变化
|
limit,
|
||||||
const handleSizeChange = (size: number) => {
|
source: source || undefined
|
||||||
pageSize.value = size
|
|
||||||
currentPage.value = 1
|
|
||||||
fetchData()
|
|
||||||
}
|
|
||||||
|
|
||||||
// 设置日期范围为最近3天
|
|
||||||
const setLast3Days = () => {
|
|
||||||
const endDate = new Date()
|
|
||||||
const startDate = new Date()
|
|
||||||
startDate.setDate(startDate.getDate() - 2) // 最近3天（包括今天）
|
|
||||||
|
|
||||||
const formatDateForPicker = (date: Date) => {
|
|
||||||
const year = date.getFullYear()
|
|
||||||
const month = String(date.getMonth() + 1).padStart(2, '0')
|
|
||||||
const day = String(date.getDate()).padStart(2, '0')
|
|
||||||
return `${year}-${month}-${day}`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
dateRange.value = [formatDateForPicker(startDate), formatDateForPicker(endDate)]
|
|
||||||
|
|
||||||
console.log('setLast3Days called, todayBids:', todayBids.value.length, 'dateRange:', dateRange.value)
|
|
||||||
|
|
||||||
// 直接计算筛选结果并显示提示（只限制开始时间，不限制结束时间）
|
|
||||||
const start = new Date(startDate)
|
|
||||||
start.setHours(0, 0, 0, 0)
|
|
||||||
|
|
||||||
let result = todayBids.value
|
|
||||||
result = result.filter(bid => {
|
|
||||||
if (!bid.publishDate) return false
|
|
||||||
const bidDate = new Date(bid.publishDate)
|
|
||||||
return bidDate >= start
|
|
||||||
})
|
})
|
||||||
|
bids.value = res.data.items
|
||||||
const totalBids = todayBids.value.length
|
total.value = res.data.total
|
||||||
const filteredCount = result.length
|
} catch (error) {
|
||||||
|
console.error('Failed to fetch bids:', error)
|
||||||
console.log('setLast3Days result, totalBids:', totalBids, 'filteredCount:', filteredCount)
|
} finally {
|
||||||
if (totalBids === 0) {
|
loading.value = false
|
||||||
ElMessage.warning('暂无数据，请先抓取数据')
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 设置日期范围为最近7天
|
|
||||||
const setLast7Days = () => {
|
|
||||||
const endDate = new Date()
|
|
||||||
const startDate = new Date()
|
|
||||||
startDate.setDate(startDate.getDate() - 6) // 最近7天（包括今天）
|
|
||||||
|
|
||||||
const formatDateForPicker = (date: Date) => {
|
|
||||||
const year = date.getFullYear()
|
|
||||||
const month = String(date.getMonth() + 1).padStart(2, '0')
|
|
||||||
const day = String(date.getDate()).padStart(2, '0')
|
|
||||||
return `${year}-${month}-${day}`
|
|
||||||
}
|
|
||||||
|
|
||||||
dateRange.value = [formatDateForPicker(startDate), formatDateForPicker(endDate)]
|
|
||||||
|
|
||||||
console.log('setLast7Days called, todayBids:', todayBids.value.length, 'dateRange:', dateRange.value)
|
|
||||||
|
|
||||||
// 直接计算筛选结果并显示提示（只限制开始时间，不限制结束时间）
|
|
||||||
const start = new Date(startDate)
|
|
||||||
start.setHours(0, 0, 0, 0)
|
|
||||||
|
|
||||||
let result = todayBids.value
|
|
||||||
result = result.filter(bid => {
|
|
||||||
if (!bid.publishDate) return false
|
|
||||||
const bidDate = new Date(bid.publishDate)
|
|
||||||
return bidDate >= start
|
|
||||||
})
|
|
||||||
|
|
||||||
const totalBids = todayBids.value.length
|
|
||||||
const filteredCount = result.length
|
|
||||||
|
|
||||||
console.log('setLast7Days result, totalBids:', totalBids, 'filteredCount:', filteredCount)
|
|
||||||
if (totalBids === 0) {
|
|
||||||
ElMessage.warning('暂无数据，请先抓取数据')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const formatDate = (dateString: string) => {
|
|
||||||
if (!dateString) return '-'
|
|
||||||
return new Date(dateString).toLocaleDateString()
|
|
||||||
}
|
|
||||||
|
|
||||||
// 过滤 Today's Bids，只显示包含所选关键字的项目，并且在日期范围内
|
|
||||||
const filteredTodayBids = computed(() => {
|
|
||||||
let result = todayBids.value
|
|
||||||
|
|
||||||
// 按关键字筛选
|
|
||||||
if (selectedKeywords.value.length > 0) {
|
|
||||||
result = result.filter(bid => {
|
|
||||||
return selectedKeywords.value.some(keyword =>
|
|
||||||
bid.title.toLowerCase().includes(keyword.toLowerCase())
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// 按日期范围筛选（只限制开始时间，不限制结束时间）
|
|
||||||
if (dateRange.value && dateRange.value.length === 2) {
|
|
||||||
const [startDate] = dateRange.value
|
|
||||||
result = result.filter(bid => {
|
|
||||||
if (!bid.publishDate) return false
|
|
||||||
const bidDate = new Date(bid.publishDate)
|
|
||||||
const start = new Date(startDate)
|
|
||||||
// 设置时间为当天的开始
|
|
||||||
start.setHours(0, 0, 0, 0)
|
|
||||||
return bidDate >= start
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
})
|
|
||||||
|
|
||||||
// 监听筛选结果变化并显示提示
|
|
||||||
watch(filteredTodayBids, (newFilteredBids) => {
|
|
||||||
const totalBids = todayBids.value.length
|
|
||||||
const filteredCount = newFilteredBids.length
|
|
||||||
|
|
||||||
if (totalBids > 0 && filteredCount < totalBids) {
|
|
||||||
ElMessage.info(`筛选结果：共 ${filteredCount} 条数据（总共 ${totalBids} 条）`)
|
|
||||||
}
|
|
||||||
}, { deep: true })
|
|
||||||
|
|
||||||
const fetchData = async () => {
|
const fetchData = async () => {
|
||||||
loading.value = true
|
loading.value = true
|
||||||
try {
|
try {
|
||||||
const [bidsRes, recentRes, highRes, kwRes, sourcesRes, statusRes] = await Promise.all([
|
const [bidsRes, recentRes, highRes, kwRes, sourcesRes, statusRes] = await Promise.all([
|
||||||
axios.get('/api/bids', {
|
axios.get('/api/bids', {
|
||||||
params: {
|
params: {
|
||||||
page: currentPage.value,
|
page: 1,
|
||||||
limit: pageSize.value,
|
limit: 10
|
||||||
source: selectedSource.value || undefined
|
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
axios.get('/api/bids/recent'),
|
axios.get('/api/bids/recent'),
|
||||||
@@ -411,58 +126,13 @@ const fetchData = async () => {
|
|||||||
sourceOptions.value = sourcesRes.data
|
sourceOptions.value = sourcesRes.data
|
||||||
isCrawling.value = statusRes.data.isCrawling
|
isCrawling.value = statusRes.data.isCrawling
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
ElMessage.error('Failed to fetch data')
|
console.error('Failed to fetch data:', error)
|
||||||
} finally {
|
} finally {
|
||||||
loading.value = false
|
loading.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const handleCrawl = async () => {
|
|
||||||
if (isCrawling.value) {
|
|
||||||
ElMessage.warning('Crawl is already running')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
crawling.value = true
|
|
||||||
try {
|
|
||||||
await axios.post('/api/crawler/run')
|
|
||||||
ElMessage.success('Crawl completed successfully')
|
|
||||||
fetchData() // Refresh data after crawl
|
|
||||||
} catch (error) {
|
|
||||||
ElMessage.error('Failed to run crawl task')
|
|
||||||
} finally {
|
|
||||||
crawling.value = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const handleAddKeyword = async () => {
|
|
||||||
if (!form.word) {
|
|
||||||
ElMessage.warning('Please enter a keyword')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
await axios.post('/api/keywords', form)
|
|
||||||
ElMessage.success('Keyword added')
|
|
||||||
dialogVisible.value = false
|
|
||||||
form.word = ''
|
|
||||||
form.weight = 1
|
|
||||||
fetchData()
|
|
||||||
} catch (error) {
|
|
||||||
ElMessage.error('Failed to add keyword')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const handleDeleteKeyword = async (id: string) => {
|
|
||||||
try {
|
|
||||||
await axios.delete(`/api/keywords/${id}`)
|
|
||||||
ElMessage.success('Keyword deleted')
|
|
||||||
fetchData()
|
|
||||||
} catch (error) {
|
|
||||||
ElMessage.error('Failed to delete keyword')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
loadSavedKeywords()
|
|
||||||
fetchData()
|
fetchData()
|
||||||
})
|
})
|
||||||
</script>
|
</script>
|
||||||
@@ -486,9 +156,4 @@ onMounted(() => {
|
|||||||
font-size: 18px;
|
font-size: 18px;
|
||||||
background-color: #434a50;
|
background-color: #434a50;
|
||||||
}
|
}
|
||||||
.card-header {
|
|
||||||
display: flex;
|
|
||||||
justify-content: space-between;
|
|
||||||
align-items: center;
|
|
||||||
}
|
|
||||||
</style>
|
</style>
|
||||||
78
frontend/src/components/Bids.vue
Normal file
78
frontend/src/components/Bids.vue
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
<template>
|
||||||
|
<div>
|
||||||
|
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px;">
|
||||||
|
<h2 style="margin: 0;">All Bids</h2>
|
||||||
|
<el-select v-model="selectedSource" placeholder="Filter by Source" clearable style="width: 200px" @change="handleSourceChange">
|
||||||
|
<el-option
|
||||||
|
v-for="source in sourceOptions"
|
||||||
|
:key="source"
|
||||||
|
:label="source"
|
||||||
|
:value="source"
|
||||||
|
/>
|
||||||
|
</el-select>
|
||||||
|
</div>
|
||||||
|
<el-table :data="bids" v-loading="loading" style="width: 100%">
|
||||||
|
<el-table-column prop="title" label="Title">
|
||||||
|
<template #default="scope">
|
||||||
|
<a :href="scope.row.url" target="_blank">{{ scope.row.title }}</a>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column prop="source" label="Source" width="200" />
|
||||||
|
<el-table-column prop="publishDate" label="Date" width="150">
|
||||||
|
<template #default="scope">{{ formatDate(scope.row.publishDate) }}</template>
|
||||||
|
</el-table-column>
|
||||||
|
</el-table>
|
||||||
|
<el-pagination
|
||||||
|
v-model:current-page="currentPage"
|
||||||
|
v-model:page-size="pageSize"
|
||||||
|
:page-sizes="[10, 20, 50, 100]"
|
||||||
|
:total="total"
|
||||||
|
layout="total, sizes, prev, pager, next, jumper"
|
||||||
|
@current-change="handlePageChange"
|
||||||
|
@size-change="handleSizeChange"
|
||||||
|
style="margin-top: 20px; justify-content: flex-end;"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup lang="ts">
|
||||||
|
import { ref } from 'vue'
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
bids: any[]
|
||||||
|
sourceOptions: string[]
|
||||||
|
loading: boolean
|
||||||
|
total: number
|
||||||
|
}
|
||||||
|
|
||||||
|
const props = defineProps<Props>()
|
||||||
|
|
||||||
|
const emit = defineEmits<{
|
||||||
|
fetch: [page: number, limit: number, source?: string]
|
||||||
|
}>()
|
||||||
|
|
||||||
|
const selectedSource = ref('')
|
||||||
|
const currentPage = ref(1)
|
||||||
|
const pageSize = ref(10)
|
||||||
|
|
||||||
|
const formatDate = (dateString: string) => {
|
||||||
|
if (!dateString) return '-'
|
||||||
|
return new Date(dateString).toLocaleDateString()
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleSourceChange = () => {
|
||||||
|
currentPage.value = 1
|
||||||
|
emit('fetch', currentPage.value, pageSize.value, selectedSource.value || undefined)
|
||||||
|
}
|
||||||
|
|
||||||
|
const handlePageChange = (page: number) => {
|
||||||
|
currentPage.value = page
|
||||||
|
emit('fetch', currentPage.value, pageSize.value, selectedSource.value || undefined)
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleSizeChange = (size: number) => {
|
||||||
|
pageSize.value = size
|
||||||
|
currentPage.value = 1
|
||||||
|
emit('fetch', currentPage.value, pageSize.value, selectedSource.value || undefined)
|
||||||
|
}
|
||||||
|
</script>
|
||||||
279
frontend/src/components/Dashboard.vue
Normal file
279
frontend/src/components/Dashboard.vue
Normal file
@@ -0,0 +1,279 @@
|
|||||||
|
<template>
|
||||||
|
<div>
|
||||||
|
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px;">
|
||||||
|
<h2 style="margin: 0;">Dashboard</h2>
|
||||||
|
<el-button type="primary" :loading="crawling" :disabled="isCrawling" @click="handleCrawl">
|
||||||
|
<el-icon style="margin-right: 5px"><Refresh /></el-icon>
|
||||||
|
立刻抓取
|
||||||
|
</el-button>
|
||||||
|
</div>
|
||||||
|
<el-row :gutter="20">
|
||||||
|
<el-col :span="24">
|
||||||
|
<el-card class="box-card" shadow="hover">
|
||||||
|
<template #header>
|
||||||
|
<div class="card-header">
|
||||||
|
<span>High Priority Bids</span>
|
||||||
|
<el-tag type="danger">Top 10</el-tag>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
<el-table :data="highPriorityBids" style="width: 100%" size="small">
|
||||||
|
<el-table-column prop="title" label="Title">
|
||||||
|
<template #default="scope">
|
||||||
|
<a :href="scope.row.url" target="_blank">{{ scope.row.title }}</a>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column prop="source" label="Source" width="240" />
|
||||||
|
<el-table-column prop="publishDate" label="Date" width="120">
|
||||||
|
<template #default="scope">{{ formatDate(scope.row.publishDate) }}</template>
|
||||||
|
</el-table-column>
|
||||||
|
</el-table>
|
||||||
|
</el-card>
|
||||||
|
</el-col>
|
||||||
|
</el-row>
|
||||||
|
<el-divider />
|
||||||
|
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px;">
|
||||||
|
<h3 style="margin: 0;">Today's Bids</h3>
|
||||||
|
<div style="display: flex; gap: 10px;">
|
||||||
|
<el-date-picker
|
||||||
|
v-model="dateRange"
|
||||||
|
type="daterange"
|
||||||
|
range-separator="To"
|
||||||
|
start-placeholder="Start Date"
|
||||||
|
end-placeholder="End Date"
|
||||||
|
format="YYYY-MM-DD"
|
||||||
|
value-format="YYYY-MM-DD"
|
||||||
|
clearable
|
||||||
|
style="width: 240px;"
|
||||||
|
/>
|
||||||
|
<el-button type="primary" @click="setLast3Days">3天</el-button>
|
||||||
|
<el-button type="primary" @click="setLast7Days">7天</el-button>
|
||||||
|
<el-select
|
||||||
|
v-model="selectedKeywords"
|
||||||
|
multiple
|
||||||
|
collapse-tags
|
||||||
|
collapse-tags-tooltip
|
||||||
|
placeholder="Filter by Keywords"
|
||||||
|
clearable
|
||||||
|
style="width: 300px;"
|
||||||
|
>
|
||||||
|
<el-option
|
||||||
|
v-for="keyword in keywords"
|
||||||
|
:key="keyword.id"
|
||||||
|
:label="keyword.word"
|
||||||
|
:value="keyword.word"
|
||||||
|
/>
|
||||||
|
</el-select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<el-table :data="filteredTodayBids" v-loading="loading" style="width: 100%">
|
||||||
|
<el-table-column prop="title" label="Title">
|
||||||
|
<template #default="scope">
|
||||||
|
<a :href="scope.row.url" target="_blank">{{ scope.row.title }}</a>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column prop="source" label="Source" width="220" />
|
||||||
|
<el-table-column prop="publishDate" label="Date" width="150">
|
||||||
|
<template #default="scope">{{ formatDate(scope.row.publishDate) }}</template>
|
||||||
|
</el-table-column>
|
||||||
|
</el-table>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup lang="ts">
|
||||||
|
import { ref, computed, watch } from 'vue'
|
||||||
|
import axios from 'axios'
|
||||||
|
import { ElMessage } from 'element-plus'
|
||||||
|
import { Refresh } from '@element-plus/icons-vue'
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
todayBids: any[]
|
||||||
|
highPriorityBids: any[]
|
||||||
|
keywords: any[]
|
||||||
|
loading: boolean
|
||||||
|
isCrawling: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
const props = defineProps<Props>()
|
||||||
|
|
||||||
|
const emit = defineEmits<{
|
||||||
|
crawl: []
|
||||||
|
refresh: []
|
||||||
|
}>()
|
||||||
|
|
||||||
|
const selectedKeywords = ref<string[]>([])
|
||||||
|
const dateRange = ref<[string, string] | null>(null)
|
||||||
|
const crawling = ref(false)
|
||||||
|
|
||||||
|
// 从 localStorage 加载保存的关键字
|
||||||
|
const loadSavedKeywords = () => {
|
||||||
|
const saved = localStorage.getItem('selectedKeywords')
|
||||||
|
if (saved) {
|
||||||
|
try {
|
||||||
|
selectedKeywords.value = JSON.parse(saved)
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Failed to parse saved keywords:', e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 监听关键字变化并保存到 localStorage
|
||||||
|
watch(selectedKeywords, (newKeywords) => {
|
||||||
|
localStorage.setItem('selectedKeywords', JSON.stringify(newKeywords))
|
||||||
|
}, { deep: true })
|
||||||
|
|
||||||
|
// 监听日期范围变化并显示提示
|
||||||
|
watch(dateRange, () => {
|
||||||
|
const totalBids = props.todayBids.length
|
||||||
|
const filteredCount = filteredTodayBids.value.length
|
||||||
|
|
||||||
|
if (totalBids > 0 && filteredCount < totalBids) {
|
||||||
|
ElMessage.info(`筛选结果：共 ${filteredCount} 条数据（总共 ${totalBids} 条）`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const formatDate = (dateString: string) => {
|
||||||
|
if (!dateString) return '-'
|
||||||
|
return new Date(dateString).toLocaleDateString()
|
||||||
|
}
|
||||||
|
|
||||||
|
// 过滤 Today's Bids，只显示包含所选关键字的项目，并且在日期范围内
|
||||||
|
const filteredTodayBids = computed(() => {
|
||||||
|
let result = props.todayBids
|
||||||
|
|
||||||
|
// 按关键字筛选
|
||||||
|
if (selectedKeywords.value.length > 0) {
|
||||||
|
result = result.filter(bid => {
|
||||||
|
return selectedKeywords.value.some(keyword =>
|
||||||
|
bid.title.toLowerCase().includes(keyword.toLowerCase())
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// 按日期范围筛选（只限制开始时间，不限制结束时间）
|
||||||
|
if (dateRange.value && dateRange.value.length === 2) {
|
||||||
|
const [startDate] = dateRange.value
|
||||||
|
result = result.filter(bid => {
|
||||||
|
if (!bid.publishDate) return false
|
||||||
|
const bidDate = new Date(bid.publishDate)
|
||||||
|
const start = new Date(startDate)
|
||||||
|
// 设置时间为当天的开始
|
||||||
|
start.setHours(0, 0, 0, 0)
|
||||||
|
return bidDate >= start
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
})
|
||||||
|
|
||||||
|
// 监听筛选结果变化并显示提示
|
||||||
|
watch(filteredTodayBids, (newFilteredBids) => {
|
||||||
|
const totalBids = props.todayBids.length
|
||||||
|
const filteredCount = newFilteredBids.length
|
||||||
|
|
||||||
|
if (totalBids > 0 && filteredCount < totalBids) {
|
||||||
|
ElMessage.info(`筛选结果：共 ${filteredCount} 条数据（总共 ${totalBids} 条）`)
|
||||||
|
}
|
||||||
|
}, { deep: true })
|
||||||
|
|
||||||
|
// 设置日期范围为最近3天
|
||||||
|
const setLast3Days = () => {
|
||||||
|
const endDate = new Date()
|
||||||
|
const startDate = new Date()
|
||||||
|
startDate.setDate(startDate.getDate() - 2) // 最近3天（包括今天）
|
||||||
|
|
||||||
|
const formatDateForPicker = (date: Date) => {
|
||||||
|
const year = date.getFullYear()
|
||||||
|
const month = String(date.getMonth() + 1).padStart(2, '0')
|
||||||
|
const day = String(date.getDate()).padStart(2, '0')
|
||||||
|
return `${year}-${month}-${day}`
|
||||||
|
}
|
||||||
|
|
||||||
|
dateRange.value = [formatDateForPicker(startDate), formatDateForPicker(endDate)]
|
||||||
|
|
||||||
|
console.log('setLast3Days called, todayBids:', props.todayBids.length, 'dateRange:', dateRange.value)
|
||||||
|
|
||||||
|
// 直接计算筛选结果并显示提示（只限制开始时间，不限制结束时间）
|
||||||
|
const start = new Date(startDate)
|
||||||
|
start.setHours(0, 0, 0, 0)
|
||||||
|
|
||||||
|
let result = props.todayBids
|
||||||
|
result = result.filter(bid => {
|
||||||
|
if (!bid.publishDate) return false
|
||||||
|
const bidDate = new Date(bid.publishDate)
|
||||||
|
return bidDate >= start
|
||||||
|
})
|
||||||
|
|
||||||
|
const totalBids = props.todayBids.length
|
||||||
|
const filteredCount = result.length
|
||||||
|
|
||||||
|
console.log('setLast3Days result, totalBids:', totalBids, 'filteredCount:', filteredCount)
|
||||||
|
if (totalBids === 0) {
|
||||||
|
ElMessage.warning('暂无数据，请先抓取数据')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 设置日期范围为最近7天
|
||||||
|
const setLast7Days = () => {
|
||||||
|
const endDate = new Date()
|
||||||
|
const startDate = new Date()
|
||||||
|
startDate.setDate(startDate.getDate() - 6) // 最近7天（包括今天）
|
||||||
|
|
||||||
|
const formatDateForPicker = (date: Date) => {
|
||||||
|
const year = date.getFullYear()
|
||||||
|
const month = String(date.getMonth() + 1).padStart(2, '0')
|
||||||
|
const day = String(date.getDate()).padStart(2, '0')
|
||||||
|
return `${year}-${month}-${day}`
|
||||||
|
}
|
||||||
|
|
||||||
|
dateRange.value = [formatDateForPicker(startDate), formatDateForPicker(endDate)]
|
||||||
|
|
||||||
|
console.log('setLast7Days called, todayBids:', props.todayBids.length, 'dateRange:', dateRange.value)
|
||||||
|
|
||||||
|
// 直接计算筛选结果并显示提示（只限制开始时间，不限制结束时间）
|
||||||
|
const start = new Date(startDate)
|
||||||
|
start.setHours(0, 0, 0, 0)
|
||||||
|
|
||||||
|
let result = props.todayBids
|
||||||
|
result = result.filter(bid => {
|
||||||
|
if (!bid.publishDate) return false
|
||||||
|
const bidDate = new Date(bid.publishDate)
|
||||||
|
return bidDate >= start
|
||||||
|
})
|
||||||
|
|
||||||
|
const totalBids = props.todayBids.length
|
||||||
|
const filteredCount = result.length
|
||||||
|
|
||||||
|
console.log('setLast7Days result, totalBids:', totalBids, 'filteredCount:', filteredCount)
|
||||||
|
if (totalBids === 0) {
|
||||||
|
ElMessage.warning('暂无数据，请先抓取数据')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleCrawl = async () => {
|
||||||
|
if (props.isCrawling) {
|
||||||
|
ElMessage.warning('Crawl is already running')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
crawling.value = true
|
||||||
|
try {
|
||||||
|
await axios.post('/api/crawler/run')
|
||||||
|
ElMessage.success('Crawl completed successfully')
|
||||||
|
emit('refresh') // Refresh data after crawl
|
||||||
|
} catch (error) {
|
||||||
|
ElMessage.error('Failed to run crawl task')
|
||||||
|
} finally {
|
||||||
|
crawling.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 初始化时加载保存的关键字
|
||||||
|
loadSavedKeywords()
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.card-header {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
<script setup lang="ts">
|
|
||||||
import { ref } from 'vue'
|
|
||||||
|
|
||||||
defineProps<{ msg: string }>()
|
|
||||||
|
|
||||||
const count = ref(0)
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<template>
|
|
||||||
<h1>{{ msg }}</h1>
|
|
||||||
|
|
||||||
<div class="card">
|
|
||||||
<button type="button" @click="count++">count is {{ count }}</button>
|
|
||||||
<p>
|
|
||||||
Edit
|
|
||||||
<code>components/HelloWorld.vue</code> to test HMR
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<p>
|
|
||||||
Check out
|
|
||||||
<a href="https://vuejs.org/guide/quick-start.html#local" target="_blank"
|
|
||||||
>create-vue</a
|
|
||||||
>, the official Vue + Vite starter
|
|
||||||
</p>
|
|
||||||
<p>
|
|
||||||
Learn more about IDE Support for Vue in the
|
|
||||||
<a
|
|
||||||
href="https://vuejs.org/guide/scaling-up/tooling.html#ide-support"
|
|
||||||
target="_blank"
|
|
||||||
>Vue Docs Scaling up Guide</a
|
|
||||||
>.
|
|
||||||
</p>
|
|
||||||
<p class="read-the-docs">Click on the Vite and Vue logos to learn more</p>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<style scoped>
|
|
||||||
.read-the-docs {
|
|
||||||
color: #888;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
107
frontend/src/components/Keywords.vue
Normal file
107
frontend/src/components/Keywords.vue
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
<template>
|
||||||
|
<div>
|
||||||
|
<div class="card-header" style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px;">
|
||||||
|
<h2>Keyword Management</h2>
|
||||||
|
<el-button type="primary" @click="dialogVisible = true">Add Keyword</el-button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div v-loading="loading" style="min-height: 200px;">
|
||||||
|
<el-tag
|
||||||
|
v-for="keyword in keywords"
|
||||||
|
:key="keyword.id"
|
||||||
|
closable
|
||||||
|
:type="getTagType(keyword.weight)"
|
||||||
|
@close="handleDeleteKeyword(keyword.id)"
|
||||||
|
style="margin: 5px;"
|
||||||
|
>
|
||||||
|
{{ keyword.word }}
|
||||||
|
</el-tag>
|
||||||
|
<el-empty v-if="keywords.length === 0" description="No keywords" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<el-dialog v-model="dialogVisible" title="Add Keyword" width="30%">
|
||||||
|
<el-form :model="form" label-width="120px">
|
||||||
|
<el-form-item label="Keyword">
|
||||||
|
<el-input v-model="form.word" />
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="Weight">
|
||||||
|
<el-input-number v-model="form.weight" :min="1" :max="5" />
|
||||||
|
</el-form-item>
|
||||||
|
</el-form>
|
||||||
|
<template #footer>
|
||||||
|
<span class="dialog-footer">
|
||||||
|
<el-button @click="dialogVisible = false">Cancel</el-button>
|
||||||
|
<el-button type="primary" @click="handleAddKeyword">Confirm</el-button>
|
||||||
|
</span>
|
||||||
|
</template>
|
||||||
|
</el-dialog>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup lang="ts">
|
||||||
|
import { ref, reactive } from 'vue'
|
||||||
|
import axios from 'axios'
|
||||||
|
import { ElMessage } from 'element-plus'
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
keywords: any[]
|
||||||
|
loading: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
const props = defineProps<Props>()
|
||||||
|
|
||||||
|
const emit = defineEmits<{
|
||||||
|
refresh: []
|
||||||
|
}>()
|
||||||
|
|
||||||
|
const dialogVisible = ref(false)
|
||||||
|
|
||||||
|
const form = reactive({
|
||||||
|
word: '',
|
||||||
|
weight: 1
|
||||||
|
})
|
||||||
|
|
||||||
|
// 根据 weight 获取 tag 类型
|
||||||
|
const getTagType = (weight: number) => {
|
||||||
|
if (weight >= 5) return 'danger'
|
||||||
|
if (weight >= 4) return 'warning'
|
||||||
|
if (weight >= 3) return 'primary'
|
||||||
|
if (weight >= 2) return 'success'
|
||||||
|
return 'info'
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleAddKeyword = async () => {
|
||||||
|
if (!form.word) {
|
||||||
|
ElMessage.warning('Please enter a keyword')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await axios.post('/api/keywords', form)
|
||||||
|
ElMessage.success('Keyword added')
|
||||||
|
dialogVisible.value = false
|
||||||
|
form.word = ''
|
||||||
|
form.weight = 1
|
||||||
|
emit('refresh')
|
||||||
|
} catch (error) {
|
||||||
|
ElMessage.error('Failed to add keyword')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleDeleteKeyword = async (id: string) => {
|
||||||
|
try {
|
||||||
|
await axios.delete(`/api/keywords/${id}`)
|
||||||
|
ElMessage.success('Keyword deleted')
|
||||||
|
emit('refresh')
|
||||||
|
} catch (error) {
|
||||||
|
ElMessage.error('Failed to delete keyword')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.card-header {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -137,9 +137,18 @@ export const CdtCrawler = {
|
|||||||
logger.log('Simulating human scrolling...');
|
logger.log('Simulating human scrolling...');
|
||||||
await simulateHumanScrolling(page);
|
await simulateHumanScrolling(page);
|
||||||
|
|
||||||
|
// 等待表格加载完成
|
||||||
|
logger.log('Waiting for table to load...');
|
||||||
|
await page.waitForSelector('table.layui-table', { timeout: 30000 });
|
||||||
|
|
||||||
while (currentPage <= maxPages) {
|
while (currentPage <= maxPages) {
|
||||||
|
// 等待表格数据加载
|
||||||
|
await page.waitForSelector('tbody tr', { timeout: 10000 });
|
||||||
|
|
||||||
|
// 获取当前页面的 HTML 内容
|
||||||
const content = await page.content();
|
const content = await page.content();
|
||||||
const pageResults = this.extract(content);
|
const pageResults = this.extract(content);
|
||||||
|
|
||||||
if (pageResults.length === 0) {
|
if (pageResults.length === 0) {
|
||||||
logger.warn(`No results found on page ${currentPage}, stopping.`);
|
logger.warn(`No results found on page ${currentPage}, stopping.`);
|
||||||
break;
|
break;
|
||||||
@@ -155,24 +164,45 @@ export const CdtCrawler = {
|
|||||||
logger.log('Simulating human scrolling before pagination...');
|
logger.log('Simulating human scrolling before pagination...');
|
||||||
await simulateHumanScrolling(page);
|
await simulateHumanScrolling(page);
|
||||||
|
|
||||||
// Find the "Next Page" button - layui pagination
|
// 查找下一页按钮
|
||||||
const nextButtonSelector = 'a.layui-laypage-next:not(.layui-disabled)';
|
const nextButtonSelector = 'a.layui-laypage-next:not(.layui-disabled)';
|
||||||
const nextButton = await page.$(nextButtonSelector);
|
const nextButtonExists = await page.evaluate((selector) => {
|
||||||
|
const btn = document.querySelector(selector);
|
||||||
|
return btn !== null && !btn.classList.contains('layui-disabled');
|
||||||
|
}, nextButtonSelector);
|
||||||
|
|
||||||
if (!nextButton) {
|
if (!nextButtonExists) {
|
||||||
logger.log('Next page button not found. Reached end of list.');
|
logger.log('Next page button not found or disabled. Reached end of list.');
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.log(`Navigating to page ${currentPage + 1}...`);
|
logger.log(`Navigating to page ${currentPage + 1}...`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await Promise.all([
|
// 点击下一页按钮
|
||||||
page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 60000 }),
|
await page.evaluate((selector) => {
|
||||||
nextButton.click(),
|
const btn = document.querySelector(selector) as HTMLElement;
|
||||||
]);
|
if (btn) btn.click();
|
||||||
} catch (navError) {
|
}, nextButtonSelector);
|
||||||
logger.error(`Navigation to page ${currentPage + 1} failed: ${navError.message}`);
|
|
||||||
|
// 等待 AJAX 请求完成（通过监听网络请求）
|
||||||
|
await page.waitForFunction(() => {
|
||||||
|
// 检查表格是否正在加载
|
||||||
|
const loading = document.querySelector('.layui-table-loading');
|
||||||
|
return !loading;
|
||||||
|
}, { timeout: 30000 }).catch(() => {});
|
||||||
|
|
||||||
|
// 额外等待确保数据加载完成
|
||||||
|
await new Promise(r => setTimeout(r, 2000));
|
||||||
|
|
||||||
|
// 检查是否真的翻页了（通过检查当前页码）
|
||||||
|
const currentActivePage = await page.evaluate(() => {
|
||||||
|
const activeSpan = document.querySelector('.layui-laypage-curr em:last-child');
|
||||||
|
return activeSpan ? parseInt(activeSpan.textContent || '1') : 1;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (currentActivePage <= currentPage) {
|
||||||
|
logger.log('Page did not change, stopping.');
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -188,6 +218,11 @@ export const CdtCrawler = {
|
|||||||
// Random delay between pages
|
// Random delay between pages
|
||||||
const delay = Math.floor(Math.random() * (3000 - 1000 + 1)) + 1000;
|
const delay = Math.floor(Math.random() * (3000 - 1000 + 1)) + 1000;
|
||||||
await new Promise(resolve => setTimeout(resolve, delay));
|
await new Promise(resolve => setTimeout(resolve, delay));
|
||||||
|
|
||||||
|
} catch (navError) {
|
||||||
|
logger.error(`Navigation to page ${currentPage + 1} failed: ${navError.message}`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return allResults;
|
return allResults;
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ export interface EspicResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export const EspicCrawler = {
|
export const EspicCrawler = {
|
||||||
name: '电能e招采平台',
|
name: '电能e招采平台（国电投）',
|
||||||
baseUrl: 'https://ebid.espic.com.cn',
|
baseUrl: 'https://ebid.espic.com.cn',
|
||||||
|
|
||||||
// 生成动态 URL，使用当前日期
|
// 生成动态 URL，使用当前日期
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ export class BidCrawlTask {
|
|||||||
|
|
||||||
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
|
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
|
||||||
async handleCron() {
|
async handleCron() {
|
||||||
this.logger.debug('Scheduled crawl task started');
|
// this.logger.debug('Scheduled crawl task started');
|
||||||
await this.crawlerService.crawlAll();
|
// await this.crawlerService.crawlAll();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
|
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
|
||||||
|
|||||||
Reference in New Issue
Block a user