Compare commits
4 Commits
f2630ed01c
...
2b21ddb990
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2b21ddb990 | ||
|
|
3d269ce9d1 | ||
|
|
61520e9ebf | ||
|
|
3647b9a2e5 |
3
.env
3
.env
@@ -12,3 +12,6 @@ PROXY_PORT=3211
|
||||
|
||||
# 日志级别(可选):error, warn, info, debug, verbose
|
||||
LOG_LEVEL=info
|
||||
|
||||
# OpenAI API Key (用于 AI 推荐)
|
||||
ARK_API_KEY=a63d58b6-cf56-434b-8a42-5c781ba0822a
|
||||
@@ -14,3 +14,6 @@ PROXY_PORT=6000
|
||||
|
||||
# 日志级别(可选):error, warn, info, debug, verbose
|
||||
LOG_LEVEL=info
|
||||
|
||||
# OpenAI API Key (用于 AI 推荐)
|
||||
ARK_API_KEY=your_openai_api_key_here
|
||||
2
frontend/.env
Normal file
2
frontend/.env
Normal file
@@ -0,0 +1,2 @@
|
||||
# ARK API Key (用于 AI 推荐)
|
||||
VITE_ARK_API_KEY=a63d58b6-cf56-434b-8a42-5c781ba0822a
|
||||
2
frontend/.env.example
Normal file
2
frontend/.env.example
Normal file
@@ -0,0 +1,2 @@
|
||||
# OpenAI API Key (用于 AI 推荐)
|
||||
VITE_OPENAI_API_KEY=your_openai_api_key_here
|
||||
@@ -12,6 +12,7 @@
|
||||
"@element-plus/icons-vue": "^2.3.2",
|
||||
"axios": "^1.13.2",
|
||||
"element-plus": "^2.13.1",
|
||||
"openai": "^6.16.0",
|
||||
"vue": "^3.5.24"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -15,10 +15,14 @@
|
||||
<span>Dashboard</span>
|
||||
</el-menu-item>
|
||||
<el-menu-item index="2">
|
||||
<el-icon><MagicStick /></el-icon>
|
||||
<span>Dashboard AI</span>
|
||||
</el-menu-item>
|
||||
<el-menu-item index="3">
|
||||
<el-icon><Document /></el-icon>
|
||||
<span>Bids</span>
|
||||
</el-menu-item>
|
||||
<el-menu-item index="3">
|
||||
<el-menu-item index="4">
|
||||
<el-icon><Setting /></el-icon>
|
||||
<span>Keywords</span>
|
||||
</el-menu-item>
|
||||
@@ -41,9 +45,14 @@
|
||||
@refresh="fetchData"
|
||||
/>
|
||||
|
||||
<Bids
|
||||
<DashboardAI
|
||||
v-if="activeIndex === '2'"
|
||||
:bids="bids"
|
||||
/>
|
||||
|
||||
<Bids
|
||||
v-if="activeIndex === '3'"
|
||||
:bids="bids"
|
||||
:source-options="sourceOptions"
|
||||
:loading="loading"
|
||||
:total="total"
|
||||
@@ -51,7 +60,7 @@
|
||||
/>
|
||||
|
||||
<Keywords
|
||||
v-if="activeIndex === '3'"
|
||||
v-if="activeIndex === '4'"
|
||||
:keywords="keywords"
|
||||
:loading="loading"
|
||||
@refresh="fetchData"
|
||||
@@ -64,8 +73,9 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, onMounted } from 'vue'
|
||||
import axios from 'axios'
|
||||
import { DataBoard, Document, Setting } from '@element-plus/icons-vue'
|
||||
import { DataBoard, Document, Setting, MagicStick } from '@element-plus/icons-vue'
|
||||
import Dashboard from './components/Dashboard.vue'
|
||||
import DashboardAI from './components/Dashboard-AI.vue'
|
||||
import Bids from './components/Bids.vue'
|
||||
import Keywords from './components/Keywords.vue'
|
||||
|
||||
|
||||
@@ -76,3 +76,14 @@ const handleSizeChange = (size: number) => {
|
||||
emit('fetch', currentPage.value, pageSize.value, selectedSource.value || undefined)
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
a {
|
||||
text-decoration: none;
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
a:hover {
|
||||
color: #409eff;
|
||||
}
|
||||
</style>
|
||||
|
||||
270
frontend/src/components/Dashboard-AI.vue
Normal file
270
frontend/src/components/Dashboard-AI.vue
Normal file
@@ -0,0 +1,270 @@
|
||||
<template>
|
||||
<div>
|
||||
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px;">
|
||||
<h2 style="margin: 0;">Dashboard AI</h2>
|
||||
<el-button type="primary" :loading="loading" @click="fetchAIRecommendations">
|
||||
<el-icon style="margin-right: 5px"><MagicStick /></el-icon>
|
||||
获取 AI 推荐
|
||||
</el-button>
|
||||
</div>
|
||||
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px;">
|
||||
<h3 style="margin: 0;">选择日期范围</h3>
|
||||
<div style="display: flex; gap: 10px;">
|
||||
<el-date-picker
|
||||
v-model="dateRange"
|
||||
type="daterange"
|
||||
range-separator="To"
|
||||
start-placeholder="Start Date"
|
||||
end-placeholder="End Date"
|
||||
format="YYYY-MM-DD"
|
||||
value-format="YYYY-MM-DD"
|
||||
clearable
|
||||
style="width: 240px;"
|
||||
/>
|
||||
<el-button type="primary" @click="setLast3Days">3天</el-button>
|
||||
<el-button type="primary" @click="setLast7Days">7天</el-button>
|
||||
<el-button type="success" :loading="bidsLoading" @click="fetchBidsByDateRange">
|
||||
<el-icon style="margin-right: 5px"><List /></el-icon>
|
||||
列出时间范围内所有工程
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
<el-row :gutter="20">
|
||||
<el-col :span="24">
|
||||
<el-card class="box-card" shadow="hover">
|
||||
<template #header>
|
||||
<div class="card-header">
|
||||
<span>AI 推荐项目</span>
|
||||
<el-tag type="success">{{ aiRecommendations.length }} 个推荐</el-tag>
|
||||
</div>
|
||||
</template>
|
||||
<div v-if="loading" style="text-align: center; padding: 40px;">
|
||||
<el-icon class="is-loading" :size="30"><Loading /></el-icon>
|
||||
<p style="margin-top: 10px; color: #909399;">AI 正在分析中...</p>
|
||||
</div>
|
||||
<div v-else-if="aiRecommendations.length === 0" style="text-align: center; padding: 40px; color: #909399;">
|
||||
<el-icon :size="40"><InfoFilled /></el-icon>
|
||||
<p style="margin-top: 10px;">暂无 AI 推荐项目</p>
|
||||
<p style="font-size: 12px; margin-top: 5px;">点击上方按钮获取 AI 推荐</p>
|
||||
</div>
|
||||
<div v-else>
|
||||
<el-table :data="aiRecommendations" style="width: 100%" size="small">
|
||||
<el-table-column prop="title" label="项目名称">
|
||||
<template #default="scope">
|
||||
<a :href="scope.row.url" target="_blank">{{ scope.row.title }}</a>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="source" label="来源" width="200" />
|
||||
<el-table-column prop="confidence" label="推荐度" width="120">
|
||||
<template #default="scope">
|
||||
<el-tag :type="getConfidenceType(scope.row.confidence)">
|
||||
{{ scope.row.confidence }}%
|
||||
</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
</div>
|
||||
</el-card>
|
||||
</el-col>
|
||||
</el-row>
|
||||
<el-row v-if="showAllBids" :gutter="20" style="margin-top: 20px;">
|
||||
<el-col :span="24">
|
||||
<el-card class="box-card" shadow="hover">
|
||||
<template #header>
|
||||
<div class="card-header">
|
||||
<span>时间范围内所有工程</span>
|
||||
<el-tag type="info">{{ bidsByDateRange.length }} 个工程</el-tag>
|
||||
</div>
|
||||
</template>
|
||||
<div v-if="bidsLoading" style="text-align: center; padding: 40px;">
|
||||
<el-icon class="is-loading" :size="30"><Loading /></el-icon>
|
||||
<p style="margin-top: 10px; color: #909399;">加载中...</p>
|
||||
</div>
|
||||
<div v-else-if="bidsByDateRange.length === 0" style="text-align: center; padding: 40px; color: #909399;">
|
||||
<el-icon :size="40"><InfoFilled /></el-icon>
|
||||
<p style="margin-top: 10px;">该时间范围内暂无工程</p>
|
||||
</div>
|
||||
<div v-else>
|
||||
<el-table :data="bidsByDateRange" style="width: 100%" size="small">
|
||||
<el-table-column prop="title" label="项目名称">
|
||||
<template #default="scope">
|
||||
<a :href="scope.row.url" target="_blank">{{ scope.row.title }}</a>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="source" label="来源" width="200" />
|
||||
<el-table-column prop="publishDate" label="发布日期" width="180">
|
||||
<template #default="scope">
|
||||
{{ formatDate(scope.row.publishDate) }}
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
</div>
|
||||
</el-card>
|
||||
</el-col>
|
||||
</el-row>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref } from 'vue'
|
||||
import axios from 'axios'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { MagicStick, Loading, InfoFilled, List } from '@element-plus/icons-vue'
|
||||
|
||||
|
||||
interface AIRecommendation {
|
||||
title: string
|
||||
url: string
|
||||
source: string
|
||||
confidence: number
|
||||
}
|
||||
|
||||
interface Props {
|
||||
bids: any[]
|
||||
}
|
||||
|
||||
const props = defineProps<Props>()
|
||||
|
||||
const loading = ref(false)
|
||||
const aiRecommendations = ref<AIRecommendation[]>([])
|
||||
const dateRange = ref<[string, string] | null>(null)
|
||||
const showAllBids = ref(false)
|
||||
const bidsLoading = ref(false)
|
||||
const bidsByDateRange = ref<any[]>([])
|
||||
|
||||
// 设置日期范围为最近3天
|
||||
const setLast3Days = () => {
|
||||
const endDate = new Date()
|
||||
const startDate = new Date()
|
||||
startDate.setDate(startDate.getDate() - 2) // 最近3天(包括今天)
|
||||
|
||||
const formatDateForPicker = (date: Date) => {
|
||||
const year = date.getFullYear()
|
||||
const month = String(date.getMonth() + 1).padStart(2, '0')
|
||||
const day = String(date.getDate()).padStart(2, '0')
|
||||
return `${year}-${month}-${day}`
|
||||
}
|
||||
|
||||
dateRange.value = [formatDateForPicker(startDate), formatDateForPicker(endDate)]
|
||||
fetchBidsByDateRange()
|
||||
}
|
||||
|
||||
// 设置日期范围为最近7天
|
||||
const setLast7Days = () => {
|
||||
const endDate = new Date()
|
||||
const startDate = new Date()
|
||||
startDate.setDate(startDate.getDate() - 6) // 最近7天(包括今天)
|
||||
|
||||
const formatDateForPicker = (date: Date) => {
|
||||
const year = date.getFullYear()
|
||||
const month = String(date.getMonth() + 1).padStart(2, '0')
|
||||
const day = String(date.getDate()).padStart(2, '0')
|
||||
return `${year}-${month}-${day}`
|
||||
}
|
||||
|
||||
dateRange.value = [formatDateForPicker(startDate), formatDateForPicker(endDate)]
|
||||
fetchBidsByDateRange()
|
||||
}
|
||||
|
||||
// 获取 AI 推荐项目
|
||||
const fetchAIRecommendations = async () => {
|
||||
loading.value = true
|
||||
try {
|
||||
// 准备发送给后端的数据(只包含 title)
|
||||
const bidsData = bidsByDateRange.value.map(bid => ({
|
||||
title: bid.title
|
||||
}))
|
||||
|
||||
// 调用后端 API
|
||||
const response = await axios.post('/api/ai/recommendations', {
|
||||
bids: bidsData
|
||||
})
|
||||
|
||||
// 根据 title 从 bidsByDateRange 中更新 url 和 source
|
||||
aiRecommendations.value = response.data.map((rec: any) => {
|
||||
const bid = bidsByDateRange.value.find(b => b.title === rec.title)
|
||||
return {
|
||||
title: rec.title,
|
||||
url: bid?.url || '',
|
||||
source: bid?.source || '',
|
||||
confidence: rec.confidence
|
||||
}
|
||||
})
|
||||
|
||||
ElMessage.success('AI 推荐获取成功')
|
||||
} catch (error: any) {
|
||||
ElMessage.error('获取 AI 推荐失败')
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// 获取时间范围内的所有工程
|
||||
const fetchBidsByDateRange = async () => {
|
||||
if (!dateRange.value || dateRange.value.length !== 2) {
|
||||
ElMessage.warning('请先选择日期范围')
|
||||
return
|
||||
}
|
||||
|
||||
showAllBids.value = true
|
||||
bidsLoading.value = true
|
||||
try {
|
||||
const [startDate, endDate] = dateRange.value
|
||||
|
||||
// 检查 endDate 是否是今天
|
||||
const today = new Date()
|
||||
const todayStr = `${today.getFullYear()}-${String(today.getMonth() + 1).padStart(2, '0')}-${String(today.getDate()).padStart(2, '0')}`
|
||||
|
||||
// 如果 endDate 是今天,则不传递 endDate 参数(不限制截止时间)
|
||||
const params: any = { startDate }
|
||||
if (endDate !== todayStr) {
|
||||
params.endDate = endDate
|
||||
}
|
||||
|
||||
const response = await axios.get('/api/bids/by-date-range', { params })
|
||||
bidsByDateRange.value = response.data
|
||||
ElMessage.success(`获取成功,共 ${response.data.length} 个工程`)
|
||||
} catch (error: any) {
|
||||
ElMessage.error('获取工程列表失败')
|
||||
} finally {
|
||||
bidsLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// 格式化日期,只显示年月日
|
||||
const formatDate = (dateStr: string) => {
|
||||
if (!dateStr) return ''
|
||||
const date = new Date(dateStr)
|
||||
const year = date.getFullYear()
|
||||
const month = String(date.getMonth() + 1).padStart(2, '0')
|
||||
const day = String(date.getDate()).padStart(2, '0')
|
||||
return `${year}-${month}-${day}`
|
||||
}
|
||||
|
||||
// 根据推荐度返回标签类型
|
||||
const getConfidenceType = (confidence: number) => {
|
||||
if (confidence >= 90) return 'success'
|
||||
if (confidence >= 70) return 'warning'
|
||||
return 'info'
|
||||
}
|
||||
|
||||
// 初始化时设置默认日期范围为最近3天
|
||||
setLast3Days()
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.card-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
a {
|
||||
text-decoration: none;
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
a:hover {
|
||||
color: #409eff;
|
||||
}
|
||||
</style>
|
||||
@@ -11,22 +11,31 @@
|
||||
<el-col :span="24">
|
||||
<el-card class="box-card" shadow="hover">
|
||||
<template #header>
|
||||
<div class="card-header">
|
||||
<div class="card-header" @click="toggleHighPriority" style="cursor: pointer;">
|
||||
<span>High Priority Bids</span>
|
||||
<el-tag type="danger">Top 10</el-tag>
|
||||
<div style="display: flex; align-items: center; gap: 10px;">
|
||||
<el-tag type="danger">Top 10</el-tag>
|
||||
<el-icon :style="{ transform: highPriorityCollapsed ? 'rotate(-90deg)' : 'rotate(0deg)', transition: 'transform 0.3s' }">
|
||||
<ArrowDown />
|
||||
</el-icon>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
<el-table :data="highPriorityBids" style="width: 100%" size="small">
|
||||
<el-table-column prop="title" label="Title">
|
||||
<template #default="scope">
|
||||
<a :href="scope.row.url" target="_blank">{{ scope.row.title }}</a>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="source" label="Source" width="240" />
|
||||
<el-table-column prop="publishDate" label="Date" width="120">
|
||||
<template #default="scope">{{ formatDate(scope.row.publishDate) }}</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
<el-collapse-transition>
|
||||
<div v-show="!highPriorityCollapsed">
|
||||
<el-table :data="highPriorityBids" style="width: 100%" size="small">
|
||||
<el-table-column prop="title" label="Title">
|
||||
<template #default="scope">
|
||||
<a :href="scope.row.url" target="_blank">{{ scope.row.title }}</a>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="source" label="Source" width="240" />
|
||||
<el-table-column prop="publishDate" label="Date" width="120">
|
||||
<template #default="scope">{{ formatDate(scope.row.publishDate) }}</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
</div>
|
||||
</el-collapse-transition>
|
||||
</el-card>
|
||||
</el-col>
|
||||
</el-row>
|
||||
@@ -83,7 +92,7 @@
|
||||
import { ref, computed, watch } from 'vue'
|
||||
import axios from 'axios'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { Refresh } from '@element-plus/icons-vue'
|
||||
import { Refresh, ArrowDown } from '@element-plus/icons-vue'
|
||||
|
||||
interface Props {
|
||||
todayBids: any[]
|
||||
@@ -103,6 +112,20 @@ const emit = defineEmits<{
|
||||
const selectedKeywords = ref<string[]>([])
|
||||
const dateRange = ref<[string, string] | null>(null)
|
||||
const crawling = ref(false)
|
||||
const highPriorityCollapsed = ref(false)
|
||||
const isInitialized = ref(false)
|
||||
|
||||
// 切换 High Priority Bids 的折叠状态
|
||||
const toggleHighPriority = () => {
|
||||
highPriorityCollapsed.value = !highPriorityCollapsed.value
|
||||
}
|
||||
|
||||
// 监听 highPriorityBids,当没有数据时自动折叠
|
||||
watch(() => props.highPriorityBids, (newBids) => {
|
||||
if (newBids.length === 0) {
|
||||
highPriorityCollapsed.value = true
|
||||
}
|
||||
}, { immediate: true })
|
||||
|
||||
// 从 localStorage 加载保存的关键字
|
||||
const loadSavedKeywords = () => {
|
||||
@@ -123,6 +146,12 @@ watch(selectedKeywords, (newKeywords) => {
|
||||
|
||||
// 监听日期范围变化并显示提示
|
||||
watch(dateRange, () => {
|
||||
// 初始化时不显示提示
|
||||
if (!isInitialized.value) {
|
||||
isInitialized.value = true
|
||||
return
|
||||
}
|
||||
|
||||
const totalBids = props.todayBids.length
|
||||
const filteredCount = filteredTodayBids.value.length
|
||||
|
||||
@@ -207,9 +236,7 @@ const setLast3Days = () => {
|
||||
const filteredCount = result.length
|
||||
|
||||
console.log('setLast3Days result, totalBids:', totalBids, 'filteredCount:', filteredCount)
|
||||
if (totalBids === 0) {
|
||||
ElMessage.warning('暂无数据,请先抓取数据')
|
||||
}
|
||||
// 只在手动点击按钮时显示提示,初始化时不显示
|
||||
}
|
||||
|
||||
// 设置日期范围为最近7天
|
||||
@@ -244,9 +271,7 @@ const setLast7Days = () => {
|
||||
const filteredCount = result.length
|
||||
|
||||
console.log('setLast7Days result, totalBids:', totalBids, 'filteredCount:', filteredCount)
|
||||
if (totalBids === 0) {
|
||||
ElMessage.warning('暂无数据,请先抓取数据')
|
||||
}
|
||||
// 只在手动点击按钮时显示提示,初始化时不显示
|
||||
}
|
||||
|
||||
const handleCrawl = async () => {
|
||||
@@ -268,6 +293,9 @@ const handleCrawl = async () => {
|
||||
|
||||
// 初始化时加载保存的关键字
|
||||
loadSavedKeywords()
|
||||
|
||||
// 初始化时设置默认日期范围为最近3天
|
||||
setLast3Days()
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
@@ -276,4 +304,13 @@ loadSavedKeywords()
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
a {
|
||||
text-decoration: none;
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
a:hover {
|
||||
color: #409eff;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -12,5 +12,5 @@
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedSideEffectImports": true
|
||||
},
|
||||
"include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.vue"]
|
||||
"include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.vue", "../src/ai/Prompt.ts"]
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
"test:e2e": "jest --config ./test/jest-e2e.json",
|
||||
"crawl": "ts-node -r tsconfig-paths/register src/scripts/crawl.ts",
|
||||
"update-source": "ts-node -r tsconfig-paths/register src/scripts/update-source.ts",
|
||||
"web":"npm --prefix frontend run build"
|
||||
"web": "npm --prefix frontend run build"
|
||||
},
|
||||
"dependencies": {
|
||||
"@nestjs/common": "^11.0.1",
|
||||
@@ -35,6 +35,7 @@
|
||||
"class-validator": "^0.14.3",
|
||||
"dotenv": "^16.4.7",
|
||||
"mysql2": "^3.16.0",
|
||||
"openai": "^6.16.0",
|
||||
"puppeteer": "^24.34.0",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||
|
||||
1
src/ai/Prompt.ts
Normal file
1
src/ai/Prompt.ts
Normal file
@@ -0,0 +1 @@
|
||||
export const PromptString: string = `先给我说你统计了多少个项目。我只对辽宁、山东、江苏、浙江、福建、广东、广西、河北这些地方的海上风电、海上光伏、漂浮式光伏、滩涂光伏、滩涂风电、渔光互补项目感兴趣。从我提供的这些工程里面找到我感兴趣的工程。如果没有推荐的,也要给出思考过程。`;
|
||||
20
src/ai/ai.controller.ts
Normal file
20
src/ai/ai.controller.ts
Normal file
@@ -0,0 +1,20 @@
|
||||
import { Controller, Post, Body } from '@nestjs/common';
|
||||
import { AiService } from './ai.service';
|
||||
|
||||
export class BidDataDto {
|
||||
title: string;
|
||||
}
|
||||
|
||||
export class BidsRequestDto {
|
||||
bids: BidDataDto[];
|
||||
}
|
||||
|
||||
@Controller('api/ai')
|
||||
export class AiController {
|
||||
constructor(private readonly aiService: AiService) {}
|
||||
|
||||
@Post('recommendations')
|
||||
async getRecommendations(@Body() request: BidsRequestDto) {
|
||||
return this.aiService.getRecommendations(request.bids);
|
||||
}
|
||||
}
|
||||
12
src/ai/ai.module.ts
Normal file
12
src/ai/ai.module.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
import { AiController } from './ai.controller';
|
||||
import { AiService } from './ai.service';
|
||||
|
||||
@Module({
|
||||
imports: [ConfigModule],
|
||||
controllers: [AiController],
|
||||
providers: [AiService],
|
||||
exports: [AiService],
|
||||
})
|
||||
export class AiModule {}
|
||||
83
src/ai/ai.service.ts
Normal file
83
src/ai/ai.service.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import OpenAI from 'openai';
|
||||
import { PromptString } from './Prompt';
|
||||
|
||||
export interface BidDataDto {
|
||||
title: string;
|
||||
}
|
||||
|
||||
export interface AIRecommendation {
|
||||
title: string;
|
||||
url: string;
|
||||
source: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AiService {
|
||||
private readonly logger = new Logger(AiService.name);
|
||||
private openai: OpenAI;
|
||||
|
||||
constructor(private readonly configService: ConfigService) {
|
||||
const apiKey = this.configService.get<string>('ARK_API_KEY');
|
||||
this.openai = new OpenAI({
|
||||
apiKey: apiKey || '',
|
||||
baseURL: 'https://ark.cn-beijing.volces.com/api/v3',
|
||||
timeout: 120000, // 120秒超时
|
||||
});
|
||||
}
|
||||
|
||||
async getRecommendations(bids: BidDataDto[]): Promise<AIRecommendation[]> {
|
||||
this.logger.log('开始获取 AI 推荐');
|
||||
this.logger.log(`发送给 AI 的数据数量: ${bids.length}`);
|
||||
|
||||
try {
|
||||
const prompt =PromptString+ `请根据以下投标项目标题列表,推荐最值得关注的 5-10 个项目。请以 JSON 格式返回,格式如下:
|
||||
[
|
||||
{
|
||||
"title": "项目标题",
|
||||
"confidence": 推荐度(0-100的数字)
|
||||
}
|
||||
]
|
||||
|
||||
投标项目标题列表:
|
||||
${JSON.stringify(bids.map(b => b.title), null, 2)}`;
|
||||
this.logger.log('发给AI的内容',prompt);
|
||||
const completion = await this.openai.chat.completions.create({
|
||||
model: 'doubao-seed-1-6-lite-251015',
|
||||
max_tokens: 32768,
|
||||
reasoning_effort: 'medium',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
this.logger.log('AI API 响应成功');
|
||||
|
||||
const aiContent = completion.choices[0].message.content;
|
||||
this.logger.log('AI 返回的内容:', aiContent);
|
||||
|
||||
if (!aiContent) {
|
||||
throw new Error('AI 返回内容为空');
|
||||
}
|
||||
|
||||
const jsonMatch = aiContent.match(/\[[\s\S]*\]/);
|
||||
|
||||
if (!jsonMatch) {
|
||||
throw new Error('AI 返回格式不正确');
|
||||
}
|
||||
|
||||
const recommendations = JSON.parse(jsonMatch[0]) as AIRecommendation[];
|
||||
this.logger.log(`解析后的推荐结果: ${recommendations.length} 个`);
|
||||
|
||||
return recommendations;
|
||||
} catch (error) {
|
||||
this.logger.error('获取 AI 推荐失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import { KeywordsModule } from './keywords/keywords.module';
|
||||
import { CrawlerModule } from './crawler/crawler.module';
|
||||
import { TasksModule } from './schedule/schedule.module';
|
||||
import { LoggerModule } from './common/logger/logger.module';
|
||||
import { AiModule } from './ai/ai.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -16,7 +17,7 @@ import { LoggerModule } from './common/logger/logger.module';
|
||||
ScheduleModule.forRoot(),
|
||||
ServeStaticModule.forRoot({
|
||||
rootPath: join(__dirname, '..', 'frontend', 'dist'),
|
||||
exclude: ['/api*'],
|
||||
exclude: ['/api/(.*)'],
|
||||
}),
|
||||
LoggerModule,
|
||||
DatabaseModule,
|
||||
@@ -24,6 +25,7 @@ import { LoggerModule } from './common/logger/logger.module';
|
||||
KeywordsModule,
|
||||
CrawlerModule,
|
||||
TasksModule,
|
||||
AiModule,
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
@@ -24,4 +24,9 @@ export class BidsController {
|
||||
getSources() {
|
||||
return this.bidsService.getSources();
|
||||
}
|
||||
|
||||
@Get('by-date-range')
|
||||
getByDateRange(@Query('startDate') startDate: string, @Query('endDate') endDate: string) {
|
||||
return this.bidsService.getBidsByDateRange(startDate, endDate);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,4 +87,22 @@ export class BidsService {
|
||||
.orderBy('bid.publishDate', 'DESC')
|
||||
.getMany();
|
||||
}
|
||||
|
||||
async getBidsByDateRange(startDate?: string, endDate?: string) {
|
||||
const qb = this.bidRepository.createQueryBuilder('bid');
|
||||
|
||||
if (startDate) {
|
||||
const start = new Date(startDate);
|
||||
start.setHours(0, 0, 0, 0);
|
||||
qb.andWhere('bid.publishDate >= :startDate', { startDate: start });
|
||||
}
|
||||
|
||||
if (endDate) {
|
||||
const end = new Date(endDate);
|
||||
end.setHours(23, 59, 59, 999);
|
||||
qb.andWhere('bid.publishDate <= :endDate', { endDate: end });
|
||||
}
|
||||
|
||||
return qb.orderBy('bid.publishDate', 'DESC').getMany();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface CdtResult {
|
||||
export const CdtCrawler = {
|
||||
name: '中国大唐集团电子商务平台',
|
||||
url: 'https://tang.cdt-ec.com/home/index.html',
|
||||
baseUrl: 'https://tang.cdt-ec.com',
|
||||
baseUrl: 'https://tang.cdt-ec.com/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<CdtResult[]> {
|
||||
const logger = new Logger('CdtCrawler');
|
||||
|
||||
@@ -50,7 +50,7 @@ async function simulateHumanScrolling(page: puppeteer.Page) {
|
||||
export const CeicCrawler = {
|
||||
name: '国家能源集团生态协作平台',
|
||||
url: 'https://ceic.dlnyzb.com/3001',
|
||||
baseUrl: 'https://ceic.dlnyzb.com',
|
||||
baseUrl: 'https://ceic.dlnyzb.com/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<ChdtpResult[]> {
|
||||
const logger = new Logger('CeicCrawler');
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface CgnpcResult {
|
||||
export const CgnpcCrawler = {
|
||||
name: '中广核电子商务平台',
|
||||
url: 'https://ecp.cgnpc.com.cn/zbgg.html',
|
||||
baseUrl: 'https://ecp.cgnpc.com.cn',
|
||||
baseUrl: 'https://ecp.cgnpc.com.cn/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<CgnpcResult[]> {
|
||||
const logger = new Logger('CgnpcCrawler');
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
import { ChngCrawler } from './chng_target';
|
||||
import puppeteer from 'puppeteer-extra';
|
||||
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
||||
import type { Browser, Page } from 'puppeteer';
|
||||
|
||||
// 使用 stealth 插件增强反爬虫能力
|
||||
puppeteer.use(StealthPlugin());
|
||||
|
||||
// Increase timeout to 180 seconds for slow sites and stealth mode
|
||||
jest.setTimeout(180000);
|
||||
|
||||
// 获取代理配置
|
||||
const getProxyArgs = (): string[] => {
|
||||
const proxyHost = process.env.PROXY_HOST;
|
||||
const proxyPort = process.env.PROXY_PORT;
|
||||
const proxyUsername = process.env.PROXY_USERNAME;
|
||||
const proxyPassword = process.env.PROXY_PASSWORD;
|
||||
|
||||
if (proxyHost && proxyPort) {
|
||||
const args = [`--proxy-server=${proxyHost}:${proxyPort}`];
|
||||
if (proxyUsername && proxyPassword) {
|
||||
args.push(`--proxy-auth=${proxyUsername}:${proxyPassword}`);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
// 模拟人类鼠标移动
|
||||
async function simulateHumanMouseMovement(page: Page) {
|
||||
const viewport = page.viewport();
|
||||
if (!viewport) return;
|
||||
|
||||
const movements = 5 + Math.floor(Math.random() * 5); // 5-10次随机移动
|
||||
|
||||
for (let i = 0; i < movements; i++) {
|
||||
const x = Math.floor(Math.random() * viewport.width);
|
||||
const y = Math.floor(Math.random() * viewport.height);
|
||||
|
||||
await page.mouse.move(x, y, {
|
||||
steps: 10 + Math.floor(Math.random() * 20) // 10-30步,使移动更平滑
|
||||
});
|
||||
|
||||
// 随机停顿 100-500ms
|
||||
await new Promise(r => setTimeout(r, 100 + Math.random() * 400));
|
||||
}
|
||||
}
|
||||
|
||||
// 模拟人类滚动
|
||||
async function simulateHumanScrolling(page: Page) {
|
||||
const scrollCount = 3 + Math.floor(Math.random() * 5); // 3-7次滚动
|
||||
|
||||
for (let i = 0; i < scrollCount; i++) {
|
||||
const scrollDistance = 100 + Math.floor(Math.random() * 400); // 100-500px
|
||||
|
||||
await page.evaluate((distance) => {
|
||||
window.scrollBy({
|
||||
top: distance,
|
||||
behavior: 'smooth'
|
||||
});
|
||||
}, scrollDistance);
|
||||
|
||||
// 随机停顿 500-1500ms
|
||||
await new Promise(r => setTimeout(r, 500 + Math.random() * 1000));
|
||||
}
|
||||
|
||||
// 滚动回顶部
|
||||
await page.evaluate(() => {
|
||||
window.scrollTo({ top: 0, behavior: 'smooth' });
|
||||
});
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
}
|
||||
|
||||
describe('ChngCrawler Stealth Test (Headless Mode with Stealth Plugin)', () => {
|
||||
let browser: Browser;
|
||||
|
||||
beforeAll(async () => {
|
||||
const proxyArgs = getProxyArgs();
|
||||
if (proxyArgs.length > 0) {
|
||||
console.log('Using proxy:', proxyArgs.join(' '));
|
||||
}
|
||||
|
||||
browser = await puppeteer.launch({
|
||||
headless: true, // 使用 headless 模式
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--window-size=1920,1080',
|
||||
'--disable-infobars',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-accelerated-2d-canvas',
|
||||
'--no-first-run',
|
||||
'--no-zygote',
|
||||
'--disable-gpu',
|
||||
'--disable-features=VizDisplayCompositor',
|
||||
'--disable-webgl',
|
||||
...proxyArgs,
|
||||
],
|
||||
defaultViewport: null
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (browser) {
|
||||
await browser.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('should visit the website and list all found bid information with stealth plugin', async () => {
|
||||
// 为此测试单独设置更长的超时时间
|
||||
jest.setTimeout(180000);
|
||||
console.log(`
|
||||
Starting crawl for: ${ChngCrawler.name}`);
|
||||
console.log(`Target URL: ${ChngCrawler.url}`);
|
||||
console.log('Using puppeteer-extra-plugin-stealth for anti-detection');
|
||||
console.log('Running in headless mode');
|
||||
|
||||
// 创建一个临时页面用于模拟人类行为
|
||||
const tempPage = await browser.newPage();
|
||||
await tempPage.setViewport({ width: 1920, height: 1080, deviceScaleFactor: 1 });
|
||||
|
||||
// 模拟人类鼠标移动
|
||||
console.log('Simulating human mouse movements...');
|
||||
await simulateHumanMouseMovement(tempPage);
|
||||
|
||||
// 模拟人类滚动
|
||||
console.log('Simulating human scrolling...');
|
||||
await simulateHumanScrolling(tempPage);
|
||||
|
||||
await tempPage.close();
|
||||
|
||||
const results = await ChngCrawler.crawl(browser);
|
||||
|
||||
console.log(`
|
||||
Successfully found ${results.length} items:
|
||||
`);
|
||||
console.log('----------------------------------------');
|
||||
results.forEach((item, index) => {
|
||||
console.log(`${index + 1}. [${item.publishDate.toLocaleDateString()}] ${item.title}`);
|
||||
console.log(` Link: ${item.url}`);
|
||||
console.log('----------------------------------------');
|
||||
});
|
||||
|
||||
expect(results).toBeDefined();
|
||||
expect(Array.isArray(results)).toBeTruthy();
|
||||
|
||||
if (results.length === 0) {
|
||||
console.warn('Warning: No items found. The site might have detected the crawler or content is not loading properly.');
|
||||
} else {
|
||||
const firstItem = results[0];
|
||||
expect(firstItem.title).toBeTruthy();
|
||||
expect(firstItem.url).toMatch(/^https?:\/\//);
|
||||
expect(firstItem.publishDate).toBeInstanceOf(Date);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -55,7 +55,7 @@ export interface CnncecpResult {
|
||||
export const CnncecpCrawler = {
|
||||
name: '中核集团电子采购平台',
|
||||
url: 'https://www.cnncecp.com/xzbgg/index.jhtml',
|
||||
baseUrl: 'https://www.cnncecp.com',
|
||||
baseUrl: 'https://www.cnncecp.com/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<CnncecpResult[]> {
|
||||
const logger = new Logger('CnncecpCrawler');
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface CnoocResult {
|
||||
export const CnoocCrawler = {
|
||||
name: '中海油招标平台',
|
||||
url: 'https://buy.cnooc.com.cn/cbjyweb/001/001001/moreinfo.html',
|
||||
baseUrl: 'https://buy.cnooc.com.cn',
|
||||
baseUrl: 'https://buy.cnooc.com.cn/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<CnoocResult[]> {
|
||||
const logger = new Logger('CnoocCrawler');
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface EpsResult {
|
||||
export const EpsCrawler = {
|
||||
name: '中国三峡集团电子商务平台',
|
||||
url: 'https://eps.ctg.com.cn/cms/channel/1ywgg1/index.htm',
|
||||
baseUrl: 'https://eps.ctg.com.cn',
|
||||
baseUrl: 'https://eps.ctg.com.cn/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<EpsResult[]> {
|
||||
const logger = new Logger('EpsCrawler');
|
||||
|
||||
@@ -54,7 +54,7 @@ export interface EspicResult {
|
||||
|
||||
export const EspicCrawler = {
|
||||
name: '电能e招采平台(国电投)',
|
||||
baseUrl: 'https://ebid.espic.com.cn',
|
||||
baseUrl: 'https://ebid.espic.com.cn/',
|
||||
|
||||
// 生成动态 URL,使用当前日期
|
||||
getUrl(page: number = 1): string {
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface PowerbeijingResult {
|
||||
export const PowerbeijingCrawler = {
|
||||
name: '北京京能电子商务平台',
|
||||
url: 'https://www.powerbeijing-ec.com/jncms/search/bulletin.html?dates=300&categoryId=2&tabName=%E6%8B%9B%E6%A0%87%E5%85%AC%E5%91%8A&page=1',
|
||||
baseUrl: 'https://www.powerbeijing-ec.com',
|
||||
baseUrl: 'https://www.powerbeijing-ec.com/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<PowerbeijingResult[]> {
|
||||
const logger = new Logger('PowerbeijingCrawler');
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface SdiccResult {
|
||||
export const SdiccCrawler = {
|
||||
name: '国投集团电子采购平台',
|
||||
url: 'https://www.sdicc.com.cn/cgxx/ggList',
|
||||
baseUrl: 'https://www.sdicc.com.cn',
|
||||
baseUrl: 'https://www.sdicc.com.cn/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<SdiccResult[]> {
|
||||
const logger = new Logger('SdiccCrawler');
|
||||
|
||||
@@ -50,7 +50,7 @@ async function simulateHumanScrolling(page: puppeteer.Page) {
|
||||
export const SzecpCrawler = {
|
||||
name: '华润守正采购交易平台',
|
||||
url: 'https://www.szecp.com.cn/first_zbgg/index.html',
|
||||
baseUrl: 'https://www.szecp.com.cn',
|
||||
baseUrl: 'https://www.szecp.com.cn/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<ChdtpResult[]> {
|
||||
const logger = new Logger('SzecpCrawler');
|
||||
|
||||
Reference in New Issue
Block a user