feat: 重构AI推荐功能并优化爬虫基础URL
重构前端AI推荐组件,移除本地过滤逻辑,改为从后端获取日期范围内的数据 新增AI服务模块,包含Prompt和推荐逻辑 为投标服务添加按日期范围查询接口 统一各爬虫服务的baseURL格式
This commit is contained in:
1
src/ai/Prompt.ts
Normal file
1
src/ai/Prompt.ts
Normal file
@@ -0,0 +1 @@
|
||||
export const PromptString: string = `先给我说你统计了多少个项目。我只对辽宁、山东、江苏、浙江、福建、广东、广西、河北这些地方的海上风电、海上光伏、漂浮式光伏、滩涂光伏、滩涂风电、渔光互补项目感兴趣。从我提供的这些工程里面找到我感兴趣的工程。如果没有推荐的,也要给出思考过程。`;
|
||||
@@ -3,9 +3,6 @@ import { AiService } from './ai.service';
|
||||
|
||||
export class BidDataDto {
|
||||
title: string;
|
||||
url: string;
|
||||
source: string;
|
||||
publishDate: string;
|
||||
}
|
||||
|
||||
export class BidsRequestDto {
|
||||
|
||||
83
src/ai/ai.service.ts
Normal file
83
src/ai/ai.service.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import OpenAI from 'openai';
|
||||
import { PromptString } from './Prompt';
|
||||
|
||||
export interface BidDataDto {
|
||||
title: string;
|
||||
}
|
||||
|
||||
export interface AIRecommendation {
|
||||
title: string;
|
||||
url: string;
|
||||
source: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AiService {
|
||||
private readonly logger = new Logger(AiService.name);
|
||||
private openai: OpenAI;
|
||||
|
||||
constructor(private readonly configService: ConfigService) {
|
||||
const apiKey = this.configService.get<string>('ARK_API_KEY');
|
||||
this.openai = new OpenAI({
|
||||
apiKey: apiKey || '',
|
||||
baseURL: 'https://ark.cn-beijing.volces.com/api/v3',
|
||||
timeout: 120000, // 120秒超时
|
||||
});
|
||||
}
|
||||
|
||||
async getRecommendations(bids: BidDataDto[]): Promise<AIRecommendation[]> {
|
||||
this.logger.log('开始获取 AI 推荐');
|
||||
this.logger.log(`发送给 AI 的数据数量: ${bids.length}`);
|
||||
|
||||
try {
|
||||
const prompt =PromptString+ `请根据以下投标项目标题列表,推荐最值得关注的 5-10 个项目。请以 JSON 格式返回,格式如下:
|
||||
[
|
||||
{
|
||||
"title": "项目标题",
|
||||
"confidence": 推荐度(0-100的数字)
|
||||
}
|
||||
]
|
||||
|
||||
投标项目标题列表:
|
||||
${JSON.stringify(bids.map(b => b.title), null, 2)}`;
|
||||
this.logger.log('发给AI的内容',prompt);
|
||||
const completion = await this.openai.chat.completions.create({
|
||||
model: 'doubao-seed-1-6-lite-251015',
|
||||
max_tokens: 32768,
|
||||
reasoning_effort: 'medium',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
this.logger.log('AI API 响应成功');
|
||||
|
||||
const aiContent = completion.choices[0].message.content;
|
||||
this.logger.log('AI 返回的内容:', aiContent);
|
||||
|
||||
if (!aiContent) {
|
||||
throw new Error('AI 返回内容为空');
|
||||
}
|
||||
|
||||
const jsonMatch = aiContent.match(/\[[\s\S]*\]/);
|
||||
|
||||
if (!jsonMatch) {
|
||||
throw new Error('AI 返回格式不正确');
|
||||
}
|
||||
|
||||
const recommendations = JSON.parse(jsonMatch[0]) as AIRecommendation[];
|
||||
this.logger.log(`解析后的推荐结果: ${recommendations.length} 个`);
|
||||
|
||||
return recommendations;
|
||||
} catch (error) {
|
||||
this.logger.error('获取 AI 推荐失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,4 +24,9 @@ export class BidsController {
|
||||
getSources() {
|
||||
return this.bidsService.getSources();
|
||||
}
|
||||
|
||||
@Get('by-date-range')
|
||||
getByDateRange(@Query('startDate') startDate: string, @Query('endDate') endDate: string) {
|
||||
return this.bidsService.getBidsByDateRange(startDate, endDate);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,4 +87,22 @@ export class BidsService {
|
||||
.orderBy('bid.publishDate', 'DESC')
|
||||
.getMany();
|
||||
}
|
||||
|
||||
async getBidsByDateRange(startDate?: string, endDate?: string) {
|
||||
const qb = this.bidRepository.createQueryBuilder('bid');
|
||||
|
||||
if (startDate) {
|
||||
const start = new Date(startDate);
|
||||
start.setHours(0, 0, 0, 0);
|
||||
qb.andWhere('bid.publishDate >= :startDate', { startDate: start });
|
||||
}
|
||||
|
||||
if (endDate) {
|
||||
const end = new Date(endDate);
|
||||
end.setHours(23, 59, 59, 999);
|
||||
qb.andWhere('bid.publishDate <= :endDate', { endDate: end });
|
||||
}
|
||||
|
||||
return qb.orderBy('bid.publishDate', 'DESC').getMany();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface CdtResult {
|
||||
export const CdtCrawler = {
|
||||
name: '中国大唐集团电子商务平台',
|
||||
url: 'https://tang.cdt-ec.com/home/index.html',
|
||||
baseUrl: 'https://tang.cdt-ec.com',
|
||||
baseUrl: 'https://tang.cdt-ec.com/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<CdtResult[]> {
|
||||
const logger = new Logger('CdtCrawler');
|
||||
|
||||
@@ -50,7 +50,7 @@ async function simulateHumanScrolling(page: puppeteer.Page) {
|
||||
export const CeicCrawler = {
|
||||
name: '国家能源集团生态协作平台',
|
||||
url: 'https://ceic.dlnyzb.com/3001',
|
||||
baseUrl: 'https://ceic.dlnyzb.com',
|
||||
baseUrl: 'https://ceic.dlnyzb.com/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<ChdtpResult[]> {
|
||||
const logger = new Logger('CeicCrawler');
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface CgnpcResult {
|
||||
export const CgnpcCrawler = {
|
||||
name: '中广核电子商务平台',
|
||||
url: 'https://ecp.cgnpc.com.cn/zbgg.html',
|
||||
baseUrl: 'https://ecp.cgnpc.com.cn',
|
||||
baseUrl: 'https://ecp.cgnpc.com.cn/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<CgnpcResult[]> {
|
||||
const logger = new Logger('CgnpcCrawler');
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface CnncecpResult {
|
||||
export const CnncecpCrawler = {
|
||||
name: '中核集团电子采购平台',
|
||||
url: 'https://www.cnncecp.com/xzbgg/index.jhtml',
|
||||
baseUrl: 'https://www.cnncecp.com',
|
||||
baseUrl: 'https://www.cnncecp.com/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<CnncecpResult[]> {
|
||||
const logger = new Logger('CnncecpCrawler');
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface CnoocResult {
|
||||
export const CnoocCrawler = {
|
||||
name: '中海油招标平台',
|
||||
url: 'https://buy.cnooc.com.cn/cbjyweb/001/001001/moreinfo.html',
|
||||
baseUrl: 'https://buy.cnooc.com.cn',
|
||||
baseUrl: 'https://buy.cnooc.com.cn/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<CnoocResult[]> {
|
||||
const logger = new Logger('CnoocCrawler');
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface EpsResult {
|
||||
export const EpsCrawler = {
|
||||
name: '中国三峡集团电子商务平台',
|
||||
url: 'https://eps.ctg.com.cn/cms/channel/1ywgg1/index.htm',
|
||||
baseUrl: 'https://eps.ctg.com.cn',
|
||||
baseUrl: 'https://eps.ctg.com.cn/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<EpsResult[]> {
|
||||
const logger = new Logger('EpsCrawler');
|
||||
|
||||
@@ -54,7 +54,7 @@ export interface EspicResult {
|
||||
|
||||
export const EspicCrawler = {
|
||||
name: '电能e招采平台(国电投)',
|
||||
baseUrl: 'https://ebid.espic.com.cn',
|
||||
baseUrl: 'https://ebid.espic.com.cn/',
|
||||
|
||||
// 生成动态 URL,使用当前日期
|
||||
getUrl(page: number = 1): string {
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface PowerbeijingResult {
|
||||
export const PowerbeijingCrawler = {
|
||||
name: '北京京能电子商务平台',
|
||||
url: 'https://www.powerbeijing-ec.com/jncms/search/bulletin.html?dates=300&categoryId=2&tabName=%E6%8B%9B%E6%A0%87%E5%85%AC%E5%91%8A&page=1',
|
||||
baseUrl: 'https://www.powerbeijing-ec.com',
|
||||
baseUrl: 'https://www.powerbeijing-ec.com/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<PowerbeijingResult[]> {
|
||||
const logger = new Logger('PowerbeijingCrawler');
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface SdiccResult {
|
||||
export const SdiccCrawler = {
|
||||
name: '国投集团电子采购平台',
|
||||
url: 'https://www.sdicc.com.cn/cgxx/ggList',
|
||||
baseUrl: 'https://www.sdicc.com.cn',
|
||||
baseUrl: 'https://www.sdicc.com.cn/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<SdiccResult[]> {
|
||||
const logger = new Logger('SdiccCrawler');
|
||||
|
||||
@@ -50,7 +50,7 @@ async function simulateHumanScrolling(page: puppeteer.Page) {
|
||||
export const SzecpCrawler = {
|
||||
name: '华润守正采购交易平台',
|
||||
url: 'https://www.szecp.com.cn/first_zbgg/index.html',
|
||||
baseUrl: 'https://www.szecp.com.cn',
|
||||
baseUrl: 'https://www.szecp.com.cn/',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<ChdtpResult[]> {
|
||||
const logger = new Logger('SzecpCrawler');
|
||||
|
||||
Reference in New Issue
Block a user