feat: 增强爬虫系统功能和性能管理
- 新增爬虫状态接口:GET /api/crawler/status 可实时查看爬虫运行状态 - 防止重复爬取:添加 isCrawling 标志位,避免同时运行多个爬虫任务 - 增强爬虫服务:集成所有9个爬虫平台到 BidCrawlerService - 添加执行时间限制:设置最大执行时间为1小时,防止任务无限运行 - 新增来源统计功能:GET /api/bids/sources 可查看所有招标来源平台 - 优化错误处理:完善爬虫完成后的时间统计和超时警告 - 改进控制器逻辑:更好的异常处理和状态管理 - 支持的平台包括:华能、大唐、华润、三峡、中核、中广核、电能e招采、大连能源、北京电力等9大采购平台
This commit is contained in:
@@ -1,21 +1,38 @@
|
||||
import { Controller, Post } from '@nestjs/common';
|
||||
import { Controller, Post, Get } from '@nestjs/common';
|
||||
import { BidCrawlerService } from './services/bid-crawler.service';
|
||||
|
||||
@Controller('api/crawler')
|
||||
export class CrawlerController {
|
||||
private isCrawling = false;
|
||||
|
||||
constructor(private readonly crawlerService: BidCrawlerService) {}
|
||||
|
||||
@Get('status')
|
||||
getStatus() {
|
||||
return { isCrawling: this.isCrawling };
|
||||
}
|
||||
|
||||
@Post('run')
|
||||
async runCrawl() {
|
||||
if (this.isCrawling) {
|
||||
return { message: 'Crawl is already running' };
|
||||
}
|
||||
|
||||
this.isCrawling = true;
|
||||
|
||||
// We don't await this because we want it to run in the background
|
||||
// and return immediately, or we can await if we want the user to wait.
|
||||
// and return immediately, or we can await if we want to user to wait.
|
||||
// Given the requirement "Immediate Crawl", usually implies triggering it.
|
||||
// However, for a better UI experience, we might want to wait or just trigger.
|
||||
// Let's await it so the user knows when it's done (or failed),
|
||||
// Let's await it so that user knows when it's done (or failed),
|
||||
// assuming it doesn't take too long for the mock.
|
||||
// Real crawling might take long, so background is better.
|
||||
// For this prototype, I'll await it to show completion.
|
||||
await this.crawlerService.crawlAll();
|
||||
return { message: 'Crawl completed successfully' };
|
||||
try {
|
||||
await this.crawlerService.crawlAll();
|
||||
return { message: 'Crawl completed successfully' };
|
||||
} finally {
|
||||
this.isCrawling = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user