a1badea135
- 新增爬虫状态接口：GET /api/crawler/status 可实时查看爬虫运行状态 - 防止重复爬取：添加 isCrawling 标志位，避免同时运行多个爬虫任务 - 增强爬虫服务：集成所有9个爬虫平台到 BidCrawlerService - 添加执行时间限制：设置最大执行时间为1小时，防止任务无限运行 - 新增来源统计功能：GET /api/bids/sources 可查看所有招标来源平台 - 优化错误处理：完善爬虫完成后的时间统计和超时警告 - 改进控制器逻辑：更好的异常处理和状态管理 - 支持的平台包括：华能、大唐、华润、三峡、中核、中广核、电能e招采、大连能源、北京电力等9大采购平台
108 lines
3.8 KiB
TypeScript
108 lines
3.8 KiB
TypeScript
import { Injectable, Logger } from '@nestjs/common';
|
||
import { ConfigService } from '@nestjs/config';
|
||
import * as puppeteer from 'puppeteer';
|
||
import { BidsService } from '../../bids/services/bid.service';
|
||
import { ChdtpCrawler } from './chdtp_target';
|
||
import { ChngCrawler } from './chng_target';
|
||
import { SzecpCrawler } from './szecp_target';
|
||
import { CdtCrawler } from './cdt_target';
|
||
import { EpsCrawler } from './eps_target';
|
||
import { CnncecpCrawler } from './cnncecp_target';
|
||
import { CgnpcCrawler } from './cgnpc_target';
|
||
import { CeicCrawler } from './ceic_target';
|
||
import { EspicCrawler } from './espic_target';
|
||
import { PowerbeijingCrawler } from './powerbeijing_target';
|
||
|
||
@Injectable()
|
||
export class BidCrawlerService {
|
||
private readonly logger = new Logger(BidCrawlerService.name);
|
||
|
||
constructor(
|
||
private bidsService: BidsService,
|
||
private configService: ConfigService,
|
||
) {}
|
||
|
||
async crawlAll() {
|
||
this.logger.log('Starting crawl task with Puppeteer...');
|
||
|
||
// 设置最大执行时间为1小时
|
||
const maxExecutionTime = 60 * 60 * 1000; // 1小时（毫秒）
|
||
const startTime = Date.now();
|
||
|
||
// 从环境变量读取代理配置
|
||
const proxyHost = this.configService.get<string>('PROXY_HOST');
|
||
const proxyPort = this.configService.get<string>('PROXY_PORT');
|
||
const proxyUsername = this.configService.get<string>('PROXY_USERNAME');
|
||
const proxyPassword = this.configService.get<string>('PROXY_PASSWORD');
|
||
|
||
// 构建代理参数
|
||
const args = [
|
||
'--no-sandbox',
|
||
'--disable-setuid-sandbox',
|
||
'--disable-blink-features=AutomationControlled',
|
||
'--disable-infobars',
|
||
'--window-position=0,0',
|
||
'--ignore-certifcate-errors',
|
||
'--ignore-certifcate-errors-spki-list',
|
||
];
|
||
|
||
if (proxyHost && proxyPort) {
|
||
const proxyUrl = proxyUsername && proxyPassword
|
||
? `http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}`
|
||
: `http://${proxyHost}:${proxyPort}`;
|
||
args.push(`--proxy-server=${proxyUrl}`);
|
||
this.logger.log(`Using proxy: ${proxyHost}:${proxyPort}`);
|
||
}
|
||
|
||
const browser = await puppeteer.launch({
|
||
headless: false,
|
||
args,
|
||
});
|
||
|
||
const crawlers = [ChdtpCrawler, ChngCrawler, SzecpCrawler, CdtCrawler, EpsCrawler, CnncecpCrawler, CgnpcCrawler, CeicCrawler, EspicCrawler, PowerbeijingCrawler];
|
||
|
||
try {
|
||
for (const crawler of crawlers) {
|
||
this.logger.log(`Crawling: ${crawler.name}`);
|
||
|
||
// 检查是否超时
|
||
const elapsedTime = Date.now() - startTime;
|
||
if (elapsedTime > maxExecutionTime) {
|
||
this.logger.warn(`⚠️ Crawl task exceeded maximum execution time of 1 hour. Stopping...`);
|
||
this.logger.warn(`⚠️ Total elapsed time: ${Math.floor(elapsedTime / 1000 / 60)} minutes`);
|
||
break;
|
||
}
|
||
|
||
try {
|
||
const results = await crawler.crawl(browser);
|
||
this.logger.log(`Extracted ${results.length} items from ${crawler.name}`);
|
||
|
||
for (const item of results) {
|
||
await this.bidsService.createOrUpdate({
|
||
title: item.title,
|
||
url: item.url,
|
||
publishDate: item.publishDate,
|
||
source: crawler.name,
|
||
unit: '',
|
||
});
|
||
}
|
||
} catch (err) {
|
||
this.logger.error(`Error crawling ${crawler.name}: ${err.message}`);
|
||
}
|
||
}
|
||
} catch (error) {
|
||
this.logger.error(`Crawl task failed: ${error.message}`);
|
||
} finally {
|
||
await browser.close();
|
||
|
||
const totalTime = Date.now() - startTime;
|
||
const minutes = Math.floor(totalTime / 1000 / 60);
|
||
this.logger.log(`Crawl task finished. Total time: ${minutes} minutes`);
|
||
|
||
if (totalTime > maxExecutionTime) {
|
||
this.logger.warn(`⚠️ Crawl task exceeded maximum execution time of 1 hour.`);
|
||
}
|
||
}
|
||
}
|
||
}
|