feat: 添加华能集团电子商务平台爬虫功能

- 新增 ChngCrawler 爬虫类,支持华能集团电子商务平台招标信息抓取
- 更新 BidCrawlerService,集成 ChngCrawler 到爬虫任务中
- 添加代理配置支持,从环境变量读取代理设置
- 优化爬虫逻辑,支持多个爬虫并行执行
- 新增 ChngCrawler 的单元测试文件
- 改进错误处理,单个爬虫失败不影响其他爬虫执行
- 更新 chdtp_target.ts,添加代理认证支持
This commit is contained in:
dmy
2026-01-11 18:20:43 +08:00
parent d9105797f4
commit 07a7301968
5 changed files with 350 additions and 15 deletions

View File

@@ -15,6 +15,13 @@ export const ChdtpCrawler = {
async crawl(browser: puppeteer.Browser): Promise<ChdtpResult[]> {
const logger = new Logger('ChdtpCrawler');
const page = await browser.newPage();
const username = process.env.PROXY_USERNAME;
const password = process.env.PROXY_PASSWORD;
if (username && password) {
await page.authenticate({ username, password });
}
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36');
const allResults: ChdtpResult[] = [];