feat: 添加华能集团电子商务平台爬虫功能
- 新增 ChngCrawler 爬虫类,支持华能集团电子商务平台招标信息抓取 - 更新 BidCrawlerService,集成 ChngCrawler 到爬虫任务中 - 添加代理配置支持,从环境变量读取代理设置 - 优化爬虫逻辑,支持多个爬虫并行执行 - 新增 ChngCrawler 的单元测试文件 - 改进错误处理,单个爬虫失败不影响其他爬虫执行 - 更新 chdtp_target.ts,添加代理认证支持
This commit is contained in:
@@ -15,6 +15,13 @@ export const ChdtpCrawler = {
|
||||
async crawl(browser: puppeteer.Browser): Promise<ChdtpResult[]> {
|
||||
const logger = new Logger('ChdtpCrawler');
|
||||
const page = await browser.newPage();
|
||||
|
||||
const username = process.env.PROXY_USERNAME;
|
||||
const password = process.env.PROXY_PASSWORD;
|
||||
if (username && password) {
|
||||
await page.authenticate({ username, password });
|
||||
}
|
||||
|
||||
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36');
|
||||
|
||||
const allResults: ChdtpResult[] = [];
|
||||
|
||||
Reference in New Issue
Block a user