feat: 添加华能集团电子商务平台爬虫功能
- 新增 ChngCrawler 爬虫类，支持华能集团电子商务平台招标信息抓取 - 更新 BidCrawlerService，集成 ChngCrawler 到爬虫任务中 - 添加代理配置支持，从环境变量读取代理设置 - 优化爬虫逻辑，支持多个爬虫并行执行 - 新增 ChngCrawler 的单元测试文件 - 改进错误处理，单个爬虫失败不影响其他爬虫执行 - 更新 chdtp_target.ts，添加代理认证支持
This commit is contained in:
dmy
163
src/crawler/services/chng_target.ts
Normal file
163
src/crawler/services/chng_target.ts
Normal file
@@ -0,0 +1,163 @@
|
||||
import * as puppeteer from 'puppeteer';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { ChdtpResult } from './chdtp_target';
|
||||
|
||||
export const ChngCrawler = {
|
||||
name: '华能集团电子商务平台',
|
||||
url: 'https://ec.chng.com.cn/ecmall/index.html#/purchase/home?top=0',
|
||||
baseUrl: 'https://ec.chng.com.cn/ecmall/index.html',
|
||||
|
||||
async crawl(browser: puppeteer.Browser): Promise<ChdtpResult[]> {
|
||||
const logger = new Logger('ChngCrawler');
|
||||
let page = await browser.newPage();
|
||||
|
||||
const username = process.env.PROXY_USERNAME;
|
||||
const password = process.env.PROXY_PASSWORD;
|
||||
if (username && password) {
|
||||
await page.authenticate({ username, password });
|
||||
}
|
||||
|
||||
await page.evaluateOnNewDocument(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => false });
|
||||
});
|
||||
|
||||
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36');
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
const allResults: ChdtpResult[] = [];
|
||||
let currentPage = 1;
|
||||
const maxPages = 5;
|
||||
|
||||
try {
|
||||
logger.log('Navigating to Bing...');
|
||||
await page.goto('https://cn.bing.com', { waitUntil: 'networkidle2' });
|
||||
|
||||
logger.log('Searching for target site...');
|
||||
const searchBoxSelector = 'input[name="q"]';
|
||||
await page.waitForSelector(searchBoxSelector);
|
||||
await page.type(searchBoxSelector, 'https://ec.chng.com.cn/');
|
||||
await page.keyboard.press('Enter');
|
||||
await page.waitForNavigation({ waitUntil: 'networkidle2' });
|
||||
|
||||
logger.log('Clicking search result...');
|
||||
const firstResultSelector = '#b_results .b_algo h2 a';
|
||||
await page.waitForSelector(firstResultSelector);
|
||||
|
||||
const newTargetPromise = browser.waitForTarget(target => target.opener() === page.target());
|
||||
await page.click(firstResultSelector);
|
||||
|
||||
const newTarget = await newTargetPromise;
|
||||
const newPage = await newTarget.page();
|
||||
|
||||
if (newPage) {
|
||||
await page.close();
|
||||
page = newPage;
|
||||
if (username && password) {
|
||||
await page.authenticate({ username, password });
|
||||
}
|
||||
}
|
||||
|
||||
await page.waitForNavigation({ waitUntil: 'domcontentloaded' }).catch(() => {});
|
||||
|
||||
// PAUSE 15 SECONDS as requested
|
||||
logger.log('Pausing 15 seconds before looking for "采购专栏"...');
|
||||
await new Promise(r => setTimeout(r, 15000));
|
||||
|
||||
logger.log('Looking for "采购专栏" link...');
|
||||
await page.waitForFunction(() => {
|
||||
const divs = Array.from(document.querySelectorAll('div.text'));
|
||||
return divs.some(div => div.textContent && div.textContent.includes('采购专栏'));
|
||||
}, { timeout: 60000 });
|
||||
|
||||
const purchaseTargetPromise = browser.waitForTarget(target => target.opener() === page.target(), { timeout: 15000 }).catch(() => null);
|
||||
|
||||
await page.evaluate(() => {
|
||||
const divs = Array.from(document.querySelectorAll('div.text'));
|
||||
const target = divs.find(div => div.textContent && div.textContent.includes('采购专栏')) as HTMLElement;
|
||||
if (target) target.click();
|
||||
});
|
||||
|
||||
const purchaseTarget = await purchaseTargetPromise;
|
||||
if (purchaseTarget) {
|
||||
const pPage = await purchaseTarget.page();
|
||||
if (pPage) {
|
||||
logger.log('Switched to Purchase Page tab.');
|
||||
page = pPage;
|
||||
if (username && password) {
|
||||
await page.authenticate({ username, password });
|
||||
}
|
||||
await new Promise(r => setTimeout(r, 5000));
|
||||
}
|
||||
}
|
||||
|
||||
logger.log(`Active URL: ${page.url()}`);
|
||||
|
||||
while (currentPage <= maxPages) {
|
||||
logger.log(`Processing page ${currentPage}...`);
|
||||
|
||||
// Wait for table rows to load
|
||||
await page.waitForFunction(() => {
|
||||
return document.querySelectorAll('tr.ant-table-row').length > 0;
|
||||
}, { timeout: 60000 }).catch(() => logger.warn('Content not found. Site might be slow.'));
|
||||
|
||||
const pageResults = await page.evaluate((baseUrl) => {
|
||||
// Extract from table rows
|
||||
const items = Array.from(document.querySelectorAll('tr.ant-table-row'));
|
||||
return items.map(item => {
|
||||
const titleSpan = item.querySelector('span.list-text');
|
||||
const dateCell = item.querySelector('td.ant-table-row-cell-break-word p');
|
||||
|
||||
if (titleSpan && dateCell) {
|
||||
const title = titleSpan.textContent?.trim() || '';
|
||||
const dateStr = dateCell.textContent?.trim() || '';
|
||||
|
||||
if (title.length < 5) return null; // Filter noise
|
||||
|
||||
// URL is not directly available in the table, need to construct from data-row-key
|
||||
const rowKey = item.getAttribute('data-row-key');
|
||||
const url = rowKey ? `${baseUrl}#/purchase/detail?id=${rowKey}` : '';
|
||||
|
||||
return {
|
||||
title,
|
||||
dateStr,
|
||||
url
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}).filter(i => i !== null);
|
||||
}, this.baseUrl);
|
||||
|
||||
if (pageResults.length === 0) {
|
||||
logger.warn(`No results found on page ${currentPage}. Extraction failed.`);
|
||||
break;
|
||||
}
|
||||
|
||||
allResults.push(...pageResults.map(r => ({
|
||||
title: r!.title,
|
||||
publishDate: new Date(r!.dateStr),
|
||||
url: r!.url
|
||||
})));
|
||||
|
||||
logger.log(`Extracted ${pageResults.length} items.`);
|
||||
|
||||
// Pagination: look for the "right" icon SVG
|
||||
const nextButton = await page.$('svg[data-icon="right"]');
|
||||
if (!nextButton) break;
|
||||
|
||||
await nextButton.click();
|
||||
await new Promise(r => setTimeout(r, 5000));
|
||||
currentPage++;
|
||||
}
|
||||
|
||||
return allResults;
|
||||
|
||||
} catch (error) {
|
||||
logger.error(`Crawl failed: ${error.message}`);
|
||||
return allResults;
|
||||
} finally {
|
||||
if (page) await page.close();
|
||||
}
|
||||
},
|
||||
|
||||
extract() { return []; }
|
||||
};
|
||||
Reference in New Issue
Block a user