feat: 新增北京电力交易平台爬虫
- 新增 PowerbeijingCrawler 爬虫类,支持北京电力交易平台招标信息抓取 - 目标平台:https://www.powerbeijing-ec.com,专注于北京地区电力交易招标信息 - 使用正则表达式提取招标公告,支持标题、URL和发布日期解析 - 添加完整的人类行为模拟功能(鼠标移动、滚动)降低检测风险 - 支持分页抓取,最多抓取5页数据,使用 turnPage 函数进行翻页 - 添加完整的单元测试文件,确保爬虫功能正常 - 统一的错误处理机制,单个爬虫失败不影响整体任务执行
This commit is contained in:
51
src/crawler/services/powerbeijing_target.spec.ts
Normal file
51
src/crawler/services/powerbeijing_target.spec.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import { PowerbeijingCrawler } from './powerbeijing_target';
|
||||
import * as puppeteer from 'puppeteer';
|
||||
|
||||
// Increase timeout to 60 seconds for network operations
|
||||
jest.setTimeout(60000*5);
|
||||
|
||||
describe('PowerbeijingCrawler Real Site Test', () => {
|
||||
let browser: puppeteer.Browser;
|
||||
|
||||
beforeAll(async () => {
|
||||
browser = await puppeteer.launch({
|
||||
headless: false, // Change to false to see browser UI
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (browser) {
|
||||
await browser.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('should visit website and list all found bid information', async () => {
|
||||
console.log(`\nStarting crawl for: ${PowerbeijingCrawler.name}`);
|
||||
console.log(`Target URL: ${PowerbeijingCrawler.url}`);
|
||||
|
||||
const results = await PowerbeijingCrawler.crawl(browser);
|
||||
|
||||
console.log(`\nSuccessfully found ${results.length} items:\n`);
|
||||
console.log('----------------------------------------');
|
||||
results.forEach((item, index) => {
|
||||
console.log(`${index + 1}. [${item.publishDate.toLocaleDateString()}] ${item.title}`);
|
||||
console.log(` Link: ${item.url}`);
|
||||
console.log('----------------------------------------');
|
||||
});
|
||||
|
||||
// Basic assertions to ensure crawler is working
|
||||
expect(results).toBeDefined();
|
||||
expect(Array.isArray(results)).toBeTruthy();
|
||||
// Warn but don't fail if site returns 0 items (could be empty or changed structure)
|
||||
if (results.length === 0) {
|
||||
console.warn('Warning: No items found. Check if website structure has changed or if list is currently empty.');
|
||||
} else {
|
||||
// Check data integrity of first item
|
||||
const firstItem = results[0];
|
||||
expect(firstItem.title).toBeTruthy();
|
||||
expect(firstItem.url).toMatch(/^https?:\/\//);
|
||||
expect(firstItem.publishDate).toBeInstanceOf(Date);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user