import * as puppeteer from 'puppeteer'; import { Logger } from '@nestjs/common'; // 模拟人类鼠标移动 async function simulateHumanMouseMovement(page: puppeteer.Page) { const viewport = page.viewport(); if (!viewport) return; const movements = 5 + Math.floor(Math.random() * 5); // 5-10次随机移动 for (let i = 0; i < movements; i++) { const x = Math.floor(Math.random() * viewport.width); const y = Math.floor(Math.random() * viewport.height); await page.mouse.move(x, y, { steps: 10 + Math.floor(Math.random() * 20), // 10-30步,使移动更平滑 }); // 随机停顿 100-500ms await new Promise((r) => setTimeout(r, 100 + Math.random() * 400)); } } // 模拟人类滚动 async function simulateHumanScrolling(page: puppeteer.Page) { const scrollCount = 3 + Math.floor(Math.random() * 5); // 3-7次滚动 for (let i = 0; i < scrollCount; i++) { const scrollDistance = 100 + Math.floor(Math.random() * 400); // 100-500px await page.evaluate((distance) => { window.scrollBy({ top: distance, behavior: 'smooth', }); }, scrollDistance); // 随机停顿 500-1500ms await new Promise((r) => setTimeout(r, 500 + Math.random() * 1000)); } // 滚动回顶部 await page.evaluate(() => { window.scrollTo({ top: 0, behavior: 'smooth' }); }); await new Promise((r) => setTimeout(r, 1000)); } export interface CnncecpResult { title: string; publishDate: Date; url: string; } interface CnncecpCrawlerType { name: string; url: string; baseUrl: string; extract(html: string): CnncecpResult[]; } export const CnncecpCrawler = { name: '中核集团电子采购平台', url: 'https://www.cnncecp.com/xzbgg/index.jhtml', baseUrl: 'https://www.cnncecp.com/', async crawl( this: CnncecpCrawlerType, browser: puppeteer.Browser, ): Promise { const logger = new Logger('CnncecpCrawler'); const page = await browser.newPage(); const username = process.env.PROXY_USERNAME; const password = process.env.PROXY_PASSWORD; if (username && password) { await page.authenticate({ username, password }); } await page.evaluateOnNewDocument(() => { Object.defineProperty(navigator, 'webdriver', { get: () => false }); Object.defineProperty(navigator, 'language', { get: () => 'zh-CN' }); Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5], }); }); await page.setUserAgent( 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36', ); await page.setViewport({ width: 1920, height: 1080 }); const allResults: CnncecpResult[] = []; let currentPage = 1; const maxPages = 5; try { logger.log(`Navigating to ${this.url}...`); await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); // 模拟人类行为 logger.log('Simulating human mouse movements...'); await simulateHumanMouseMovement(page); logger.log('Simulating human scrolling...'); await simulateHumanScrolling(page); while (currentPage <= maxPages) { logger.log(`Processing page ${currentPage}...`); const content = await page.content(); const pageResults = this.extract(content); if (pageResults.length === 0) { logger.warn(`No results found on page ${currentPage}, stopping.`); break; } allResults.push(...pageResults); logger.log( `Extracted ${pageResults.length} items from page ${currentPage}`, ); // 模拟人类行为 - 翻页前 logger.log('Simulating human mouse movements before pagination...'); await simulateHumanMouseMovement(page); logger.log('Simulating human scrolling before pagination...'); await simulateHumanScrolling(page); // 查找下一页按钮 const nextButtonSelector = 'a[href*="index_"]'; const nextButton = await page.$(nextButtonSelector); if (!nextButton) { logger.log('Next page button not found. Reached end of list.'); break; } logger.log(`Navigating to page ${currentPage + 1}...`); try { // 点击下一页按钮 await nextButton.click(); await new Promise((r) => setTimeout(r, 3000)); // 等待页面加载 } catch (navError) { const navErrorMessage = navError instanceof Error ? navError.message : String(navError); logger.error( `Navigation to page ${currentPage + 1} failed: ${navErrorMessage}`, ); break; } currentPage++; // 模拟人类行为 - 翻页后 logger.log('Simulating human mouse movements after pagination...'); await simulateHumanMouseMovement(page); logger.log('Simulating human scrolling after pagination...'); await simulateHumanScrolling(page); // Random delay between pages const delay = Math.floor(Math.random() * (3000 - 1000 + 1)) + 1000; await new Promise((resolve) => setTimeout(resolve, delay)); } return allResults; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); logger.error(`Failed to crawl ${this.name}: ${errorMessage}`); return allResults; } finally { await page.close(); } }, extract(this: CnncecpCrawlerType, html: string): CnncecpResult[] { const results: CnncecpResult[] = []; /** * Regex groups for cnncecp.com: * 1: Date (发布时间,格式:2026-01-11) * 2: URL (href属性) * 3: Title (a标签文本) * * HTML结构示例: *
  • * 2026-01-11 * 文件下载截止:2025-12-08 23:59:00 * [变更公告] * 中核四0四有限公司2026-2028年度质量流量控制器等采购项目(二次)变更公告 *
  • */ const regex = /
  • [\s\S]*?\s*(\d{4}-\d{2}-\d{2})\s*<\/span>[\s\S]*?]*href="([^"]*)"[^>]*>([^<]*)<\/a>[\s\S]*?<\/li>/gs; let match: RegExpExecArray | null; while ((match = regex.exec(html)) !== null) { const dateStr = match[1]?.trim() ?? ''; const url = match[2]?.trim() ?? ''; const title = match[3]?.trim() ?? ''; if (title && url) { const fullUrl = url.startsWith('http') ? url : this.baseUrl + url; results.push({ title, publishDate: dateStr ? new Date(dateStr) : new Date(), url: fullUrl.replace(/\/\//g, '/'), }); } } return results; }, };