feat: 增强华能电商平台爬虫的反检测能力和数据结构

- 在 BidItem 实体中添加 priority 和 unit 字段，扩展数据结构
- 将爬虫浏览器模式改为非无头模式(headless: false)便于调试
- 为 ChngCrawler 添加人类行为模拟功能：
  * 模拟鼠标随机移动，增加移动步数和停顿时间
  * 模拟人类滚动行为，包括随机滚动距离和停顿
  * 添加 navigator 属性伪装，包括语言、插件等
- 在关键节点添加截图功能(bing.png, newPage.png, huaneng.png)用于调试
- 优化反检测策略，降低被目标网站识别为机器人的风险

src/bids/entities/bid-item.entity.ts

@@ -20,6 +20,12 @@ export class BidItem {
   @Column({ default: false })
   isRead: boolean;
 
+  @Column({ default: 0 })
+  priority: number;
+
+  @Column({ nullable: true })
+  unit: string;
+
   @CreateDateColumn()
   createdAt: Date;
 

src/crawler/services/bid-crawler.service.ts

@@ -43,7 +43,7 @@ export class BidCrawlerService {
     }
 
     const browser = await puppeteer.launch({
-      headless: true,
+      headless: false,
       args,
     });
 

src/crawler/services/chng_target.ts

@@ -2,6 +2,51 @@ import * as puppeteer from 'puppeteer';
 import { Logger } from '@nestjs/common';
 import { ChdtpResult } from './chdtp_target';
 
+// 模拟人类鼠标移动
+async function simulateHumanMouseMovement(page: puppeteer.Page) {
+  const viewport = page.viewport();
+  if (!viewport) return;
+
+  const movements = 5 + Math.floor(Math.random() * 5); // 5-10次随机移动
+
+  for (let i = 0; i < movements; i++) {
+    const x = Math.floor(Math.random() * viewport.width);
+    const y = Math.floor(Math.random() * viewport.height);
+    
+    await page.mouse.move(x, y, {
+      steps: 10 + Math.floor(Math.random() * 20) // 10-30步，使移动更平滑
+    });
+    
+    // 随机停顿 100-500ms
+    await new Promise(r => setTimeout(r, 100 + Math.random() * 400));
+  }
+}
+
+// 模拟人类滚动
+async function simulateHumanScrolling(page: puppeteer.Page) {
+  const scrollCount = 3 + Math.floor(Math.random() * 5); // 3-7次滚动
+
+  for (let i = 0; i < scrollCount; i++) {
+    const scrollDistance = 100 + Math.floor(Math.random() * 400); // 100-500px
+    
+    await page.evaluate((distance) => {
+      window.scrollBy({
+        top: distance,
+        behavior: 'smooth'
+      });
+    }, scrollDistance);
+
+    // 随机停顿 500-1500ms
+    await new Promise(r => setTimeout(r, 500 + Math.random() * 1000));
+  }
+
+  // 滚动回顶部
+  await page.evaluate(() => {
+    window.scrollTo({ top: 0, behavior: 'smooth' });
+  });
+  await new Promise(r => setTimeout(r, 1000));
+}
+
 export const ChngCrawler = {
   name: '华能集团电子商务平台',
   url: 'https://ec.chng.com.cn/ecmall/index.html#/purchase/home?top=0',
@@ -10,7 +55,8 @@ export const ChngCrawler = {
   async crawl(browser: puppeteer.Browser): Promise<ChdtpResult[]> {
     const logger = new Logger('ChngCrawler');
     let page = await browser.newPage();
-
+    // await page.setViewport({ width: 1920, height: 1080, deviceScaleFactor: 1 });
+    // await page.setViewport({ deviceScaleFactor: 1 });
     const username = process.env.PROXY_USERNAME;
     const password = process.env.PROXY_PASSWORD;
     if (username && password) {
@@ -19,6 +65,8 @@ export const ChngCrawler = {
     
     await page.evaluateOnNewDocument(() => {
       Object.defineProperty(navigator, 'webdriver', { get: () => false });
+      Object.defineProperty(navigator, 'language', {	get: () => "zh-CN"});
+      Object.defineProperty(navigator, 'plugins', {	get: () => [1,2,3,4,5]});
     });
 
     await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36');
@@ -40,6 +88,7 @@ export const ChngCrawler = {
       await page.waitForNavigation({ waitUntil: 'networkidle2' });
       
       logger.log('Clicking search result...');
+      await page.screenshot({ path: 'bing.png' });
       const firstResultSelector = '#b_results .b_algo h2 a';
       await page.waitForSelector(firstResultSelector);
       
@@ -50,18 +99,33 @@ export const ChngCrawler = {
       const newPage = await newTarget.page();
       
       if (newPage) {
+        await newPage.screenshot({ path: 'newPage.png' });
         await page.close();
         page = newPage;
         if (username && password) {
           await page.authenticate({ username, password });
         }
       }
+       // 模拟人类行为
+      logger.log('Simulating human mouse movements...');
+      await simulateHumanMouseMovement(page);
+      
+      logger.log('Simulating human scrolling...');
+      await simulateHumanScrolling(page);
       
       await page.waitForNavigation({ waitUntil: 'domcontentloaded' }).catch(() => {});
+       // 模拟人类行为
+      logger.log('Simulating human mouse movements...');
+      await simulateHumanMouseMovement(page);
+      
+      logger.log('Simulating human scrolling...');
+      await simulateHumanScrolling(page);
+      
       
       // PAUSE 15 SECONDS as requested
       logger.log('Pausing 15 seconds before looking for "采购专栏"...');
       await new Promise(r => setTimeout(r, 15000));
+      await page.screenshot({ path: 'huaneng.png' });
 
       logger.log('Looking for "采购专栏" link...');
       await page.waitForFunction(() => {
@@ -92,6 +156,13 @@ export const ChngCrawler = {
       
       logger.log(`Active URL: ${page.url()}`);
       
+      // 模拟人类行为
+      logger.log('Simulating human mouse movements...');
+      await simulateHumanMouseMovement(page);
+      
+      logger.log('Simulating human scrolling...');
+      await simulateHumanScrolling(page);
+      
       while (currentPage <= maxPages) {
         logger.log(`Processing page ${currentPage}...`);