diff --git a/src/bids/entities/bid-item.entity.ts b/src/bids/entities/bid-item.entity.ts index 7b793f8..5e012d1 100644 --- a/src/bids/entities/bid-item.entity.ts +++ b/src/bids/entities/bid-item.entity.ts @@ -20,6 +20,12 @@ export class BidItem { @Column({ default: false }) isRead: boolean; + @Column({ default: 0 }) + priority: number; + + @Column({ nullable: true }) + unit: string; + @CreateDateColumn() createdAt: Date; diff --git a/src/crawler/services/bid-crawler.service.ts b/src/crawler/services/bid-crawler.service.ts index 8f4d973..9aaf147 100644 --- a/src/crawler/services/bid-crawler.service.ts +++ b/src/crawler/services/bid-crawler.service.ts @@ -43,7 +43,7 @@ export class BidCrawlerService { } const browser = await puppeteer.launch({ - headless: true, + headless: false, args, }); diff --git a/src/crawler/services/chng_target.ts b/src/crawler/services/chng_target.ts index d51de6e..8eb8710 100644 --- a/src/crawler/services/chng_target.ts +++ b/src/crawler/services/chng_target.ts @@ -2,6 +2,51 @@ import * as puppeteer from 'puppeteer'; import { Logger } from '@nestjs/common'; import { ChdtpResult } from './chdtp_target'; +// 模拟人类鼠标移动 +async function simulateHumanMouseMovement(page: puppeteer.Page) { + const viewport = page.viewport(); + if (!viewport) return; + + const movements = 5 + Math.floor(Math.random() * 5); // 5-10次随机移动 + + for (let i = 0; i < movements; i++) { + const x = Math.floor(Math.random() * viewport.width); + const y = Math.floor(Math.random() * viewport.height); + + await page.mouse.move(x, y, { + steps: 10 + Math.floor(Math.random() * 20) // 10-30步,使移动更平滑 + }); + + // 随机停顿 100-500ms + await new Promise(r => setTimeout(r, 100 + Math.random() * 400)); + } +} + +// 模拟人类滚动 +async function simulateHumanScrolling(page: puppeteer.Page) { + const scrollCount = 3 + Math.floor(Math.random() * 5); // 3-7次滚动 + + for (let i = 0; i < scrollCount; i++) { + const scrollDistance = 100 + Math.floor(Math.random() * 400); // 100-500px + + await page.evaluate((distance) => { + window.scrollBy({ + top: distance, + behavior: 'smooth' + }); + }, scrollDistance); + + // 随机停顿 500-1500ms + await new Promise(r => setTimeout(r, 500 + Math.random() * 1000)); + } + + // 滚动回顶部 + await page.evaluate(() => { + window.scrollTo({ top: 0, behavior: 'smooth' }); + }); + await new Promise(r => setTimeout(r, 1000)); +} + export const ChngCrawler = { name: '华能集团电子商务平台', url: 'https://ec.chng.com.cn/ecmall/index.html#/purchase/home?top=0', @@ -10,7 +55,8 @@ export const ChngCrawler = { async crawl(browser: puppeteer.Browser): Promise { const logger = new Logger('ChngCrawler'); let page = await browser.newPage(); - + // await page.setViewport({ width: 1920, height: 1080, deviceScaleFactor: 1 }); + // await page.setViewport({ deviceScaleFactor: 1 }); const username = process.env.PROXY_USERNAME; const password = process.env.PROXY_PASSWORD; if (username && password) { @@ -19,6 +65,8 @@ export const ChngCrawler = { await page.evaluateOnNewDocument(() => { Object.defineProperty(navigator, 'webdriver', { get: () => false }); + Object.defineProperty(navigator, 'language', { get: () => "zh-CN"}); + Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5]}); }); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36'); @@ -40,6 +88,7 @@ export const ChngCrawler = { await page.waitForNavigation({ waitUntil: 'networkidle2' }); logger.log('Clicking search result...'); + await page.screenshot({ path: 'bing.png' }); const firstResultSelector = '#b_results .b_algo h2 a'; await page.waitForSelector(firstResultSelector); @@ -50,18 +99,33 @@ export const ChngCrawler = { const newPage = await newTarget.page(); if (newPage) { + await newPage.screenshot({ path: 'newPage.png' }); await page.close(); page = newPage; if (username && password) { await page.authenticate({ username, password }); } } + // 模拟人类行为 + logger.log('Simulating human mouse movements...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling...'); + await simulateHumanScrolling(page); await page.waitForNavigation({ waitUntil: 'domcontentloaded' }).catch(() => {}); + // 模拟人类行为 + logger.log('Simulating human mouse movements...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling...'); + await simulateHumanScrolling(page); + // PAUSE 15 SECONDS as requested logger.log('Pausing 15 seconds before looking for "采购专栏"...'); await new Promise(r => setTimeout(r, 15000)); + await page.screenshot({ path: 'huaneng.png' }); logger.log('Looking for "采购专栏" link...'); await page.waitForFunction(() => { @@ -92,6 +156,13 @@ export const ChngCrawler = { logger.log(`Active URL: ${page.url()}`); + // 模拟人类行为 + logger.log('Simulating human mouse movements...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling...'); + await simulateHumanScrolling(page); + while (currentPage <= maxPages) { logger.log(`Processing page ${currentPage}...`);