import * as puppeteer from 'puppeteer'; import { Logger } from '@nestjs/common'; import { ChdtpResult } from './chdtp_target'; // 模拟人类鼠标移动 async function simulateHumanMouseMovement(page: puppeteer.Page) { const viewport = page.viewport(); if (!viewport) return; const movements = 5 + Math.floor(Math.random() * 5); // 5-10次随机移动 for (let i = 0; i < movements; i++) { const x = Math.floor(Math.random() * viewport.width); const y = Math.floor(Math.random() * viewport.height); await page.mouse.move(x, y, { steps: 10 + Math.floor(Math.random() * 20) // 10-30步,使移动更平滑 }); // 随机停顿 100-500ms await new Promise(r => setTimeout(r, 100 + Math.random() * 400)); } } // 模拟人类滚动 async function simulateHumanScrolling(page: puppeteer.Page) { const scrollCount = 3 + Math.floor(Math.random() * 5); // 3-7次滚动 for (let i = 0; i < scrollCount; i++) { const scrollDistance = 100 + Math.floor(Math.random() * 400); // 100-500px await page.evaluate((distance) => { window.scrollBy({ top: distance, behavior: 'smooth' }); }, scrollDistance); // 随机停顿 500-1500ms await new Promise(r => setTimeout(r, 500 + Math.random() * 1000)); } // 滚动回顶部 await page.evaluate(() => { window.scrollTo({ top: 0, behavior: 'smooth' }); }); await new Promise(r => setTimeout(r, 1000)); } export const ChngCrawler = { name: '华能集团电子商务平台', url: 'https://ec.chng.com.cn/ecmall/index.html#/purchase/home?top=0', baseUrl: 'https://ec.chng.com.cn/ecmall/index.html', async crawl(browser: puppeteer.Browser): Promise { const logger = new Logger('ChngCrawler'); let page = await browser.newPage(); // await page.setViewport({ width: 1920, height: 1080, deviceScaleFactor: 1 }); // await page.setViewport({ deviceScaleFactor: 1 }); const username = process.env.PROXY_USERNAME; const password = process.env.PROXY_PASSWORD; if (username && password) { await page.authenticate({ username, password }); } await page.evaluateOnNewDocument(() => { Object.defineProperty(navigator, 'webdriver', { get: () => false }); Object.defineProperty(navigator, 'language', { get: () => "zh-CN"}); Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5]}); }); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36'); await page.setViewport({ width: 1920, height: 1080 }); const allResults: ChdtpResult[] = []; let currentPage = 1; const maxPages = 5; try { logger.log('Navigating to Bing...'); await page.goto('https://cn.bing.com', { waitUntil: 'networkidle2' }); logger.log('Searching for target site...'); const searchBoxSelector = 'input[name="q"]'; await page.waitForSelector(searchBoxSelector); await page.type(searchBoxSelector, 'https://ec.chng.com.cn/'); await page.keyboard.press('Enter'); await page.waitForNavigation({ waitUntil: 'networkidle2' }); logger.log('Clicking search result...'); await page.screenshot({ path: 'bing.png' }); const firstResultSelector = '#b_results .b_algo h2 a'; await page.waitForSelector(firstResultSelector); const newTargetPromise = browser.waitForTarget(target => target.opener() === page.target()); await page.click(firstResultSelector); const newTarget = await newTargetPromise; const newPage = await newTarget.page(); if (newPage) { await newPage.screenshot({ path: 'newPage.png' }); await page.close(); page = newPage; if (username && password) { await page.authenticate({ username, password }); } } // 模拟人类行为 logger.log('Simulating human mouse movements...'); await simulateHumanMouseMovement(page); logger.log('Simulating human scrolling...'); await simulateHumanScrolling(page); await page.waitForNavigation({ waitUntil: 'domcontentloaded' }).catch(() => {}); // 模拟人类行为 logger.log('Simulating human mouse movements...'); await simulateHumanMouseMovement(page); logger.log('Simulating human scrolling...'); await simulateHumanScrolling(page); // PAUSE 15 SECONDS as requested logger.log('Pausing 15 seconds before looking for "采购专栏"...'); await new Promise(r => setTimeout(r, 15000)); await page.screenshot({ path: 'huaneng.png' }); logger.log('Looking for "采购专栏" link...'); await page.waitForFunction(() => { const divs = Array.from(document.querySelectorAll('div.text')); return divs.some(div => div.textContent && div.textContent.includes('采购专栏')); }, { timeout: 60000 }); const purchaseTargetPromise = browser.waitForTarget(target => target.opener() === page.target(), { timeout: 15000 }).catch(() => null); await page.evaluate(() => { const divs = Array.from(document.querySelectorAll('div.text')); const target = divs.find(div => div.textContent && div.textContent.includes('采购专栏')) as HTMLElement; if (target) target.click(); }); const purchaseTarget = await purchaseTargetPromise; if (purchaseTarget) { const pPage = await purchaseTarget.page(); if (pPage) { logger.log('Switched to Purchase Page tab.'); page = pPage; if (username && password) { await page.authenticate({ username, password }); } await new Promise(r => setTimeout(r, 5000)); } } logger.log(`Active URL: ${page.url()}`); // 模拟人类行为 logger.log('Simulating human mouse movements...'); await simulateHumanMouseMovement(page); logger.log('Simulating human scrolling...'); await simulateHumanScrolling(page); while (currentPage <= maxPages) { logger.log(`Processing page ${currentPage}...`); // Wait for table rows to load await page.waitForFunction(() => { return document.querySelectorAll('tr.ant-table-row').length > 0; }, { timeout: 60000 }).catch(() => logger.warn('Content not found. Site might be slow.')); const pageResults = await page.evaluate((baseUrl) => { // Extract from table rows const items = Array.from(document.querySelectorAll('tr.ant-table-row')); return items.map(item => { const titleSpan = item.querySelector('span.list-text'); const dateCell = item.querySelector('td.ant-table-row-cell-break-word p'); if (titleSpan && dateCell) { const title = titleSpan.textContent?.trim() || ''; const dateStr = dateCell.textContent?.trim() || ''; if (title.length < 5) return null; // Filter noise // URL is not directly available in the table, need to construct from data-row-key const rowKey = item.getAttribute('data-row-key'); const url = rowKey ? `${baseUrl}#/purchase/detail?id=${rowKey}` : ''; return { title, dateStr, url }; } return null; }).filter(i => i !== null); }, this.baseUrl); if (pageResults.length === 0) { logger.warn(`No results found on page ${currentPage}. Extraction failed.`); break; } allResults.push(...pageResults.map(r => ({ title: r!.title, publishDate: new Date(r!.dateStr), url: r!.url }))); logger.log(`Extracted ${pageResults.length} items.`); // Pagination: look for the "right" icon SVG const nextButton = await page.$('svg[data-icon="right"]'); if (!nextButton) break; await nextButton.click(); await new Promise(r => setTimeout(r, 5000)); currentPage++; } return allResults; } catch (error) { logger.error(`Crawl failed: ${error.message}`); return allResults; } finally { if (page) await page.close(); } }, extract() { return []; } };