import * as puppeteer from 'puppeteer'; import { Logger } from '@nestjs/common'; import { ChdtpResult } from './chdtp_target'; export const ChngCrawler = { name: '华能集团电子商务平台', url: 'https://ec.chng.com.cn/ecmall/index.html#/purchase/home?top=0', baseUrl: 'https://ec.chng.com.cn/ecmall/index.html', async crawl(browser: puppeteer.Browser): Promise { const logger = new Logger('ChngCrawler'); let page = await browser.newPage(); const username = process.env.PROXY_USERNAME; const password = process.env.PROXY_PASSWORD; if (username && password) { await page.authenticate({ username, password }); } await page.evaluateOnNewDocument(() => { Object.defineProperty(navigator, 'webdriver', { get: () => false }); }); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36'); await page.setViewport({ width: 1920, height: 1080 }); const allResults: ChdtpResult[] = []; let currentPage = 1; const maxPages = 5; try { logger.log('Navigating to Bing...'); await page.goto('https://cn.bing.com', { waitUntil: 'networkidle2' }); logger.log('Searching for target site...'); const searchBoxSelector = 'input[name="q"]'; await page.waitForSelector(searchBoxSelector); await page.type(searchBoxSelector, 'https://ec.chng.com.cn/'); await page.keyboard.press('Enter'); await page.waitForNavigation({ waitUntil: 'networkidle2' }); logger.log('Clicking search result...'); const firstResultSelector = '#b_results .b_algo h2 a'; await page.waitForSelector(firstResultSelector); const newTargetPromise = browser.waitForTarget(target => target.opener() === page.target()); await page.click(firstResultSelector); const newTarget = await newTargetPromise; const newPage = await newTarget.page(); if (newPage) { await page.close(); page = newPage; if (username && password) { await page.authenticate({ username, password }); } } await page.waitForNavigation({ waitUntil: 'domcontentloaded' }).catch(() => {}); // PAUSE 15 SECONDS as requested logger.log('Pausing 15 seconds before looking for "采购专栏"...'); await new Promise(r => setTimeout(r, 15000)); logger.log('Looking for "采购专栏" link...'); await page.waitForFunction(() => { const divs = Array.from(document.querySelectorAll('div.text')); return divs.some(div => div.textContent && div.textContent.includes('采购专栏')); }, { timeout: 60000 }); const purchaseTargetPromise = browser.waitForTarget(target => target.opener() === page.target(), { timeout: 15000 }).catch(() => null); await page.evaluate(() => { const divs = Array.from(document.querySelectorAll('div.text')); const target = divs.find(div => div.textContent && div.textContent.includes('采购专栏')) as HTMLElement; if (target) target.click(); }); const purchaseTarget = await purchaseTargetPromise; if (purchaseTarget) { const pPage = await purchaseTarget.page(); if (pPage) { logger.log('Switched to Purchase Page tab.'); page = pPage; if (username && password) { await page.authenticate({ username, password }); } await new Promise(r => setTimeout(r, 5000)); } } logger.log(`Active URL: ${page.url()}`); while (currentPage <= maxPages) { logger.log(`Processing page ${currentPage}...`); // Wait for table rows to load await page.waitForFunction(() => { return document.querySelectorAll('tr.ant-table-row').length > 0; }, { timeout: 60000 }).catch(() => logger.warn('Content not found. Site might be slow.')); const pageResults = await page.evaluate((baseUrl) => { // Extract from table rows const items = Array.from(document.querySelectorAll('tr.ant-table-row')); return items.map(item => { const titleSpan = item.querySelector('span.list-text'); const dateCell = item.querySelector('td.ant-table-row-cell-break-word p'); if (titleSpan && dateCell) { const title = titleSpan.textContent?.trim() || ''; const dateStr = dateCell.textContent?.trim() || ''; if (title.length < 5) return null; // Filter noise // URL is not directly available in the table, need to construct from data-row-key const rowKey = item.getAttribute('data-row-key'); const url = rowKey ? `${baseUrl}#/purchase/detail?id=${rowKey}` : ''; return { title, dateStr, url }; } return null; }).filter(i => i !== null); }, this.baseUrl); if (pageResults.length === 0) { logger.warn(`No results found on page ${currentPage}. Extraction failed.`); break; } allResults.push(...pageResults.map(r => ({ title: r!.title, publishDate: new Date(r!.dateStr), url: r!.url }))); logger.log(`Extracted ${pageResults.length} items.`); // Pagination: look for the "right" icon SVG const nextButton = await page.$('svg[data-icon="right"]'); if (!nextButton) break; await nextButton.click(); await new Promise(r => setTimeout(r, 5000)); currentPage++; } return allResults; } catch (error) { logger.error(`Crawl failed: ${error.message}`); return allResults; } finally { if (page) await page.close(); } }, extract() { return []; } };