From 6d626a0946cebaceafa3f2b5cb96db14f9f2e918 Mon Sep 17 00:00:00 2001 From: dmy Date: Sun, 11 Jan 2026 22:34:38 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E5=A4=9A=E4=B8=AA?= =?UTF-8?q?=E7=94=B5=E5=8A=9B=E9=9B=86=E5=9B=A2=E9=87=87=E8=B4=AD=E5=B9=B3?= =?UTF-8?q?=E5=8F=B0=E7=88=AC=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增4个电力集团采购平台爬虫： * 中国大唐集团电子商务平台 (CdtCrawler) * 大连能源采购平台 (CeicCrawler) * 华润守正采购交易平台 (SzecpCrawler) - 更新 BidCrawlerService，集成新的爬虫到爬虫任务中 - 添加环境变量示例文件 .env.example，包含数据库和代理配置 - 优化 .env 文件，添加代理配置示例 - 为所有新爬虫添加完整的单元测试文件 - 使用与现有爬虫相同的反检测策略（人类行为模拟） - 支持分页抓取，每个平台最多抓取5页数据 - 统一的错误处理机制，单个爬虫失败不影响其他爬虫执行 --- .env | 6 +- .env.example | 13 ++ src/crawler/services/bid-crawler.service.ts | 4 +- src/crawler/services/cdt_target.spec.ts | 51 +++++ src/crawler/services/cdt_target.ts | 229 ++++++++++++++++++++ src/crawler/services/ceic_target.spec.ts | 61 ++++++ src/crawler/services/ceic_target.ts | 168 ++++++++++++++ src/crawler/services/chng_target.spec.ts | 74 ++++++- src/crawler/services/szecp_target.spec.ts | 61 ++++++ src/crawler/services/szecp_target.ts | 170 +++++++++++++++ 10 files changed, 833 insertions(+), 4 deletions(-) create mode 100644 .env.example create mode 100644 src/crawler/services/cdt_target.spec.ts create mode 100644 src/crawler/services/cdt_target.ts create mode 100644 src/crawler/services/ceic_target.spec.ts create mode 100644 src/crawler/services/ceic_target.ts create mode 100644 src/crawler/services/szecp_target.spec.ts create mode 100644 src/crawler/services/szecp_target.ts diff --git a/.env b/.env index ebc534b..c759777 100644 --- a/.env +++ b/.env @@ -4,4 +4,8 @@ DATABASE_PORT=23306 DATABASE_USERNAME=root DATABASE_PASSWORD=410491 DATABASE_NAME=bidding -DATABASE_SYNCHRONIZE=true \ No newline at end of file +DATABASE_SYNCHRONIZE=true + +# 代理配置（可选） +PROXY_HOST=127.0.0.1 +PROXY_PORT=3211 \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..e514bc9 --- /dev/null +++ b/.env.example @@ -0,0 +1,13 @@ +DATABASE_TYPE=mariadb +DATABASE_HOST=localhost +DATABASE_PORT=3306 +DATABASE_USERNAME=root +DATABASE_PASSWORD=root +DATABASE_NAME=bidding +DATABASE_SYNCHRONIZE=true + +# 代理配置（可选） +PROXY_HOST=127.0.0.1 +PROXY_PORT=6000 +# PROXY_USERNAME= +# PROXY_PASSWORD= \ No newline at end of file diff --git a/src/crawler/services/bid-crawler.service.ts b/src/crawler/services/bid-crawler.service.ts index 9aaf147..3ee3499 100644 --- a/src/crawler/services/bid-crawler.service.ts +++ b/src/crawler/services/bid-crawler.service.ts @@ -4,6 +4,8 @@ import * as puppeteer from 'puppeteer'; import { BidsService } from '../../bids/services/bid.service'; import { ChdtpCrawler } from './chdtp_target'; import { ChngCrawler } from './chng_target'; +import { SzecpCrawler } from './szecp_target'; +import { CdtCrawler } from './cdt_target'; @Injectable() export class BidCrawlerService { @@ -47,7 +49,7 @@ export class BidCrawlerService { args, }); - const crawlers = [ChdtpCrawler, ChngCrawler]; + const crawlers = [ChdtpCrawler, ChngCrawler, SzecpCrawler, CdtCrawler]; try { for (const crawler of crawlers) { diff --git a/src/crawler/services/cdt_target.spec.ts b/src/crawler/services/cdt_target.spec.ts new file mode 100644 index 0000000..cf73eb1 --- /dev/null +++ b/src/crawler/services/cdt_target.spec.ts @@ -0,0 +1,51 @@ +import { CdtCrawler } from './cdt_target'; +import * as puppeteer from 'puppeteer'; + +// Increase timeout to 60 seconds for network operations +jest.setTimeout(60000*5); + +describe('CdtCrawler Real Site Test', () => { + let browser: puppeteer.Browser; + + beforeAll(async () => { + browser = await puppeteer.launch({ + headless: false, // Change to false to see browser UI + args: ['--no-sandbox', '--disable-setuid-sandbox'], + }); + }); + + afterAll(async () => { + if (browser) { + await browser.close(); + } + }); + + it('should visit website and list all found bid information', async () => { + console.log(`\nStarting crawl for: ${CdtCrawler.name}`); + console.log(`Target URL: ${CdtCrawler.url}`); + + const results = await CdtCrawler.crawl(browser); + + console.log(`\nSuccessfully found ${results.length} items:\n`); + console.log('----------------------------------------'); + results.forEach((item, index) => { + console.log(`${index + 1}. [${item.publishDate.toLocaleDateString()}] ${item.title}`); + console.log(` Link: ${item.url}`); + console.log('----------------------------------------'); + }); + + // Basic assertions to ensure crawler is working + expect(results).toBeDefined(); + expect(Array.isArray(results)).toBeTruthy(); + // Warn but don't fail if site returns 0 items (could be empty or changed structure) + if (results.length === 0) { + console.warn('Warning: No items found. Check if website structure has changed or if list is currently empty.'); + } else { + // Check data integrity of first item + const firstItem = results[0]; + expect(firstItem.title).toBeTruthy(); + expect(firstItem.url).toMatch(/^https?:\/\//); + expect(firstItem.publishDate).toBeInstanceOf(Date); + } + }); +}); diff --git a/src/crawler/services/cdt_target.ts b/src/crawler/services/cdt_target.ts new file mode 100644 index 0000000..0b55b39 --- /dev/null +++ b/src/crawler/services/cdt_target.ts @@ -0,0 +1,229 @@ +import * as puppeteer from 'puppeteer'; +import { Logger } from '@nestjs/common'; + +// 模拟人类鼠标移动 +async function simulateHumanMouseMovement(page: puppeteer.Page) { + const viewport = page.viewport(); + if (!viewport) return; + + const movements = 5 + Math.floor(Math.random() * 5); // 5-10次随机移动 + + for (let i = 0; i < movements; i++) { + const x = Math.floor(Math.random() * viewport.width); + const y = Math.floor(Math.random() * viewport.height); + + await page.mouse.move(x, y, { + steps: 10 + Math.floor(Math.random() * 20) // 10-30步，使移动更平滑 + }); + + // 随机停顿 100-500ms + await new Promise(r => setTimeout(r, 100 + Math.random() * 400)); + } +} + +// 模拟人类滚动 +async function simulateHumanScrolling(page: puppeteer.Page) { + const scrollCount = 3 + Math.floor(Math.random() * 5); // 3-7次滚动 + + for (let i = 0; i < scrollCount; i++) { + const scrollDistance = 100 + Math.floor(Math.random() * 400); // 100-500px + + await page.evaluate((distance) => { + window.scrollBy({ + top: distance, + behavior: 'smooth' + }); + }, scrollDistance); + + // 随机停顿 500-1500ms + await new Promise(r => setTimeout(r, 500 + Math.random() * 1000)); + } + + // 滚动回顶部 + await page.evaluate(() => { + window.scrollTo({ top: 0, behavior: 'smooth' }); + }); + await new Promise(r => setTimeout(r, 1000)); +} + +export interface CdtResult { + title: string; + publishDate: Date; + url: string; +} + +export const CdtCrawler = { + name: '中国大唐集团电子商务平台', + url: 'https://tang.cdt-ec.com/home/index.html', + baseUrl: 'https://tang.cdt-ec.com', + + async crawl(browser: puppeteer.Browser): Promise { + const logger = new Logger('CdtCrawler'); + const page = await browser.newPage(); + + const username = process.env.PROXY_USERNAME; + const password = process.env.PROXY_PASSWORD; + if (username && password) { + await page.authenticate({ username, password }); + } + + await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36'); + + const allResults: CdtResult[] = []; + let currentPage = 1; + const maxPages = 5; + + try { + logger.log(`Navigating to ${this.url}...`); + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + + // 模拟人类行为 + logger.log('Simulating human mouse movements...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling...'); + await simulateHumanScrolling(page); + + // 点击"招标公告"标签 + logger.log('Looking for "招标公告" tab...'); + await page.waitForFunction(() => { + const tabs = Array.from(document.querySelectorAll('span.notice-tab')); + return tabs.some(tab => tab.textContent && tab.textContent.includes('招标公告')); + }, { timeout: 30000 }); + + await page.evaluate(() => { + const tabs = Array.from(document.querySelectorAll('span.notice-tab')); + const target = tabs.find(tab => tab.textContent && tab.textContent.includes('招标公告')) as HTMLElement; + if (target) target.click(); + }); + + logger.log('Clicked "招标公告" tab.'); + await new Promise(r => setTimeout(r, 2000)); + + // 模拟人类行为 + logger.log('Simulating human mouse movements...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling...'); + await simulateHumanScrolling(page); + + // 点击"招标公告"下的"更多+"链接 + logger.log('Looking for "更多+" link under "招标公告"...'); + await page.waitForFunction(() => { + const titles = Array.from(document.querySelectorAll('span.h-notice-title')); + return titles.some(title => title.textContent && title.textContent.includes('招标公告')); + }, { timeout: 30000 }); + + await page.evaluate(() => { + const titles = Array.from(document.querySelectorAll('span.h-notice-title')); + const targetTitle = titles.find(title => title.textContent && title.textContent.includes('招标公告')); + if (targetTitle) { + const parent = targetTitle.parentElement; + if (parent) { + const moreLink = parent.querySelector('a.h-notice-more') as HTMLElement; + if (moreLink) moreLink.click(); + } + } + }); + + logger.log('Clicked "更多+" link under "招标公告".'); + await page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 60000 }).catch(() => {}); + await new Promise(r => setTimeout(r, 3000)); + + // 模拟人类行为 + logger.log('Simulating human mouse movements...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling...'); + await simulateHumanScrolling(page); + + while (currentPage <= maxPages) { + const content = await page.content(); + const pageResults = this.extract(content); + if (pageResults.length === 0) { + logger.warn(`No results found on page ${currentPage}, stopping.`); + break; + } + + allResults.push(...pageResults); + logger.log(`Extracted ${pageResults.length} items from page ${currentPage}`); + + // 模拟人类行为 - 翻页前 + logger.log('Simulating human mouse movements before pagination...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling before pagination...'); + await simulateHumanScrolling(page); + + // Find the "Next Page" button - layui pagination + const nextButtonSelector = 'a.layui-laypage-next:not(.layui-disabled)'; + const nextButton = await page.$(nextButtonSelector); + + if (!nextButton) { + logger.log('Next page button not found. Reached end of list.'); + break; + } + + logger.log(`Navigating to page ${currentPage + 1}...`); + + try { + await Promise.all([ + page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 60000 }), + nextButton.click(), + ]); + } catch (navError) { + logger.error(`Navigation to page ${currentPage + 1} failed: ${navError.message}`); + break; + } + + currentPage++; + + // 模拟人类行为 - 翻页后 + logger.log('Simulating human mouse movements after pagination...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling after pagination...'); + await simulateHumanScrolling(page); + + // Random delay between pages + const delay = Math.floor(Math.random() * (3000 - 1000 + 1)) + 1000; + await new Promise(resolve => setTimeout(resolve, delay)); + } + + return allResults; + + } catch (error) { + logger.error(`Failed to crawl ${this.name}: ${error.message}`); + return allResults; + } finally { + await page.close(); + } + }, + + extract(html: string): CdtResult[] { + const results: CdtResult[] = []; + /** + * Regex groups for tang.cdt-ec.com: + * 1: URL + * 2: Title (项目名称) + * 3: Date (发布时间) + */ + const regex = /]*data-index="[^"]*"[^>]*>[\s\S]*?]*class="layui-table-link"[^>]*href="([^"]*)"[^>]*>([^<]*)<\/a>[\s\S]*?]*data-field="publish_time"[^>]*>[\s\S]*?]*class="layui-table-cell[^"]*"[^>]*>([^<]*)<\/div>[\s\S]*?<\/td>[\s\S]*?<\/tr>/gs; + + let match; + while ((match = regex.exec(html)) !== null) { + const url = match[1]?.trim(); + const title = match[2]?.trim(); + const dateStr = match[3]?.trim(); + + if (title && url) { + results.push({ + title, + publishDate: dateStr ? new Date(dateStr) : new Date(), + url: url.startsWith('http') ? url : this.baseUrl + url + }); + } + } + return results; + } +}; diff --git a/src/crawler/services/ceic_target.spec.ts b/src/crawler/services/ceic_target.spec.ts new file mode 100644 index 0000000..7de5f81 --- /dev/null +++ b/src/crawler/services/ceic_target.spec.ts @@ -0,0 +1,61 @@ +import { CeicCrawler } from './ceic_target'; +import * as puppeteer from 'puppeteer'; + +// Increase timeout to 120 seconds for manual inspection and slow sites +jest.setTimeout(120000); + +describe('CeicCrawler Real Site Test', () => { + let browser: puppeteer.Browser; + + beforeAll(async () => { + browser = await puppeteer.launch({ + headless: false, // Run in non-headless mode + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-blink-features=AutomationControlled', + '--window-size=1920,1080', + '--disable-infobars', + ], + defaultViewport: null + }); + }); + + afterAll(async () => { + if (browser) { + // Keep open for a few seconds after test to see result + await new Promise(r => setTimeout(r, 50000)); + await browser.close(); + } + }); + + it('should visit website and list all found bid information', async () => { + console.log(` +Starting crawl for: ${CeicCrawler.name}`); + console.log(`Target URL: ${CeicCrawler.url}`); + + const results = await CeicCrawler.crawl(browser); + + console.log(` +Successfully found ${results.length} items: +`); + console.log('----------------------------------------'); + results.forEach((item, index) => { + console.log(`${index + 1}. [${item.publishDate.toLocaleDateString()}] ${item.title}`); + console.log(` Link: ${item.url}`); + console.log('----------------------------------------'); + }); + + expect(results).toBeDefined(); + expect(Array.isArray(results)).toBeTruthy(); + + if (results.length === 0) { + console.warn('Warning: No items found. Observe browser window to see if content is loading or if there is a verification challenge.'); + } else { + const firstItem = results[0]; + expect(firstItem.title).toBeTruthy(); + expect(firstItem.url).toMatch(/^https?:\/\//); + expect(firstItem.publishDate).toBeInstanceOf(Date); + } + }); +}); diff --git a/src/crawler/services/ceic_target.ts b/src/crawler/services/ceic_target.ts new file mode 100644 index 0000000..b7266b7 --- /dev/null +++ b/src/crawler/services/ceic_target.ts @@ -0,0 +1,168 @@ +import * as puppeteer from 'puppeteer'; +import { Logger } from '@nestjs/common'; +import { ChdtpResult } from './chdtp_target'; + +// 模拟人类鼠标移动 +async function simulateHumanMouseMovement(page: puppeteer.Page) { + const viewport = page.viewport(); + if (!viewport) return; + + const movements = 5 + Math.floor(Math.random() * 5); // 5-10次随机移动 + + for (let i = 0; i < movements; i++) { + const x = Math.floor(Math.random() * viewport.width); + const y = Math.floor(Math.random() * viewport.height); + + await page.mouse.move(x, y, { + steps: 10 + Math.floor(Math.random() * 20) // 10-30步，使移动更平滑 + }); + + // 随机停顿 100-500ms + await new Promise(r => setTimeout(r, 100 + Math.random() * 400)); + } +} + +// 模拟人类滚动 +async function simulateHumanScrolling(page: puppeteer.Page) { + const scrollCount = 3 + Math.floor(Math.random() * 5); // 3-7次滚动 + + for (let i = 0; i < scrollCount; i++) { + const scrollDistance = 100 + Math.floor(Math.random() * 400); // 100-500px + + await page.evaluate((distance) => { + window.scrollBy({ + top: distance, + behavior: 'smooth' + }); + }, scrollDistance); + + // 随机停顿 500-1500ms + await new Promise(r => setTimeout(r, 500 + Math.random() * 1000)); + } + + // 滚动回顶部 + await page.evaluate(() => { + window.scrollTo({ top: 0, behavior: 'smooth' }); + }); + await new Promise(r => setTimeout(r, 1000)); +} + +export const CeicCrawler = { + name: '大连能源采购平台', + url: 'https://ceic.dlnyzb.com/3001', + baseUrl: 'https://ceic.dlnyzb.com', + + async crawl(browser: puppeteer.Browser): Promise { + const logger = new Logger('CeicCrawler'); + const page = await browser.newPage(); + + const username = process.env.PROXY_USERNAME; + const password = process.env.PROXY_PASSWORD; + if (username && password) { + await page.authenticate({ username, password }); + } + + await page.evaluateOnNewDocument(() => { + Object.defineProperty(navigator, 'webdriver', { get: () => false }); + Object.defineProperty(navigator, 'language', { get: () => 'zh-CN' }); + Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] }); + }); + + await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36'); + await page.setViewport({ width: 1920, height: 1080 }); + + const allResults: ChdtpResult[] = []; + let currentPage = 1; + const maxPages = 5; + + try { + logger.log(`Navigating to ${this.url}...`); + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + + // 模拟人类行为 + logger.log('Simulating human mouse movements...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling...'); + await simulateHumanScrolling(page); + + while (currentPage <= maxPages) { + logger.log(`Processing page ${currentPage}...`); + + // Wait for content to load - MUI list items + await page.waitForFunction(() => { + return document.querySelectorAll('li.MuiListItem-root').length > 0; + }, { timeout: 60000 }).catch(() => logger.warn('Content not found. Site might be slow.')); + + const pageResults = await page.evaluate(() => { + const results: { title: string; dateStr: string; url: string }[] = []; + + // Extract from MUI list items + const listItems = Array.from(document.querySelectorAll('li.MuiListItem-root')); + listItems.forEach(item => { + // Find the title link + const titleLink = item.querySelector('a.css-1vdw90h'); + const title = titleLink?.textContent?.trim() || ''; + const href = titleLink?.getAttribute('href') || ''; + + // Find the publish date - look for text containing "发布时间：" + const paragraphs = Array.from(item.querySelectorAll('p')); + let dateStr = ''; + for (const p of paragraphs) { + const text = p.textContent || ''; + if (text.includes('发布时间：')) { + dateStr = text.replace('发布时间：', '').trim(); + break; + } + } + + if (title.length >= 5 && href) { + results.push({ title, dateStr, url: href }); + } + }); + + return results; + }); + + if (pageResults.length === 0) { + logger.warn(`No results found on page ${currentPage}. Extraction failed.`); + break; + } + + allResults.push(...pageResults.map(r => ({ + title: r.title, + publishDate: r.dateStr ? new Date(r.dateStr) : new Date(), + url: r.url + }))); + + logger.log(`Extracted ${pageResults.length} items.`); + + // Pagination: look for next page button in MUI pagination + const nextButton = await page.$('a[aria-label="Go to next page"]'); + if (!nextButton) break; + + await nextButton.click(); + await new Promise(r => setTimeout(r, 3000)); + + // 模拟人类行为 + logger.log('Simulating human mouse movements...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling...'); + await simulateHumanScrolling(page); + + currentPage++; + } + + return allResults; + + } catch (error) { + logger.error(`Crawl failed: ${error.message}`); + return allResults; + } finally { + if (page) await page.close(); + } + }, + + extract() { return []; } +}; diff --git a/src/crawler/services/chng_target.spec.ts b/src/crawler/services/chng_target.spec.ts index 105aa8b..23be984 100644 --- a/src/crawler/services/chng_target.spec.ts +++ b/src/crawler/services/chng_target.spec.ts @@ -4,6 +4,51 @@ import * as puppeteer from 'puppeteer'; // Increase timeout to 120 seconds for manual inspection and slow sites jest.setTimeout(120000); +// 模拟人类鼠标移动 +async function simulateHumanMouseMovement(page: puppeteer.Page) { + const viewport = page.viewport(); + if (!viewport) return; + + const movements = 5 + Math.floor(Math.random() * 5); // 5-10次随机移动 + + for (let i = 0; i < movements; i++) { + const x = Math.floor(Math.random() * viewport.width); + const y = Math.floor(Math.random() * viewport.height); + + await page.mouse.move(x, y, { + steps: 10 + Math.floor(Math.random() * 20) // 10-30步，使移动更平滑 + }); + + // 随机停顿 100-500ms + await new Promise(r => setTimeout(r, 100 + Math.random() * 400)); + } +} + +// 模拟人类滚动 +async function simulateHumanScrolling(page: puppeteer.Page) { + const scrollCount = 3 + Math.floor(Math.random() * 5); // 3-7次滚动 + + for (let i = 0; i < scrollCount; i++) { + const scrollDistance = 100 + Math.floor(Math.random() * 400); // 100-500px + + await page.evaluate((distance) => { + window.scrollBy({ + top: distance, + behavior: 'smooth' + }); + }, scrollDistance); + + // 随机停顿 500-1500ms + await new Promise(r => setTimeout(r, 500 + Math.random() * 1000)); + } + + // 滚动回顶部 + await page.evaluate(() => { + window.scrollTo({ top: 0, behavior: 'smooth' }); + }); + await new Promise(r => setTimeout(r, 1000)); +} + describe('ChngCrawler Real Site Test', () => { let browser: puppeteer.Browser; @@ -14,9 +59,20 @@ describe('ChngCrawler Real Site Test', () => { '--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled', - '--window-size=1920,1080' + '--window-size=1920,1080', + "--disable-infobars", + // "--headless=new", + // '--disable-dev-shm-usage', + // '--disable-accelerated-2d-canvas', + // '--no-first-run', + // '--no-zygote', + // '--disable-gpu', + // '--disable-features=VizDisplayCompositor', + // '--disable-webgl', + // '--disable-javascript', ], defaultViewport: null + }); }); @@ -24,7 +80,7 @@ describe('ChngCrawler Real Site Test', () => { if (browser) { // Keep open for a few seconds after test to see result await new Promise(r => setTimeout(r, 50000)); - // await browser.close(); + await browser.close(); } }); @@ -33,6 +89,20 @@ describe('ChngCrawler Real Site Test', () => { Starting crawl for: ${ChngCrawler.name}`); console.log(`Target URL: ${ChngCrawler.url}`); + // 创建一个临时页面用于模拟人类行为 + const tempPage = await browser.newPage(); + await tempPage.setViewport({ width: 1920, height: 1080, deviceScaleFactor: 1 }); + + // 模拟人类鼠标移动 + console.log('Simulating human mouse movements...'); + await simulateHumanMouseMovement(tempPage); + + // 模拟人类滚动 + console.log('Simulating human scrolling...'); + await simulateHumanScrolling(tempPage); + + await tempPage.close(); + const results = await ChngCrawler.crawl(browser); console.log(` diff --git a/src/crawler/services/szecp_target.spec.ts b/src/crawler/services/szecp_target.spec.ts new file mode 100644 index 0000000..ad0abb2 --- /dev/null +++ b/src/crawler/services/szecp_target.spec.ts @@ -0,0 +1,61 @@ +import { SzecpCrawler } from './szecp_target'; +import * as puppeteer from 'puppeteer'; + +// Increase timeout to 120 seconds for manual inspection and slow sites +jest.setTimeout(120000); + +describe('SzecpCrawler Real Site Test', () => { + let browser: puppeteer.Browser; + + beforeAll(async () => { + browser = await puppeteer.launch({ + headless: false, // Run in non-headless mode + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-blink-features=AutomationControlled', + '--window-size=1920,1080', + '--disable-infobars', + ], + defaultViewport: null + }); + }); + + afterAll(async () => { + if (browser) { + // Keep open for a few seconds after test to see result + await new Promise(r => setTimeout(r, 50000)); + await browser.close(); + } + }); + + it('should visit website and list all found bid information', async () => { + console.log(` +Starting crawl for: ${SzecpCrawler.name}`); + console.log(`Target URL: ${SzecpCrawler.url}`); + + const results = await SzecpCrawler.crawl(browser); + + console.log(` +Successfully found ${results.length} items: +`); + console.log('----------------------------------------'); + results.forEach((item, index) => { + console.log(`${index + 1}. [${item.publishDate.toLocaleDateString()}] ${item.title}`); + console.log(` Link: ${item.url}`); + console.log('----------------------------------------'); + }); + + expect(results).toBeDefined(); + expect(Array.isArray(results)).toBeTruthy(); + + if (results.length === 0) { + console.warn('Warning: No items found. Observe browser window to see if content is loading or if there is a verification challenge.'); + } else { + const firstItem = results[0]; + expect(firstItem.title).toBeTruthy(); + expect(firstItem.url).toMatch(/^https?:\/\//); + expect(firstItem.publishDate).toBeInstanceOf(Date); + } + }); +}); diff --git a/src/crawler/services/szecp_target.ts b/src/crawler/services/szecp_target.ts new file mode 100644 index 0000000..19f2a5b --- /dev/null +++ b/src/crawler/services/szecp_target.ts @@ -0,0 +1,170 @@ +import * as puppeteer from 'puppeteer'; +import { Logger } from '@nestjs/common'; +import { ChdtpResult } from './chdtp_target'; + +// 模拟人类鼠标移动 +async function simulateHumanMouseMovement(page: puppeteer.Page) { + const viewport = page.viewport(); + if (!viewport) return; + + const movements = 5 + Math.floor(Math.random() * 5); // 5-10次随机移动 + + for (let i = 0; i < movements; i++) { + const x = Math.floor(Math.random() * viewport.width); + const y = Math.floor(Math.random() * viewport.height); + + await page.mouse.move(x, y, { + steps: 10 + Math.floor(Math.random() * 20) // 10-30步，使移动更平滑 + }); + + // 随机停顿 100-500ms + await new Promise(r => setTimeout(r, 100 + Math.random() * 400)); + } +} + +// 模拟人类滚动 +async function simulateHumanScrolling(page: puppeteer.Page) { + const scrollCount = 3 + Math.floor(Math.random() * 5); // 3-7次滚动 + + for (let i = 0; i < scrollCount; i++) { + const scrollDistance = 100 + Math.floor(Math.random() * 400); // 100-500px + + await page.evaluate((distance) => { + window.scrollBy({ + top: distance, + behavior: 'smooth' + }); + }, scrollDistance); + + // 随机停顿 500-1500ms + await new Promise(r => setTimeout(r, 500 + Math.random() * 1000)); + } + + // 滚动回顶部 + await page.evaluate(() => { + window.scrollTo({ top: 0, behavior: 'smooth' }); + }); + await new Promise(r => setTimeout(r, 1000)); +} + +export const SzecpCrawler = { + name: '华润守正采购交易平台', + url: 'https://www.szecp.com.cn/first_zbgg/index.html', + baseUrl: 'https://www.szecp.com.cn', + + async crawl(browser: puppeteer.Browser): Promise { + const logger = new Logger('SzecpCrawler'); + const page = await browser.newPage(); + + const username = process.env.PROXY_USERNAME; + const password = process.env.PROXY_PASSWORD; + if (username && password) { + await page.authenticate({ username, password }); + } + + await page.evaluateOnNewDocument(() => { + Object.defineProperty(navigator, 'webdriver', { get: () => false }); + Object.defineProperty(navigator, 'language', { get: () => 'zh-CN' }); + Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] }); + }); + + await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36'); + await page.setViewport({ width: 1920, height: 1080 }); + + const allResults: ChdtpResult[] = []; + let currentPage = 1; + const maxPages = 5; + + try { + logger.log(`Navigating to ${this.url}...`); + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + + // 模拟人类行为 + logger.log('Simulating human mouse movements...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling...'); + await simulateHumanScrolling(page); + + // Wait for search button to be available and click it + logger.log('Clicking search button...'); + await page.waitForSelector('.szb-zbcgSearch-key-v1', { timeout: 60000 }); + await page.click('.szb-zbcgSearch-key-v1'); + await new Promise(r => setTimeout(r, 3000)); // Wait for results to load + + while (currentPage <= maxPages) { + logger.log(`Processing page ${currentPage}...`); + + // Wait for content to load + await page.waitForFunction(() => { + return document.querySelectorAll('.szb-zbcgTable-other').length > 0; + }, { timeout: 60000 }).catch(() => logger.warn('Content not found. Site might be slow.')); + + const pageResults = await page.evaluate((baseUrl) => { + // Extract from table rows + const items = Array.from(document.querySelectorAll('.szb-zbcgTable-other')); + return items.map(item => { + const divs = item.querySelectorAll('div'); + if (divs.length >= 5) { + const titleLink = divs[1].querySelector('a'); + const title = titleLink?.textContent?.trim() || ''; + const dateStr = divs[4].textContent?.trim() || ''; + const href = titleLink?.getAttribute('href') || ''; + + if (title.length < 5) return null; // Filter noise + + // Construct full URL if href is relative + const url = href.startsWith('http') ? href : `${baseUrl}${href}`; + + return { + title, + dateStr, + url + }; + } + return null; + }).filter(i => i !== null); + }, this.baseUrl); + + if (pageResults.length === 0) { + logger.warn(`No results found on page ${currentPage}. Extraction failed.`); + break; + } + + allResults.push(...pageResults.map(r => ({ + title: r!.title, + publishDate: new Date(r!.dateStr), + url: r!.url + }))); + + logger.log(`Extracted ${pageResults.length} items.`); + + // Pagination: look for next page link + const nextButton = await page.$('.pagination li a[page="+"]'); + if (!nextButton) break; + + await nextButton.click(); + await new Promise(r => setTimeout(r, 3000)); + + // 模拟人类行为 + logger.log('Simulating human mouse movements...'); + await simulateHumanMouseMovement(page); + + logger.log('Simulating human scrolling...'); + await simulateHumanScrolling(page); + + currentPage++; + } + + return allResults; + + } catch (error) { + logger.error(`Crawl failed: ${error.message}`); + return allResults; + } finally { + if (page) await page.close(); + } + }, + + extract() { return []; } +};