diff --git a/src/crawler/services/cdt_target.ts b/src/crawler/services/cdt_target.ts index c7a67d0..ea5372b 100644 --- a/src/crawler/services/cdt_target.ts +++ b/src/crawler/services/cdt_target.ts @@ -46,6 +46,56 @@ async function simulateHumanScrolling(page: puppeteer.Page) { await new Promise((r) => setTimeout(r, 1000)); } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + export interface CdtResult { title: string; publishDate: Date; @@ -87,7 +137,14 @@ export const CdtCrawler = { try { logger.log(`Navigating to ${this.url}...`); - await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + await delayRetry( + async () => { + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + }, + 3, + 5000, + logger, + ); // 模拟人类行为 logger.log('Simulating human mouse movements...'); diff --git a/src/crawler/services/ceic_target.ts b/src/crawler/services/ceic_target.ts index c860b28..1dc822e 100644 --- a/src/crawler/services/ceic_target.ts +++ b/src/crawler/services/ceic_target.ts @@ -47,6 +47,56 @@ async function simulateHumanScrolling(page: puppeteer.Page) { await new Promise((r) => setTimeout(r, 1000)); } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + interface CeicCrawlerType { name: string; url: string; @@ -90,7 +140,14 @@ export const CeicCrawler = { try { logger.log(`Navigating to ${this.url}...`); - await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + await delayRetry( + async () => { + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + }, + 3, + 5000, + logger, + ); // 模拟人类行为 logger.log('Simulating human mouse movements...'); diff --git a/src/crawler/services/cgnpc_target.ts b/src/crawler/services/cgnpc_target.ts index 9b046df..4557f51 100644 --- a/src/crawler/services/cgnpc_target.ts +++ b/src/crawler/services/cgnpc_target.ts @@ -46,6 +46,56 @@ async function simulateHumanScrolling(page: puppeteer.Page) { await new Promise((r) => setTimeout(r, 1000)); } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + export interface CgnpcResult { title: string; publishDate: Date; @@ -96,7 +146,14 @@ export const CgnpcCrawler = { try { logger.log(`Navigating to ${this.url}...`); - await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + await delayRetry( + async () => { + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + }, + 3, + 5000, + logger, + ); // 模拟人类行为 logger.log('Simulating human mouse movements...'); diff --git a/src/crawler/services/chdtp_target.ts b/src/crawler/services/chdtp_target.ts index 410d7a7..f920f03 100644 --- a/src/crawler/services/chdtp_target.ts +++ b/src/crawler/services/chdtp_target.ts @@ -14,6 +14,56 @@ interface ChdtpCrawlerType { extract(html: string): ChdtpResult[]; } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + export const ChdtpCrawler = { name: '华电集团电子商务平台 ', url: 'https://www.chdtp.com/webs/queryWebZbgg.action?zbggType=1', @@ -42,7 +92,14 @@ export const ChdtpCrawler = { try { logger.log(`Navigating to ${this.url}...`); - await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + await delayRetry( + async () => { + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + }, + 3, + 5000, + logger, + ); while (currentPage <= maxPages) { const content = await page.content(); diff --git a/src/crawler/services/chng_target.ts b/src/crawler/services/chng_target.ts index b3dfacd..3ee5db1 100644 --- a/src/crawler/services/chng_target.ts +++ b/src/crawler/services/chng_target.ts @@ -71,6 +71,56 @@ async function simulateHumanScrolling(page: puppeteer.Page) { } } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + interface ChngCrawlerType { name: string; url: string; @@ -115,7 +165,14 @@ export const ChngCrawler = { try { logger.log('Navigating to Bing...'); - await page.goto('https://cn.bing.com', { waitUntil: 'networkidle2' }); + await delayRetry( + async () => { + await page.goto('https://cn.bing.com', { waitUntil: 'networkidle2' }); + }, + 3, + 5000, + logger, + ); logger.log('Searching for target site...'); const searchBoxSelector = 'input[name="q"]'; diff --git a/src/crawler/services/cnncecp_target.ts b/src/crawler/services/cnncecp_target.ts index 8ed8746..e59b760 100644 --- a/src/crawler/services/cnncecp_target.ts +++ b/src/crawler/services/cnncecp_target.ts @@ -46,6 +46,56 @@ async function simulateHumanScrolling(page: puppeteer.Page) { await new Promise((r) => setTimeout(r, 1000)); } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + export interface CnncecpResult { title: string; publishDate: Date; @@ -96,7 +146,14 @@ export const CnncecpCrawler = { try { logger.log(`Navigating to ${this.url}...`); - await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + await delayRetry( + async () => { + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + }, + 3, + 5000, + logger, + ); // 模拟人类行为 logger.log('Simulating human mouse movements...'); diff --git a/src/crawler/services/cnooc_target.ts b/src/crawler/services/cnooc_target.ts index 12d1b2a..7aa742d 100644 --- a/src/crawler/services/cnooc_target.ts +++ b/src/crawler/services/cnooc_target.ts @@ -46,6 +46,56 @@ async function simulateHumanScrolling(page: puppeteer.Page) { await new Promise((r) => setTimeout(r, 1000)); } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + export interface CnoocResult { title: string; publishDate: Date; @@ -96,7 +146,14 @@ export const CnoocCrawler = { try { logger.log(`Navigating to ${this.url}...`); - await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + await delayRetry( + async () => { + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + }, + 3, + 5000, + logger, + ); // 模拟人类行为 logger.log('Simulating human mouse movements...'); diff --git a/src/crawler/services/eps_target.ts b/src/crawler/services/eps_target.ts index bc175ec..970ae90 100644 --- a/src/crawler/services/eps_target.ts +++ b/src/crawler/services/eps_target.ts @@ -46,6 +46,56 @@ async function simulateHumanScrolling(page: puppeteer.Page) { await new Promise((r) => setTimeout(r, 1000)); } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + export interface EpsResult { title: string; publishDate: Date; @@ -96,7 +146,14 @@ export const EpsCrawler = { try { logger.log(`Navigating to ${this.url}...`); - await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + await delayRetry( + async () => { + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + }, + 3, + 5000, + logger, + ); // 模拟人类行为 logger.log('Simulating human mouse movements...'); diff --git a/src/crawler/services/espic_target.ts b/src/crawler/services/espic_target.ts index f7d31c4..07068c2 100644 --- a/src/crawler/services/espic_target.ts +++ b/src/crawler/services/espic_target.ts @@ -46,6 +46,56 @@ async function simulateHumanScrolling(page: puppeteer.Page) { await new Promise((r) => setTimeout(r, 1000)); } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + export interface EspicResult { title: string; publishDate: Date; @@ -106,7 +156,14 @@ export const EspicCrawler = { try { const url = this.getUrl(currentPage); logger.log(`Navigating to ${url}...`); - await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 }); + await delayRetry( + async () => { + await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 }); + }, + 3, + 5000, + logger, + ); // 等待 WAF 验证通过 logger.log('Waiting for WAF verification...'); diff --git a/src/crawler/services/powerbeijing_target.ts b/src/crawler/services/powerbeijing_target.ts index 825512e..8702f9a 100644 --- a/src/crawler/services/powerbeijing_target.ts +++ b/src/crawler/services/powerbeijing_target.ts @@ -46,6 +46,56 @@ async function simulateHumanScrolling(page: puppeteer.Page) { await new Promise((r) => setTimeout(r, 1000)); } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + export interface PowerbeijingResult { title: string; publishDate: Date; @@ -96,7 +146,14 @@ export const PowerbeijingCrawler = { try { logger.log(`Navigating to ${this.url}...`); - await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + await delayRetry( + async () => { + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + }, + 3, + 5000, + logger, + ); // 模拟人类行为 logger.log('Simulating human mouse movements...'); diff --git a/src/crawler/services/sdicc_target.ts b/src/crawler/services/sdicc_target.ts index 75c307b..8a7c464 100644 --- a/src/crawler/services/sdicc_target.ts +++ b/src/crawler/services/sdicc_target.ts @@ -46,6 +46,56 @@ async function simulateHumanScrolling(page: puppeteer.Page) { await new Promise((r) => setTimeout(r, 1000)); } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + export interface SdiccResult { title: string; publishDate: Date; @@ -96,7 +146,14 @@ export const SdiccCrawler = { try { logger.log(`Navigating to ${this.url}...`); - await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + await delayRetry( + async () => { + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + }, + 3, + 5000, + logger, + ); // 模拟人类行为 logger.log('Simulating human mouse movements...'); diff --git a/src/crawler/services/szecp_target.ts b/src/crawler/services/szecp_target.ts index 16b7ce2..2fe9c59 100644 --- a/src/crawler/services/szecp_target.ts +++ b/src/crawler/services/szecp_target.ts @@ -47,6 +47,56 @@ async function simulateHumanScrolling(page: puppeteer.Page) { await new Promise((r) => setTimeout(r, 1000)); } +// 检查错误是否为代理隧道连接失败 +function isTunnelConnectionFailedError(error: unknown): boolean { + if (error instanceof Error) { + return ( + error.message.includes('net::ERR_TUNNEL_CONNECTION_FAILED') || + error.message.includes('ERR_TUNNEL_CONNECTION_FAILED') + ); + } + return false; +} + +// 延迟重试函数 +async function delayRetry( + operation: () => Promise, + maxRetries: number = 3, + delayMs: number = 5000, + logger?: Logger, +): Promise { + let lastError: Error | unknown; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await operation(); + return; + } catch (error) { + lastError = error; + + if (isTunnelConnectionFailedError(error)) { + if (attempt < maxRetries) { + const delay = delayMs * attempt; // 递增延迟 + logger?.warn( + `代理隧道连接失败,第 ${attempt} 次尝试失败,${delay / 1000} 秒后重试...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger?.error( + `代理隧道连接失败,已达到最大重试次数 ${maxRetries} 次`, + ); + throw error; + } + } else { + // 非代理错误,直接抛出 + throw error; + } + } + } + + throw lastError; +} + interface SzecpCrawlerType { name: string; url: string; @@ -90,7 +140,14 @@ export const SzecpCrawler = { try { logger.log(`Navigating to ${this.url}...`); - await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + await delayRetry( + async () => { + await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 }); + }, + 3, + 5000, + logger, + ); // 模拟人类行为 logger.log('Simulating human mouse movements...'); diff --git a/uni-app-version/.env b/uni-app-version/.env new file mode 100644 index 0000000..0ac80d8 --- /dev/null +++ b/uni-app-version/.env @@ -0,0 +1,8 @@ +# API 配置 +VITE_API_BASE_URL=http://localhost:3000 + +# 应用配置 +VITE_APP_TITLE=投标项目查看器`nVITE_APP_VERSION=1.0.0 + +# 刷新间隔(毫秒) +VITE_AUTO_REFRESH_INTERVAL=300000 diff --git a/uni-app-version/.env.development b/uni-app-version/.env.development new file mode 100644 index 0000000..72abd04 --- /dev/null +++ b/uni-app-version/.env.development @@ -0,0 +1,2 @@ +# 开发环境配置`nVITE_API_BASE_URL=http://localhost:3000 +VITE_APP_TITLE=投标项目查看?开? diff --git a/uni-app-version/.npmrc b/uni-app-version/.npmrc new file mode 100644 index 0000000..b2613db --- /dev/null +++ b/uni-app-version/.npmrc @@ -0,0 +1,3 @@ +@ +legacy-peer-deps=true +strict-peer-dependencies=false diff --git a/uni-app-version/index.html b/uni-app-version/index.html new file mode 100644 index 0000000..c346991 --- /dev/null +++ b/uni-app-version/index.html @@ -0,0 +1,17 @@ + + + + + + 投标项目查看器 + + + + +
+ + + diff --git a/uni-app-version/package.json b/uni-app-version/package.json new file mode 100644 index 0000000..38ea327 --- /dev/null +++ b/uni-app-version/package.json @@ -0,0 +1,45 @@ +{ + "name": "bidding-looker-uniapp", + "version": "1.0.0", + "description": "投标项目查看器 - uni-app 版本", + "main": "main.ts", + "type": "module", + "scripts": { + "dev:h5": "uni", + "build:h5": "uni build", + "dev:mp-weixin": "uni -p mp-weixin", + "build:mp-weixin": "uni build -p mp-weixin", + "dev:app": "uni -p app", + "build:app": "uni build -p app", + "type-check": "vue-tsc --noEmit" + }, + "keywords": [ + "uni-app", + "bidding", + "looker", + "typescript", + "tailwindcss" + ], + "author": "", + "license": "MIT", + "dependencies": { + "vue": "^3.4.21" + }, + "devDependencies": { + "@dcloudio/types": "^3.4.8", + "@dcloudio/uni-app": "3.0.0-alpha-4020920240929001", + "@dcloudio/uni-app-plus": "3.0.0-alpha-4020920240929001", + "@dcloudio/uni-components": "3.0.0-alpha-4020920240929001", + "@dcloudio/uni-h5": "3.0.0-alpha-4020920240929001", + "@dcloudio/uni-mp-weixin": "3.0.0-alpha-4020920240929001", + "@dcloudio/vite-plugin-uni": "3.0.0-alpha-4020920240929001", + "@types/node": "^20.14.0", + "autoprefixer": "^10.4.20", + "postcss": "^8.4.49", + "sass": "^1.77.0", + "tailwindcss": "^3.4.17", + "typescript": "^5.3.3", + "vite": "5.2.13", + "vue-tsc": "^2.0.0" + } +} diff --git a/uni-app-version/postcss.config.js b/uni-app-version/postcss.config.js new file mode 100644 index 0000000..2b75bd8 --- /dev/null +++ b/uni-app-version/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {} + } +} diff --git a/uni-app-version/src/App.vue b/uni-app-version/src/App.vue new file mode 100644 index 0000000..3482a7f --- /dev/null +++ b/uni-app-version/src/App.vue @@ -0,0 +1,36 @@ + + + diff --git a/uni-app-version/src/components/AiRecommendations.vue b/uni-app-version/src/components/AiRecommendations.vue new file mode 100644 index 0000000..435acc1 --- /dev/null +++ b/uni-app-version/src/components/AiRecommendations.vue @@ -0,0 +1,144 @@ + + + + + diff --git a/uni-app-version/src/components/CrawlInfo.vue b/uni-app-version/src/components/CrawlInfo.vue new file mode 100644 index 0000000..cf282db --- /dev/null +++ b/uni-app-version/src/components/CrawlInfo.vue @@ -0,0 +1,148 @@ + + + + + diff --git a/uni-app-version/src/components/PinnedBids.vue b/uni-app-version/src/components/PinnedBids.vue new file mode 100644 index 0000000..f897a26 --- /dev/null +++ b/uni-app-version/src/components/PinnedBids.vue @@ -0,0 +1,147 @@ + + + + + diff --git a/uni-app-version/src/env.d.ts b/uni-app-version/src/env.d.ts new file mode 100644 index 0000000..2ca4de8 --- /dev/null +++ b/uni-app-version/src/env.d.ts @@ -0,0 +1,18 @@ +/// + +declare module '*.vue' { + import { DefineComponent } from 'vue' + const component: DefineComponent<{}, {}, any> + export default component +} + +interface ImportMetaEnv { + readonly VITE_API_BASE_URL: string + readonly VITE_APP_TITLE: string + readonly VITE_APP_VERSION: string + readonly VITE_AUTO_REFRESH_INTERVAL: string +} + +interface ImportMeta { + readonly env: ImportMetaEnv +} diff --git a/uni-app-version/src/main.ts b/uni-app-version/src/main.ts new file mode 100644 index 0000000..ffcc69d --- /dev/null +++ b/uni-app-version/src/main.ts @@ -0,0 +1,9 @@ +import { createSSRApp } from 'vue' +import App from './App.vue' + +export function createApp() { + const app = createSSRApp(App) + return { + app + } +} diff --git a/uni-app-version/src/manifest.json b/uni-app-version/src/manifest.json new file mode 100644 index 0000000..04578ab --- /dev/null +++ b/uni-app-version/src/manifest.json @@ -0,0 +1,73 @@ +{ + "name": "bidding-looker", + "appid": "__UNI__BIDDING_LOOKER", + "description": "投标项目查看器", + "versionName": "1.0.0", + "versionCode": "100", + "transformPx": false, + "app-plus": { + "usingComponents": true, + "nvueStyleCompiler": "uni-app", + "compilerVersion": 3, + "splashscreen": { + "alwaysShowBeforeRender": true, + "waiting": true, + "autoclose": true, + "delay": 0 + }, + "modules": {}, + "distribute": { + "android": { + "permissions": [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + ] + }, + "ios": {}, + "sdkConfigs": {} + } + }, + "quickapp": {}, + "mp-weixin": { + "appid": "", + "setting": { + "urlCheck": false + }, + "usingComponents": true + }, + "mp-alipay": { + "usingComponents": true + }, + "mp-baidu": { + "usingComponents": true + }, + "mp-toutiao": { + "usingComponents": true + }, + "uniStatistics": { + "enable": false + }, + "vueVersion": "3", + "h5": { + "router": { + "mode": "hash", + "base": "/" + }, + "devServer": { + "port": 8080, + "disableHostCheck": true + } + } +} diff --git a/uni-app-version/src/pages.json b/uni-app-version/src/pages.json new file mode 100644 index 0000000..a8f231a --- /dev/null +++ b/uni-app-version/src/pages.json @@ -0,0 +1,26 @@ +{ + "easycom": { + "autoscan": true, + "custom": { + "^u-(.*)": "uview-ui/components/u-$1/u-$1.vue" + } + }, + "pages": [ + { + "path": "pages/index/index", + "style": { + "navigationBarTitleText": "投标项目查看器", + "navigationBarBackgroundColor": "#ffffff", + "navigationBarTextStyle": "black", + "enablePullDownRefresh": true + } + } + ], + "globalStyle": { + "navigationBarTextStyle": "black", + "navigationBarTitleText": "投标项目查看器", + "navigationBarBackgroundColor": "#ffffff", + "backgroundColor": "#f5f5f5" + }, + "tabBar": {} +} diff --git a/uni-app-version/src/pages/index/index.vue b/uni-app-version/src/pages/index/index.vue new file mode 100644 index 0000000..f6acdaf --- /dev/null +++ b/uni-app-version/src/pages/index/index.vue @@ -0,0 +1,194 @@ + + + + + diff --git a/uni-app-version/src/uni.scss b/uni-app-version/src/uni.scss new file mode 100644 index 0000000..0f1d3c2 --- /dev/null +++ b/uni-app-version/src/uni.scss @@ -0,0 +1,40 @@ +/** + * uni-app 全局样式变量 + */ + +/* 颜色变量 */ +$uni-color-primary: #3498db; +$uni-color-success: #27ae60; +$uni-color-warning: #f39c12; +$uni-color-error: #e74c3c; +$uni-color-info: #909399; + +/* 文字颜色 */ +$uni-text-color: #333333; +$uni-text-color-grey: #666666; +$uni-text-color-placeholder: #999999; + +/* 背景颜色 */ +$uni-bg-color: #ffffff; +$uni-bg-color-grey: #f5f5f5; +$uni-bg-color-hover: #f9f9f9; + +/* 边框颜色 */ +$uni-border-color: #e0e0e0; + +/* 字体大小 */ +$uni-font-size-xs: 20rpx; +$uni-font-size-sm: 24rpx; +$uni-font-size-base: 28rpx; +$uni-font-size-lg: 32rpx; + +/* 间距 */ +$uni-spacing-xs: 8rpx; +$uni-spacing-sm: 16rpx; +$uni-spacing-base: 24rpx; +$uni-spacing-lg: 32rpx; + +/* 圆角 */ +$uni-border-radius-sm: 6rpx; +$uni-border-radius-base: 8rpx; +$uni-border-radius-lg: 12rpx; diff --git a/uni-app-version/tailwind.config.js b/uni-app-version/tailwind.config.js new file mode 100644 index 0000000..0909304 --- /dev/null +++ b/uni-app-version/tailwind.config.js @@ -0,0 +1,25 @@ +/** @type {import('tailwindcss').Config} */ +export default { + content: [ + './index.html', + './pages/**/*.{vue,js,ts,jsx,tsx}', + './components/**/*.{vue,js,ts,jsx,tsx}', + './App.vue' + ], + theme: { + extend: { + colors: { + primary: '#3498db', + success: '#27ae60', + warning: '#f39c12', + error: '#e74c3c', + info: '#909399' + } + } + }, + plugins: [], + // uni-app 配置 + corePlugins: { + preflight: false + } +} diff --git a/uni-app-version/tsconfig.json b/uni-app-version/tsconfig.json new file mode 100644 index 0000000..c1417d1 --- /dev/null +++ b/uni-app-version/tsconfig.json @@ -0,0 +1,38 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "ESNext", + "moduleResolution": "bundler", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "skipLibCheck": true, + "allowJs": true, + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "isolatedModules": true, + "sourceMap": true, + "baseUrl": ".", + "paths": { + "@/*": ["./*"] + }, + "types": [ + "@dcloudio/types", + "vite/client" + ] + }, + "include": [ + "**/*.ts", + "**/*.d.ts", + "**/*.tsx", + "**/*.vue" + ], + "exclude": [ + "node_modules", + "dist", + "unpackage" + ] +} diff --git a/uni-app-version/vite.config.ts b/uni-app-version/vite.config.ts new file mode 100644 index 0000000..2c9b4d9 --- /dev/null +++ b/uni-app-version/vite.config.ts @@ -0,0 +1,18 @@ +import { defineConfig } from 'vite' +import uniPlugin from '@dcloudio/vite-plugin-uni' + +// Handle both ESM default export and CJS module.exports +const uni = (uniPlugin as any).default || uniPlugin + +export default defineConfig({ + plugins: [uni()], + server: { + port: 8080, + proxy: { + '/api': { + target: 'http://localhost:3000', + changeOrigin: true + } + } + } +})