feat: 为爬虫测试添加代理支持并通过环境变量配置

添加dotenv依赖,创建jest配置文件和setup文件
修改所有爬虫测试文件以支持通过环境变量配置代理
将jest配置从package.json移动到独立文件
This commit is contained in:
dmy
2026-01-12 15:19:54 +08:00
parent f1ec37143c
commit b1435523e8
17 changed files with 325 additions and 27 deletions

View File

@@ -4,10 +4,32 @@ import * as puppeteer from 'puppeteer';
// Increase timeout to 120 seconds for manual inspection and slow sites
jest.setTimeout(120000);
// 获取代理配置
const getProxyArgs = (): string[] => {
const proxyHost = process.env.PROXY_HOST;
const proxyPort = process.env.PROXY_PORT;
const proxyUsername = process.env.PROXY_USERNAME;
const proxyPassword = process.env.PROXY_PASSWORD;
if (proxyHost && proxyPort) {
const args = [`--proxy-server=${proxyHost}:${proxyPort}`];
if (proxyUsername && proxyPassword) {
args.push(`--proxy-auth=${proxyUsername}:${proxyPassword}`);
}
return args;
}
return [];
};
describe('CeicCrawler Real Site Test', () => {
let browser: puppeteer.Browser;
beforeAll(async () => {
const proxyArgs = getProxyArgs();
if (proxyArgs.length > 0) {
console.log('Using proxy:', proxyArgs.join(' '));
}
browser = await puppeteer.launch({
headless: false, // Run in non-headless mode
args: [
@@ -16,6 +38,7 @@ describe('CeicCrawler Real Site Test', () => {
'--disable-blink-features=AutomationControlled',
'--window-size=1920,1080',
'--disable-infobars',
...proxyArgs,
],
defaultViewport: null
});