feat: 为爬虫测试添加代理支持并通过环境变量配置
添加dotenv依赖,创建jest配置文件和setup文件 修改所有爬虫测试文件以支持通过环境变量配置代理 将jest配置从package.json移动到独立文件
This commit is contained in:
@@ -4,13 +4,35 @@ import * as puppeteer from 'puppeteer';
|
||||
// Increase timeout to 60 seconds for network operations
|
||||
jest.setTimeout(60000);
|
||||
|
||||
// 获取代理配置
|
||||
const getProxyArgs = (): string[] => {
|
||||
const proxyHost = process.env.PROXY_HOST;
|
||||
const proxyPort = process.env.PROXY_PORT;
|
||||
const proxyUsername = process.env.PROXY_USERNAME;
|
||||
const proxyPassword = process.env.PROXY_PASSWORD;
|
||||
|
||||
if (proxyHost && proxyPort) {
|
||||
const args = [`--proxy-server=${proxyHost}:${proxyPort}`];
|
||||
if (proxyUsername && proxyPassword) {
|
||||
args.push(`--proxy-auth=${proxyUsername}:${proxyPassword}`);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
describe('ChdtpCrawler Real Site Test', () => {
|
||||
let browser: puppeteer.Browser;
|
||||
|
||||
beforeAll(async () => {
|
||||
const proxyArgs = getProxyArgs();
|
||||
if (proxyArgs.length > 0) {
|
||||
console.log('Using proxy:', proxyArgs.join(' '));
|
||||
}
|
||||
|
||||
browser = await puppeteer.launch({
|
||||
headless: true, // Change to false to see the browser UI
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox', ...proxyArgs],
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user