feat: 为爬虫测试添加代理支持并通过环境变量配置
添加dotenv依赖,创建jest配置文件和setup文件 修改所有爬虫测试文件以支持通过环境变量配置代理 将jest配置从package.json移动到独立文件
This commit is contained in:
@@ -4,10 +4,32 @@ import * as puppeteer from 'puppeteer';
|
||||
// Increase timeout to 120 seconds for manual inspection and slow sites
|
||||
jest.setTimeout(120000);
|
||||
|
||||
// 获取代理配置
|
||||
const getProxyArgs = (): string[] => {
|
||||
const proxyHost = process.env.PROXY_HOST;
|
||||
const proxyPort = process.env.PROXY_PORT;
|
||||
const proxyUsername = process.env.PROXY_USERNAME;
|
||||
const proxyPassword = process.env.PROXY_PASSWORD;
|
||||
|
||||
if (proxyHost && proxyPort) {
|
||||
const args = [`--proxy-server=${proxyHost}:${proxyPort}`];
|
||||
if (proxyUsername && proxyPassword) {
|
||||
args.push(`--proxy-auth=${proxyUsername}:${proxyPassword}`);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
describe('CeicCrawler Real Site Test', () => {
|
||||
let browser: puppeteer.Browser;
|
||||
|
||||
beforeAll(async () => {
|
||||
const proxyArgs = getProxyArgs();
|
||||
if (proxyArgs.length > 0) {
|
||||
console.log('Using proxy:', proxyArgs.join(' '));
|
||||
}
|
||||
|
||||
browser = await puppeteer.launch({
|
||||
headless: false, // Run in non-headless mode
|
||||
args: [
|
||||
@@ -16,6 +38,7 @@ describe('CeicCrawler Real Site Test', () => {
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--window-size=1920,1080',
|
||||
'--disable-infobars',
|
||||
...proxyArgs,
|
||||
],
|
||||
defaultViewport: null
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user