60 lines
1.9 KiB
TypeScript
60 lines
1.9 KiB
TypeScript
|
import { ChngCrawler } from './chng_target';
|
||
|
|
import * as puppeteer from 'puppeteer';
|
||
|
|
|
||
|
|
// Increase timeout to 120 seconds for manual inspection and slow sites
|
||
|
|
jest.setTimeout(120000);
|
||
|
|
|
||
|
|
describe('ChngCrawler Real Site Test', () => {
|
||
|
|
let browser: puppeteer.Browser;
|
||
|
|
|
||
|
|
beforeAll(async () => {
|
||
|
|
browser = await puppeteer.launch({
|
||
|
|
headless: false, // Run in non-headless mode
|
||
|
|
args: [
|
||
|
|
'--no-sandbox',
|
||
|
|
'--disable-setuid-sandbox',
|
||
|
|
'--disable-blink-features=AutomationControlled',
|
||
|
|
'--window-size=1920,1080'
|
||
|
|
],
|
||
|
|
defaultViewport: null
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
afterAll(async () => {
|
||
|
|
if (browser) {
|
||
|
|
// Keep open for a few seconds after test to see result
|
||
|
|
await new Promise(r => setTimeout(r, 50000));
|
||
|
|
// await browser.close();
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
it('should visit the website and list all found bid information', async () => {
|
||
|
|
console.log(`
|
||
|
|
Starting crawl for: ${ChngCrawler.name}`);
|
||
|
|
console.log(`Target URL: ${ChngCrawler.url}`);
|
||
|
|
|
||
|
|
const results = await ChngCrawler.crawl(browser);
|
||
|
|
|
||
|
|
console.log(`
|
||
|
|
Successfully found ${results.length} items:
|
||
|
|
`);
|
||
|
|
console.log('----------------------------------------');
|
||
|
|
results.forEach((item, index) => {
|
||
|
|
console.log(`${index + 1}. [${item.publishDate.toLocaleDateString()}] ${item.title}`);
|
||
|
|
console.log(` Link: ${item.url}`);
|
||
|
|
console.log('----------------------------------------');
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(results).toBeDefined();
|
||
|
|
expect(Array.isArray(results)).toBeTruthy();
|
||
|
|
|
||
|
|
if (results.length === 0) {
|
||
|
|
console.warn('Warning: No items found. Observe the browser window to see if content is loading or if there is a verification challenge.');
|
||
|
|
} else {
|
||
|
|
const firstItem = results[0];
|
||
|
|
expect(firstItem.title).toBeTruthy();
|
||
|
|
expect(firstItem.url).toMatch(/^https?:\/\//);
|
||
|
|
expect(firstItem.publishDate).toBeInstanceOf(Date);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|