diff --git a/frontend/src/components/Bids.vue b/frontend/src/components/Bids.vue index 5cf15a1..a8fe8cf 100644 --- a/frontend/src/components/Bids.vue +++ b/frontend/src/components/Bids.vue @@ -76,3 +76,14 @@ const handleSizeChange = (size: number) => { emit('fetch', currentPage.value, pageSize.value, selectedSource.value || undefined) } + + diff --git a/frontend/src/components/Dashboard.vue b/frontend/src/components/Dashboard.vue index d87ec1c..ff0db3b 100644 --- a/frontend/src/components/Dashboard.vue +++ b/frontend/src/components/Dashboard.vue @@ -11,22 +11,31 @@ - - - - - - - - - + +
+ + + + + + + + + +
+ @@ -83,7 +92,7 @@ import { ref, computed, watch } from 'vue' import axios from 'axios' import { ElMessage } from 'element-plus' -import { Refresh } from '@element-plus/icons-vue' +import { Refresh, ArrowDown } from '@element-plus/icons-vue' interface Props { todayBids: any[] @@ -103,6 +112,19 @@ const emit = defineEmits<{ const selectedKeywords = ref([]) const dateRange = ref<[string, string] | null>(null) const crawling = ref(false) +const highPriorityCollapsed = ref(false) + +// 切换 High Priority Bids 的折叠状态 +const toggleHighPriority = () => { + highPriorityCollapsed.value = !highPriorityCollapsed.value +} + +// 监听 highPriorityBids，当没有数据时自动折叠 +watch(() => props.highPriorityBids, (newBids) => { + if (newBids.length === 0) { + highPriorityCollapsed.value = true + } +}, { immediate: true }) // 从 localStorage 加载保存的关键字 const loadSavedKeywords = () => { @@ -207,9 +229,7 @@ const setLast3Days = () => { const filteredCount = result.length console.log('setLast3Days result, totalBids:', totalBids, 'filteredCount:', filteredCount) - if (totalBids === 0) { - ElMessage.warning('暂无数据，请先抓取数据') - } + // 只在手动点击按钮时显示提示，初始化时不显示 } // 设置日期范围为最近7天 @@ -244,9 +264,7 @@ const setLast7Days = () => { const filteredCount = result.length console.log('setLast7Days result, totalBids:', totalBids, 'filteredCount:', filteredCount) - if (totalBids === 0) { - ElMessage.warning('暂无数据，请先抓取数据') - } + // 只在手动点击按钮时显示提示，初始化时不显示 } const handleCrawl = async () => { @@ -268,6 +286,9 @@ const handleCrawl = async () => { // 初始化时加载保存的关键字 loadSavedKeywords() + +// 初始化时设置默认日期范围为最近3天 +setLast3Days() diff --git a/src/crawler/services/chng_target_stealth.spec.ts b/src/crawler/services/chng_target_stealth.spec.ts deleted file mode 100644 index e918f18..0000000 --- a/src/crawler/services/chng_target_stealth.spec.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { ChngCrawler } from './chng_target'; -import puppeteer from 'puppeteer-extra'; -import StealthPlugin from 'puppeteer-extra-plugin-stealth'; -import type { Browser, Page } from 'puppeteer'; - -// 使用 stealth 插件增强反爬虫能力 -puppeteer.use(StealthPlugin()); - -// Increase timeout to 180 seconds for slow sites and stealth mode -jest.setTimeout(180000); - -// 获取代理配置 -const getProxyArgs = (): string[] => { - const proxyHost = process.env.PROXY_HOST; - const proxyPort = process.env.PROXY_PORT; - const proxyUsername = process.env.PROXY_USERNAME; - const proxyPassword = process.env.PROXY_PASSWORD; - - if (proxyHost && proxyPort) { - const args = [`--proxy-server=${proxyHost}:${proxyPort}`]; - if (proxyUsername && proxyPassword) { - args.push(`--proxy-auth=${proxyUsername}:${proxyPassword}`); - } - return args; - } - return []; -}; - -// 模拟人类鼠标移动 -async function simulateHumanMouseMovement(page: Page) { - const viewport = page.viewport(); - if (!viewport) return; - - const movements = 5 + Math.floor(Math.random() * 5); // 5-10次随机移动 - - for (let i = 0; i < movements; i++) { - const x = Math.floor(Math.random() * viewport.width); - const y = Math.floor(Math.random() * viewport.height); - - await page.mouse.move(x, y, { - steps: 10 + Math.floor(Math.random() * 20) // 10-30步，使移动更平滑 - }); - - // 随机停顿 100-500ms - await new Promise(r => setTimeout(r, 100 + Math.random() * 400)); - } -} - -// 模拟人类滚动 -async function simulateHumanScrolling(page: Page) { - const scrollCount = 3 + Math.floor(Math.random() * 5); // 3-7次滚动 - - for (let i = 0; i < scrollCount; i++) { - const scrollDistance = 100 + Math.floor(Math.random() * 400); // 100-500px - - await page.evaluate((distance) => { - window.scrollBy({ - top: distance, - behavior: 'smooth' - }); - }, scrollDistance); - - // 随机停顿 500-1500ms - await new Promise(r => setTimeout(r, 500 + Math.random() * 1000)); - } - - // 滚动回顶部 - await page.evaluate(() => { - window.scrollTo({ top: 0, behavior: 'smooth' }); - }); - await new Promise(r => setTimeout(r, 1000)); -} - -describe('ChngCrawler Stealth Test (Headless Mode with Stealth Plugin)', () => { - let browser: Browser; - - beforeAll(async () => { - const proxyArgs = getProxyArgs(); - if (proxyArgs.length > 0) { - console.log('Using proxy:', proxyArgs.join(' ')); - } - - browser = await puppeteer.launch({ - headless: true, // 使用 headless 模式 - args: [ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-blink-features=AutomationControlled', - '--window-size=1920,1080', - '--disable-infobars', - '--disable-dev-shm-usage', - '--disable-accelerated-2d-canvas', - '--no-first-run', - '--no-zygote', - '--disable-gpu', - '--disable-features=VizDisplayCompositor', - '--disable-webgl', - ...proxyArgs, - ], - defaultViewport: null - }); - }); - - afterAll(async () => { - if (browser) { - await browser.close(); - } - }); - - it('should visit the website and list all found bid information with stealth plugin', async () => { - // 为此测试单独设置更长的超时时间 - jest.setTimeout(180000); - console.log(` -Starting crawl for: ${ChngCrawler.name}`); - console.log(`Target URL: ${ChngCrawler.url}`); - console.log('Using puppeteer-extra-plugin-stealth for anti-detection'); - console.log('Running in headless mode'); - - // 创建一个临时页面用于模拟人类行为 - const tempPage = await browser.newPage(); - await tempPage.setViewport({ width: 1920, height: 1080, deviceScaleFactor: 1 }); - - // 模拟人类鼠标移动 - console.log('Simulating human mouse movements...'); - await simulateHumanMouseMovement(tempPage); - - // 模拟人类滚动 - console.log('Simulating human scrolling...'); - await simulateHumanScrolling(tempPage); - - await tempPage.close(); - - const results = await ChngCrawler.crawl(browser); - - console.log(` -Successfully found ${results.length} items: -`); - console.log('----------------------------------------'); - results.forEach((item, index) => { - console.log(`${index + 1}. [${item.publishDate.toLocaleDateString()}] ${item.title}`); - console.log(` Link: ${item.url}`); - console.log('----------------------------------------'); - }); - - expect(results).toBeDefined(); - expect(Array.isArray(results)).toBeTruthy(); - - if (results.length === 0) { - console.warn('Warning: No items found. The site might have detected the crawler or content is not loading properly.'); - } else { - const firstItem = results[0]; - expect(firstItem.title).toBeTruthy(); - expect(firstItem.url).toMatch(/^https?:\/\//); - expect(firstItem.publishDate).toBeInstanceOf(Date); - } - }); -});