diff --git a/src/bids/controllers/bid.controller.ts b/src/bids/controllers/bid.controller.ts index 8c55a6f..53fab05 100644 --- a/src/bids/controllers/bid.controller.ts +++ b/src/bids/controllers/bid.controller.ts @@ -14,4 +14,9 @@ export class BidsController { getHighPriority() { return this.bidsService.getHighPriorityCorrected(); } + + @Get('sources') + getSources() { + return this.bidsService.getSources(); + } } diff --git a/src/bids/services/bid.service.ts b/src/bids/services/bid.service.ts index 341d537..1918ec0 100644 --- a/src/bids/services/bid.service.ts +++ b/src/bids/services/bid.service.ts @@ -65,4 +65,14 @@ export class BidsService { createdAt: LessThan(thirtyDaysAgo), }); } + + async getSources() { + const result = await this.bidRepository + .createQueryBuilder('bid') + .select('DISTINCT bid.source') + .where('bid.source IS NOT NULL') + .orderBy('bid.source', 'ASC') + .getRawMany(); + return result.map((item: any) => item.source); + } } diff --git a/src/crawler/crawler.controller.ts b/src/crawler/crawler.controller.ts index 74f8dcd..1b59178 100644 --- a/src/crawler/crawler.controller.ts +++ b/src/crawler/crawler.controller.ts @@ -1,21 +1,38 @@ -import { Controller, Post } from '@nestjs/common'; +import { Controller, Post, Get } from '@nestjs/common'; import { BidCrawlerService } from './services/bid-crawler.service'; @Controller('api/crawler') export class CrawlerController { + private isCrawling = false; + constructor(private readonly crawlerService: BidCrawlerService) {} + @Get('status') + getStatus() { + return { isCrawling: this.isCrawling }; + } + @Post('run') async runCrawl() { + if (this.isCrawling) { + return { message: 'Crawl is already running' }; + } + + this.isCrawling = true; + // We don't await this because we want it to run in the background - // and return immediately, or we can await if we want the user to wait. + // and return immediately, or we can await if we want to user to wait. // Given the requirement "Immediate Crawl", usually implies triggering it. // However, for a better UI experience, we might want to wait or just trigger. - // Let's await it so the user knows when it's done (or failed), + // Let's await it so that user knows when it's done (or failed), // assuming it doesn't take too long for the mock. // Real crawling might take long, so background is better. // For this prototype, I'll await it to show completion. - await this.crawlerService.crawlAll(); - return { message: 'Crawl completed successfully' }; + try { + await this.crawlerService.crawlAll(); + return { message: 'Crawl completed successfully' }; + } finally { + this.isCrawling = false; + } } } diff --git a/src/crawler/services/bid-crawler.service.ts b/src/crawler/services/bid-crawler.service.ts index 46b94dc..44026f7 100644 --- a/src/crawler/services/bid-crawler.service.ts +++ b/src/crawler/services/bid-crawler.service.ts @@ -9,6 +9,9 @@ import { CdtCrawler } from './cdt_target'; import { EpsCrawler } from './eps_target'; import { CnncecpCrawler } from './cnncecp_target'; import { CgnpcCrawler } from './cgnpc_target'; +import { CeicCrawler } from './ceic_target'; +import { EspicCrawler } from './espic_target'; +import { PowerbeijingCrawler } from './powerbeijing_target'; @Injectable() export class BidCrawlerService { @@ -22,6 +25,10 @@ export class BidCrawlerService { async crawlAll() { this.logger.log('Starting crawl task with Puppeteer...'); + // 设置最大执行时间为1小时 + const maxExecutionTime = 60 * 60 * 1000; // 1小时(毫秒) + const startTime = Date.now(); + // 从环境变量读取代理配置 const proxyHost = this.configService.get('PROXY_HOST'); const proxyPort = this.configService.get('PROXY_PORT'); @@ -52,11 +59,20 @@ export class BidCrawlerService { args, }); - const crawlers = [ChdtpCrawler, ChngCrawler, SzecpCrawler, CdtCrawler, EpsCrawler, CnncecpCrawler, CgnpcCrawler]; + const crawlers = [ChdtpCrawler, ChngCrawler, SzecpCrawler, CdtCrawler, EpsCrawler, CnncecpCrawler, CgnpcCrawler, CeicCrawler, EspicCrawler, PowerbeijingCrawler]; try { for (const crawler of crawlers) { this.logger.log(`Crawling: ${crawler.name}`); + + // 检查是否超时 + const elapsedTime = Date.now() - startTime; + if (elapsedTime > maxExecutionTime) { + this.logger.warn(`⚠️ Crawl task exceeded maximum execution time of 1 hour. Stopping...`); + this.logger.warn(`⚠️ Total elapsed time: ${Math.floor(elapsedTime / 1000 / 60)} minutes`); + break; + } + try { const results = await crawler.crawl(browser); this.logger.log(`Extracted ${results.length} items from ${crawler.name}`); @@ -78,7 +94,14 @@ export class BidCrawlerService { this.logger.error(`Crawl task failed: ${error.message}`); } finally { await browser.close(); - this.logger.log('Crawl task finished.'); + + const totalTime = Date.now() - startTime; + const minutes = Math.floor(totalTime / 1000 / 60); + this.logger.log(`Crawl task finished. Total time: ${minutes} minutes`); + + if (totalTime > maxExecutionTime) { + this.logger.warn(`⚠️ Crawl task exceeded maximum execution time of 1 hour.`); + } } } }