feat: 全面优化爬虫系统和数据处理能力
- 增强数据重试机制:对数据为0的爬虫自动重试,提高数据完整性 - 优化前端筛选逻辑:改进日期筛选,只限制开始时间,更灵活的数据查看 - 新增最近数据接口:添加 /api/bids/recent 获取30天内最新招标数据 - 改进统计展示:实时显示筛选结果数量,优化用户体验 - 完善日志系统:确保日志目录自动创建,避免启动错误 - 增强独立脚本:使用自定义logger,完善错误处理和程序关闭 - 优化主程序:集成自定义日志服务,统一日志格式 - 扩展npm脚本:新增 web 命令用于构建前端 - 改进大唐爬虫:延长等待时间到60秒,提高页面加载成功率 - 优化数据筛选:今日招标改为使用独立接口,提升性能
This commit is contained in:
@@ -1,14 +1,18 @@
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { AppModule } from '../app.module';
|
||||
import { BidCrawlerService } from '../crawler/services/bid-crawler.service';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { CustomLogger } from '../common/logger/logger.service';
|
||||
|
||||
async function runCrawler() {
|
||||
const logger = new Logger('CrawlScript');
|
||||
const app = await NestFactory.createApplicationContext(AppModule);
|
||||
|
||||
// 设置自定义 logger,使 NestJS 框架日志也输出到文件
|
||||
const logger = await app.resolve(CustomLogger);
|
||||
app.useLogger(logger);
|
||||
logger.setContext('CrawlScript');
|
||||
|
||||
try {
|
||||
const app = await NestFactory.createApplicationContext(AppModule);
|
||||
const crawlerService = app.get(BidCrawlerService);
|
||||
const crawlerService = await app.resolve(BidCrawlerService);
|
||||
|
||||
logger.log('Starting crawler...');
|
||||
await crawlerService.crawlAll();
|
||||
@@ -18,6 +22,7 @@ async function runCrawler() {
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
logger.error('Crawler failed:', error);
|
||||
await app.close();
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user