feat: 添加单源爬取功能并优化数据库同步

新增单源爬取功能,支持在界面上单独更新每个数据源
添加数据库同步脚本,支持主从数据库结构同步和数据同步
优化华能集团爬虫的页面导航和稳定性
新增系统托盘功能,支持最小化到托盘
This commit is contained in:
dmy
2026-01-14 16:25:01 +08:00
parent bcd7af4e69
commit 3f6d10061d
8 changed files with 691 additions and 41 deletions

View File

@@ -1,15 +1,19 @@
import { Controller, Post, Get } from '@nestjs/common';
import { Controller, Post, Get, Param, Body } from '@nestjs/common';
import { BidCrawlerService } from './services/bid-crawler.service';
@Controller('api/crawler')
export class CrawlerController {
private isCrawling = false;
private crawlingSources = new Set<string>();
constructor(private readonly crawlerService: BidCrawlerService) {}
@Get('status')
getStatus() {
return { isCrawling: this.isCrawling };
return {
isCrawling: this.isCrawling,
crawlingSources: Array.from(this.crawlingSources)
};
}
@Post('run')
@@ -20,12 +24,12 @@ export class CrawlerController {
this.isCrawling = true;
// We don't await this because we want it to run in the background
// We don't await this because we want it to run in the background
// and return immediately, or we can await if we want to user to wait.
// Given the requirement "Immediate Crawl", usually implies triggering it.
// However, for a better UI experience, we might want to wait or just trigger.
// Let's await it so that user knows when it's done (or failed),
// assuming it doesn't take too long for the mock.
// Let's await it so that user knows when it's done (or failed),
// assuming it doesn't take too long for the mock.
// Real crawling might take long, so background is better.
// For this prototype, I'll await it to show completion.
try {
@@ -35,4 +39,20 @@ export class CrawlerController {
this.isCrawling = false;
}
}
@Post('crawl/:sourceName')
async crawlSingleSource(@Param('sourceName') sourceName: string) {
if (this.crawlingSources.has(sourceName)) {
return { message: `Source ${sourceName} is already being crawled` };
}
this.crawlingSources.add(sourceName);
try {
const result = await this.crawlerService.crawlSingleSource(sourceName);
return result;
} finally {
this.crawlingSources.delete(sourceName);
}
}
}