feat: 添加单源爬取功能并优化数据库同步
新增单源爬取功能,支持在界面上单独更新每个数据源 添加数据库同步脚本,支持主从数据库结构同步和数据同步 优化华能集团爬虫的页面导航和稳定性 新增系统托盘功能,支持最小化到托盘
This commit is contained in:
@@ -1,15 +1,19 @@
|
||||
import { Controller, Post, Get } from '@nestjs/common';
|
||||
import { Controller, Post, Get, Param, Body } from '@nestjs/common';
|
||||
import { BidCrawlerService } from './services/bid-crawler.service';
|
||||
|
||||
@Controller('api/crawler')
|
||||
export class CrawlerController {
|
||||
private isCrawling = false;
|
||||
private crawlingSources = new Set<string>();
|
||||
|
||||
constructor(private readonly crawlerService: BidCrawlerService) {}
|
||||
|
||||
@Get('status')
|
||||
getStatus() {
|
||||
return { isCrawling: this.isCrawling };
|
||||
return {
|
||||
isCrawling: this.isCrawling,
|
||||
crawlingSources: Array.from(this.crawlingSources)
|
||||
};
|
||||
}
|
||||
|
||||
@Post('run')
|
||||
@@ -20,12 +24,12 @@ export class CrawlerController {
|
||||
|
||||
this.isCrawling = true;
|
||||
|
||||
// We don't await this because we want it to run in the background
|
||||
// We don't await this because we want it to run in the background
|
||||
// and return immediately, or we can await if we want to user to wait.
|
||||
// Given the requirement "Immediate Crawl", usually implies triggering it.
|
||||
// However, for a better UI experience, we might want to wait or just trigger.
|
||||
// Let's await it so that user knows when it's done (or failed),
|
||||
// assuming it doesn't take too long for the mock.
|
||||
// Let's await it so that user knows when it's done (or failed),
|
||||
// assuming it doesn't take too long for the mock.
|
||||
// Real crawling might take long, so background is better.
|
||||
// For this prototype, I'll await it to show completion.
|
||||
try {
|
||||
@@ -35,4 +39,20 @@ export class CrawlerController {
|
||||
this.isCrawling = false;
|
||||
}
|
||||
}
|
||||
|
||||
@Post('crawl/:sourceName')
|
||||
async crawlSingleSource(@Param('sourceName') sourceName: string) {
|
||||
if (this.crawlingSources.has(sourceName)) {
|
||||
return { message: `Source ${sourceName} is already being crawled` };
|
||||
}
|
||||
|
||||
this.crawlingSources.add(sourceName);
|
||||
|
||||
try {
|
||||
const result = await this.crawlerService.crawlSingleSource(sourceName);
|
||||
return result;
|
||||
} finally {
|
||||
this.crawlingSources.delete(sourceName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user