bidding_watcher/src/crawler/services/chdtp_target.ts

import * as puppeteer from 'puppeteer';
import { Logger } from '@nestjs/common';

export interface ChdtpResult {
  title: string;
  publishDate: Date;
  url: string; // Necessary for system uniqueness
}

export const ChdtpCrawler = {
  name: '中国华能集团',
  url: 'https://www.chdtp.com/webs/queryWebZbgg.action?zbggType=1',
  baseUrl: 'https://www.chdtp.com/webs/',

  async crawl(browser: puppeteer.Browser): Promise<ChdtpResult[]> {
    const logger = new Logger('ChdtpCrawler');
    const page = await browser.newPage();
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36');

    const allResults: ChdtpResult[] = [];
    let currentPage = 1;
    const maxPages = 5; // Safety limit to prevent infinite loops during testing

    try {
      logger.log(`Navigating to ${this.url}...`);
      await page.goto(this.url, { waitUntil: 'networkidle2', timeout: 60000 });

      while (currentPage <= maxPages) {
        const content = await page.content();
        const pageResults = this.extract(content);
        
        if (pageResults.length === 0) {
          logger.warn(`No results found on page ${currentPage}, stopping.`);
          break;
        }

        allResults.push(...pageResults);
        logger.log(`Extracted ${pageResults.length} items from page ${currentPage}`);

        // Find the "Next Page" button
        // Using partial match for src to be robust against path variations
        const nextButtonSelector = 'input[type="image"][src*="page-next.png"]';
        const nextButton = await page.$(nextButtonSelector);

        if (!nextButton) {
          logger.log('Next page button not found. Reached end of list.');
          break;
        }

        // Optional: Check if the button is disabled (though image inputs usually aren't "disabled" in the same way)
        // For this specific site, we'll try to click.

        logger.log(`Navigating to page ${currentPage + 1}...`);
        
        try {
          await Promise.all([
            page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 60000 }),
            nextButton.click(),
          ]);
        } catch (navError) {
          logger.error(`Navigation to page ${currentPage + 1} failed: ${navError.message}`);
          break;
        }

        currentPage++;
        
        // Random delay between pages
        const delay = Math.floor(Math.random() * (3000 - 1000 + 1)) + 1000;
        await new Promise(resolve => setTimeout(resolve, delay));
      }

      return allResults;

    } catch (error) {
      logger.error(`Failed to crawl ${this.name}: ${error.message}`);
      return allResults; // Return what we have so far
    } finally {
      await page.close();
    }
  },

  extract(html: string): ChdtpResult[] {
    const results: ChdtpResult[] = [];
    /**
     * Regex groups for chdtp.com:
     * 1: Status
     * 2: URL suffix
     * 3: Title
     * 4: Business Type
     * 5: Date
     */
    const regex = /<tr[^>]*>\s*<td class="td_1">.*?<span[^>]*>\s*(.*?)\s*<\/span>.*?<\/td>\s*<td class="td_2">\s*<a[^>]*href="javascript:toGetContent\('(.*?)'\)" title="(.*?)">.*?<\/a><\/td>\s*<td class="td_3">\s*<a[^>]*>\s*(.*?)\s*<\/a>\s*<\/td>\s*<td class="td_4"><span>\[(.*?)\]<\/span><\/td>/gs;

    let match;
    while ((match = regex.exec(html)) !== null) {
      const urlSuffix = match[2]?.trim();
      const title = match[3]?.trim();
      const dateStr = match[5]?.trim();

      if (title && urlSuffix) {
        results.push({
          title,
          publishDate: dateStr ? new Date(dateStr) : new Date(),
          url: this.baseUrl + urlSuffix
        });
      }
    }
    return results;
  }
};