萬盛學電腦網

您现在的位置: 萬盛學電腦網 >> 程式設計 >  >> 網路程式設計 >> PHP 程式設計

PHP 打造百度詞典爬蟲:從單詞庫導入到多維度釋義結構化採集實戰

需要離線詞典資料、構建語言學習 App、或進行 NLP 訓練資料準備?百度詞典擁有豐富的釋義、音標、發音、例句、同反義詞、短語搭配。本文提供基於 PHP cURL + DOMDocument 的高穩定性採集器完整實作,支援 13.5 萬單詞庫批次採集、斷點續傳、頻率控制、資料清洗與結構化儲存。

專案架構設計


baidu_dict/
├── src/
│   ├── DictClient.php      # HTTP 客戶端 (cURL 封裝、重試、限流)
│   ├── Parser.php          # HTML 解析器 (DOMDocument + XPath)
│   ├── WordRepository.php  # 儲存抽象層 (MySQL/Redis/CSV/Elasticsearch)
│   └── Collector.php       # 採集調度器 (隊列、進度、錯誤處理)
├── data/
│   └── words_135k.txt      # 單詞庫 (每行一單詞)
├── config.php              # 配置
└── run.php                 # 入口腳本

核心類別實作

1. DictClient - 穩健的 HTTP 客戶端

<?php
namespace BaiduDict;

class DictClient {
    private \CurlHandle $ch;
    private int $requestCount = 0;
    private float $lastRequestTime = 0;
    private float $minInterval = 0.5; // 最小請求間隔 (秒)

    public function __construct(
        private string $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
        private int $timeout = 30,
        private int $maxRetries = 3
    ) {
        $this->ch = curl_init();
        curl_setopt_array($this->ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_MAXREDIRS => 5,
            CURLOPT_AUTOREFERER => true,
            CURLOPT_USERAGENT => $this->userAgent,
            CURLOPT_SSL_VERIFYPEER => true,
            CURLOPT_SSL_VERIFYHOST => 2,
            CURLOPT_TIMEOUT => $this->timeout,
            CURLOPT_CONNECTTIMEOUT => 10,
            CURLOPT_ENCODING => 'gzip, deflate, br',
            CURLOPT_COOKIEJAR => '', // 啟用記憶體 Cookie 管理
            CURLOPT_COOKIEFILE => '',
        ]);
    }

    public function get(string $url): string {
        $this->throttle();
        
        for ($attempt = 1; $attempt <= $this->maxRetries; $attempt++) {
            curl_setopt($this->ch, CURLOPT_URL, $url);
            curl_setopt($this->ch, CURLOPT_HTTPGET, true);
            curl_setopt($this->ch, CURLOPT_POST, false);
            
            $html = curl_exec($this->ch);
            $errno = curl_errno($this->ch);
            $httpCode = curl_getinfo($this->ch, CURLINFO_HTTP_CODE);
            
            if ($errno === 0 && $httpCode === 200) {
                $this->requestCount++;
                return $html;
            }
            
            $delay = $attempt * 2 + random_int(0, 1000) / 1000;
            error_log("Request failed (attempt $attempt): cURL $errno, HTTP $httpCode, url: $url. Retry in ${delay}s");
            sleep($delay);
        }
        throw new \RuntimeException("Max retries exceeded for $url");
    }

    private function throttle(): void {
        $elapsed = microtime(true) - $this->lastRequestTime;
        if ($elapsed < $this->minInterval) {
            usleep(($this->minInterval - $elapsed) * 1_000_000);
        }
        $this->lastRequestTime = microtime(true);
    }

    public function getRequestCount(): int { return $this->requestCount; }
    public function __destruct() { curl_close($this->ch); }
}

2. Parser - 結構化解析百度詞典 HTML

<?php
namespace BaiduDict;

class Parser {
    public function parse(string $html, string $word): array {
        $dom = new \DOMDocument();
        libxml_use_internal_errors(true);
        $dom->loadHTML('' . $html);
        libxml_clear_errors();
        $xpath = new \DOMXPath($dom);

        $result = [
            'word' => $word,
            'symbol' => '',      // 音標
            'pronunciation' => [], // 發音 URL
            'examples' => [],     // 例句
            'definitions' => [],  // 釋義
            'synonyms' => [],     // 同義詞
            'antonyms' => [],     // 反義詞
            'phrases' => [],      // 短語
        ];

        // 音標
        if ($nodes = $xpath->query('//div[@class="headword"]//span[@class="phonetic"]')) {
            $result['symbol'] = trim($nodes[0]->textContent);
        }

        // 發音 (通常有美式/英式)
        foreach ($xpath->query('//div[@class="headword"]//a[@class="play"]/@data-rel') as $attr) {
            $result['pronunciation'][] = 'https:' . $attr->value;
        }

        // 簡明釋義
        foreach ($xpath->query('//div[@class="tab-content"]//li[@class="clearfix"]') as $li) {
            $pos = $xpath->query('.//span[@class="pos"]', $li);
            $def = $xpath->query('.//span[@class="def"]', $li);
            if ($pos->length && $def->length) {
                $result['definitions'][] = [
                    'pos' => trim($pos[0]->textContent),
                    'def' => trim($def[0]->textContent),
                ];
            }
        }

        // 例句 (雙語)
        foreach ($xpath->query('//div[@id="sentence-pair"]//li') as $li) {
            $en = $xpath->query('.//p[@class="sen-en"]', $li);
            $cn = $xpath->query('.//p[@class="sen-cn"]', $li);
            if ($en->length && $cn->length) {
                $result['examples'][] = [
                    'en' => trim($en[0]->textContent),
                    'cn' => trim($cn[0]->textContent),
                ];
            }
        }

        // 同反義詞
        foreach ($xpath->query('//div[@class="synonyms"]//a') as $a) {
            $result['synonyms'][] = trim($a->textContent);
        }
        foreach ($xpath->query('//div[@class="antonyms"]//a') as $a) {
            $result['antonyms'][] = trim($a->textContent);
        }

        // 短語搭配
        foreach ($xpath->query('//div[@class="phrase"]//li') as $li) {
            $phrase = $xpath->query('.//a', $li);
            $trans = $xpath->query('.//span', $li);
            if ($phrase->length) {
                $result['phrases'][] = [
                    'phrase' => trim($phrase[0]->textContent),
                    'translation' => $trans->length ? trim($trans[0]->textContent) : '',
                ];
            }
        }

        return $result;
    }
}

3. Collector - 採集調度與持久化

<?php
namespace BaiduDict;

class Collector {
    public function __construct(
        private DictClient $client,
        private Parser $parser,
        private WordRepository $repo,
        private int $batchSize = 100
    ) {}

    public function collectFromFile(string $wordFile, int $offset = 0, int $limit = 0): void {
        $handle = fopen($wordFile, 'r');
        if (!$handle) throw new \RuntimeException("Cannot open $wordFile");
        
        // 跳過 offset
        for ($i = 0; $i < $offset && !feof($handle); $i++) fgets($handle);
        
        $batch = [];
        $count = 0;
        $processed = 0;
        
        while (($line = fgets($handle)) !== false) {
            $word = trim($line);
            if ($word === '') continue;
            
            try {
                $url = "https://dict.baidu.com/s?wd=" . urlencode($word);
                $html = $this->client->get($url);
                $data = $this->parser->parse($html, $word);
                $batch[] = $data;
                $processed++;
                
                if (count($batch) >= $this->batchSize) {
                    $this->repo->saveBatch($batch);
                    $batch = [];
                    echo "Saved batch. Total processed: $processed\n";
                }
                
                if ($limit > 0 && $processed >= $limit) break;
            } catch (\Throwable $e) {
                error_log("Failed to collect '$word': {$e->getMessage()}");
                // 記錄失敗單詞供重試
                file_put_contents('failed_words.log', $word . "\n", FILE_APPEND);
            }
        }
        
        if ($batch) $this->repo->saveBatch($batch);
        fclose($handle);
        echo "Collection completed. Total: $processed\n";
    }
}

4. 儲存庫介面與 MySQL 實作

<?php
namespace BaiduDict;

interface WordRepository {
    public function saveBatch(array $words): void;
}

class MySqlRepository implements WordRepository {
    public function __construct(private \PDO $pdo) {
        $this->pdo->exec("CREATE TABLE IF NOT EXISTS dict_words (
            id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
            word VARCHAR(100) NOT NULL UNIQUE,
            symbol VARCHAR(100),
            pronunciation JSON,
            examples JSON,
            definitions JSON,
            synonyms JSON,
            antonyms JSON,
            phrases JSON,
            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
            INDEX idx_word (word)
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4");
    }

    public function saveBatch(array $words): void {
        $sql = "INSERT INTO dict_words (word, symbol, pronunciation, examples, definitions, synonyms, antonyms, phrases)
                VALUES " . implode(',', array_fill(0, count($words), '(?,?,?,?,?,?,?,?)')) . "
                ON DUPLICATE KEY UPDATE
                symbol=VALUES(symbol), pronunciation=VALUES(pronunciation), examples=VALUES(examples),
                definitions=VALUES(definitions), synonyms=VALUES(synonyms), antonyms=VALUES(antonyms), phrases=VALUES(phrases)";
        
        $stmt = $this->pdo->prepare($sql);
        $params = [];
        foreach ($words as $w) {
            $params[] = $w['word'];
            $params[] = $w['symbol'];
            $params[] = json_encode($w['pronunciation'], JSON_UNESCAPED_UNICODE);
            $params[] = json_encode($w['examples'], JSON_UNESCAPED_UNICODE);
            $params[] = json_encode($w['definitions'], JSON_UNESCAPED_UNICODE);
            $params[] = json_encode($w['synonyms'], JSON_UNESCAPED_UNICODE);
            $params[] = json_encode($w['antonyms'], JSON_UNESCAPED_UNICODE);
            $params[] = json_encode($w['phrases'], JSON_UNESCAPED_UNICODE);
        }
        $stmt->execute($params);
    }
}

執行入口

<?php
// run.php
require 'vendor/autoload.php';
require 'config.php';

use BaiduDict\{DictClient, Parser, Collector, MySqlRepository};

$pdo = new PDO("mysql:host={$config['db_host']};dbname={$config['db_name']};charset=utf8mb4", 
               $config['db_user'], $config['db_pass'], [
    PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
]);

$client = new DictClient();
$parser = new Parser();
$repo = new MySqlRepository($pdo);
$collector = new Collector($client, $parser, $repo, batchSize: 50);

// 斷點續傳:從第 10000 條開始,採集 5000 條
$collector->collectFromFile('data/words_135k.txt', offset: 10000, limit: 5000);

echo "Total requests: {$client->getRequestCount()}\n";

反爬應對與合規建議

  • 頻率控制:minInterval >= 0.5s,隨機抖動 ±0.2s
  • IP 輪換:代理池 (住宅代理 > 資料中心代理),失敗自動切換
  • 瀏覽器指紋:完整 Header、Cookie 持久化、User-Agent 輪換
  • 驗證碼處理:遇到驗證碼頁面暫停、人工介入或接打碼平台
  • 遵守 robots.txt:Disallow: /s? 可能限制搜尋頁,評估法律風險
  • 資料使用合規:僅供個人學習/研究,商業用途需授權

資料驗證與品質監控


// 採集後抽樣驗證腳本
$pdo->query("SELECT word, definitions FROM dict_words WHERE definitions = '[]' OR definitions IS NULL LIMIT 100");
// 檢查關鍵欄位完整率
$stmt = $pdo->query("SELECT 
    COUNT(*) as total,
    SUM(symbol != '') as has_symbol,
    SUM(JSON_LENGTH(pronunciation) > 0) as has_pron,
    SUM(JSON_LENGTH(definitions) > 0) as has_def
    FROM dict_words");

結語

構建穩健的詞典採集器,關鍵在於 模組化解耦 (Client/Parser/Repo)、異常熔斷與重試、斷點續傳能力、頻率與指紋控制。本文提供的架構可直接擴展至其他詞典站點 (有道、牛津、劍橋)、百科類站點或電商商品採集。記住:技術手段服務於合規數據獲取,尊重知識產權與站點服務穩定。

copyright © 萬盛學電腦網 all rights reserved