需要離線詞典資料、構建語言學習 App、或進行 NLP 訓練資料準備?百度詞典擁有豐富的釋義、音標、發音、例句、同反義詞、短語搭配。本文提供基於 PHP cURL + DOMDocument 的高穩定性採集器完整實作,支援 13.5 萬單詞庫批次採集、斷點續傳、頻率控制、資料清洗與結構化儲存。
baidu_dict/
├── src/
│ ├── DictClient.php # HTTP 客戶端 (cURL 封裝、重試、限流)
│ ├── Parser.php # HTML 解析器 (DOMDocument + XPath)
│ ├── WordRepository.php # 儲存抽象層 (MySQL/Redis/CSV/Elasticsearch)
│ └── Collector.php # 採集調度器 (隊列、進度、錯誤處理)
├── data/
│ └── words_135k.txt # 單詞庫 (每行一單詞)
├── config.php # 配置
└── run.php # 入口腳本
<?php
namespace BaiduDict;
class DictClient {
private \CurlHandle $ch;
private int $requestCount = 0;
private float $lastRequestTime = 0;
private float $minInterval = 0.5; // 最小請求間隔 (秒)
public function __construct(
private string $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
private int $timeout = 30,
private int $maxRetries = 3
) {
$this->ch = curl_init();
curl_setopt_array($this->ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 5,
CURLOPT_AUTOREFERER => true,
CURLOPT_USERAGENT => $this->userAgent,
CURLOPT_SSL_VERIFYPEER => true,
CURLOPT_SSL_VERIFYHOST => 2,
CURLOPT_TIMEOUT => $this->timeout,
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_ENCODING => 'gzip, deflate, br',
CURLOPT_COOKIEJAR => '', // 啟用記憶體 Cookie 管理
CURLOPT_COOKIEFILE => '',
]);
}
public function get(string $url): string {
$this->throttle();
for ($attempt = 1; $attempt <= $this->maxRetries; $attempt++) {
curl_setopt($this->ch, CURLOPT_URL, $url);
curl_setopt($this->ch, CURLOPT_HTTPGET, true);
curl_setopt($this->ch, CURLOPT_POST, false);
$html = curl_exec($this->ch);
$errno = curl_errno($this->ch);
$httpCode = curl_getinfo($this->ch, CURLINFO_HTTP_CODE);
if ($errno === 0 && $httpCode === 200) {
$this->requestCount++;
return $html;
}
$delay = $attempt * 2 + random_int(0, 1000) / 1000;
error_log("Request failed (attempt $attempt): cURL $errno, HTTP $httpCode, url: $url. Retry in ${delay}s");
sleep($delay);
}
throw new \RuntimeException("Max retries exceeded for $url");
}
private function throttle(): void {
$elapsed = microtime(true) - $this->lastRequestTime;
if ($elapsed < $this->minInterval) {
usleep(($this->minInterval - $elapsed) * 1_000_000);
}
$this->lastRequestTime = microtime(true);
}
public function getRequestCount(): int { return $this->requestCount; }
public function __destruct() { curl_close($this->ch); }
}
<?php
namespace BaiduDict;
class Parser {
public function parse(string $html, string $word): array {
$dom = new \DOMDocument();
libxml_use_internal_errors(true);
$dom->loadHTML('' . $html);
libxml_clear_errors();
$xpath = new \DOMXPath($dom);
$result = [
'word' => $word,
'symbol' => '', // 音標
'pronunciation' => [], // 發音 URL
'examples' => [], // 例句
'definitions' => [], // 釋義
'synonyms' => [], // 同義詞
'antonyms' => [], // 反義詞
'phrases' => [], // 短語
];
// 音標
if ($nodes = $xpath->query('//div[@class="headword"]//span[@class="phonetic"]')) {
$result['symbol'] = trim($nodes[0]->textContent);
}
// 發音 (通常有美式/英式)
foreach ($xpath->query('//div[@class="headword"]//a[@class="play"]/@data-rel') as $attr) {
$result['pronunciation'][] = 'https:' . $attr->value;
}
// 簡明釋義
foreach ($xpath->query('//div[@class="tab-content"]//li[@class="clearfix"]') as $li) {
$pos = $xpath->query('.//span[@class="pos"]', $li);
$def = $xpath->query('.//span[@class="def"]', $li);
if ($pos->length && $def->length) {
$result['definitions'][] = [
'pos' => trim($pos[0]->textContent),
'def' => trim($def[0]->textContent),
];
}
}
// 例句 (雙語)
foreach ($xpath->query('//div[@id="sentence-pair"]//li') as $li) {
$en = $xpath->query('.//p[@class="sen-en"]', $li);
$cn = $xpath->query('.//p[@class="sen-cn"]', $li);
if ($en->length && $cn->length) {
$result['examples'][] = [
'en' => trim($en[0]->textContent),
'cn' => trim($cn[0]->textContent),
];
}
}
// 同反義詞
foreach ($xpath->query('//div[@class="synonyms"]//a') as $a) {
$result['synonyms'][] = trim($a->textContent);
}
foreach ($xpath->query('//div[@class="antonyms"]//a') as $a) {
$result['antonyms'][] = trim($a->textContent);
}
// 短語搭配
foreach ($xpath->query('//div[@class="phrase"]//li') as $li) {
$phrase = $xpath->query('.//a', $li);
$trans = $xpath->query('.//span', $li);
if ($phrase->length) {
$result['phrases'][] = [
'phrase' => trim($phrase[0]->textContent),
'translation' => $trans->length ? trim($trans[0]->textContent) : '',
];
}
}
return $result;
}
}
<?php
namespace BaiduDict;
class Collector {
public function __construct(
private DictClient $client,
private Parser $parser,
private WordRepository $repo,
private int $batchSize = 100
) {}
public function collectFromFile(string $wordFile, int $offset = 0, int $limit = 0): void {
$handle = fopen($wordFile, 'r');
if (!$handle) throw new \RuntimeException("Cannot open $wordFile");
// 跳過 offset
for ($i = 0; $i < $offset && !feof($handle); $i++) fgets($handle);
$batch = [];
$count = 0;
$processed = 0;
while (($line = fgets($handle)) !== false) {
$word = trim($line);
if ($word === '') continue;
try {
$url = "https://dict.baidu.com/s?wd=" . urlencode($word);
$html = $this->client->get($url);
$data = $this->parser->parse($html, $word);
$batch[] = $data;
$processed++;
if (count($batch) >= $this->batchSize) {
$this->repo->saveBatch($batch);
$batch = [];
echo "Saved batch. Total processed: $processed\n";
}
if ($limit > 0 && $processed >= $limit) break;
} catch (\Throwable $e) {
error_log("Failed to collect '$word': {$e->getMessage()}");
// 記錄失敗單詞供重試
file_put_contents('failed_words.log', $word . "\n", FILE_APPEND);
}
}
if ($batch) $this->repo->saveBatch($batch);
fclose($handle);
echo "Collection completed. Total: $processed\n";
}
}
<?php
namespace BaiduDict;
interface WordRepository {
public function saveBatch(array $words): void;
}
class MySqlRepository implements WordRepository {
public function __construct(private \PDO $pdo) {
$this->pdo->exec("CREATE TABLE IF NOT EXISTS dict_words (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
word VARCHAR(100) NOT NULL UNIQUE,
symbol VARCHAR(100),
pronunciation JSON,
examples JSON,
definitions JSON,
synonyms JSON,
antonyms JSON,
phrases JSON,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_word (word)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4");
}
public function saveBatch(array $words): void {
$sql = "INSERT INTO dict_words (word, symbol, pronunciation, examples, definitions, synonyms, antonyms, phrases)
VALUES " . implode(',', array_fill(0, count($words), '(?,?,?,?,?,?,?,?)')) . "
ON DUPLICATE KEY UPDATE
symbol=VALUES(symbol), pronunciation=VALUES(pronunciation), examples=VALUES(examples),
definitions=VALUES(definitions), synonyms=VALUES(synonyms), antonyms=VALUES(antonyms), phrases=VALUES(phrases)";
$stmt = $this->pdo->prepare($sql);
$params = [];
foreach ($words as $w) {
$params[] = $w['word'];
$params[] = $w['symbol'];
$params[] = json_encode($w['pronunciation'], JSON_UNESCAPED_UNICODE);
$params[] = json_encode($w['examples'], JSON_UNESCAPED_UNICODE);
$params[] = json_encode($w['definitions'], JSON_UNESCAPED_UNICODE);
$params[] = json_encode($w['synonyms'], JSON_UNESCAPED_UNICODE);
$params[] = json_encode($w['antonyms'], JSON_UNESCAPED_UNICODE);
$params[] = json_encode($w['phrases'], JSON_UNESCAPED_UNICODE);
}
$stmt->execute($params);
}
}
<?php
// run.php
require 'vendor/autoload.php';
require 'config.php';
use BaiduDict\{DictClient, Parser, Collector, MySqlRepository};
$pdo = new PDO("mysql:host={$config['db_host']};dbname={$config['db_name']};charset=utf8mb4",
$config['db_user'], $config['db_pass'], [
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
]);
$client = new DictClient();
$parser = new Parser();
$repo = new MySqlRepository($pdo);
$collector = new Collector($client, $parser, $repo, batchSize: 50);
// 斷點續傳:從第 10000 條開始,採集 5000 條
$collector->collectFromFile('data/words_135k.txt', offset: 10000, limit: 5000);
echo "Total requests: {$client->getRequestCount()}\n";
minInterval >= 0.5s,隨機抖動 ±0.2sDisallow: /s? 可能限制搜尋頁,評估法律風險
// 採集後抽樣驗證腳本
$pdo->query("SELECT word, definitions FROM dict_words WHERE definitions = '[]' OR definitions IS NULL LIMIT 100");
// 檢查關鍵欄位完整率
$stmt = $pdo->query("SELECT
COUNT(*) as total,
SUM(symbol != '') as has_symbol,
SUM(JSON_LENGTH(pronunciation) > 0) as has_pron,
SUM(JSON_LENGTH(definitions) > 0) as has_def
FROM dict_words");
構建穩健的詞典採集器,關鍵在於 模組化解耦 (Client/Parser/Repo)、異常熔斷與重試、斷點續傳能力、頻率與指紋控制。本文提供的架構可直接擴展至其他詞典站點 (有道、牛津、劍橋)、百科類站點或電商商品採集。記住:技術手段服務於合規數據獲取,尊重知識產權與站點服務穩定。