抓取網頁標題、描述、關鍵字與正文內容,是搜尋引擎、內容聚合、SEO 分析的基礎功能。單純正則匹配 HTML 極其脆弱,現代方案採用 cURL 獲取原始碼 + DOMDocument 結構化解析,兼容性與容錯性大幅提升。
<?php
class PageContentExtractor
{
private int $timeout = 10;
private string $userAgent = 'Mozilla/5.0 (compatible; PHPBot/1.0; +https://example.com/bot)';
public function extract(string $url): array
{
$html = $this->fetch($url);
if (!$html) {
return ['error' => 'FETCH_FAILED', 'url' => $url];
}
$dom = $this->parseHtml($html);
if (!$dom) {
return ['error' => 'PARSE_FAILED', 'url' => $url];
}
return [
'url' => $url,
'title' => $this->extractTitle($dom),
'description' => $this->extractMeta($dom, 'description'),
'keywords' => $this->extractMeta($dom, 'keywords'),
'canonical' => $this->extractCanonical($dom),
'og' => $this->extractOpenGraph($dom),
'body_text' => $this->extractBodyText($dom),
'html' => $html, // 可選:保留原始碼供除錯
];
}
private function fetch(string $url): ?string
{
$ch = curl_init($url);
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 5,
CURLOPT_TIMEOUT => $this->timeout,
CURLOPT_CONNECTTIMEOUT => 5,
CURLOPT_USERAGENT => $this->userAgent,
CURLOPT_SSL_VERIFYPEER => true, // 生產環境建議開啟
CURLOPT_SSL_VERIFYHOST => 2,
CURLOPT_ENCODING => 'gzip,deflate', // 自動解壓
]);
$html = curl_exec($ch);
$errno = curl_errno($ch);
$info = curl_getinfo($ch);
curl_close($ch);
if ($errno || ($info['http_code'] ?? 0) !== 200) {
return null;
}
// 檢測編碼並轉 UTF-8
return $this->convertToUtf8($html, $info['content_type'] ?? '');
}
private function convertToUtf8(string $html, string $contentType): string
{
// 1. 從 HTTP 標頭解析 charset
if (preg_match('/charset=([^;\s]+)/i', $contentType, $m)) {
$encoding = strtoupper(trim($m[1]));
if ($encoding !== 'UTF-8' && function_exists('mb_convert_encoding')) {
return mb_convert_encoding($html, 'UTF-8', $encoding);
}
return $html;
}
// 2. 從 HTML meta 標籤解析
if (preg_match('/]+charset=["\']?([^"\'>\s]+)/i', $html, $m)) {
$encoding = strtoupper(trim($m[1]));
if ($encoding !== 'UTF-8' && function_exists('mb_convert_encoding')) {
return mb_convert_encoding($html, 'UTF-8', $encoding);
}
}
// 3. 讓 mb_detect_encoding 猜測
if (function_exists('mb_detect_encoding')) {
$detected = mb_detect_encoding($html, 'UTF-8, GBK, BIG5, ISO-8859-1', true);
if ($detected && $detected !== 'UTF-8') {
return mb_convert_encoding($html, 'UTF-8', $detected);
}
}
return $html;
}
private function parseHtml(string $html): ?DOMDocument
{
libxml_use_internal_errors(true);
$dom = new DOMDocument();
// 處理 HTML5 標籤兼容
$html = '' . $html;
$loaded = $dom->loadHTML($html, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
libxml_clear_errors();
return $loaded ? $dom : null;
}
private function extractTitle(DOMDocument $dom): ?string
{
$titles = $dom->getElementsByTagName('title');
return $titles->length ? trim($titles->item(0)->textContent) : null;
}
private function extractMeta(DOMDocument $dom, string $name): ?string
{
// 標準 meta name
$xpath = new DOMXPath($dom);
$nodes = $xpath->query("//meta[@name='{$name}' or @property='og:{$name}']/@content");
if ($nodes->length) {
return trim($nodes->item(0)->value);
}
return null;
}
private function extractCanonical(DOMDocument $dom): ?string
{
$xpath = new DOMXPath($dom);
$nodes = $xpath->query("//link[@rel='canonical']/@href");
return $nodes->length ? trim($nodes->item(0)->value) : null;
}
private function extractOpenGraph(DOMDocument $dom): array
{
$xpath = new DOMXPath($dom);
$nodes = $xpath->query("//meta[starts-with(@property, 'og:')]");
$og = [];
foreach ($nodes as $node) {
$prop = $node->getAttribute('property');
$content = $node->getAttribute('content');
$og[$prop] = $content;
}
return $og;
}
private function extractBodyText(DOMDocument $dom): string
{
$xpath = new DOMXPath($dom);
// 移除 script/style/noscript/iframe
$removeQuery = "//script | //style | //noscript | //iframe | //svg | //comment()";
foreach ($xpath->query($removeQuery) as $node) {
$node->parentNode->removeChild($node);
}
$body = $dom->getElementsByTagName('body')->item(0);
if (!$body) {
return '';
}
// 取得文字內容,保留段落結構
$text = '';
foreach ($xpath->query('.//p | .//h1 | .//h2 | .//h3 | .//h4 | .//li | .//div', $body) as $node) {
$t = trim(preg_replace('/\s+/', ' ', $node->textContent));
if (strlen($t) > 20) { // 過濾太短片段
$text .= $t . "\n\n";
}
}
return trim($text);
}
}
?>
<?php
$extractor = new PageContentExtractor();
$result = $extractor->extract('https://www.example.com/article/123');
if (isset($result['error'])) {
echo "抓取失敗:{$result['error']}\n";
} else {
echo "標題:{$result['title']}\n";
echo "描述:{$result['description']}\n";
echo "關鍵字:{$result['keywords']}\n";
echo "正文前 500 字:\n" . mb_substr($result['body_text'], 0, 500) . "\n";
// OpenGraph 社群分享資料
if ($result['og']) {
echo "OG 圖片:{$result['og']['og:image'] ?? '無'}\n";
}
}
?>
| 策略 | 實現方式 | 適用場景 |
|---|---|---|
| 隨機 User-Agent | 維護 UA 池輪換 | 基礎防禦 |
| Cookie 持久化 | CURLOPT_COOKIEJAR/FILE | 需登入頁面 |
| 請求頻率控制 | Token Bucket / 漏桶演算法 | 大規模抓取 |
| 代理 IP 池 | CURLOPT_PROXY + 驗證可用性 | 高頻/嚴格反爬 |
| Headless Chrome | Puppeteer/Playwright + PHP 橋接 | JS 渲染頁面 |
robots.txt(檢查 User-agent: * 與 Disallow)X-Robots-Tag: noindex, nofollowCrawl-delay,避免影響目標站點服務curl_multi_* 並發抓取多個 URLCURLOPT_ENCODING 自動解壓 gzip/deflate/brCURLOPT_TCP_KEEPALIVE)現代網頁內容提取的關鍵在於:穩健的編碼處理 + 結構化 DOM 解析 + 完善的異常處理 + 遵守爬蟲禮儀。上述類別提供了可直接生產使用的基礎框架,可根據具體業務擴展 Schema.org、JSON-LD、微格式等結構化資料解析。