萬盛學電腦網

您现在的位置: 萬盛學電腦網 >> 程式設計 >  >> 網路程式設計 >> PHP 程式設計

PHP 網頁內容抓取:基於 cURL 與 DOM 解析的標題、Meta、正文提取實戰

從簡單抓取到結構化解析

抓取網頁標題、描述、關鍵字與正文內容,是搜尋引擎、內容聚合、SEO 分析的基礎功能。單純正則匹配 HTML 極其脆弱,現代方案採用 cURL 獲取原始碼 + DOMDocument 結構化解析,兼容性與容錯性大幅提升。

核心抓取類實現

<?php
class PageContentExtractor
{
    private int $timeout = 10;
    private string $userAgent = 'Mozilla/5.0 (compatible; PHPBot/1.0; +https://example.com/bot)';

    public function extract(string $url): array
    {
        $html = $this->fetch($url);
        if (!$html) {
            return ['error' => 'FETCH_FAILED', 'url' => $url];
        }

        $dom = $this->parseHtml($html);
        if (!$dom) {
            return ['error' => 'PARSE_FAILED', 'url' => $url];
        }

        return [
            'url' => $url,
            'title' => $this->extractTitle($dom),
            'description' => $this->extractMeta($dom, 'description'),
            'keywords' => $this->extractMeta($dom, 'keywords'),
            'canonical' => $this->extractCanonical($dom),
            'og' => $this->extractOpenGraph($dom),
            'body_text' => $this->extractBodyText($dom),
            'html' => $html, // 可選:保留原始碼供除錯
        ];
    }

    private function fetch(string $url): ?string
    {
        $ch = curl_init($url);
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_MAXREDIRS => 5,
            CURLOPT_TIMEOUT => $this->timeout,
            CURLOPT_CONNECTTIMEOUT => 5,
            CURLOPT_USERAGENT => $this->userAgent,
            CURLOPT_SSL_VERIFYPEER => true, // 生產環境建議開啟
            CURLOPT_SSL_VERIFYHOST => 2,
            CURLOPT_ENCODING => 'gzip,deflate', // 自動解壓
        ]);

        $html = curl_exec($ch);
        $errno = curl_errno($ch);
        $info = curl_getinfo($ch);
        curl_close($ch);

        if ($errno || ($info['http_code'] ?? 0) !== 200) {
            return null;
        }

        // 檢測編碼並轉 UTF-8
        return $this->convertToUtf8($html, $info['content_type'] ?? '');
    }

    private function convertToUtf8(string $html, string $contentType): string
    {
        // 1. 從 HTTP 標頭解析 charset
        if (preg_match('/charset=([^;\s]+)/i', $contentType, $m)) {
            $encoding = strtoupper(trim($m[1]));
            if ($encoding !== 'UTF-8' && function_exists('mb_convert_encoding')) {
                return mb_convert_encoding($html, 'UTF-8', $encoding);
            }
            return $html;
        }

        // 2. 從 HTML meta 標籤解析
        if (preg_match('/]+charset=["\']?([^"\'>\s]+)/i', $html, $m)) {
            $encoding = strtoupper(trim($m[1]));
            if ($encoding !== 'UTF-8' && function_exists('mb_convert_encoding')) {
                return mb_convert_encoding($html, 'UTF-8', $encoding);
            }
        }

        // 3. 讓 mb_detect_encoding 猜測
        if (function_exists('mb_detect_encoding')) {
            $detected = mb_detect_encoding($html, 'UTF-8, GBK, BIG5, ISO-8859-1', true);
            if ($detected && $detected !== 'UTF-8') {
                return mb_convert_encoding($html, 'UTF-8', $detected);
            }
        }

        return $html;
    }

    private function parseHtml(string $html): ?DOMDocument
    {
        libxml_use_internal_errors(true);
        $dom = new DOMDocument();
        // 處理 HTML5 標籤兼容
        $html = '' . $html;
        $loaded = $dom->loadHTML($html, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
        libxml_clear_errors();
        return $loaded ? $dom : null;
    }

    private function extractTitle(DOMDocument $dom): ?string
    {
        $titles = $dom->getElementsByTagName('title');
        return $titles->length ? trim($titles->item(0)->textContent) : null;
    }

    private function extractMeta(DOMDocument $dom, string $name): ?string
    {
        // 標準 meta name
        $xpath = new DOMXPath($dom);
        $nodes = $xpath->query("//meta[@name='{$name}' or @property='og:{$name}']/@content");
        if ($nodes->length) {
            return trim($nodes->item(0)->value);
        }
        return null;
    }

    private function extractCanonical(DOMDocument $dom): ?string
    {
        $xpath = new DOMXPath($dom);
        $nodes = $xpath->query("//link[@rel='canonical']/@href");
        return $nodes->length ? trim($nodes->item(0)->value) : null;
    }

    private function extractOpenGraph(DOMDocument $dom): array
    {
        $xpath = new DOMXPath($dom);
        $nodes = $xpath->query("//meta[starts-with(@property, 'og:')]");
        $og = [];
        foreach ($nodes as $node) {
            $prop = $node->getAttribute('property');
            $content = $node->getAttribute('content');
            $og[$prop] = $content;
        }
        return $og;
    }

    private function extractBodyText(DOMDocument $dom): string
    {
        $xpath = new DOMXPath($dom);
        // 移除 script/style/noscript/iframe
        $removeQuery = "//script | //style | //noscript | //iframe | //svg | //comment()";
        foreach ($xpath->query($removeQuery) as $node) {
            $node->parentNode->removeChild($node);
        }

        $body = $dom->getElementsByTagName('body')->item(0);
        if (!$body) {
            return '';
        }

        // 取得文字內容,保留段落結構
        $text = '';
        foreach ($xpath->query('.//p | .//h1 | .//h2 | .//h3 | .//h4 | .//li | .//div', $body) as $node) {
            $t = trim(preg_replace('/\s+/', ' ', $node->textContent));
            if (strlen($t) > 20) { // 過濾太短片段
                $text .= $t . "\n\n";
            }
        }
        return trim($text);
    }
}
?>

使用範例

<?php
$extractor = new PageContentExtractor();
$result = $extractor->extract('https://www.example.com/article/123');

if (isset($result['error'])) {
    echo "抓取失敗:{$result['error']}\n";
} else {
    echo "標題:{$result['title']}\n";
    echo "描述:{$result['description']}\n";
    echo "關鍵字:{$result['keywords']}\n";
    echo "正文前 500 字:\n" . mb_substr($result['body_text'], 0, 500) . "\n";
    
    // OpenGraph 社群分享資料
    if ($result['og']) {
        echo "OG 圖片:{$result['og']['og:image'] ?? '無'}\n";
    }
}
?>

反爬蟲應對策略

策略實現方式適用場景
隨機 User-Agent維護 UA 池輪換基礎防禦
Cookie 持久化CURLOPT_COOKIEJAR/FILE需登入頁面
請求頻率控制Token Bucket / 漏桶演算法大規模抓取
代理 IP 池CURLOPT_PROXY + 驗證可用性高頻/嚴格反爬
Headless ChromePuppeteer/Playwright + PHP 橋接JS 渲染頁面

法律與道德邊界

  1. 遵守 robots.txt(檢查 User-agent: * 與 Disallow)
  2. 尊重 X-Robots-Tag: noindex, nofollow
  3. 設定合理 Crawl-delay,避免影響目標站點服務
  4. 僅抓取公開內容,不繞過登入/付費牆
  5. 資料使用符合著作權法與網站服務條款

效能優化建議

  • 使用 curl_multi_* 並發抓取多個 URL
  • 啟用 CURLOPT_ENCODING 自動解壓 gzip/deflate/br
  • 針對固定站點緩存 DNS、重用連線(CURLOPT_TCP_KEEPALIVE)
  • 大量抓取考慮 Swoole/RoadRunner 協程化

總結

現代網頁內容提取的關鍵在於:穩健的編碼處理 + 結構化 DOM 解析 + 完善的異常處理 + 遵守爬蟲禮儀。上述類別提供了可直接生產使用的基礎框架,可根據具體業務擴展 Schema.org、JSON-LD、微格式等結構化資料解析。

copyright © 萬盛學電腦網 all rights reserved