網頁資料擷取 (Web Scraping) 是常見需求。本文介紹兩種核心擷取技巧:字串切割 (`explode`/`strstr`) 與正則表達式 (`preg_match`),並提供可重用的工具函式與最佳實務。
<?php
/**
* 字串切割擷取:以起始/結束字串為界
* 效能優於正則,適用於固定格式標記
*/
function str_between(string $haystack, string $start, string $end, int $offset = 0): ?string {
$startPos = strpos($haystack, $start, $offset);
if ($startPos === false) return null;
$startPos += strlen($start);
$endPos = strpos($haystack, $end, $startPos);
if ($endPos === false) return null;
return substr($haystack, $startPos, $endPos - $startPos);
}
/**
* 正則擷取:支援複雜模式
* 回傳第一個捕獲群組或完整匹配
*/
function preg_between(string $haystack, string $pattern, int $group = 1): ?string {
if (preg_match($pattern, $haystack, $matches)) {
return $matches[$group] ?? $matches[0];
}
return null;
}
/**
* 正則擷取所有匹配
*/
function preg_between_all(string $haystack, string $pattern, int $group = 1): array {
$results = [];
if (preg_match_all($pattern, $haystack, $matches)) {
return $matches[$group] ?? $matches[0];
}
return [];
}
?>
<?php
// 取得網頁內容 (處理編碼)
$url = 'https://example.com/article/123';
$html = file_get_contents($url);
// 偵測編碼並轉 UTF-8
$encoding = mb_detect_encoding($html, ['UTF-8', 'BIG5', 'GB2312'], true);
if ($encoding && $encoding !== 'UTF-8') {
$html = mb_convert_encoding($html, 'UTF-8', $encoding);
}
// 1. 擷取標題 ...
$title = str_between($html, '', ' ');
// 2. 擷取作者:正則匹配 作者名
$author = preg_between($html, '/]*>([^<]+)<\/a>/');
// 3. 擷取文章內容:...
$content = str_between($html, '', '');
// 4. 擷取所有圖片連結
$images = preg_between_all($html, '/
]*src=["\']([^"\']+)["\'][^>]*>/i');
echo "標題: $title\n";
echo "作者: $author\n";
echo "內容長度: " . strlen($content) . "\n";
echo "圖片數: " . count($images) . "\n";
?>
<?php
$dom = new DOMDocument();
libxml_use_internal_errors(true); // 抑制 HTML5 解析警告
$dom->loadHTML($html);
libxml_clear_errors();
$xpath = new DOMXPath($dom);
// 擷取標題
$titleNode = $xpath->query('//title')->item(0);
$title = $titleNode ? $titleNode->textContent : '';
// 擷取所有連結
$links = [];
foreach ($xpath->query('//a[@href]') as $link) {
$links[] = [
'url' => $link->getAttribute('href'),
'text' => trim($link->textContent),
];
}
// 擷取特定 class 內容
$contents = [];
foreach ($xpath->query('//div[contains(@class, "content")]') as $node) {
$contents[] = trim($node->textContent);
}
?>
<?php
class SimpleScraper {
private array $headers = [
'User-Agent' => 'Mozilla/5.0 (compatible; PHPScraper/1.0)',
'Accept' => 'text/html,application/xhtml+xml',
'Accept-Language' => 'zh-TW,zh;q=0.9,en;q=0.8',
];
private int $timeout = 30;
public function fetch(string $url): ?string {
$ch = curl_init($url);
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 5,
CURLOPT_TIMEOUT => $this->timeout,
CURLOPT_HTTPHEADER => array_map(fn($k, $v) => "$k: $v", array_keys($this->headers), $this->headers),
CURLOPT_SSL_VERIFYPEER => true,
CURLOPT_ENCODING => 'gzip,deflate',
]);
$content = curl_exec($ch);
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
if ($httpCode !== 200 || $content === false) {
return null;
}
return $content;
}
public function extract(string $html, string $selector, string $attr = 'text'): ?string {
$dom = new DOMDocument();
libxml_use_internal_errors(true);
$dom->loadHTML('' . $html);
libxml_clear_errors();
$xpath = new DOMXPath($dom);
$node = $xpath->query($selector)->item(0);
if (!$node) return null;
return $attr === 'text' ? trim($node->textContent) : $node->getAttribute($attr);
}
public function extractAll(string $html, string $selector, string $attr = 'text'): array {
$dom = new DOMDocument();
libxml_use_internal_errors(true);
$dom->loadHTML('' . $html);
libxml_clear_errors();
$xpath = new DOMXPath($dom);
$results = [];
foreach ($xpath->query($selector) as $node) {
$results[] = $attr === 'text' ? trim($node->textContent) : $node->getAttribute($attr);
}
return $results;
}
}
// 使用
$scraper = new SimpleScraper();
$html = $scraper->fetch('https://example.com');
if ($html) {
$title = $scraper->extract($html, '//title');
$articles = $scraper->extractAll($html, '//article//h2/a', 'href');
print_r($articles);
}
?>
| 原則 | 說明 |
|---|---|
| 遵守 robots.txt | 檢查 User-agent: * 規則,尊重 Disallow 路徑 |
| 限制請求頻率 | 每秒 ≤ 1 請求,隨機延遲 1-3 秒,避免造成伺服器負擔 |
| 識別自我 | 設定有意義的 User-Agent,包含聯絡資訊 |
| 不繞過防護 | 不破解 CAPTCHA、不繞過 WAF、不暴力嘗試 |
| 資料使用合法 | 確認授權條款 (ToS)、著作權、個資法 (GDPR/PDPA) |
| 優先使用官方 API | 若網站提供 API,應優先使用而非爬蟲 |
<?php
// 伺服器端:偵測與阻擋爬蟲
function isBot(): bool {
$ua = $_SERVER['HTTP_USER_AGENT'] ?? '';
$botPatterns = [
'/bot/i', '/crawler/i', '/spider/i', '/slurp/i',
'/facebookexternalhit/i', '/twitterbot/i', '/linkedinbot/i',
'/python-requests/i', '/curl/i', '/wget/i', '/go-http-client/i',
];
foreach ($botPatterns as $pattern) {
if (preg_match($pattern, $ua)) return true;
}
// 行為分析:請求頻率、無 Cookie、無 Referer、無 JS 執行能力
return false;
}
// 中介軟體攔截
if (isBot() && !isAllowedBot()) {
http_response_code(403);
exit('Forbidden: Bot access denied');
}
?>