User-Agent (UA) 是 HTTP 請求標頭,識別客戶端瀏覽器類型、版本、作業系統、渲染引擎等。網站常依據 UA 判斷是否為正常瀏覽器請求,拒絕無 UA 或爬蟲特徵的請求。
file_get_contents() 預設不發送 User-Agent,導致許多防爬網站回傳空內容或 403。
ini_set('user_agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
// 後續 file_get_contents 會自動帶上此 UA
$html = file_get_contents('https://example.com');
$opts = [
'http' => [
'method' => 'GET',
'header' => implode("\r\n", [
'Host: zh.wikipedia.org',
'Accept-Language: zh-TW,zh;q=0.9,en;q=0.8',
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Connection: keep-alive',
]),
'timeout' => 30,
'ignore_errors' => true, // 即使 4xx/5xx 也取得內容
],
'ssl' => [
'verify_peer' => false, // 開發環境可關閉驗證
'verify_peer_name' => false,
]
];
$context = stream_context_create($opts);
$html = file_get_contents('https://zh.wikipedia.org/', false, $context);
| 瀏覽器 | User-Agent 範例 |
|---|---|
| Chrome 120 (Win10) | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 |
| Firefox 121 (Win10) | Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0 |
| Safari 17 (macOS) | Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15 |
| Edge 120 (Win10) | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0 |
| iPhone Safari | Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1 |
| Googlebot | Mozilla/5.0 (compatible; Googlebot/2.1; +https://www.google.com/bot.html) |
$ch = curl_init();
curl_setopt_array($ch, [
CURLOPT_URL => 'https://example.com',
CURLOPT_RETURNTRANSFER => true,
CURLOPT_USERAGENT => 'Mozilla/5.0 ...',
CURLOPT_HTTPHEADER => [
'Accept-Language: zh-TW,zh;q=0.9',
'Accept: text/html,application/xhtml+xml',
],
CURLOPT_TIMEOUT => 30,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_SSL_VERIFYPEER => false, // 僅開發用
]);
$html = curl_exec($ch);
curl_close($ch);
robots.txt 與網站爬蟲協議