抓取網頁原始碼是爬蟲、整合舊系統、SEO 分析的基礎。.NET Framework 使用 `HttpWebRequest`,.NET Core / .NET 5+ 強烈建議使用 `HttpClient`。本文提供兩種版本完整範例,含編碼自動偵測、User-Agent 偽裝、逾時與錯誤處理。
public static string GetHtmlHttpWebRequest(string url, string preferredEncoding = null, int timeoutMs = 30000)
{
var request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "GET";
request.Timeout = timeoutMs;
request.ReadWriteTimeout = timeoutMs;
request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
request.KeepAlive = true;
request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
using (var response = (HttpWebResponse)request.GetResponse())
using (var stream = response.GetResponseStream())
{
Encoding encoding = null;
// 1. 嘗試從 Content-Type 取得 charset
var contentType = response.ContentType;
if (!string.IsNullOrEmpty(contentType))
{
var match = Regex.Match(contentType, @"charset\s*=\s*([\w-]+)", RegexOptions.IgnoreCase);
if (match.Success) encoding = Encoding.GetEncoding(match.Groups[1].Value);
}
// 2. 若無則用指定編碼
if (encoding == null && !string.IsNullOrEmpty(preferredEncoding))
encoding = Encoding.GetEncoding(preferredEncoding);
// 3. 最後嘗試從 HTML meta 標籤偵測 (需先讀取位元組)
if (encoding == null)
{
// 簡易偵測:讀前 2048 位元組找 charset
// 實務建議用 Ude.NetStandard (chardet) 套件
encoding = Encoding.UTF8; // 預設
}
using (var reader = new StreamReader(stream, encoding, detectEncodingFromByteOrderMarks: true))
{
return reader.ReadToEnd();
}
}
}
// 靜態單例 HttpClient (避免 Socket 耗盡)
private static readonly HttpClient _httpClient = new HttpClient(new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
UseCookies = false
})
{
Timeout = TimeSpan.FromSeconds(30)
};
static HttpHelper()
{
_httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36");
_httpClient.DefaultRequestHeaders.Accept.ParseAdd("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
}
public static async Task<string> GetHtmlAsync(string url, CancellationToken ct = default)
{
using (var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, ct))
{
response.EnsureSuccessStatusCode(); // 非 2xx 擲出
// 取得編碼:優先 Content-Type charset,其次 UTF-8
var contentType = response.Content.Headers.ContentType;
Encoding encoding = Encoding.UTF8;
if (contentType?.CharSet != null)
{
try { encoding = Encoding.GetEncoding(contentType.CharSet); } catch { /* ignore */ }
}
// 讀取位元組並解碼
var bytes = await response.Content.ReadAsByteArrayAsync(ct);
// 可選:用 Ude.NetStandard 偵測編碼
// var detected = CharsetDetector.DetectFromBytes(bytes); encoding = Encoding.GetEncoding(detected.Charset);
return encoding.GetString(bytes);
}
}
若回應無 `Content-Type charset`,需解析 HTML `` 或 ``。建議引用 Ude.NetStandard (chardet 移植) 或 HtmlAgilityPack 解析:
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(htmlBytes, detectEncodingFromByteOrderMarks: true); // 自動偵測 BOM + meta
return doc.DocumentNode.OuterHtml; // 已正確解碼
var handler = new HttpClientHandler
{
AllowAutoRedirect = true,
CookieContainer = new CookieContainer(), // 自動處理 Cookie
UseCookies = true
};
// 設定 Referer
_httpClient.DefaultRequestHeaders.Referrer = new Uri("https://www.google.com/");
using (var stream = await response.Content.ReadAsStreamAsync(ct))
using (var reader = new StreamReader(stream, encoding))
{
// 逐行處理或寫入檔案
while (!reader.EndOfStream)
{
var line = await reader.ReadLineAsync();
ProcessLine(line);
}
}