HttpClient 是 .NET Core / .NET 5+ 標準 HTTP 客戶端,支援非同步、連線池、自動解壓縮。
using System;
using System.Net.Http;
using System.Threading.Tasks;
public static class WebScraper
{
private static readonly HttpClient _client = new HttpClient
{
Timeout = TimeSpan.FromSeconds(30)
};
public static async Task GetHtmlAsync(string url)
{
try
{
HttpResponseMessage response = await _client.GetAsync(url);
response.EnsureSuccessStatusCode(); // 非 2xx 拋例外
return await response.Content.ReadAsStringAsync();
}
catch (HttpRequestException ex)
{
Console.Error.WriteLine($"請求失敗: {ex.Message}");
return null;
}
catch (TaskCanceledException)
{
Console.Error.WriteLine("請求逾時");
return null;
}
}
}
// 使用
string html = await WebScraper.GetHtmlAsync("https://www.example.com");
public static async Task GetHtmlWithAuthAsync(string url, string username, string password)
{
var byteArray = System.Text.Encoding.ASCII.GetBytes($"{username}:{password}");
_client.DefaultRequestHeaders.Authorization =
new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
return await GetHtmlAsync(url);
}
public static async Task GetHtmlWithEncodingAsync(string url)
{
HttpResponseMessage resp = await _client.GetAsync(url);
resp.EnsureSuccessStatusCode();
// 1. 先以位元組讀取
byte[] bytes = await resp.Content.ReadAsByteArrayAsync();
// 2. 從 Content-Type 標頭解析 charset
string charset = "utf-8"; // 預設
if (resp.Content.Headers.ContentType?.CharSet != null)
{
charset = resp.Content.Headers.ContentType.CharSet.Trim('"');
}
// 3. 若無標頭,嘗試從 HTML meta 標籤偵測
string raw = System.Text.Encoding.GetEncoding("iso-8859-1").GetString(bytes); // 先用 Latin1 保留原始位元組
if (charset.Equals("iso-8859-1", StringComparison.OrdinalIgnoreCase) ||
string.IsNullOrEmpty(resp.Content.Headers.ContentType?.CharSet))
{
// 簡易 meta charset 擷取
var match = System.Text.RegularExpressions.Regex.Match(raw, @"]+charset=[""']?([\w-]+)", RegexOptions.IgnoreCase);
if (match.Success) charset = match.Groups[1].Value;
}
// 4. 正確解碼
try
{
var encoding = System.Text.Encoding.GetEncoding(charset);
return encoding.GetString(bytes);
}
catch
{
return System.Text.Encoding.UTF8.GetString(bytes); // 回退 UTF-8
}
}
// WebClient (已過時,不建議新專案使用)
using (var wc = new System.Net.WebClient())
{
wc.Encoding = System.Text.Encoding.UTF8; // 強制編碼
wc.Credentials = new System.Net.NetworkCredential("user", "pass");
string html = wc.DownloadString("https://example.com");
}
// HttpWebRequest (底層控制)
var req = (HttpWebRequest)WebRequest.Create("https://example.com");
req.Credentials = new NetworkCredential("user", "pass");
req.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
using (var resp = (HttpWebResponse)req.GetResponse())
using (var stream = resp.GetResponseStream())
using (var reader = new System.IO.StreamReader(stream, System.Text.Encoding.GetEncoding(resp.CharacterSet)))
{
string html = reader.ReadToEnd();
}
// NuGet: HtmlAgilityPack
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
var title = doc.DocumentNode.SelectSingleNode("//title")?.InnerText;
var links = doc.DocumentNode.SelectNodes("//a[@href]")
?.Select(n => n.GetAttributeValue("href", ""))
.Where(h => !string.IsNullOrEmpty(h)) ?? Enumerable.Empty();
| 項目 | 建議 |
|---|---|
| HttpClient 生命週期 | 單例或 DI 註冊,避免 Socket 耗盡 |
| Timeout | 設定合理 Timeout (預設 100s 太長) |
| User-Agent | 設定合理 UA,避免被反爬蟲封鎖 |
| Cookie 管理 | 使用 CookieContainer + HttpClientHandler |
| 重試策略 | Polly: WaitAndRetryAsync(3, i => TimeSpan.FromSeconds(Math.Pow(2,i))) |
| 速率限制 | SemaphoreSlim 或 Token Bucket 控制並發 |
| 法律合規 | 遵守 robots.txt、ToS、GDPR、個資法 |