萬盛學電腦網

您现在的位置: 萬盛學電腦網 >> 程式設計 >  >> 網路程式設計 >> 程式語言綜合

C# 讀取網頁原始碼完整實作:HttpClient、驗證、編碼處理

現代做法:HttpClient (推薦)

HttpClient 是 .NET Core / .NET 5+ 標準 HTTP 客戶端,支援非同步、連線池、自動解壓縮。

基本 GET 請求

using System;
using System.Net.Http;
using System.Threading.Tasks;

public static class WebScraper
{
    private static readonly HttpClient _client = new HttpClient
    {
        Timeout = TimeSpan.FromSeconds(30)
    };

    public static async Task GetHtmlAsync(string url)
    {
        try
        {
            HttpResponseMessage response = await _client.GetAsync(url);
            response.EnsureSuccessStatusCode(); // 非 2xx 拋例外
            return await response.Content.ReadAsStringAsync();
        }
        catch (HttpRequestException ex)
        {
            Console.Error.WriteLine($"請求失敗: {ex.Message}");
            return null;
        }
        catch (TaskCanceledException)
        {
            Console.Error.WriteLine("請求逾時");
            return null;
        }
    }
}

// 使用
string html = await WebScraper.GetHtmlAsync("https://www.example.com");

支援 Basic Authentication

public static async Task GetHtmlWithAuthAsync(string url, string username, string password)
{
    var byteArray = System.Text.Encoding.ASCII.GetBytes($"{username}:{password}");
    _client.DefaultRequestHeaders.Authorization = 
        new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));

    return await GetHtmlAsync(url);
}

編碼自動偵測與處理

public static async Task GetHtmlWithEncodingAsync(string url)
{
    HttpResponseMessage resp = await _client.GetAsync(url);
    resp.EnsureSuccessStatusCode();

    // 1. 先以位元組讀取
    byte[] bytes = await resp.Content.ReadAsByteArrayAsync();

    // 2. 從 Content-Type 標頭解析 charset
    string charset = "utf-8"; // 預設
    if (resp.Content.Headers.ContentType?.CharSet != null)
    {
        charset = resp.Content.Headers.ContentType.CharSet.Trim('"');
    }

    // 3. 若無標頭,嘗試從 HTML meta 標籤偵測
    string raw = System.Text.Encoding.GetEncoding("iso-8859-1").GetString(bytes); // 先用 Latin1 保留原始位元組
    if (charset.Equals("iso-8859-1", StringComparison.OrdinalIgnoreCase) || 
        string.IsNullOrEmpty(resp.Content.Headers.ContentType?.CharSet))
    {
        // 簡易 meta charset 擷取
        var match = System.Text.RegularExpressions.Regex.Match(raw, @"]+charset=[""']?([\w-]+)", RegexOptions.IgnoreCase);
        if (match.Success) charset = match.Groups[1].Value;
    }

    // 4. 正確解碼
    try
    {
        var encoding = System.Text.Encoding.GetEncoding(charset);
        return encoding.GetString(bytes);
    }
    catch
    {
        return System.Text.Encoding.UTF8.GetString(bytes); // 回退 UTF-8
    }
}

舊版 WebClient / WebRequest (僅維護舊專案參考)

// WebClient (已過時,不建議新專案使用)
using (var wc = new System.Net.WebClient())
{
    wc.Encoding = System.Text.Encoding.UTF8; // 強制編碼
    wc.Credentials = new System.Net.NetworkCredential("user", "pass");
    string html = wc.DownloadString("https://example.com");
}

// HttpWebRequest (底層控制)
var req = (HttpWebRequest)WebRequest.Create("https://example.com");
req.Credentials = new NetworkCredential("user", "pass");
req.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
using (var resp = (HttpWebResponse)req.GetResponse())
using (var stream = resp.GetResponseStream())
using (var reader = new System.IO.StreamReader(stream, System.Text.Encoding.GetEncoding(resp.CharacterSet)))
{
    string html = reader.ReadToEnd();
}

進階:HtmlAgilityPack 解析 HTML

// NuGet: HtmlAgilityPack
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);

var title = doc.DocumentNode.SelectSingleNode("//title")?.InnerText;
var links = doc.DocumentNode.SelectNodes("//a[@href]")
    ?.Select(n => n.GetAttributeValue("href", ""))
    .Where(h => !string.IsNullOrEmpty(h)) ?? Enumerable.Empty();

最佳實踐清單

項目建議
HttpClient 生命週期單例或 DI 註冊,避免 Socket 耗盡
Timeout設定合理 Timeout (預設 100s 太長)
User-Agent設定合理 UA,避免被反爬蟲封鎖
Cookie 管理使用 CookieContainer + HttpClientHandler
重試策略Polly: WaitAndRetryAsync(3, i => TimeSpan.FromSeconds(Math.Pow(2,i)))
速率限制SemaphoreSlim 或 Token Bucket 控制並發
法律合規遵守 robots.txt、ToS、GDPR、個資法
copyright © 萬盛學電腦網 all rights reserved