public static string GetPageSource(string url, Encoding encoding, int timeoutMs = 20000)
{
var request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = "Mozilla/5.0 (compatible; MyBot/1.0)";
request.Timeout = timeoutMs;
request.AllowAutoRedirect = true;
request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
try
{
using (var response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode != HttpStatusCode.OK) return string.Empty;
if (response.ContentLength > 1024 * 1024) return string.Empty; // 限制 1MB
using (var stream = response.GetResponseStream())
using (var reader = new StreamReader(stream, encoding, detectEncodingFromByteOrderMarks: true))
{
return reader.ReadToEnd();
}
}
}
catch (WebException ex)
{
// 記錄 ex.Response?.ResponseUri 等診斷資訊
return string.Empty;
}
}
HTTP 回應標頭 `Content-Type: text/html; charset=utf-8` 優先;無標頭則解析 HTML `` 或 ``。
public static (string Html, Encoding Encoding) GetPageWithAutoDetect(string url, int timeoutMs = 20000)
{
var request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = "Mozilla/5.0 (compatible; MyBot/1.0)";
request.Timeout = timeoutMs;
request.AllowAutoRedirect = true;
request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
using (var response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode != HttpStatusCode.OK) return (string.Empty, Encoding.UTF8);
// 1. 從 Header 取得編碼
Encoding encoding = null;
var contentType = response.ContentType;
if (!string.IsNullOrEmpty(contentType))
{
var match = Regex.Match(contentType, @"charset\s*=\s*([\w-]+)", RegexOptions.IgnoreCase);
if (match.Success) encoding = GetEncodingSafe(match.Groups[1].Value);
}
// 2. 先以預設編碼讀取前 4KB 偵測 meta 標籤
using (var stream = response.GetResponseStream())
using (var ms = new MemoryStream())
{
stream.CopyTo(ms);
var bytes = ms.ToArray();
if (encoding == null)
{
// 偵測 BOM
encoding = DetectBomEncoding(bytes);
}
if (encoding == null)
{
// 解析前 4KB HTML 找 meta charset
var preview = Encoding.ASCII.GetString(bytes, 0, Math.Min(bytes.Length, 4096));
encoding = DetectMetaCharset(preview) ?? Encoding.UTF8; // 預設 UTF-8
}
// 3. 完整解碼
string html = encoding.GetString(bytes);
return (html, encoding);
}
}
}
private static Encoding GetEncodingSafe(string name)
{
try { return Encoding.GetEncoding(name); } catch { return null; }
}
private static Encoding DetectBomEncoding(byte[] bytes)
{
if (bytes.Length >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF) return Encoding.UTF8;
if (bytes.Length >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF) return Encoding.BigEndianUnicode;
if (bytes.Length >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE) return Encoding.Unicode;
return null;
}
private static Encoding DetectMetaCharset(string html)
{
var match = Regex.Match(html, @"]+charset\s*=\s*[""']?([\w-]+)", RegexOptions.IgnoreCase);
if (match.Success) return GetEncodingSafe(match.Groups[1].Value);
match = Regex.Match(html, @"]+http-equiv\s*=\s*[""']?Content-Type[""']?[^>]+charset\s*=\s*([\w-]+)", RegexOptions.IgnoreCase);
if (match.Success) return GetEncodingSafe(match.Groups[1].Value);
return null;
}
private static readonly HttpClient _http = new HttpClient
{
Timeout = TimeSpan.FromSeconds(20)
};
public static async Task GetPageAsync(string url)
{
_http.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (compatible; MyBot/1.0)");
var response = await _http.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
response.EnsureSuccessStatusCode();
// HttpClient 內建自動偵測 BOM + Content-Type charset
return await response.Content.ReadAsStringAsync();
}