萬盛學電腦網

您现在的位置: 萬盛學電腦網 >> 程式設計 >  >> 網路程式設計 >> 程式語言綜合

Python urllib2 實現 HTTP 斷點續傳下載:Range 頭部實戰

原理:HTTP Range 請求

HTTP/1.1 定義 Range 請求頭,允許客戶端請求資源的部分內容。伺服器支援時返回 206 Partial Content 及 Content-Range、Content-Length。斷點續傳核心流程:

  1. HEAD 請求獲取 Content-Length、Accept-Ranges: bytes 確認支援。
  2. 檢查本地臨時文件大小 → 確定下一個位元組偏移量。
  3. Range: bytes=START- 請求剩餘部分。
  4. 追加寫入文件,更新進度,重複直到完成。
  5. 完成後重命名臨時文件為目標文件(原子性)。

最小可行示例(單片段)


import urllib.request

url = "https://www.python.org/"
req = urllib.request.Request(url)
req.add_header('Range', 'bytes=0-19')  # 前 20 字節

with urllib.request.urlopen(req) as resp:
    print(f"Status: {resp.status}")          # 206
    print(f"Content-Range: {resp.headers.get('Content-Range')}")
    print(f"Content-Length: {resp.headers.get('Content-Length')}")
    data = resp.read()
    print(f"Received {len(data)} bytes: {data[:50]}")

生產級斷點續傳下載器


import os
import sys
import time
import urllib.request
import urllib.error
from pathlib import Path
from typing import Optional

class ResumableDownloader:
    def __init__(self, url: str, dest: Path, chunk_size: int = 8192, max_retries: int = 3):
        self.url = url
        self.dest = dest
        self.tmp = dest.with_suffix(dest.suffix + ".part")
        self.chunk_size = chunk_size
        self.max_retries = max_retries
        self.total_size: Optional[int] = None

    def _head(self) -> bool:
        """HEAD 請求獲取總大小與範圍支援"""
        req = urllib.request.Request(self.url, method='HEAD')
        try:
            with urllib.request.urlopen(req, timeout=10) as resp:
                self.total_size = int(resp.headers['Content-Length'])
                return resp.headers.get('Accept-Ranges', '').lower() == 'bytes'
        except Exception as e:
            print(f"HEAD failed: {e}")
            return False

    def _download_range(self, start: int) -> bool:
        """下載單個範圍,返回是否成功"""
        end = start + self.chunk_size - 1
        if self.total_size and end >= self.total_size:
            end = self.total_size - 1
        range_header = f"bytes={start}-{end}"
        req = urllib.request.Request(self.url, headers={'Range': range_header})
        try:
            with urllib.request.urlopen(req, timeout=30) as resp:
                if resp.status not in (200, 206):
                    print(f"Unexpected status {resp.status}")
                    return False
                with open(self.tmp, 'r+b') as f:
                    f.seek(start)
                    while True:
                        chunk = resp.read(self.chunk_size)
                        if not chunk: break
                        f.write(chunk)
                return True
        except urllib.error.HTTPError as e:
            if e.code == 416:  # Range Not Satisfiable
                print("Range not satisfiable, download may be complete")
                return True
            print(f"HTTP {e.code}: {e.reason}")
        except Exception as e:
            print(f"Error: {e}")
        return False

    def run(self) -> bool:
        if not self._head():
            print("Server does not support range requests, falling back to full download")
            return self._full_download()

        # 確定起始位置
        start = self.tmp.stat().st_size if self.tmp.exists() else 0
        if self.total_size and start >= self.total_size:
            print("Already complete")
            return self._finalize()

        print(f"Downloading {self.url} -> {self.dest} ({self.total_size:,} bytes)")
        print(f"Resuming from byte {start:,}")

        retries = 0
        while self.total_size is None or start < self.total_size:
            if self._download_range(start):
                start = self.tmp.stat().st_size
                retries = 0
                progress = (start / self.total_size * 100) if self.total_size else 0
                print(f"\rProgress: {progress:.1f}% ({start:,}/{self.total_size:,})", end="")
            else:
                retries += 1
                if retries > self.max_retries:
                    print(f"\nMax retries exceeded")
                    return False
                time.sleep(2 ** retries)  # 指數退避

        print()
        return self._finalize()

    def _full_download(self) -> bool:
        """不支援 Range 時全量下載"""
        try:
            with urllib.request.urlopen(self.url, timeout=30) as resp, open(self.tmp, 'wb') as f:
                self.total_size = int(resp.headers.get('Content-Length', 0))
                while True:
                    chunk = resp.read(self.chunk_size)
                    if not chunk: break
                    f.write(chunk)
            return self._finalize()
        except Exception as e:
            print(f"Full download failed: {e}")
            return False

    def _finalize(self) -> bool:
        try:
            if self.dest.exists(): self.dest.unlink()
            self.tmp.rename(self.dest)
            print(f"Saved to {self.dest}")
            return True
        except Exception as e:
            print(f"Finalize failed: {e}")
            return False

if __name__ == "__main__":
    if len(sys.argv) < 3:
        print("Usage: python downloader.py  ")
        sys.exit(1)
    downloader = ResumableDownloader(sys.argv[1], Path(sys.argv[2]))
    success = downloader.run()
    sys.exit(0 if success else 1)

關鍵細節說明

  • 臨時文件 .part:避免部分下載文件被誤用,完成後原子重命名。
  • 二進制模式 r+b/wb:Windows 下必須,防止換行符轉換。
  • 416 Range Not Satisfiable:請求範圍超出文件大小,視為完成。
  • 指數退避重試:網絡抖動自動恢復。
  • 進度顯示:\r 回車符原地刷新。

進階擴展方向

  1. 多線程分片下載:將文件分為 N 段,並行下載合併(需線程安全寫入或分文件後合併)。
  2. 校驗完整性:下載後驗證 MD5/SHA256(伺服器提供 Content-MD5 或單獨校驗文件)。
  3. 限速:time.sleep(chunk_size / rate_limit) 實現流控。
  4. 代理/認證:urllib.request.ProxyHandler, HTTPBasicAuthHandler。
  5. 異步版本:aiohttp + aiofiles 實現高併發非阻塞下載器。

總結

利用 urllib.request(Py3)或 urllib2(Py2)配合 Range 頭部實現斷點續傳並不複雜,關鍵在於**正確處理臨時文件、範圍計算、錯誤重試與原子落盤**。上述 ResumableDownloader 類已可直接用於生產環境,並易於擴展多線程、校驗、限速等企業級功能。

copyright © 萬盛學電腦網 all rights reserved