頻繁創建/銷毀線程開銷大(內核對象、棧內存、上下文切換),且無限制創建會耗盡系統資源導致崩潰。線程池核心價值:
+----------------+ put(task) +------------------+
| Producer | ------------------> | Task Queue | (thread-safe, blocking)
| (Main Thread) | | (Queue.Queue) |
+----------------+ +--------+---------+
|
+---------------------+---------------------+
| | |
+-----v-----+ +-----v-----+ +-----v-----+
| Worker 1 | | Worker 2 | ... | Worker N |
| (Thread) | | (Thread) | | (Thread) |
+-----------+ +-----------+ +-----------+
| | |
get(task) get(task) get(task)
| | |
run(task) run(task) run(task)
import threading
import queue
import time
import signal
import os
from typing import Callable, Any, Optional
class Task:
"""任務封裝"""
__slots__ = ('func', 'args', 'kwargs')
def __init__(self, func: Callable, *args, **kwargs):
self.func = func
self.args = args
self.kwargs = kwargs
class ThreadPool:
"""簡易線程池:固定線程數 + 隊列 + 優雅關閉"""
def __init__(self, num_workers: int, max_queue_size: int = 0):
self.task_queue = queue.Queue(maxsize=max_queue_size) # 0=無界
self.workers = []
self._shutdown = False
self._shutdown_lock = threading.Lock()
for i in range(num_workers):
t = threading.Thread(target=self._worker_loop, name=f'Worker-{i}', daemon=True)
t.start()
self.workers.append(t)
def _worker_loop(self) -> None:
while True:
try:
task = self.task_queue.get(timeout=0.5) # 定期檢查關閉標誌
if task is None: # 停止信號
self.task_queue.task_done()
break
try:
task.func(*task.args, **task.kwargs)
except Exception as e:
# 生產環境應接入日誌/監控
print(f"[{threading.current_thread().name}] 任務異常: {e}")
finally:
self.task_queue.task_done()
except queue.Empty:
with self._shutdown_lock:
if self._shutdown:
break
def submit(self, func: Callable, *args, **kwargs) -> None:
"""提交任務(非阻塞,隊列滿時阻塞)"""
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError("線程池已關閉")
self.task_queue.put(Task(func, *args, **kwargs))
def shutdown(self, wait: bool = True) -> None:
"""優雅關閉"""
with self._shutdown_lock:
if self._shutdown:
return
self._shutdown = True
# 發送停止信號給每個工作線程
for _ in self.workers:
self.task_queue.put(None)
if wait:
for t in self.workers:
t.join()
def __enter__(self) -> 'ThreadPool':
return self
def __exit__(self, *args) -> None:
self.shutdown(wait=True)
# 測試
def demo_task(name: str, delay: float):
print(f"[{threading.current_thread().name}] 開始 {name}")
time.sleep(delay)
print(f"[{threading.current_thread().name}] 完成 {name}")
if __name__ == '__main__':
with ThreadPool(3) as pool:
for i in range(10):
pool.submit(demo_task, f"Task-{i}", 0.5)
print("所有任務完成")
concurrent.futures.ThreadPoolExecutorPython 3.2+ 內建的**標準、高性能、功能完整**線程池實現,**強烈推薦直接使用**。
from concurrent.futures import ThreadPoolExecutor, as_completed, Future
import time
import logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(threadName)s] %(message)s')
logger = logging.getLogger(__name__)
def io_task(name: str, delay: float) -> str:
time.sleep(delay)
return f"Result of {name}"
def main():
# 固定 4 個工作線程
with ThreadPoolExecutor(max_workers=4, thread_name_prefix='MyPool') as executor:
# 1. 提交任務獲取 Future
futures = {executor.submit(io_task, f'Task-{i}', 0.5): i for i in range(8)}
# 2. 異步獲取結果(按完成順序)
for future in as_completed(futures):
idx = futures[future]
try:
result = future.result(timeout=10)
logger.info(f"Task-{idx} 完成: {result}")
except Exception as e:
logger.error(f"Task-{idx} 失敗: {e}")
# 3. map 批量提交(保持輸入順序返回結果)
results = executor.map(io_task, [f'Map-{i}' for i in range(5)], [0.2]*5)
for i, r in enumerate(results):
logger.info(f"Map-{i}: {r}")
# 4. 等待所有任務完成(可選)
# executor.shutdown(wait=True) # with 塊自動調用
if __name__ == '__main__':
main()
| 特性 | 說明 |
|---|---|
| Future 對象 | 代表異步計算結果,支持 result(), exception(), cancel(), done(), add_done_callback() |
| 異常傳播 | 工作線程異常自動捕獲,調用 future.result() 時重新拋出 |
| 超時控制 | future.result(timeout), as_completed(fs, timeout) |
| 回調機制 | future.add_done_callback(fn) 完成時自動觸發 |
| 線程名稱前綴 | thread_name_prefix 便於日誌追蹤 |
| 上下文管理器 | with 塊自動 shutdown(wait=True) |
| 可擴展 | 繼承 Executor 實現進程池 ProcessPoolExecutor 同接口 |