urllib.request)目標頁面包含列表項,每項需提取:

# -*- coding: utf-8 -*-
import time
from bs4 import BeautifulSoup
import urllib2
import pymongo
import re
import datetime
# MongoDB 連接(生產環境建議使用連接池與認證)
connection = pymongo.Connection('192.168.1.2', 27017)
db = connection.test_hq # 數據庫
collection = db.test # 集合
def update():
datas = {}
# 遍歷列表頁所有項目
for item in soup.find_all("div", class_="item"):
# 基礎 ID 提取(從鏈接解析)
datas['_id'] = str(item.h2.a['href']).split('/')[-1].split('.')[0]
datas['title'] = item.h2.get_text(strip=True)
# 二次請求獲取詳情內容
url2 = item.h2.a['href']
html2 = urllib2.urlopen(url2, timeout=10)
html_doc2 = html2.read()
soup2 = BeautifulSoup(html_doc2, 'html.parser')
datas['content'] = soup2.find(attrs={"name": "description"})['content']
# 股票信息提取(中文名 + 數字 ID)
stock_text = item.find(class_="stocks").get_text()
datas['stock_name'] = re.findall(u"[\u4e00-\u9fa5]+", stock_text)
datas['stock_id'] = re.findall(r"\d+", stock_text)
# 時間解析與時區轉換(UTC+8 轉 UTC)
date_str = re.search(r"\w+.*\w+", item.find(class_="fl date").span.get_text()).group()
dt = datetime.datetime.strptime(date_str, '%Y-%m-%d %H:%M') - datetime.timedelta(hours=8)
datas['update_time'] = dt
# 圍觀數
datas['onlooker'] = int(re.search(r"\d+", item.find(class_="icons ic-wg").get_text()).group())
# 寫入 MongoDB(upsert 防重複)
collection.save(datas)
def get_data():
global soup
latest_id = str(soup.h2.a['href']).split('/')[-1].split('.')[0]
# 讀取上次記錄的最新 ID
try:
with open('update.txt', 'r') as f:
last_id = f.readline().strip()
except IOError:
last_id = ''
if latest_id == last_id:
print '目前無更新:', latest_id
return
# 更新記錄並執行抓取
with open('update.txt', 'w') as f:
f.write(latest_id)
update()
# 主循環:每 30 秒輪詢一次
if __name__ == '__main__':
while True:
url = 'https://www.ipython.me/qingbao/'
html = urllib2.urlopen(url, timeout=10)
html_doc = html.read()
soup = BeautifulSoup(html_doc, 'html.parser')
get_data()
time.sleep(30)

gevent/asyncio + aiohttp 替換同步 urllib2,提升 10-50 倍吞吐_id + update_one(upsert=True) 替代 save()# Python 3 關鍵變更
import urllib.request as urllib2 # 模組重命名
# print 語句 -> 函數
# unicode 字面量默認,移除 u"" 前綴
# pymongo.Connection -> pymongo.MongoClient
這個案例展示了從頁面分析、數據提取、清洗轉換到持久化存儲的完整爬蟲流程,是數據採集項目的經典範本。