目標:從騰訊校友介面 (api.pengyou.qq.com) 批次抓取全省市大學校名稱、代碼、拼音首字母,寫入本地資料庫供自動完成/下拉選單使用。
<?php
// config.php
return [
'db' => [
'host' => '127.0.0.1',
'user' => 'collector',
'pass' => 'secret',
'name' => 'school_db',
'charset' => 'utf8mb4',
],
'source' => [
'base_url' => 'https://api.pengyou.qq.com/index.php',
'params' => [
'mod' => 'school',
'act' => 'selector',
'schooltype' => 0, // 大學
'country' => 0,
],
'provinces' => range(11, 82), // 省份代碼範圍
'prefixes' => range('a', 'z'), // 拼音首字母
],
'rate_limit' => [
'min_delay_ms' => 500,
'max_delay_ms' => 1500,
'retry_times' => 3,
],
];
// collector.php
require_once 'vendor/autoload.php';
$config = require 'config.php';
$pdo = new PDO(
"mysql:host={$config['db']['host']};dbname={$config['db']['name']};charset={$config['db']['charset']}",
$config['db']['user'],
$config['db']['pass'],
[PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC]
);
// 建表 (若不存在)
$pdo->exec("CREATE TABLE IF NOT EXISTS schools (
id INT AUTO_INCREMENT PRIMARY KEY,
area_id VARCHAR(10) NOT NULL,
name VARCHAR(100) NOT NULL,
type TINYINT DEFAULT 0,
prefix CHAR(1) NOT NULL,
UNIQUE KEY uniq_area_name (area_id, name)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4");
$client = new GuzzleHttp\Client(['timeout' => 10, 'http_errors' => false]);
foreach ($config['source']['provinces'] as $province) {
$batch = [];
foreach ($config['source']['prefixes'] as $prefix) {
$params = array_merge($config['source']['params'], [
'province' => $province,
'prefix' => $prefix,
]);
$url = $config['source']['base_url'] . '?' . http_build_query($params);
// 指數退避重試
$attempt = 0;
$response = null;
while ($attempt < $config['rate_limit']['retry_times']) {
$response = $client->get($url);
if ($response->getStatusCode() === 200) break;
$attempt++;
usleep(mt_rand($config['rate_limit']['min_delay_ms'], $config['rate_limit']['max_delay_ms']) * 1000);
}
if ($response->getStatusCode() !== 200) {
error_log("Failed: $url after {$config['rate_limit']['retry_times']} retries");
continue;
}
$html = $response->getBody()->getContents();
// 解析 JavaScript 呼叫格式: 網頁特效:choose_school(12345,'學校名稱')
if (preg_match_all('#網頁特效:choose_school\((\d+),\s*\'([^\']+)\'\)#i', $html, $matches)) {
foreach ($matches[2] as $idx => $name) {
$areaId = $matches[1][$idx] . '0000'; // 補齊區域代碼
$batch[] = "('{$areaId}', '{$pdo->quote($name)}', 0, '{$prefix}')";
}
}
// 禮貌延遲
usleep(mt_rand($config['rate_limit']['min_delay_ms'], $config['rate_limit']['max_delay_ms']) * 1000);
}
if (!empty($batch)) {
$sql = "INSERT IGNORE INTO schools (area_id, name, type, prefix) VALUES " . implode(',', $batch);
$pdo->exec($sql);
echo "Province {$province}0000: inserted " . $pdo->lastInsertId() . " records\n";
}
}
?>
file_get_contents,改用 Guzzle / Symfony HttpClient,支援超時、重試、並發、Cookie Jar、User-Agent 旋轉。preg_match_all 擷取參數。若結構複雜,考慮 V8Js 執行 JS 或無頭瀏覽器 (Puppeteer/Playwright)。robots.txt、RateLimit 標頭UNIQUE KEY + INSERT IGNORE / ON DUPLICATE KEY UPDATE。robots.txt、服務條款,評估資料著作權風險,必要時申請授權或使用官方 API。