利用 Shell 腳本每日自動分析前一天 Nginx 存取日誌,提取百度蜘蛛爬取的 404 路徑,生成 death.txt 供百度站長平台死鏈提交工具抓取,實現死鏈自動收集與提交。
+https://www.baidu.com/search/spider.html)death.txt(完整 URL 格式)deathlink.sh#!/bin/bash
# Desc: Death Chain File Script
# Author: ZhangGe
UA="+https://www.baidu.com/search/spider.html"
DATE=$(date +%Y-%m-%d -d "1 day ago")
logfile="/home/wwwlogs/yourdomain_${DATE}.log"
deathfile="/home/wwwroot/yourdomain/death.txt"
website="https://yourdomain"
for url in $(cat ${logfile} | grep -i "${UA}" | awk '{print $7 " " $9}' | grep " 404" | awk '{print $1}')
do
grep -q "$url" ${deathfile} || echo "${website}${url}" >> ${deathfile}
done
/root/deathlink.sh,賦予執行權限 chmod +x。crontab -e,加入:0 1 * * * /root/deathlink.sh >/dev/null 2>&1 (每天凌晨 1 點執行)https://yourdomain/death.txt 確認內容。若尚未配置日誌切割,可用單一腳本同時完成:
#!/bin/bash
LOGS_PATH=/home/wwwlogs
UA="+https://www.baidu.com/search/spider.html"
DOMAIN=yourdomain
DATE=$(date +%Y-%m-%d -d "1 day ago")
logfile="${LOGS_PATH}/${DOMAIN}_${DATE}.log"
deathfile="/home/wwwroot/${DOMAIN}/death.txt"
website="https://${DOMAIN}"
# 1. 切割日誌
mv ${LOGS_PATH}/${DOMAIN}.log ${logfile}
kill -USR1 $(ps axu | grep "nginx: master process" | grep -v grep | awk '{print $2}')
# 可選:刪除 30 天前舊日誌
find ${LOGS_PATH} -mtime +30 -name "*20[1-9][3-9]*" | xargs rm -f
# 2. 生成死鏈
for url in $(cat ${logfile} | grep -i "${UA}" | awk '{print $7 " " $9}' | grep " 404" | awk '{print $1}')
do
grep -q "$url" ${deathfile} || echo "${website}${url}" >> ${deathfile}
done