利用mysql general log日志找出查詢次數最多的SQL句子

我們知道不管是apache,iis及mysql都可以通過日志能判斷程序性能與錯誤定位了，今天我們來介紹利用mysql general log日志找出查詢次數最多的SQL句子

查詢最多的sql語句

開啟general log
 
mysql> show  variables like '%general%';
+------------------+-------------------------------------+
| Variable_name | Value |
+------------------+-------------------------------------+
| general_log | OFF |
| general_log_file | /usr/local/mysql/data/localhost.log |
+------------------+-------------------------------------+
mysql> set global general_log = "ON";
 
analysis-general-log.py腳本
 
#!/usr/bin/python
 
# sort and count mysql general log
# Author: Jason
# Created: UTC 2015-02-15 17:51:53
 
import re
import sys
import os
 
if len(sys.argv) == 2:
    logPath = sys.argv[1]
    if not os.path.exists(logPath):
        print ("file " + logPath + " does not exists.")
        sys.exit(1)
else:
    print ("Usage: " + sys.argv[0] + " logPath")
    sys.exit(1)
 
logFo = open(logPath)
match = 0
 
for line in logFo:
    line = re.sub(r"＼n","",line)
    if match == 0:
        # match line begin with numbers
        lineMatch = re.match(r"＼s+[0-9]+＼s+.*",line,flags=re.I)
        if lineMatch:
            lineTmp = lineMatch.group(0)
            match = match + 1
            continue
 
    elif match == 1:
        # match line begin with numbers
        lineMatch = re.match(r"＼s+[0-9]+＼s+.*",line,flags=re.I)
        if lineMatch:
            # match only query
            lineMatchQuery = re.match(r".*Query＼s+(.*)",lineTmp,flags=re.I)
            if lineMatchQuery:
                lineTmp = lineMatchQuery.group(1)
                # remove extra space
                lineTmp = re.sub(r"＼s+", " ",lineTmp)
                # replace values (value) to values (x)
                lineTmp = re.sub(r"values＼s*＼(.*?＼)", "values (x)",lineTmp,flags=re.I)
                # replace filed = 'value' to filed = 'x'
                lineTmp = re.sub(r"(=|>|<|>=|<=)＼s*('|＼").*?＼2","＼＼1 'x'",lineTmp)
                # replace filed = value to filed = x
                lineTmp = re.sub(r"(=|>|<|>=|<=)＼s*[0-9]+","＼＼1 x",lineTmp)
                # replace like 'value' to like 'x'
                lineTmp = re.sub(r"like＼s+('|＼").*?＼1","like 'x'",lineTmp,flags=re.I)
                # replace in (value) to in (x)
                lineTmp = re.sub(r"in＼s+＼(.*?＼)","in (x)",lineTmp,flags=re.I)
                # replace limit x,y to limit
                lineTmp = re.sub(r"limit.*","limit",lineTmp,flags=re.I)
               
                print (lineTmp)
 
            match = 1
            lineTmp = lineMatch.group(0)
        else:  
            lineTmp += line
            match = 1
 
logFo.close()

使用方法：

analysis-general-log.py general.log | sort | uniq -c | sort -nr

1032 SELECT * FROM wp_comments WHERE ( comment_approved = 'x' OR comment_approved = 'x' ) AND comment_post_ID = x ORDER BY comment_date_gmt DESC
653 SELECT post_id, meta_key, meta_value FROM wp_postmeta WHERE post_id in (x) ORDER BY meta_id ASC
527 SELECT FOUND_ROWS()
438 SELECT t.*, tt.* FROM wp_terms AS t INNER JOIN wp_term_taxonomy AS tt ON t.term_id = tt.term_id WHERE tt.taxonomy = 'x' AND t.term_id = x limit
341 SELECT option_value FROM wp_options WHERE option_name = 'x' limit
329 SELECT t.*, tt.*, tr.object_id FROM wp_terms AS t INNER JOIN wp_term_taxonomy AS tt ON tt.term_id = t.term_id INNER JOIN wp_term_relationships AS tr ON tr.term_taxonomy_id = tt.term_taxonomy_id WHERE tt.taxonomy in (x) AND tr.object_id in (x) ORDER BY t.name ASC
311 SELECT wp_posts.* FROM wp_posts WHERE 1= x AND wp_posts.ID in (x) AND wp_posts.post_type = 'x' AND ((wp_posts.post_status = 'x')) ORDER BY wp_posts.post_date DESC
219 SELECT wp_posts.* FROM wp_posts WHERE ID in (x)
218 SELECT tr.object_id FROM wp_term_relationships AS tr INNER JOIN wp_term_taxonomy AS tt ON tr.term_taxonomy_id = tt.term_taxonomy_id WHERE tt.taxonomy in (x) AND tt.term_id in (x) ORDER BY tr.object_id ASC
217 SELECT wp_posts.* FROM wp_posts WHERE 1= x AND wp_posts.ID in (x) AND wp_posts.post_type = 'x' AND ((wp_posts.post_status = 'x')) ORDER BY wp_posts.menu_order ASC
202 SELECT SQL_CALC_FOUND_ROWS wp_posts.ID FROM wp_posts WHERE 1= x AND wp_posts.post_type = 'x' AND (wp_posts.post_status = 'x') ORDER BY wp_posts.post_date DESC limit
118 SET NAMES utf8
115 SET SESSION sql_mode= 'x'
115 SELECT @@SESSION.sql_mode
112 SELECT option_name, option_value FROM wp_options WHERE autoload = 'x'
111 SELECT user_id, meta_key, meta_value FROM wp_usermeta WHERE user_id in (x) ORDER BY umeta_id ASC
108 SELECT YEAR(min(post_date_gmt)) AS firstdate, YEAR(max(post_date_gmt)) AS lastdate FROM wp_posts WHERE post_status = 'x'
108 SELECT t.*, tt.* FROM wp_terms AS t INNER JOIN wp_term_taxonomy AS tt ON t.term_id = tt.term_id WHERE tt.taxonomy in (x) AND tt.count > x ORDER BY tt.count DESC limit
107 SELECT t.*, tt.* FROM wp_terms AS t INNER JOIN wp_term_taxonomy AS tt ON t.term_id = tt.term_id WHERE tt.taxonomy in (x) AND t.term_id in (x) ORDER BY t.name ASC
107 SELECT * FROM wp_users WHERE ID = 'x'
106 SELECT SQL_CALC_FOUND_ROWS wp_posts.ID FROM wp_posts WHERE 1= x AND wp_posts.post_type = 'x' AND (wp_posts.post_status = 'x') AND post_date > 'x' ORDER BY wp_posts.post_date DESC limit
106 SELECT SQL_CALC_FOUND_ROWS wp_posts.ID FROM wp_posts WHERE 1= x AND wp_posts.post_type = 'x' AND (wp_posts.post_status = 'x') AND post_date > 'x' ORDER BY RAND() DESC limit
105 SELECT SQL_CALC_FOUND_ROWS wp_posts.ID FROM wp_posts WHERE 1= x AND wp_posts.post_type = 'x' AND (wp_posts.post_status = 'x') AND post_date > 'x' ORDER BY wp_posts.comment_count DESC limit

mysql general log日志清除技巧

mysql general log日志不能直接刪除，間接方法

USE mysql;
CREATE TABLE gn2 LIKE general_log;
RENAME TABLE general_log TO oldLogs, gn2 TO general_log;