コード
apache_log_trans_to_image.py
# -*- coding:utf-8 -*- import argparse import apache_log_trans_to_image_lib as alti parser = argparse.ArgumentParser(description='apache log to graph') parser.add_argument('log', help='log file', type=argparse.FileType('r')) parser.add_argument('--hash', help='define hash type', type=str, choices=['md5', 'sha256'], default='md5') parser.add_argument('--unit', help='unit of urls', type=int, default=2) args = parser.parse_args() if __name__ == '__main__': data = alti.get_data(args.log, args.unit) data = alti.change_data_for_graph(data, args.hash) alti.output_graph(data, args.unit, args.hash)
apache_log_trans_to_image_lib.py
# -*- coding:utf-8 -*- def get_data(log, unit): import apache_log_parser import itertools def chk_key(line): required_key = ('request_url_path', 'remote_host') for key in required_key: if not key in line: return False return True def chk_ext(line): request_url_path = line['request_url_path'] except_ext = ('gif', 'jpg', 'png', 'ico', 'css', 'js', 'woff', 'ttf', 'svg') ext = request_url_path.split('.')[-1].lower() if ext in except_ext: return False return True data = dict() parser = apache_log_parser.make_parser('%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"') for line in log: line = line.strip() line = parser(line) if not chk_key(line): continue if not chk_ext(line): continue host = line['remote_host'] request_url_path = line['request_url_path'] if host in data: data[host].append(request_url_path) else: data[host] = [request_url_path] for host,request_url_path_list in data.items(): request_url_path_list = list(itertools.zip_longest(*[iter(request_url_path_list)]*unit)) request_url_path_list[-1] = [request_url_path for request_url_path in request_url_path_list[-1] if request_url_path is not None] data[host] = request_url_path_list return data def change_data_for_graph(data, h): changed_data = dict() for host,request_url_path_list in data.items(): units_of_nums = list() for part in request_url_path_list: units_of_nums.append([trans_str_to_num(url, h) for url in part]) changed_data[host] = units_of_nums return changed_data def trans_str_to_num(s, h): import hashlib import re s = s.encode('UTF-8') if h == 'md5': m = hashlib.md5() if h == 'sha256': m = hashlib.sha256() m.update(s) h = m.hexdigest() # hは16進数32桁 # 4桁づつ、リストにする。 # https://stackoverflow.com/questions/13673060/split-string-into-strings-by-length nums = [ int(i, 16) for i in re.split('(.{4})', h)[1::2] ] return nums def output_graph(data, unit, h): import numpy as np import matplotlib.pyplot as plt if h == 'md5': xtick = 8 if h == 'sha256': xtick = 16 # https://stackoverflow.com/questions/24943991/change-grid-interval-and-specify-tick-labels-in-matplotlib for ip,units_of_nums in data.items(): seq = 0 for unit_of_nums in units_of_nums: nums = list() for num in unit_of_nums: nums.extend(num) x, y = (range(len(nums)), nums) fig, ax = plt.subplots() major_xticks = np.arange(0, unit*xtick+1, xtick) minor_xticks = np.arange(0, unit*xtick+1, 1) major_yticks = np.arange(0, 65535+1, 10000) minor_yticks = np.arange(0, 65535+1, 1000) ax.set_xticks(major_xticks) ax.set_xticks(minor_xticks, minor=True) ax.set_yticks(major_yticks) ax.set_yticks(minor_yticks, minor=True) ax.grid(which='both') ax.grid(which='minor', alpha=0.2) ax.grid(which='major', alpha=0.8) plt.plot(x, y, color='red', lw=0.5) # 目盛を表示する場合、以下をコメントアウト plt.yticks(color='None') plt.xticks(color='None') # plt.xlim([0, unit*xtick]) plt.ylim([0, 65535]) plt.savefig(ip + '_' + str(seq) + '.png') plt.close() seq += 1
アウトプット
以下ログの場合(--unitはデフォルト10)
192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:34:54 +0900] "GET /wp/wp-admin/update-core.php HTTP/1.1" 200 16579 "http://192.168.56.101/wp/wp-admin/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:34:54 +0900] "GET /wp/wp-admin/update-core.php HTTP/1.1" 200 16579 "http://192.168.56.101/wp/wp-admin/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:34:54 +0900] "GET /wp/wp-admin/update-core.php HTTP/1.1" 200 16579 "http://192.168.56.101/wp/wp-admin/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36"
192.168.56.1_0.png
以下ログの場合(--unitはデフォルト10)
192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36" 192.168.56.1 - - [08/Jul/2018:12:24:05 +0900] "GET /wp/wp-admin/install.php HTTP/1.1" 500 3606 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36"
