ログ画像化(図表示)

コード

apache_log_trans_to_image.py

# -*- coding:utf-8 -*-
import argparse
import apache_log_trans_to_image_lib as alti

parser = argparse.ArgumentParser(description='apache log to graph')
parser.add_argument('log', help='log file', type=argparse.FileType('r'))
parser.add_argument('--unit', help='unit of urls', type=int, default=2)
args = parser.parse_args()


if __name__ == '__main__':
    data = alti.get_data(args.log, args.unit)
    data = alti.change_data_for_graph(data)
    alti.output_graph(data, args.unit)

apache_log_trans_to_image_lib.py

# -*- coding:utf-8 -*-


def get_data(log, unit):
    import apache_log_parser
    import itertools

    def chk_key(line):
        required_key = ('request_url_path', 'remote_host')
        for key in required_key:
            if not key in line:
                return False
        return True

    def chk_ext(line):
        request_url_path = line['request_url_path']
        except_ext = ('gif', 'jpg', 'png', 'ico', 'css', 'js', 'woff', 'ttf', 'svg')
        ext = request_url_path.split('.')[-1].lower()
        if ext in except_ext:
            return False
        return True

    data = dict()
    parser = apache_log_parser.make_parser('%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"')

    for line in log:
        line = line.strip()
        line = parser(line)
        if not chk_key(line):
            continue
        if not chk_ext(line):
            continue
        host = line['remote_host']
        request_url_path = line['request_url_path']
        if host in data:
            data[host].append(request_url_path)
        else:
            data[host] = [request_url_path]

    for host,request_url_path_list in data.items():
        request_url_path_list = list(itertools.zip_longest(*[iter(request_url_path_list)]*unit))
        request_url_path_list[-1] = [request_url_path for request_url_path in request_url_path_list[-1] if request_url_path is not None]
        data[host] = request_url_path_list

    return data


def change_data_for_graph(data):
    changed_data = dict()
    for host,request_url_path_list in data.items():
        units_of_nums = list()
        for part in request_url_path_list:
            units_of_nums.append([trans_str_to_num(url) for url in part])
        changed_data[host] = units_of_nums
    return changed_data


def trans_str_to_num(s):
    import hashlib
    import re
    s = s.encode('UTF-8')
    m = hashlib.md5()
    m.update(s)
    h = m.hexdigest()
    # hは16進数32桁
    # 4桁づつ、リストにする。
    # https://stackoverflow.com/questions/13673060/split-string-into-strings-by-length
    nums = [ int(i, 16) for i in re.split('(.{4})', h)[1::2] ]
    return nums


def output_graph(data, unit):
    import matplotlib.pyplot as plt
    import numpy as np
    from matplotlib.ticker import MultipleLocator, FormatStrFormatter
    import math

    majorLocator = MultipleLocator(20000)
    majorFormatter = FormatStrFormatter('%d')
    minorLocator = MultipleLocator(4000)

    def fillplot(x, y, pos, fig):
        # http://nihaoshijie.hatenadiary.jp/entry/2017/12/19/013413
        # https://www.mathpython.com/ja/python-number-pow/
        # https://hibiki-press.tech/learn_prog/python/round_ceil_floor/903
        w = h = math.ceil(pow(unit, 1 / 2))
        ax = fig.add_subplot(w, h, pos)
        ax.xaxis.set_major_locator(majorLocator)
        ax.xaxis.set_major_formatter(majorFormatter)
        ax.xaxis.set_minor_locator(minorLocator)
        ax.yaxis.set_major_locator(majorLocator)
        ax.yaxis.set_major_formatter(majorFormatter)
        ax.yaxis.set_minor_locator(minorLocator)
        ax.grid(which='both')
        ax.grid(which='minor', alpha=0.2)
        ax.grid(which='major', alpha=0.8)
        # 目盛の数字を表示する時は以下をコメントアウト
        plt.yticks(color='None')
        plt.xticks(color='None')
        #
        plt.xlim([-65535, 65535])
        plt.ylim([-65535, 65535])
        ax.fill(x, y, color='r')

    for ip,units_of_nums in data.items():
        f_seq = 0
        for unit_of_nums in units_of_nums:
            g_seq = 0
            fig = plt.figure(figsize=(8, 8))
            for nums in unit_of_nums:
                x = list()
                y = list()

                # 右上
                x.append(nums[0])
                y.append(nums[1])

                # 右下
                x.append(nums[2])
                y.append(-nums[3])

                # 左下
                x.append(-nums[4])
                y.append(-nums[5])

                # 左上
                x.append(-nums[6])
                y.append(nums[7])

                fillplot(x, y, g_seq + 1, fig)
                g_seq += 1
            f_seq += 1
            plt.savefig(ip + '_' + str(f_seq) + '.png')
            plt.close()
            #plt.show()

アウトプット

--unit 16の場合

192.168.56.1_1.png

--unit 10の場合

192.168.56.1_1.png