Back

sensu插件nginx-access

2015-12-25 10:37:19 Sensu zheng

sensu日志分析插件。

nginx日志格式:

# -*- coding: utf-8 -*-

import os
import time
import json
import socket
import hashlib
from collections import defaultdict
from sensu_plugin import SensuPluginMetricGraphite

class NginxAccessMetrics(SensuPluginMetricGraphite):
    """
    This is a script for check nginx access 
    """
    def __init__(self):
        self.last_metrics_dir = '/tmp/.nginx_access_metrics'
        if not os.path.isdir(self.last_metrics_dir):
            os.mkdir(self.last_metrics_dir)
        super(NginxAccessMetrics, self).__init__()

    def setup(self):
        self.parser.add_argument(
            '-l',
            '--log',
            required=True,
            help='nginx access log'
        )
        self.parser.add_argument(
            '-s',
            '--schema',
            default='{0}.nginx.access'.format(socket.gethostname()),
            help='metrics schema'
        )

    def _get_offset_db(self):
        _db_file = os.path.join(
            self.last_metrics_dir, '.' + hashlib.md5(self.options.log).hexdigest())
        self.offset_db = _db_file

    def _get_offset(self):
        _ret = {
            'path': self.options.log,
            'inode': 0,
            'offset': 0
        }
        if os.path.isfile(self.offset_db):
            _offset_info = open(self.offset_db, 'r').read().strip().split()
            if len(_offset_info) == 3:
                _ret['inode'] = int(_offset_info[1])
                _ret['offset'] = int(_offset_info[2])
        return _ret

    def _write_offset(self, inode, offset):
        with open(self.offset_db, 'w') as fd:
            fd.write('{0} {1} {2}'.format(self.options.log, inode, offset))

    def _get_timestamp(self, time_str):
        ctime = time_str.split('+')[0]
        return time.mktime(time.strptime(ctime,'%Y-%m-%dT%H:%M:%S'))

    def _get_log_fds(self, inode, offset):
        _ret = []
        if inode == 0:
            inode = os.stat(self.options.log).st_ino
            fd = open(self.options.log, 'r')
            _ret.append((fd, inode))
        else:
            if os.stat(self.options.log).st_ino == inode:
                fd = open(self.options.log, 'r')
                fd.seek(offset)
                _ret.append((fd, inode))
            else:
                # Log rotate?
                log_dir = os.path.dirname(os.path.abspath(self.options.log))
                for _each in os.listdir(log_dir):
                    _each = os.path.join(log_dir, _each)
                    if os.stat(_each).st_ino == inode:
                        fd = open(_each, 'r')
                        fd.seek(offset)
                        _ret.append((fd, inode))
                        break
                # Add new log fd
                inode = os.stat(self.options.log).st_ino
                fd = open(self.options.log, 'r')
                _ret.append((fd, inode))
        return _ret

    def run(self):
        bandwidth = 0
        start_time = None
        end_time = None
        domain_map = defaultdict(dict)
        self._get_offset_db()
        current_offset = self._get_offset()
        fds = self._get_log_fds(current_offset['inode'], current_offset['offset'])

        for _each_fd, _each_ino in fds:
            for eachLine in _each_fd:
                try:
                    json_Line = json.loads(eachLine.decode('unicode-escape'), strict=False) 
                    if start_time == None:
                        start_time = json_Line['@timestamp']
                        end_time = json_Line['@timestamp']
                    if start_time > json_Line['@timestamp']:
                        start_time = json_Line['@timestamp']
                    elif end_time < json_Line['@timestamp']:
                        end_time = json_Line['@timestamp']
                    domain = json_Line['@fields']['host']
                    status_code = int(json_Line['@fields']['status'])
                    bits_sent = int(json_Line['@fields']['bytes_sent']) * 8
                except Exception:
                    continue
                if domain not in domain_map:
                    domain_map[domain].update(dict(
                        status=defaultdict(int),
                        bandwidth=0
                    ))
                domain_map[domain]['status']['total'] += 1
                domain_map[domain]['status'][status_code] += 1
                domain_map[domain]['bandwidth'] += bits_sent

        if start_time and end_time:
            start_time = self._get_timestamp(start_time)
            end_time = self._get_timestamp(end_time)
            end_time += 1
            interval = end_time - start_time
        # # Write offset
        self._write_offset(_each_ino, _each_fd.tell())

        if domain_map:
            for domain, info in domain_map.iteritems():
                for status_code, status_value in info['status'].iteritems():
                    self.output("{0},domain={1},type=status_code,staut_code={2}".format(
                        self.options.schema, domain, status_code), status_value/interval)
                self.output("{0},domain={1},type={2}".format(
                    self.options.schema, domain, 'bandwidth'), info['bandwidth']/interval)
        self.ok()

if __name__ == "__main__":
    NginxAccessMetrics()

In: http://www.opdev.cn/sensu/sensu-nginx-access.html


打赏!感谢你的支持,我会继续努力!

Looking for something?

About.Me

img zheng
90后,男,耳机发烧友,古典、新世纪、美剧迷。现就职于某教育公司,任运维开发工程师。
QQ: 773889242 Email: bbaobelief@163.com

小鱼儿