Mini Shell

# vim: set ts=4 sw=4 expandtab syntax=python:
"""

ngxstats.influx
Realtime Nginx stats aggregation tool
InfluxDB aggregate stats writer process

Copyright (c) 2019-2020 InMotion Hosting, Inc.
https://www.inmotionhosting.com/

@author J. Hipps <jacobh@inmotionhosting.com>

"""

import os
import logging
import multiprocessing
import platform
from urllib.parse import urlparse
import urllib3

import arrow
import toml
from influxdb import InfluxDBClient
from setproctitle import setproctitle

from ngxstats.util import gconf, check_alive

# disable warnings
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

logger = logging.getLogger('ngxstats')


def start(inq):
    """
    InfluxDB writer
    """
    ppid = os.getppid()
    logger.info(
        "influx: subservice started. pid = %d / ppid = %d",
        multiprocessing.current_process().pid,
        ppid,
    )
    try:
        setproctitle("ngxstats: influx")
    except Exception:
        pass

    if gconf.influx_use_telegraf_config is True:
        tconf = get_telegraf_config()
    else:
        tconf = {
            'database': gconf.influx_database,
            'host': gconf.influx_host,
            'port': gconf.influx_port,
            'username': gconf.influx_username,
            'password': gconf.influx_password,
            'ssl': gconf.influx_ssl,
            'verify_ssl': gconf.influx_verify_ssl,
        }

    # Connect to InfluxDB
    try:
        flux = InfluxDBClient(**tconf)
    except Exception as e:
        logger.error("Failed to connect to InfluxDB: %s", str(e))
        return None

    # Main processing loop
    while True:
        # Check if master is still alive
        if not check_alive(ppid):
            logger.info("influx: master %d has died. aborting.", ppid)
            os._exit(0)  # pylint: disable=protected-access

        # Grab the next batch, hot off the queue
        # This will block for 1 second, check if the parent is alive,
        # then block again until it receives a measurement
        try:
            tbatch = inq.get(1)
            logger.debug("influx: received new measurement")
        except multiprocessing.TimeoutError:
            continue

        # Construct query
        jbody = [
            {
                'measurement': gconf.influx_measurement,
                'tags': {
                    'host': platform.node(),
                },
                'time': arrow.get().isoformat(),
                'fields': tbatch,
            }
        ]
        logger.debug("influx: aggro jbody = %s", jbody)

        # Send to InfluxDB server
        try:
            flux.write_points(jbody)
            logger.debug("influx: wrote measurement to server")
        except Exception as e:
            logger.error("influx: write_points failed: %s", str(e))


def get_telegraf_config(confpath='/etc/telegraf/telegraf.conf'):
    """
    Determine InfluxDB server and credentials from local Telegraf config
    """
    oconf = {}
    try:
        with open(confpath, encoding='utf-8') as f:
            tconf = toml.load(f)
    except Exception as e:
        logger.debug("Unable to read local Telegraf configuration: %s", e)
        return None

    try:
        # Determine InfluxDB server
        sraw = tconf['outputs']['influxdb'][0]['urls']
        if isinstance(sraw, list):
            dburl = urlparse(sraw[0])
        else:
            dburl = urlparse(sraw)

        oconf['host'] = dburl.hostname
        oconf['port'] = dburl.port
        oconf['ssl'] = dburl.scheme == 'https'
        oconf['verify_ssl'] = bool(
            tconf['outputs']['influxdb'][0].get('insecure_skip_verify', False)
        )

        # Database, User, Password
        oconf['database'] = tconf['outputs']['influxdb'][0]['database']
        oconf['username'] = tconf['outputs']['influxdb'][0]['username']
        oconf['password'] = tconf['outputs']['influxdb'][0]['password']
    except Exception as e:
        logger.error("Failed to parse Telegraf config: %s", str(e))
        return None
    return oconf