Mini Shell

Direktori : /opt/imh-python/lib/python3.9/site-packages/ngxutil/
Upload File :
Current File : //opt/imh-python/lib/python3.9/site-packages/ngxutil/info.py

# vim: set ts=4 sw=4 expandtab syntax=python:
"""

ngxutil.info
Implements the --info mode
Makes a request to the provided URL and provides
helpful information about cache info, cookies, errors, etc.

@author J. Hipps <jacobh@inmotionhosting.com>

"""

import os
import re
import logging
import subprocess
from urllib.parse import urlparse

import requests
import arrow
import yaml
from yaml import CDumper, CLoader

from ngxutil.vts import parse_vts
from ngxutil.cache import find_cache_item_url
from ngxutil.util import *

logger = logging.getLogger('ngxutil')

# Disable insecure warnings in newer versions of Requests module
if 'packages' in requests.__dict__:
    # pylint: disable=no-member,no-name-in-module,import-error,wrong-import-position
    from requests.packages.urllib3.exceptions import InsecureRequestWarning, InsecurePlatformWarning
    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
    requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)


def check_url(url):
    """
    Fetch @url and analyze the headers and contents
    """
    start_time = arrow.now().shift(seconds=-1)

    try:
        r = requests.get(url, verify=False)
    except Exception as e:
        logger.error("Failed to fetch URL [%s]: %s", url, str(e))
        logger.error("Please NGINX is running, and that it is listening on ports 80 and 443")
        return None

    # Parse domain from URL and determine owner
    # We need this to determine the cache zone and other info
    domain = urlparse(url).netloc
    uri = urlparse(url).path
    duser = get_domain_owner(domain)
    if not duser:
        return None

    # Load domain's ngxconf config
    uconf = get_ngxconf_config(duser['owner'], duser['vhost'])
    udata = get_userdata(duser['owner'], duser['vhost'])

    # Status line
    try:
        st_color = {'1': "white", '2': "green", '3': "blue", '4': "yellow", '5': "red" }[str(r.status_code)[0]]
    except:
        st_color = "red"
    print(strcolor('white', "GET %s" % (url)))
    print("Elapsed: %2.03fs" % r.elapsed.total_seconds())
    print("---")
    print(strcolor(st_color, "%s %s" % (r.status_code, r.reason)))

    # XPC header
    xpc = r.headers.get('X-Proxy-Cache', "N/A")
    try:
        xpc_color = {'HIT': "green", 'MISS': "red", 'EXPIRED': "blue",
                     'BYPASS': "yellow", 'DISABLED': "red",
                     'STATIC/TYPE': "green", 'STATIC/PATH': "green" }[str(xpc)]
    except:
        xpc_color = "red"
    print("Cache status: %s" % (strcolor(xpc_color, xpc)))

    # Cache zone check
    ccdata = find_cache_item_url(duser['owner'], url)
    if ccdata:
        if ccdata['expired']:
            ccvalid = strcolor('yellow', "EXPIRED")
        else:
            ccvalid = strcolor('green', "VALID")
        print("* %s -- expires %s (%s) [key: %s] " % (strcolor('green', "IN CACHE"),
              ccdata['expiry_txt'], ccvalid, ccdata['key']))
        print("Cache object: %s (%s)" % (ccdata['path'], format_size(ccdata['stat'].st_size)))
    else:
        print("* " + strcolor('red', "NOT IN CACHE"))

    # Check for matching config patterns
    rgx_matches = check_uri_matches(uri, uconf)
    if rgx_matches:
        print("* " + strcolor('green', "URI REGEX MATCH FROM CONFIG"))
        map(lambda x: print("\t" + strcolor('cyan', x)), rgx_matches)
    else:
        if re.match(r'.+\.(ico|jpe?g|gif|png|bmp|svg|tiff|'
                    r'exe|dmg|zip|rar|7z|docx?|xlsx?|js|css|'
                    r'less|sass|scss|ttf|woff2?|mp3|mp4|mkv|'
                    r'avi|mov|mpe?g|aac|wav|flac)$', uri, re.I):
            if uconf.get('accel_static_content'):
                print("* " + strcolor('yellow', "Static content regex match (ACCELERATED)") + " -- served by NGINX")
                if r.status_code == 404:
                    print("  TIP: Expecting this file to be served by a script or Apache htaccess rewrite?")
                    print("       If so, add a new force_passthru rule to the domain's ngxconf config")
            else:
                print("* " + strcolor('red', "Static content regex match (BYPASSED)") + " -- served by Apache")

    print("---")

    # Redirect
    redir_loc = r.headers.get('Location')
    if redir_loc:
        print("* REDIRECT TO --> %s" % (strcolor('cyan', redir_loc)))

    # Content-Encoding/gzip headers
    gzenc = r.headers.get('Content-Encoding')
    if gzenc and gzenc != 'identity':
        print("* Content compressed/encoding via %s" % (strcolor('green', gzenc)))
    else:
        print("* " + strcolor('yellow', "No compression or content encoding"))

    if "Accept-Encoding" in r.headers.get('Vary', ''):
        print("* " + strcolor('green', "Vary on Accept-Encoding enabled"))
    else:
        print("* " + strcolor('red', "Vary on Accept-Encoding is NOT enabled"))

    # Client cache-control headers
    cch_list = ('Cache-Control', 'Expires', 'Pragma')
    if set(cch_list).intersection(r.headers.keys()):
        print("* Client cache-control headers present:")
        for thead, tval in r.headers.items():
            if thead in cch_list:
                vcolor = ''
                if thead.lower() in ('cache-control', 'pragma'):
                    if re.search(r'(no-cache|no-store|max\-age=0)', tval, re.I):
                        vcolor = 'red'
                    else:
                        vcolor = 'green'
                elif thead.lower() == 'expires':
                    if tval == '-1' or '1970' in tval:
                        vcolor = 'red'
                    else:
                        vcolor = 'green'
                print("\t%s: %s" % (thead, strcolor(vcolor, tval)))
        if uconf.get('cache_honor_cc'):
            print("* " + strcolor('yellow', "Cache-Control headers will be used to determine NGINX cache time") + \
                  " (cache_honor_cc == true)")
        if uconf.get('cache_honor_expires'):
            print("* " + strcolor('yellow', "Expires headers will be used to determine NGINX cache time") + \
                  " (cache_honor_expires == true)")
    else:
        print("* " + strcolor('yellow', "Client cache-control headers NOT present"))

    # H2/Upgrade headers
    h2upgrade = r.headers.get('Upgrade')
    if h2upgrade:
        print("* " + strcolor('yellow', "Upgrade header present: ") + strcolor('cyan', h2upgrade))
        if 'h2' in h2upgrade and check_rpm_installed('ea-apache24-mod_http2'):
            print("* " + strcolor('red', "HTTP/2 Upgrade header present; Apache mod_http2 installed"))
            print("  TIP: Using Apache mod_http2 in combination with NGINX is not recommended,")
            print("       as it can cause incorrect behavior in certain versions of Apple Safari.")
            print("       Run `yum -y remove ea-apache24-mod_http2` to resolve")
    print("---")

    # Cookies
    if len(r.cookies):
        print("* " + strcolor('red', "One or more cookies are being set by the server"))
        if uconf.get('cache_honor_cookies', True):
            print("  TIP: Cookies will prevent the page from being cached, unless cache_honor_cookies")
            print("       is set to false in the ngxconf config for this domain (default: true).")
            print("       If session management is necessary, offloading these requests to an AJAX")
            print("       endpoint via client-side Javascript will provide the best performance.")
        else:
            print("* " + strcolor('yellow', "Cookies are ignored for cache determination (cache_honor_cookies == false)"))
        for ckname, ckvalue in dict(r.cookies).items():
            print("\t%s = '%s'" % (ckname, ckvalue))
    else:
        print("* " + strcolor('green', "No cookies are being set by the server"))
    print("---")

    # Show additional information if a 4xx or 5xx error is encountered
    if str(r.status_code)[0] in ('4', '5'):
        eaphp = udata.get('phpversion', 'ea-php70')

        # Show process status for certain 5xx error codes
        if str(r.status_code) in ('502', '503', '504'):
            show_running('Apache', '/var/run/apache2/httpd.pid')
            show_running('NGINX', '/var/run/nginx.pid')
            eapid = '/opt/cpanel/%s/root/usr/var/run/php-fpm/php-fpm.pid' % (eaphp)
            show_running('PHP-FPM (%s)' % (eaphp), eapid)
            print("---")

        # Show log excerpts from PHP-FPM, Apache, and NGINX error logs
        print("\n*** ERROR LOG EXCERPTS ***\n")
        for tline in get_phpfpm_log(eaphp, start_time):
            print("\t[fpm/%s] %s" % (eaphp, tline['msg']))

        for tline in get_apache_log(start_time):
            print("\t[apache] %s" % (tline['msg']))

        for tline in get_nginx_log(start_time):
            print("\t[nginx] %s" % (tline['msg']))

        print("\n---")


def check_uri_matches(uri, uconf, ukey=None):
    """
    Checks @ukey (if None, all keys) for matches of @uri against @uconf
    """
    matchlist = []
    for tkey, tlist in uconf.items():
        if ukey is not None and ukey != tkey:
            continue

        if isinstance(tlist, list):
            for trgx in tlist:
                if re.match(r'^' + trgx, uri, re.I):
                    matchlist.append("%s -> %s" % (tkey, trgx))

    return matchlist

def get_ngxconf_config(user, domain):
    """
    Parse ngxconf config for the specified @user and @domain
    """
    udir = '/home/%s/.imh/nginx' % (user)
    tfile = '%s/%s.yml' % (udir, domain.lower().replace('.', '_'))

    try:
        with open(tfile, 'r') as f:
            uconf = yaml.load(f, Loader=CLoader)
        logger.debug("Parsed ngxconf configuration for domain from %s", tfile)
    except Exception as e:
        logger.error("Failed to parse ngxconf config for %s/%s: %s", user, domain, str(e))
        return None

    return uconf

def get_userdata(user, domain):
    """
    Parse cPanel userdata for specified domain
    """
    upath = '/var/cpanel/userdata/%s/%s' % (user, domain)

    try:
        with open(upath, 'r') as f:
            udata = yaml.load(f, Loader=CLoader)
        logger.debug("Parsed cPanel userdata for %s:%s", user, domain)
    except Exception as e:
        logger.error("Failed to parse cPanel userdata from %s: %s", upath, str(e))
        return None
    return udata

def get_phpfpm_log(phpver, start, lastlines=50):
    """
    Find matching lines from PHP-FPM error log
    """
    lpath = '/opt/cpanel/%s/root/usr/var/log/php-fpm/error.log' % (phpver)

    try:
        with open(lpath, 'r') as f:
            rawlog = f.readlines()[-lastlines:]
            logger.debug("Read %d lines from %s (requested %s)", len(rawlog), lpath, lastlines)
    except Exception as e:
        logger.error("Failed to read [%s]: %s", lpath, str(e))
        return None

    # parse timestamp from lines
    logout = []
    for tline in rawlog:
        lmatch = re.match(r'^\[(?P<timestr>[^\]]+)\] (?P<level>[A-Z]+): (?P<msg>.+)$', tline)
        if lmatch:
            tinfo = lmatch.groupdict()
            tinfo['timestamp'] = arrow.get(tinfo['timestr'], 'DD-MMM-YYYY HH:mm:ss', tzinfo='local')
            if tinfo['timestamp'] > start:
                logout.append(tinfo)
        else:
            logger.debug("Failed to parse line from PHP-FPM error log: '%s'", tline)

    return logout

def get_apache_log(start, lastlines=50):
    """
    Find matching lines from Apache error log
    """
    lpath = '/usr/local/apache/logs/error_log'

    try:
        with open(lpath, 'r') as f:
            rawlog = f.readlines()[-lastlines:]
            logger.debug("Read %d lines from %s (requested %s)", len(rawlog), lpath, lastlines)
    except Exception as e:
        logger.error("Failed to read [%s]: %s", lpath, str(e))
        return None

    # parse timestamp from lines
    logout = []
    for tline in rawlog:
        lmatch = re.match(r'^\[(?P<timestr>[^\]]+)\] \[(?P<module>[^:]*):(?P<level>[^\]]+)\] '
                          r'\[pid (?P<pid>[0-9]+):tid (?P<tid>[0-9]+)\] (?P<msg>.+)$', tline)
        if lmatch:
            tinfo = lmatch.groupdict()
            tinfo['timestamp'] = arrow.get(tinfo['timestr'], 'ddd MMM DD HH:mm:ss.SSSSSS YYYY', tzinfo='local')
            if tinfo['timestamp'] > start:
                logout.append(tinfo)
        else:
            logger.debug("Failed to parse line from Apache error log: '%s'", tline)

    return logout

def get_nginx_log(start, lastlines=50):
    """
    Find matching lines from Nginx error log
    """
    lpath = '/var/log/nginx/error.log'

    try:
        with open(lpath, 'r') as f:
            rawlog = f.readlines()[-lastlines:]
            logger.debug("Read %d lines from %s (requested %s)", len(rawlog), lpath, lastlines)
    except Exception as e:
        logger.error("Failed to read [%s]: %s", lpath, str(e))
        return None

    # parse timestamp from lines
    logout = []
    for tline in rawlog:
        lmatch = re.match(r'^(?P<timestr>.+?) \[(?P<level>[^\]]+)\] (?P<pid>[0-9]+)#(?P<tid>[0-9]+): '
                          r'\*(?P<conn_id>[0-9]+) (?P<msg>.+)$', tline)
        if lmatch:
            tinfo = lmatch.groupdict()
            tinfo['timestamp'] = arrow.get(tinfo['timestr'], 'YYYY/MM/DD HH:mm:ss', tzinfo='local')
            if tinfo['timestamp'] >= start:
                logout.append(tinfo)
        else:
            logger.debug("Failed to parse line from Nginx error log: '%s'", tline)

    return logout

def show_running(name, pidfile):
    """
    Print info about running process @name, with PID
    from @pidfile
    """
    xpid = check_running(pidfile)
    if xpid:
        print("* %s -> %s (%s)" % (name, strcolor('green', "RUNNING"), strcolor('white', xpid)))
    else:
        print("* %s -> %s" % (name, strcolor('red', "NOT RUNNING")))

def check_running(pidfile):
    """
    Check if process is running by sending SIG0 to PID
    specified in @pidfile
    """
    try:
        with open(pidfile, 'r') as f:
            praw = f.read()
            pid = int(praw.strip())
    except Exception as e:
        logger.debug("Failed to read pidfile at %s: %s", pidfile, str(e))
        return None

    try:
        os.kill(pid, 0)
    except OSError as e:
        if e.errno == 3:
            logger.debug("Process %d is not running (pidfile: %s)", pid, pidfile)
            return None
    except Exception as e:
        logger.warning("Encountered an error when checking PID %d (pidfile: %s): %s",
                       pid, pidfile, str(e))
        return None

    return pid

def check_rpm_installed(pkgname):
    """
    Checks to see if an RPM is installed
    """
    try:
        outstr = subprocess.check_output(['/bin/rpm', '-qa', pkgname])
        if outstr.strip().lower() == pkgname.lower():
            return True
        else:
            return False
    except Exception as e:
        logger.debug("Failed to run RPM command: %s", str(e))
        return False

Zerion Mini Shell 1.0