Mini Shell
# vim: set ts=4 sw=4 expandtab syntax=python:
"""
ngxutil.info
Implements the --info mode
Makes a request to the provided URL and provides
helpful information about cache info, cookies, errors, etc.
@author J. Hipps <jacobh@inmotionhosting.com>
"""
import os
import re
import logging
import subprocess
from urllib.parse import urlparse
import requests
import arrow
import yaml
from yaml import CDumper, CLoader
from ngxutil.vts import parse_vts
from ngxutil.cache import find_cache_item_url
from ngxutil.util import *
logger = logging.getLogger('ngxutil')
# Disable insecure warnings in newer versions of Requests module
if 'packages' in requests.__dict__:
# pylint: disable=no-member,no-name-in-module,import-error,wrong-import-position
from requests.packages.urllib3.exceptions import InsecureRequestWarning, InsecurePlatformWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
def check_url(url):
"""
Fetch @url and analyze the headers and contents
"""
start_time = arrow.now().shift(seconds=-1)
try:
r = requests.get(url, verify=False)
except Exception as e:
logger.error("Failed to fetch URL [%s]: %s", url, str(e))
logger.error("Please NGINX is running, and that it is listening on ports 80 and 443")
return None
# Parse domain from URL and determine owner
# We need this to determine the cache zone and other info
domain = urlparse(url).netloc
uri = urlparse(url).path
duser = get_domain_owner(domain)
if not duser:
return None
# Load domain's ngxconf config
uconf = get_ngxconf_config(duser['owner'], duser['vhost'])
udata = get_userdata(duser['owner'], duser['vhost'])
# Status line
try:
st_color = {'1': "white", '2': "green", '3': "blue", '4': "yellow", '5': "red" }[str(r.status_code)[0]]
except:
st_color = "red"
print(strcolor('white', "GET %s" % (url)))
print("Elapsed: %2.03fs" % r.elapsed.total_seconds())
print("---")
print(strcolor(st_color, "%s %s" % (r.status_code, r.reason)))
# XPC header
xpc = r.headers.get('X-Proxy-Cache', "N/A")
try:
xpc_color = {'HIT': "green", 'MISS': "red", 'EXPIRED': "blue",
'BYPASS': "yellow", 'DISABLED': "red",
'STATIC/TYPE': "green", 'STATIC/PATH': "green" }[str(xpc)]
except:
xpc_color = "red"
print("Cache status: %s" % (strcolor(xpc_color, xpc)))
# Cache zone check
ccdata = find_cache_item_url(duser['owner'], url)
if ccdata:
if ccdata['expired']:
ccvalid = strcolor('yellow', "EXPIRED")
else:
ccvalid = strcolor('green', "VALID")
print("* %s -- expires %s (%s) [key: %s] " % (strcolor('green', "IN CACHE"),
ccdata['expiry_txt'], ccvalid, ccdata['key']))
print("Cache object: %s (%s)" % (ccdata['path'], format_size(ccdata['stat'].st_size)))
else:
print("* " + strcolor('red', "NOT IN CACHE"))
# Check for matching config patterns
rgx_matches = check_uri_matches(uri, uconf)
if rgx_matches:
print("* " + strcolor('green', "URI REGEX MATCH FROM CONFIG"))
map(lambda x: print("\t" + strcolor('cyan', x)), rgx_matches)
else:
if re.match(r'.+\.(ico|jpe?g|gif|png|bmp|svg|tiff|'
r'exe|dmg|zip|rar|7z|docx?|xlsx?|js|css|'
r'less|sass|scss|ttf|woff2?|mp3|mp4|mkv|'
r'avi|mov|mpe?g|aac|wav|flac)$', uri, re.I):
if uconf.get('accel_static_content'):
print("* " + strcolor('yellow', "Static content regex match (ACCELERATED)") + " -- served by NGINX")
if r.status_code == 404:
print(" TIP: Expecting this file to be served by a script or Apache htaccess rewrite?")
print(" If so, add a new force_passthru rule to the domain's ngxconf config")
else:
print("* " + strcolor('red', "Static content regex match (BYPASSED)") + " -- served by Apache")
print("---")
# Redirect
redir_loc = r.headers.get('Location')
if redir_loc:
print("* REDIRECT TO --> %s" % (strcolor('cyan', redir_loc)))
# Content-Encoding/gzip headers
gzenc = r.headers.get('Content-Encoding')
if gzenc and gzenc != 'identity':
print("* Content compressed/encoding via %s" % (strcolor('green', gzenc)))
else:
print("* " + strcolor('yellow', "No compression or content encoding"))
if "Accept-Encoding" in r.headers.get('Vary', ''):
print("* " + strcolor('green', "Vary on Accept-Encoding enabled"))
else:
print("* " + strcolor('red', "Vary on Accept-Encoding is NOT enabled"))
# Client cache-control headers
cch_list = ('Cache-Control', 'Expires', 'Pragma')
if set(cch_list).intersection(r.headers.keys()):
print("* Client cache-control headers present:")
for thead, tval in r.headers.items():
if thead in cch_list:
vcolor = ''
if thead.lower() in ('cache-control', 'pragma'):
if re.search(r'(no-cache|no-store|max\-age=0)', tval, re.I):
vcolor = 'red'
else:
vcolor = 'green'
elif thead.lower() == 'expires':
if tval == '-1' or '1970' in tval:
vcolor = 'red'
else:
vcolor = 'green'
print("\t%s: %s" % (thead, strcolor(vcolor, tval)))
if uconf.get('cache_honor_cc'):
print("* " + strcolor('yellow', "Cache-Control headers will be used to determine NGINX cache time") + \
" (cache_honor_cc == true)")
if uconf.get('cache_honor_expires'):
print("* " + strcolor('yellow', "Expires headers will be used to determine NGINX cache time") + \
" (cache_honor_expires == true)")
else:
print("* " + strcolor('yellow', "Client cache-control headers NOT present"))
# H2/Upgrade headers
h2upgrade = r.headers.get('Upgrade')
if h2upgrade:
print("* " + strcolor('yellow', "Upgrade header present: ") + strcolor('cyan', h2upgrade))
if 'h2' in h2upgrade and check_rpm_installed('ea-apache24-mod_http2'):
print("* " + strcolor('red', "HTTP/2 Upgrade header present; Apache mod_http2 installed"))
print(" TIP: Using Apache mod_http2 in combination with NGINX is not recommended,")
print(" as it can cause incorrect behavior in certain versions of Apple Safari.")
print(" Run `yum -y remove ea-apache24-mod_http2` to resolve")
print("---")
# Cookies
if len(r.cookies):
print("* " + strcolor('red', "One or more cookies are being set by the server"))
if uconf.get('cache_honor_cookies', True):
print(" TIP: Cookies will prevent the page from being cached, unless cache_honor_cookies")
print(" is set to false in the ngxconf config for this domain (default: true).")
print(" If session management is necessary, offloading these requests to an AJAX")
print(" endpoint via client-side Javascript will provide the best performance.")
else:
print("* " + strcolor('yellow', "Cookies are ignored for cache determination (cache_honor_cookies == false)"))
for ckname, ckvalue in dict(r.cookies).items():
print("\t%s = '%s'" % (ckname, ckvalue))
else:
print("* " + strcolor('green', "No cookies are being set by the server"))
print("---")
# Show additional information if a 4xx or 5xx error is encountered
if str(r.status_code)[0] in ('4', '5'):
eaphp = udata.get('phpversion', 'ea-php70')
# Show process status for certain 5xx error codes
if str(r.status_code) in ('502', '503', '504'):
show_running('Apache', '/var/run/apache2/httpd.pid')
show_running('NGINX', '/var/run/nginx.pid')
eapid = '/opt/cpanel/%s/root/usr/var/run/php-fpm/php-fpm.pid' % (eaphp)
show_running('PHP-FPM (%s)' % (eaphp), eapid)
print("---")
# Show log excerpts from PHP-FPM, Apache, and NGINX error logs
print("\n*** ERROR LOG EXCERPTS ***\n")
for tline in get_phpfpm_log(eaphp, start_time):
print("\t[fpm/%s] %s" % (eaphp, tline['msg']))
for tline in get_apache_log(start_time):
print("\t[apache] %s" % (tline['msg']))
for tline in get_nginx_log(start_time):
print("\t[nginx] %s" % (tline['msg']))
print("\n---")
def check_uri_matches(uri, uconf, ukey=None):
"""
Checks @ukey (if None, all keys) for matches of @uri against @uconf
"""
matchlist = []
for tkey, tlist in uconf.items():
if ukey is not None and ukey != tkey:
continue
if isinstance(tlist, list):
for trgx in tlist:
if re.match(r'^' + trgx, uri, re.I):
matchlist.append("%s -> %s" % (tkey, trgx))
return matchlist
def get_ngxconf_config(user, domain):
"""
Parse ngxconf config for the specified @user and @domain
"""
udir = '/home/%s/.imh/nginx' % (user)
tfile = '%s/%s.yml' % (udir, domain.lower().replace('.', '_'))
try:
with open(tfile, 'r') as f:
uconf = yaml.load(f, Loader=CLoader)
logger.debug("Parsed ngxconf configuration for domain from %s", tfile)
except Exception as e:
logger.error("Failed to parse ngxconf config for %s/%s: %s", user, domain, str(e))
return None
return uconf
def get_userdata(user, domain):
"""
Parse cPanel userdata for specified domain
"""
upath = '/var/cpanel/userdata/%s/%s' % (user, domain)
try:
with open(upath, 'r') as f:
udata = yaml.load(f, Loader=CLoader)
logger.debug("Parsed cPanel userdata for %s:%s", user, domain)
except Exception as e:
logger.error("Failed to parse cPanel userdata from %s: %s", upath, str(e))
return None
return udata
def get_phpfpm_log(phpver, start, lastlines=50):
"""
Find matching lines from PHP-FPM error log
"""
lpath = '/opt/cpanel/%s/root/usr/var/log/php-fpm/error.log' % (phpver)
try:
with open(lpath, 'r') as f:
rawlog = f.readlines()[-lastlines:]
logger.debug("Read %d lines from %s (requested %s)", len(rawlog), lpath, lastlines)
except Exception as e:
logger.error("Failed to read [%s]: %s", lpath, str(e))
return None
# parse timestamp from lines
logout = []
for tline in rawlog:
lmatch = re.match(r'^\[(?P<timestr>[^\]]+)\] (?P<level>[A-Z]+): (?P<msg>.+)$', tline)
if lmatch:
tinfo = lmatch.groupdict()
tinfo['timestamp'] = arrow.get(tinfo['timestr'], 'DD-MMM-YYYY HH:mm:ss', tzinfo='local')
if tinfo['timestamp'] > start:
logout.append(tinfo)
else:
logger.debug("Failed to parse line from PHP-FPM error log: '%s'", tline)
return logout
def get_apache_log(start, lastlines=50):
"""
Find matching lines from Apache error log
"""
lpath = '/usr/local/apache/logs/error_log'
try:
with open(lpath, 'r') as f:
rawlog = f.readlines()[-lastlines:]
logger.debug("Read %d lines from %s (requested %s)", len(rawlog), lpath, lastlines)
except Exception as e:
logger.error("Failed to read [%s]: %s", lpath, str(e))
return None
# parse timestamp from lines
logout = []
for tline in rawlog:
lmatch = re.match(r'^\[(?P<timestr>[^\]]+)\] \[(?P<module>[^:]*):(?P<level>[^\]]+)\] '
r'\[pid (?P<pid>[0-9]+):tid (?P<tid>[0-9]+)\] (?P<msg>.+)$', tline)
if lmatch:
tinfo = lmatch.groupdict()
tinfo['timestamp'] = arrow.get(tinfo['timestr'], 'ddd MMM DD HH:mm:ss.SSSSSS YYYY', tzinfo='local')
if tinfo['timestamp'] > start:
logout.append(tinfo)
else:
logger.debug("Failed to parse line from Apache error log: '%s'", tline)
return logout
def get_nginx_log(start, lastlines=50):
"""
Find matching lines from Nginx error log
"""
lpath = '/var/log/nginx/error.log'
try:
with open(lpath, 'r') as f:
rawlog = f.readlines()[-lastlines:]
logger.debug("Read %d lines from %s (requested %s)", len(rawlog), lpath, lastlines)
except Exception as e:
logger.error("Failed to read [%s]: %s", lpath, str(e))
return None
# parse timestamp from lines
logout = []
for tline in rawlog:
lmatch = re.match(r'^(?P<timestr>.+?) \[(?P<level>[^\]]+)\] (?P<pid>[0-9]+)#(?P<tid>[0-9]+): '
r'\*(?P<conn_id>[0-9]+) (?P<msg>.+)$', tline)
if lmatch:
tinfo = lmatch.groupdict()
tinfo['timestamp'] = arrow.get(tinfo['timestr'], 'YYYY/MM/DD HH:mm:ss', tzinfo='local')
if tinfo['timestamp'] >= start:
logout.append(tinfo)
else:
logger.debug("Failed to parse line from Nginx error log: '%s'", tline)
return logout
def show_running(name, pidfile):
"""
Print info about running process @name, with PID
from @pidfile
"""
xpid = check_running(pidfile)
if xpid:
print("* %s -> %s (%s)" % (name, strcolor('green', "RUNNING"), strcolor('white', xpid)))
else:
print("* %s -> %s" % (name, strcolor('red', "NOT RUNNING")))
def check_running(pidfile):
"""
Check if process is running by sending SIG0 to PID
specified in @pidfile
"""
try:
with open(pidfile, 'r') as f:
praw = f.read()
pid = int(praw.strip())
except Exception as e:
logger.debug("Failed to read pidfile at %s: %s", pidfile, str(e))
return None
try:
os.kill(pid, 0)
except OSError as e:
if e.errno == 3:
logger.debug("Process %d is not running (pidfile: %s)", pid, pidfile)
return None
except Exception as e:
logger.warning("Encountered an error when checking PID %d (pidfile: %s): %s",
pid, pidfile, str(e))
return None
return pid
def check_rpm_installed(pkgname):
"""
Checks to see if an RPM is installed
"""
try:
outstr = subprocess.check_output(['/bin/rpm', '-qa', pkgname])
if outstr.strip().lower() == pkgname.lower():
return True
else:
return False
except Exception as e:
logger.debug("Failed to run RPM command: %s", str(e))
return False
Zerion Mini Shell 1.0