Mini Shell
# vim: set ts=4 sw=4 expandtab syntax=python:
"""
ngxconf.fpm
New FPM Management Interface
Scalable FPM management system with supervisord
Copyright (c) 2018-2020 InMotion Hosting, Inc.
http://www.inmotionhosting.com/
@author J. Hipps <jacobh@inmotionhosting.com>
Update process:
1. builder.rebuild_user() --> fpm.update_user()
a. If the user is new, a master group ID is chosen based on their
UID and PHP version
b. If changes are detected, then the cached config is updated
c. If necessary, the user's cPanel userdata is updated to
include the correct FPM socket path
IF changes are detected:
2. cli._main() --> fpm.commit()
a. Master group configration is read in full. If any user/domain
in a master group has changed, its configs are updated.
aa. fpm.commit() --> fpm.render_config()
ab. fpm.commit() --> masters.render_config()
b. All affected master groups are reloaded
ba. fpm.commit() --> masters.update_config()
3. A list of changed domains/users is returned to caller.
FPM and Supervisord configs are now updated
4. cli._main() rebuilds/reloads Apache config, then reloads NGINX
"""
# pylint: disable=invalid-name
import os
import pwd
import logging
import json
from glob import glob
import arrow
import yaml
from yaml import CLoader, CDumper
from ngxconf.util import (gconf, get_min_uid, load_template,
cp_get_package_data, cp_get_userinfo, DomainCache)
from ngxconf.masters import Supervisor
from ngxconf import __version__, __date__
logger = logging.getLogger('ngxconf')
vcache = {}
MIN_UID = get_min_uid()
def get_master_id(user, phpver, msize=1, dedicated=False):
"""
Return master_id for @user,
with a master size of @msize
"""
if dedicated:
ext = user
else:
ext = str(int((pwd.getpwnam(user).pw_uid - MIN_UID) / msize))
return phpver + '-' + ext
def get_mgroup_mtime(gid, vhost):
"""
Return mtime of mgroup (@gid) cache for @vhost
"""
cachepath = os.path.join(gconf.fpm_cache_path, gid, vhost)
try:
mtime = int(os.stat(cachepath).st_mtime)
except Exception as e:
logger.warning("mgroup cache for %s does not exist: %s", gid, str(e))
mtime = 0
return mtime
def update_user(username, udata, uconf, uinfo, force=False):
"""
Update FPM configuration for a single user
@username - Username
@udata - User data (cPanel userdata)
@uconf - User ngxconf configuration
"""
global vcache
# Get valid domains
vlist = [x for x in list(udata.keys()) if not x.startswith('_') and not uconf.get(x, {}).get('_exclude', False)]
# Determine Master Group (mgroup) ID based on PHP versions for each domain
uplist = []
for thost in vlist:
needs_update = False
phpver = udata[thost].get('phpversion', gconf.fallback_phpver)
if phpver not in gconf.fpm_versions:
logger.warning("Invalid PHP version '%s' defined for %s:%s. Falling back to system default",
phpver, username, thost)
phpver = gconf.fallback_phpver
package_data = cp_get_package_data(uinfo.get('PLAN'))
mid = get_master_id(udata[thost]['user'], phpver, gconf.fpm_master_density,
package_data.get('IMH_FPM_DEDICATED', False))
# Determine FPM socket path based on selected mgroup ID
msockpath = os.path.join(gconf.fpm_socket_path, thost.replace('.', '_') + '.sock')
# Update cPanel userdata, if necessary, with correct FPM sock path
needs_cpdata_update = False
try:
if force or udata[thost]['ngxconf']['fpm_socket'] != msockpath:
udata[thost]['ngxconf']['fpm_socket'] = msockpath
needs_update = True
needs_cpdata_update = True
if udata[thost].get('_ssl'):
if force or udata[thost]['_ssl']['ngxconf']['fpm_socket'] != msockpath:
udata[thost]['_ssl']['ngxconf']['fpm_socket'] = msockpath
needs_update = True
needs_cpdata_update = True
except:
udata[thost]['ngxconf'] = {'fpm_socket': msockpath}
if udata[thost].get('_ssl'):
udata[thost]['_ssl']['ngxconf'] = {'fpm_socket': msockpath}
needs_update = True
needs_cpdata_update = True
if needs_cpdata_update:
update_cpanel_userdata(username, thost, udata[thost])
# Check if userdata has changed, or if forced
mgroup_mtime = get_mgroup_mtime(mid, thost)
if force is True or udata[thost]['_mtime'] > mgroup_mtime or \
(udata[thost].get('_ssl') and udata[thost]['_ssl']['_mtime'] > mgroup_mtime) or \
uinfo['_mtime'] > mgroup_mtime:
needs_update = True
# Build FPM data
# Include udata (cPanel), upkg (cPanel Package), uconf (ngxconf user prefs)
fpmdata = {
'masterid': mid,
'phpversion': phpver,
'udata': udata[thost],
'uconf': uconf[thost],
'upkg': package_data
}
vcache[thost] = fpmdata
# Write updated FPM mgroup config for user domains
if needs_update is True:
uplist.append(thost)
cache_write(fpmdata, mid, thost)
logger.debug("Finished update_user run for %s -- %d of %d domains require updating", username, len(uplist), len(vlist))
return uplist
def commit(force=False, single_user=None):
"""
Apply all changes
All FPM and Supervisord configuration files are updated,
and any PHP-FPM masters that have changed will be reloaded
(or started, if the master is new).
@force - If True, ALL configs are re-rendered, and all FPM masters are
hard-restarted
"""
# Load template and domain list
domlist = DomainCache()
template = load_template('user_fpm')
supervisor = Supervisor()
# Read FPM mgroup config for all users/domains
change_list = []
group_changeset = set()
needs_reload = False
for tgpath in glob(os.path.join(gconf.fpm_cache_path, '*')):
fconfig = {}
tgroup = os.path.basename(tgpath)
tgroup_file = os.path.join(gconf.fpm_conf_path, tgroup + '.conf')
try:
tgroup_mtime = os.stat(tgroup_file).st_mtime
except Exception as e:
#logger.debug("FPM config at %s could not be read: %s", tgroup_file, str(e))
tgroup_mtime = 0
if not os.path.isdir(tgpath):
logger.warning("'%s' is not a directory!", tgpath)
continue
# Check for vhost updates
needs_update = False
last_phpver = None
for thpath in glob(os.path.join(gconf.fpm_cache_path, tgroup, '*')):
thost = os.path.basename(thpath)
thost_file = os.path.join(gconf.fpm_cache_path, tgroup, thost)
thost_mtime = os.stat(thost_file).st_mtime
# Check mtime for changes
if thost_mtime > tgroup_mtime:
needs_update = True
group_changeset.add(tgroup)
logger.debug("mgroup %s update triggered by %s change", tgroup, thost)
# Get cached master group ID
try:
cached_mid = vcache[thost]['masterid']
except:
cached_mid = None
pass
# Check if domain still exists on the server
# If not, remove it and trigger an update
if not domlist.exists(thost) or cached_mid != tgroup:
if not single_user:
if cached_mid != tgroup:
logger.debug("%s: cached masterid mismatch [%s != %s]", thost, cached_mid, tgroup)
try:
os.unlink(thost_file)
logger.info("Purged mgroup %s cache for non-existant domain %s", tgroup, thost)
needs_update = True
group_changeset.add(tgroup)
except Exception as e:
logger.error("Failed to purge stale cache file %s: %s", thost_file, str(e))
continue
# Read cached config [default: /var/ngxconf/cache/TGROUP/THOST]
try:
with open(thost_file) as f:
fconfig[thost] = json.load(f)
except Exception as e:
logger.error("Unable to read cached mgroup vhost config %s: %s", thost_file, str(e))
continue
# Update change list
try:
last_phpver = fconfig[thost]['phpversion']
except:
pass
change_list.append(thost)
# Trigger removal of mgroup if it contains no domains
if len(fconfig) == 0 and not single_user:
logger.debug("Removing empty mgroup %s", tgroup)
cp_fpm = os.path.join(gconf.fpm_conf_path, tgroup + '.conf')
cp_cache = os.path.join(gconf.fpm_cache_path, tgroup)
cp_supv = os.path.join(gconf.supervisord_conf_path, tgroup + '.conf')
try:
os.unlink(cp_fpm)
except Exception as e:
logger.warning("Failed to remove stale mgroup FPM config [%s]: %s", cp_fpm, str(e))
try:
os.rmdir(cp_cache)
except Exception as e:
logger.warning("Failed to remove stale mgroup cache dir [%s]: %s", cp_cache, str(e))
try:
os.unlink(cp_supv)
except Exception as e:
logger.warning("Failed to remove stale mgroup supervisord config [%s]: %s", cp_supv, str(e))
logger.info("Removed empty mgroup %s", tgroup)
needs_reload = True
continue
# Update FPM & Supervisord if triggered
if needs_update:
# Render new FPM config
render_config(template, gconf.fpm_conf_path, tgroup, fconfig)
# Render new Supervisord config
try:
php_binary = gconf.fpm_versions[last_phpver]
except:
logger.error("No PHP binary for version '%s' defined in ngxconf global config", last_phpver)
logger.error("SKIPPING MASTER GROUP: %s", tgroup)
continue
supervisor.render_config(tgroup, php_binary, gconf.supervisord_conf_path)
needs_reload = True
# Reload Supervisord config if triggered
if needs_reload:
supervisor.update_config(list(group_changeset))
return change_list
def render_config(template, confpath, pgname, pdata):
"""
Render PHP-FPM master/pool configuration for user/pod
"""
outconf = os.path.join(confpath, pgname + '.conf')
pidpath = os.path.join(gconf.fpm_pid_path, pgname + '.pid')
# render with Jinja2
try:
ttext = template.render(tstamp=arrow.now().format(), groupid=pgname,
groupset=gconf.fpm_master_config, pools=pdata,
pidpath=pidpath,
ngxconf_ver="%s (%s)" % (__version__, __date__))
except Exception as e:
logger.error("Failed to render FPM template for %s: %s", pgname, str(e))
return False
# write new include file
try:
with open(outconf, 'w') as f:
f.write(ttext)
logger.debug("Rendered FPM config for %s to %s", pgname, outconf)
except Exception as e:
logger.error("Failed to render template to file %s: %s", outconf, str(e))
return False
return True
def cache_write(fdata, gid=None, vhost=None):
"""
Cache FPM configuration data
@gid is mgroup ID and @vhost is vhost. None for main cache file
"""
if gid is None:
cachepath = os.path.join(gconf.fpm_cache_path, "fpm.cache")
cname = "primary"
else:
cachedir = os.path.join(gconf.fpm_cache_path, gid)
cachepath = os.path.join(cachedir, vhost)
cname = gid + "/" + vhost
# Ensure directory exists
if not os.path.exists(cachedir):
try:
os.makedirs(cachedir, 0o0750)
logger.debug("Create cache dir: %s", cachedir)
except Exception as e:
logger.error("Failed to create cache directory [%s]: %s", cachedir, str(e))
return False
# Write cache file
try:
with open(cachepath, 'w') as f:
json.dump(fdata, f)
except Exception as e:
logger.error("Failed to write %s cache data to %s: %s", cname, cachepath, str(e))
return False
logger.debug("Wrote %s cache to %s", cname, cachepath)
return True
def cache_read(gid=None, vhost=None):
"""
Read cached FPM configuration data
@gid is mgroup ID and @vhost is vhost. None for main cache file
"""
if gid is None:
cachepath = os.path.join(gconf.fpm_cache_path, "fpm.cache")
cname = "primary"
else:
cachepath = os.path.join(gconf.fpm_cache_path, gid, vhost)
cname = gid + "/" + vhost
try:
with open() as f:
cdata = json.load(f)
except Exception as e:
logger.error("Failed to load %s cache data from %s: %s", cname, cachepath, str(e))
return None
logger.debug("Loaded %s cache data from %s", cname, cachepath)
return cdata
def update_cpanel_userdata(user, domain, udata):
"""
Write updated cPanel userdata files
@domain is the canonical vhost name
@udata is userdata for a single domain
"""
udata_path = os.path.join('/var/cpanel/userdata', user, domain)
try:
with open(udata_path, 'w') as f:
fudata = {k: udata[k] for k in [x for x in list(udata.keys()) if not x.startswith('_')]}
yaml.dump(fudata, stream=f, Dumper=CDumper, default_flow_style=False)
logger.debug("Wrote updated cPanel userdata file for %s:%s", user, domain)
except Exception as e:
logger.error("Failed to write updated cPanel userdata file at %s: %s", udata_path, str(e))
return False
ucache_path = udata_path + '.cache'
try:
os.unlink(ucache_path)
except Exception as e:
logger.warning("Failed to purge cache file [%s]: %s", ucache_path, str(e))
if udata.get('_ssl'):
udata_path += '_SSL'
try:
with open(udata_path, 'w') as f:
fudata = {k: udata['_ssl'][k] for k in [x for x in list(udata['_ssl'].keys()) if not x.startswith('_')]}
yaml.dump(fudata, stream=f, Dumper=CDumper, default_flow_style=False)
logger.debug("Wrote updated cPanel SSL userdata file for %s:%s", user, domain)
except Exception as e:
logger.error("Failed to write updated SSL cPanel userdata file at %s: %s", udata_path, str(e))
return False
ucache_path = udata_path + '.cache'
try:
os.unlink(ucache_path)
except Exception as e:
logger.warning("Failed to purge SSL cache file [%s]: %s", ucache_path, str(e))
return True
Zerion Mini Shell 1.0