Mini Shell

# vim: set ts=4 sw=4 expandtab syntax=python:
"""

ngxconf.fpm
New FPM Management Interface
Scalable FPM management system with supervisord

Copyright (c) 2018-2020 InMotion Hosting, Inc.
http://www.inmotionhosting.com/

@author J. Hipps <jacobh@inmotionhosting.com>


Update process:

1. builder.rebuild_user() --> fpm.update_user()
    a. If the user is new, a master group ID is chosen based on their
       UID and PHP version
    b. If changes are detected, then the cached config is updated
    c. If necessary, the user's cPanel userdata is updated to
       include the correct FPM socket path

IF changes are detected:

2. cli._main() --> fpm.commit()
    a. Master group configration is read in full. If any user/domain
       in a master group has changed, its configs are updated.
        aa. fpm.commit() --> fpm.render_config()
        ab. fpm.commit() --> masters.render_config()
    b. All affected master groups are reloaded
        ba. fpm.commit() --> masters.update_config()
3. A list of changed domains/users is returned to caller.
   FPM and Supervisord configs are now updated
4. cli._main() rebuilds/reloads Apache config, then reloads NGINX

"""
# pylint: disable=invalid-name

import os
import pwd
import logging
import json
from glob import glob

import arrow
import yaml
from yaml import CLoader, CDumper

from ngxconf.util import (gconf, get_min_uid, load_template,
                          cp_get_package_data, cp_get_userinfo, DomainCache)
from ngxconf.masters import Supervisor
from ngxconf import __version__, __date__

logger = logging.getLogger('ngxconf')
vcache = {}
MIN_UID = get_min_uid()


def get_master_id(user, phpver, msize=1, dedicated=False):
    """
    Return master_id for @user,
    with a master size of @msize
    """
    if dedicated:
        ext = user
    else:
        ext = str(int((pwd.getpwnam(user).pw_uid - MIN_UID) / msize))
    return phpver + '-' + ext

def get_mgroup_mtime(gid, vhost):
    """
    Return mtime of mgroup (@gid) cache for @vhost
    """
    cachepath = os.path.join(gconf.fpm_cache_path, gid, vhost)
    try:
        mtime = int(os.stat(cachepath).st_mtime)
    except Exception as e:
        logger.warning("mgroup cache for %s does not exist: %s", gid, str(e))
        mtime = 0
    return mtime

def update_user(username, udata, uconf, uinfo, force=False):
    """
    Update FPM configuration for a single user

    @username - Username
    @udata - User data (cPanel userdata)
    @uconf - User ngxconf configuration
    """
    global vcache

    # Get valid domains
    vlist = [x for x in list(udata.keys()) if not x.startswith('_') and not uconf.get(x, {}).get('_exclude', False)]

    # Determine Master Group (mgroup) ID based on PHP versions for each domain
    uplist = []
    for thost in vlist:
        needs_update = False
        phpver = udata[thost].get('phpversion', gconf.fallback_phpver)
        if phpver not in gconf.fpm_versions:
            logger.warning("Invalid PHP version '%s' defined for %s:%s. Falling back to system default",
                           phpver, username, thost)
            phpver = gconf.fallback_phpver

        package_data = cp_get_package_data(uinfo.get('PLAN'))

        mid = get_master_id(udata[thost]['user'], phpver, gconf.fpm_master_density,
                            package_data.get('IMH_FPM_DEDICATED', False))

        # Determine FPM socket path based on selected mgroup ID
        msockpath = os.path.join(gconf.fpm_socket_path, thost.replace('.', '_') + '.sock')

        # Update cPanel userdata, if necessary, with correct FPM sock path
        needs_cpdata_update = False
        try:
            if force or udata[thost]['ngxconf']['fpm_socket'] != msockpath:
                udata[thost]['ngxconf']['fpm_socket'] = msockpath
                needs_update = True
                needs_cpdata_update = True
            if udata[thost].get('_ssl'):
                if force or udata[thost]['_ssl']['ngxconf']['fpm_socket'] != msockpath:
                    udata[thost]['_ssl']['ngxconf']['fpm_socket'] = msockpath
                    needs_update = True
                    needs_cpdata_update = True
        except:
            udata[thost]['ngxconf'] = {'fpm_socket': msockpath}
            if udata[thost].get('_ssl'):
                udata[thost]['_ssl']['ngxconf'] = {'fpm_socket': msockpath}
            needs_update = True
            needs_cpdata_update = True

        if needs_cpdata_update:
            update_cpanel_userdata(username, thost, udata[thost])

        # Check if userdata has changed, or if forced
        mgroup_mtime = get_mgroup_mtime(mid, thost)
        if force is True or udata[thost]['_mtime'] > mgroup_mtime or \
           (udata[thost].get('_ssl') and udata[thost]['_ssl']['_mtime'] > mgroup_mtime) or \
            uinfo['_mtime'] > mgroup_mtime:
            needs_update = True

        # Build FPM data
        # Include udata (cPanel), upkg (cPanel Package), uconf (ngxconf user prefs)
        fpmdata = {
            'masterid': mid,
            'phpversion': phpver,
            'udata': udata[thost],
            'uconf': uconf[thost],
            'upkg': package_data
        }
        vcache[thost] = fpmdata

        # Write updated FPM mgroup config for user domains
        if needs_update is True:
            uplist.append(thost)
            cache_write(fpmdata, mid, thost)

    logger.debug("Finished update_user run for %s -- %d of %d domains require updating", username, len(uplist), len(vlist))
    return uplist

def commit(force=False, single_user=None):
    """
    Apply all changes

    All FPM and Supervisord configuration files are updated,
    and any PHP-FPM masters that have changed will be reloaded
    (or started, if the master is new).

    @force - If True, ALL configs are re-rendered, and all FPM masters are
             hard-restarted
    """
    # Load template and domain list
    domlist = DomainCache()
    template = load_template('user_fpm')
    supervisor = Supervisor()

    # Read FPM mgroup config for all users/domains
    change_list = []
    group_changeset = set()
    needs_reload = False
    for tgpath in glob(os.path.join(gconf.fpm_cache_path, '*')):
        fconfig = {}
        tgroup = os.path.basename(tgpath)
        tgroup_file = os.path.join(gconf.fpm_conf_path, tgroup + '.conf')
        try:
            tgroup_mtime = os.stat(tgroup_file).st_mtime
        except Exception as e:
            #logger.debug("FPM config at %s could not be read: %s", tgroup_file, str(e))
            tgroup_mtime = 0

        if not os.path.isdir(tgpath):
            logger.warning("'%s' is not a directory!", tgpath)
            continue

        # Check for vhost updates
        needs_update = False
        last_phpver = None
        for thpath in glob(os.path.join(gconf.fpm_cache_path, tgroup, '*')):
            thost = os.path.basename(thpath)
            thost_file = os.path.join(gconf.fpm_cache_path, tgroup, thost)
            thost_mtime = os.stat(thost_file).st_mtime

            # Check mtime for changes
            if thost_mtime > tgroup_mtime:
                needs_update = True
                group_changeset.add(tgroup)
                logger.debug("mgroup %s update triggered by %s change", tgroup, thost)

            # Get cached master group ID
            try:
                cached_mid = vcache[thost]['masterid']
            except:
                cached_mid = None
                pass

            # Check if domain still exists on the server
            # If not, remove it and trigger an update
            if not domlist.exists(thost) or cached_mid != tgroup:
                if not single_user:
                    if cached_mid != tgroup:
                        logger.debug("%s: cached masterid mismatch [%s != %s]", thost, cached_mid, tgroup)
                    try:
                        os.unlink(thost_file)
                        logger.info("Purged mgroup %s cache for non-existant domain %s", tgroup, thost)
                        needs_update = True
                        group_changeset.add(tgroup)
                    except Exception as e:
                        logger.error("Failed to purge stale cache file %s: %s", thost_file, str(e))
                    continue

            # Read cached config [default: /var/ngxconf/cache/TGROUP/THOST]
            try:
                with open(thost_file) as f:
                    fconfig[thost] = json.load(f)
            except Exception as e:
                logger.error("Unable to read cached mgroup vhost config %s: %s", thost_file, str(e))
                continue

            # Update change list
            try:
                last_phpver = fconfig[thost]['phpversion']
            except:
                pass
            change_list.append(thost)

        # Trigger removal of mgroup if it contains no domains
        if len(fconfig) == 0 and not single_user:
            logger.debug("Removing empty mgroup %s", tgroup)
            cp_fpm = os.path.join(gconf.fpm_conf_path, tgroup + '.conf')
            cp_cache = os.path.join(gconf.fpm_cache_path, tgroup)
            cp_supv = os.path.join(gconf.supervisord_conf_path, tgroup + '.conf')

            try:
                os.unlink(cp_fpm)
            except Exception as e:
                logger.warning("Failed to remove stale mgroup FPM config [%s]: %s", cp_fpm, str(e))

            try:
                os.rmdir(cp_cache)
            except Exception as e:
                logger.warning("Failed to remove stale mgroup cache dir [%s]: %s", cp_cache, str(e))

            try:
                os.unlink(cp_supv)
            except Exception as e:
                logger.warning("Failed to remove stale mgroup supervisord config [%s]: %s", cp_supv, str(e))

            logger.info("Removed empty mgroup %s", tgroup)
            needs_reload = True
            continue

        # Update FPM & Supervisord if triggered
        if needs_update:
            # Render new FPM config
            render_config(template, gconf.fpm_conf_path, tgroup, fconfig)

            # Render new Supervisord config
            try:
                php_binary = gconf.fpm_versions[last_phpver]
            except:
                logger.error("No PHP binary for version '%s' defined in ngxconf global config", last_phpver)
                logger.error("SKIPPING MASTER GROUP: %s", tgroup)
                continue

            supervisor.render_config(tgroup, php_binary, gconf.supervisord_conf_path)
            needs_reload = True

    # Reload Supervisord config if triggered
    if needs_reload:
        supervisor.update_config(list(group_changeset))

    return change_list

def render_config(template, confpath, pgname, pdata):
    """
    Render PHP-FPM master/pool configuration for user/pod
    """
    outconf = os.path.join(confpath, pgname + '.conf')
    pidpath = os.path.join(gconf.fpm_pid_path, pgname + '.pid')

    # render with Jinja2
    try:
        ttext = template.render(tstamp=arrow.now().format(), groupid=pgname,
                                groupset=gconf.fpm_master_config, pools=pdata,
                                pidpath=pidpath,
                                ngxconf_ver="%s (%s)" % (__version__, __date__))
    except Exception as e:
        logger.error("Failed to render FPM template for %s: %s", pgname, str(e))
        return False

    # write new include file
    try:
        with open(outconf, 'w') as f:
            f.write(ttext)
        logger.debug("Rendered FPM config for %s to %s", pgname, outconf)
    except Exception as e:
        logger.error("Failed to render template to file %s: %s", outconf, str(e))
        return False

    return True

def cache_write(fdata, gid=None, vhost=None):
    """
    Cache FPM configuration data
    @gid is mgroup ID and @vhost is vhost. None for main cache file
    """
    if gid is None:
        cachepath = os.path.join(gconf.fpm_cache_path, "fpm.cache")
        cname = "primary"
    else:
        cachedir = os.path.join(gconf.fpm_cache_path, gid)
        cachepath = os.path.join(cachedir, vhost)
        cname = gid + "/" + vhost

    # Ensure directory exists
    if not os.path.exists(cachedir):
        try:
            os.makedirs(cachedir, 0o0750)
            logger.debug("Create cache dir: %s", cachedir)
        except Exception as e:
            logger.error("Failed to create cache directory [%s]: %s", cachedir, str(e))
            return False

    # Write cache file
    try:
        with open(cachepath, 'w') as f:
            json.dump(fdata, f)
    except Exception as e:
        logger.error("Failed to write %s cache data to %s: %s", cname, cachepath, str(e))
        return False

    logger.debug("Wrote %s cache to %s", cname, cachepath)
    return True

def cache_read(gid=None, vhost=None):
    """
    Read cached FPM configuration data
    @gid is mgroup ID and @vhost is vhost. None for main cache file
    """
    if gid is None:
        cachepath = os.path.join(gconf.fpm_cache_path, "fpm.cache")
        cname = "primary"
    else:
        cachepath = os.path.join(gconf.fpm_cache_path, gid, vhost)
        cname = gid + "/" + vhost

    try:
        with open() as f:
            cdata = json.load(f)
    except Exception as e:
        logger.error("Failed to load %s cache data from %s: %s", cname, cachepath, str(e))
        return None

    logger.debug("Loaded %s cache data from %s", cname, cachepath)
    return cdata

def update_cpanel_userdata(user, domain, udata):
    """
    Write updated cPanel userdata files
    @domain is the canonical vhost name
    @udata is userdata for a single domain
    """
    udata_path = os.path.join('/var/cpanel/userdata', user, domain)
    try:
        with open(udata_path, 'w') as f:
            fudata = {k: udata[k] for k in [x for x in list(udata.keys()) if not x.startswith('_')]}
            yaml.dump(fudata, stream=f, Dumper=CDumper, default_flow_style=False)
        logger.debug("Wrote updated cPanel userdata file for %s:%s", user, domain)
    except Exception as e:
        logger.error("Failed to write updated cPanel userdata file at %s: %s", udata_path, str(e))
        return False

    ucache_path = udata_path + '.cache'
    try:
        os.unlink(ucache_path)
    except Exception as e:
        logger.warning("Failed to purge cache file [%s]: %s", ucache_path, str(e))

    if udata.get('_ssl'):
        udata_path += '_SSL'
        try:
            with open(udata_path, 'w') as f:
                fudata = {k: udata['_ssl'][k] for k in [x for x in list(udata['_ssl'].keys()) if not x.startswith('_')]}
                yaml.dump(fudata, stream=f, Dumper=CDumper, default_flow_style=False)
            logger.debug("Wrote updated cPanel SSL userdata file for %s:%s", user, domain)
        except Exception as e:
            logger.error("Failed to write updated SSL cPanel userdata file at %s: %s", udata_path, str(e))
            return False

        ucache_path = udata_path + '.cache'
        try:
            os.unlink(ucache_path)
        except Exception as e:
            logger.warning("Failed to purge SSL cache file [%s]: %s", ucache_path, str(e))

    return True