Mini Shell

# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
import logging
import json
import time
import typing
from typing import TypedDict, NamedTuple, Self
from collections import ChainMap

from clcommon.clproc import ProcLve
from lveapi import PyLve
from lvectllib import LVE_DEFAULT
from lvestat import LVEStat

from lvestats.plugins.generic.burster.config import read_raw_config, get_boolean
from lvestats.plugins.generic.burster.overload import OverloadChecker, read_times_from_proc

EFFECTIVE_NORMAL_LIMITS_PATH = '/var/run/cloudlinux/effective-normal-limits'

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('BurstWatcher')

# Separate debug log handler
handler = logging.FileHandler('/var/log/lvestats-burstwatcher.log')
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
handler.setLevel(logging.DEBUG)
logger.addHandler(handler)


class Service:
    def __init__(self) -> None:
        self._pylve = PyLve()
        if not self._pylve.initialize():
            raise RuntimeError('pylve initialization failed')

        config = _Config.read_from_config_file()
        self._enabled = config.enabled
        self._overload_checker = OverloadChecker(
            idle_time_threshold=config.idle_time_threshold,
            max_samples_number=config.max_samples,
            get_stats=read_times_from_proc
        )
        self._current_bursting_lves = {}
        self._effective_normal_limits = {}

    def run(self) -> None:
        cnt = 0
        while True:
            time.sleep(1)
            if not self._enabled:
                continue

            if self._is_server_overloaded():
                self._load_effective_normal_limits()
                self._get_current_bursting_lves()
                self._disable_bursting()

            elif cnt % 15 == 0:
                self._load_effective_normal_limits()
                cnt = 0
            cnt += 1

    def _load_effective_normal_limits(self):
        try:
            with open(EFFECTIVE_NORMAL_LIMITS_PATH, 'r', encoding='UTF-8') as f:
                loaded_effective_normal_limits = json.loads(f.read())
                if len(loaded_effective_normal_limits) > 0:
                    self._effective_normal_limits = loaded_effective_normal_limits
        except (OSError, IOError, json.JSONDecodeError):
            logger.exception('Can`t load cached effective normal limits from file: %s', EFFECTIVE_NORMAL_LIMITS_PATH)
        logger.debug('Loaded effective normal limits for %s lves', len(self._effective_normal_limits))

    def _is_server_overloaded(self) -> bool:
        checker_result = self._overload_checker()
        logger.debug('Curent server load: %s', checker_result.server_load)
        return checker_result.is_overloaded

    def _get_current_bursting_lves(self):
        proc_lve = ProcLve()
        lve_stats = [LVEStat(line, proc_lve.version()) for line in proc_lve.lines(without_limits=True)]

        self._current_bursting_lves = {
            str(stat.id) for stat in lve_stats
            if stat.cpu != self._effective_normal_limits.get(str(stat.id), {}).get('cpu') and stat.id != 0
        }
        logger.debug('Current bursting lves: %s', self._current_bursting_lves)

    def _disable_bursting(self):
        for _id in self._current_bursting_lves:
            if _id == '0':
                logger.debug('Skipping lve_id=0')
                continue

            self._disable_bursting_for_lve(_id)

        logger.debug('Disabled bursting server-wide')

    def _disable_bursting_for_lve(self, id_: str):
        logger.debug('Disabling bursting for LVE %s', id_)
        lve_limits = self._effective_normal_limits.get(id_, {})
        default_limits = self._effective_normal_limits.get('0', {})

        try:
            lve_settings = self._pylve.liblve_settings()

            lve_settings.ls_io = int(lve_limits.get('io', default_limits.get('io', LVE_DEFAULT['io'])))
            lve_settings.ls_cpu = int(lve_limits.get('cpu', default_limits.get('cpu', LVE_DEFAULT['cpu'])))
            lve_settings.ls_cpus = int(lve_limits.get('ncpu', default_limits.get('ncpu', LVE_DEFAULT['ncpu'])))
            lve_settings.ls_memory = int(lve_limits.get('mem', default_limits.get('mem', LVE_DEFAULT['mem'])))
            lve_settings.ls_enters = int(lve_limits.get('ep', default_limits.get('ep', LVE_DEFAULT['ep'])))
            lve_settings.ls_memory_phy = int(lve_limits.get('pmem', default_limits.get('pmem', LVE_DEFAULT['pmem'])))
            lve_settings.ls_nproc = int(lve_limits.get('nproc', default_limits.get('nproc', LVE_DEFAULT['nproc'])))
            lve_settings.ls_iops = int(lve_limits.get('iops', default_limits.get('iops', LVE_DEFAULT['iops'])))

            self._pylve.lve_setup(
                int(id_), lve_settings, err_msg=f"Can't setup lve with id {id_}; error code {{code}}"
            )

        except RuntimeWarning as rw:
            logging.exception(rw)


class _RawConfig(TypedDict, total=False):
    bursting_enabled: bool
    bursting_critical_load_threshold: str
    bursting_critical_load_samples_num: str


class _Config(NamedTuple):
    @classmethod
    def read_from_config_file(cls) -> Self:
        try:
            raw_config = typing.cast(_RawConfig, read_raw_config(_logger=logger))
        except FileNotFoundError:
            logger.warning('Config file not found!')
            raw_config = {}

        default_params = {
            "enabled": False,
            "idle_time_threshold": 0.05,
            "max_samples": 15,
        }
        external_params = {}

        error_by_raw_key = {}
        for raw_key, extractor, param in [
            ('bursting_enabled', get_boolean, 'enabled'),
            ('bursting_critical_load_threshold', float, 'idle_time_threshold'),
            ('bursting_critical_load_samples_num', int, 'max_samples'),
        ]:
            try:
                raw_value = raw_config[raw_key]
            except KeyError:
                continue

            try:
                value = extractor(raw_value)
            except ValueError as e:
                error_by_raw_key[raw_key] = str(e)
                continue

            external_params[param] = value

        if error_by_raw_key:
            logger.warning(
                "Failed to parse some config keys: \n%s",
                "\n".join(f"* {k}: {e}" for k, e in error_by_raw_key.items())
            )

        self = _Config(**ChainMap(external_params, default_params))

        if (defaults_used := default_params.keys() - external_params.keys()):
            logger.info('Using default values for: %s', defaults_used)

        return self

    enabled: bool
    idle_time_threshold: float
    max_samples: int