D7net
Home
Console
Upload
information
Create File
Create Folder
About
Tools
:
/
opt
/
cloudlinux
/
venv
/
lib
/
python3.11
/
site-packages
/
lvestats
/
burstwatcher
/
Filename :
service.py
back
Copy
# coding=utf-8 # # Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT import logging import json import time import typing from typing import TypedDict, NamedTuple, Self from collections import ChainMap from clcommon.clproc import ProcLve from lveapi import PyLve from lvectllib import LVE_DEFAULT from lvestat import LVEStat from lvestats.plugins.generic.burster.config import read_raw_config, get_boolean from lvestats.plugins.generic.burster.overload import OverloadChecker, read_times_from_proc EFFECTIVE_NORMAL_LIMITS_PATH = '/var/run/cloudlinux/effective-normal-limits' logging.basicConfig(level=logging.INFO) logger = logging.getLogger('BurstWatcher') # Separate debug log handler handler = logging.FileHandler('/var/log/lvestats-burstwatcher.log') handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) handler.setLevel(logging.DEBUG) logger.addHandler(handler) class Service: def __init__(self) -> None: self._pylve = PyLve() if not self._pylve.initialize(): raise RuntimeError('pylve initialization failed') config = _Config.read_from_config_file() self._enabled = config.enabled self._overload_checker = OverloadChecker( idle_time_threshold=config.idle_time_threshold, max_samples_number=config.max_samples, get_stats=read_times_from_proc ) self._current_bursting_lves = {} self._effective_normal_limits = {} def run(self) -> None: cnt = 0 while True: time.sleep(1) if not self._enabled: continue if self._is_server_overloaded(): self._load_effective_normal_limits() self._get_current_bursting_lves() self._disable_bursting() elif cnt % 15 == 0: self._load_effective_normal_limits() cnt = 0 cnt += 1 def _load_effective_normal_limits(self): try: with open(EFFECTIVE_NORMAL_LIMITS_PATH, 'r', encoding='UTF-8') as f: loaded_effective_normal_limits = json.loads(f.read()) if len(loaded_effective_normal_limits) > 0: self._effective_normal_limits = loaded_effective_normal_limits except (OSError, IOError, json.JSONDecodeError): logger.exception('Can`t load cached effective normal limits from file: %s', EFFECTIVE_NORMAL_LIMITS_PATH) logger.debug('Loaded effective normal limits for %s lves', len(self._effective_normal_limits)) def _is_server_overloaded(self) -> bool: checker_result = self._overload_checker() logger.debug('Curent server load: %s', checker_result.server_load) return checker_result.is_overloaded def _get_current_bursting_lves(self): proc_lve = ProcLve() lve_stats = [LVEStat(line, proc_lve.version()) for line in proc_lve.lines(without_limits=True)] self._current_bursting_lves = { str(stat.id) for stat in lve_stats if stat.cpu != self._effective_normal_limits.get(str(stat.id), {}).get('cpu') and stat.id != 0 } logger.debug('Current bursting lves: %s', self._current_bursting_lves) def _disable_bursting(self): for _id in self._current_bursting_lves: if _id == '0': logger.debug('Skipping lve_id=0') continue self._disable_bursting_for_lve(_id) logger.debug('Disabled bursting server-wide') def _disable_bursting_for_lve(self, id_: str): logger.debug('Disabling bursting for LVE %s', id_) lve_limits = self._effective_normal_limits.get(id_, {}) default_limits = self._effective_normal_limits.get('0', {}) try: lve_settings = self._pylve.liblve_settings() lve_settings.ls_io = int(lve_limits.get('io', default_limits.get('io', LVE_DEFAULT['io']))) lve_settings.ls_cpu = int(lve_limits.get('cpu', default_limits.get('cpu', LVE_DEFAULT['cpu']))) lve_settings.ls_cpus = int(lve_limits.get('ncpu', default_limits.get('ncpu', LVE_DEFAULT['ncpu']))) lve_settings.ls_memory = int(lve_limits.get('mem', default_limits.get('mem', LVE_DEFAULT['mem']))) lve_settings.ls_enters = int(lve_limits.get('ep', default_limits.get('ep', LVE_DEFAULT['ep']))) lve_settings.ls_memory_phy = int(lve_limits.get('pmem', default_limits.get('pmem', LVE_DEFAULT['pmem']))) lve_settings.ls_nproc = int(lve_limits.get('nproc', default_limits.get('nproc', LVE_DEFAULT['nproc']))) lve_settings.ls_iops = int(lve_limits.get('iops', default_limits.get('iops', LVE_DEFAULT['iops']))) self._pylve.lve_setup( int(id_), lve_settings, err_msg=f"Can't setup lve with id {id_}; error code {{code}}" ) except RuntimeWarning as rw: logging.exception(rw) class _RawConfig(TypedDict, total=False): bursting_enabled: bool bursting_critical_load_threshold: str bursting_critical_load_samples_num: str class _Config(NamedTuple): @classmethod def read_from_config_file(cls) -> Self: try: raw_config = typing.cast(_RawConfig, read_raw_config(_logger=logger)) except FileNotFoundError: logger.warning('Config file not found!') raw_config = {} default_params = { "enabled": False, "idle_time_threshold": 0.05, "max_samples": 15, } external_params = {} error_by_raw_key = {} for raw_key, extractor, param in [ ('bursting_enabled', get_boolean, 'enabled'), ('bursting_critical_load_threshold', float, 'idle_time_threshold'), ('bursting_critical_load_samples_num', int, 'max_samples'), ]: try: raw_value = raw_config[raw_key] except KeyError: continue try: value = extractor(raw_value) except ValueError as e: error_by_raw_key[raw_key] = str(e) continue external_params[param] = value if error_by_raw_key: logger.warning( "Failed to parse some config keys: \n%s", "\n".join(f"* {k}: {e}" for k, e in error_by_raw_key.items()) ) self = _Config(**ChainMap(external_params, default_params)) if (defaults_used := default_params.keys() - external_params.keys()): logger.info('Using default values for: %s', defaults_used) return self enabled: bool idle_time_threshold: float max_samples: int