# -*- coding: utf-8 -*-
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
"""
This module contains FaultDetector class, aimed to track throttling
for incoming requests
"""
import logging
import time
from datetime import datetime, timedelta
from threading import current_thread, RLock
from typing import Tuple, Any
from .constants import drop_after, check_period, throttling_threshold
class FaultDetector:
"""
Fault Detector class
"""
def __init__(self):
self.logger = logging.getLogger('fault_detector')
self.throttling_stat = dict()
self.lock = RLock()
self.time_marker_file = '/usr/share/alt-php-xray/flush.latest'
def __call__(self, t_key: Any, cpu_value: int) -> Tuple[bool, int]:
"""
Perform fault detection
"""
with self.lock:
try:
saved_cpu, _ = self.throttling_stat[t_key]
except KeyError:
hitting_limits, throttled_time = False, 0
else:
hitting_limits, throttled_time = self._was_throttled_since(
saved_cpu, cpu_value)
self._flush_entry(t_key)
return hitting_limits, throttled_time
@property
def timestamp(self) -> int:
"""
Current Unix timestamp
"""
return int(time.time())
@property
def _drop_after(self) -> timedelta:
"""
Drop after value as datetime.timedelta
"""
return self.timedelta_in_minutes(drop_after)
@property
def _check_period(self) -> timedelta:
"""
Check period value as datetime.timedelta
"""
return self.timedelta_in_minutes(check_period)
@property
def should_flush(self) -> bool:
"""
If throttling_stat should be flushed right now
regarding check period value
"""
try:
with open(self.time_marker_file) as latest_info:
latest_flush_time = int(latest_info.read().strip())
return self.current_delta(latest_flush_time) > self._check_period
except (OSError, ValueError):
with self.lock:
self._update_latest_flush()
return False
@staticmethod
def timedelta_in_minutes(mins: int) -> timedelta:
"""
Get timedelta object for given number of minutes
"""
return timedelta(minutes=mins)
def current_delta(self, time_stamp: int) -> timedelta:
"""
Calculate timedelta for given time_stamp from current timestamp
"""
def _cast(ts: int) -> datetime:
"""
Make a datetime object from ts (int)
"""
return datetime.fromtimestamp(ts)
return _cast(self.timestamp) - _cast(time_stamp)
def save(self, key: Any, cpu: int) -> None:
"""
Save throttling entry for given key
"""
stamp = self.timestamp
with self.lock:
self.logger.debug('[%s] Adding new throttling entry %s: (%i, %i)',
current_thread().name, key, cpu, stamp)
self.throttling_stat[key] = (cpu, stamp)
self.logger.debug('[%s] Throttling entries total %i',
current_thread().name,
len(self.throttling_stat))
def _flush_entry(self, key: Any) -> None:
"""
Remove throttling entry by given key
"""
with self.lock:
self.logger.debug('[%s] Abandon throttling entry %s',
current_thread().name, key)
self.throttling_stat.pop(key, 0)
self.logger.debug('[%s] Throttling entries total %i',
current_thread().name,
len(self.throttling_stat))
def _was_throttled_since(self,
initial_value: int,
current_value: int) -> Tuple[bool, int]:
"""
Check current throttling time of given lve_id vs given initial_value
Return: tuple(fact of throttling, throttling time in ms)
"""
# initial values of throttling time are in nanoseconds!
spent_time = (current_value - initial_value) // 1000000 # cast to ms
throttling_fact = spent_time > throttling_threshold
self.logger.debug('[%s] Throttling checked with %i: %s, %i',
current_thread().name, throttling_threshold,
throttling_fact, spent_time)
return throttling_fact, spent_time if spent_time > 0 else 0
def flush(self) -> None:
"""
Clear throttling stat dict
"""
if self.should_flush:
with self.lock:
self.logger.debug('[%s] Flush started, total entries %i',
current_thread().name,
len(self.throttling_stat))
for key in self._expired_entries():
self._flush_entry(key)
self._update_latest_flush()
self.logger.debug('[%s] Flush finished, total entries %i',
current_thread().name,
len(self.throttling_stat))
def _update_latest_flush(self) -> None:
"""
Write current timestamp into time marker file
"""
try:
with open(self.time_marker_file, 'w') as latest_info:
latest_info.write(str(self.timestamp))
except OSError:
self.logger.error('Failed to save timestamp of latest flush')
def _expired_entries(self) -> list:
"""
Collect expired throttling stat entries keys
regarding the drop after value
"""
expired = list()
for key, value in self.throttling_stat.items():
_, timestamp = value
_delta = self.current_delta(timestamp)
if _delta > self._drop_after:
self.logger.debug('[%s] Expired entry detected %i: %s elapsed',
current_thread().name, key, _delta)
expired.append(key)
return expired
|